src/tex2lyx/text.cpp

   1 /**
   2  * \file tex2lyx/text.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  * \author Uwe Stöhr
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 // {[(
  14
  15 #include <config.h>
  16
  17 #include "tex2lyx.h"
  18
  19 #include "Context.h"
  20 #include "Encoding.h"
  21 #include "FloatList.h"
  22 #include "LaTeXPackages.h"
  23 #include "Layout.h"
  24 #include "Length.h"
  25 #include "Preamble.h"
  26
  27 #include "insets/ExternalTemplate.h"
  28
  29 #include "support/lassert.h"
  30 #include "support/convert.h"
  31 #include "support/FileName.h"
  32 #include "support/filetools.h"
  33 #include "support/lstrings.h"
  34 #include "support/lyxtime.h"
  35
  36 #include <algorithm>
  37 #include <iostream>
  38 #include <map>
  39 #include <sstream>
  40 #include <vector>
  41
  42 using namespace std;
  43 using namespace lyx::support;
  44
  45 namespace lyx {
  46
  47
  48 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  49                 Context const & context, InsetLayout const * layout)
  50 {
  51         bool const forcePlainLayout =
  52                 layout ? layout->forcePlainLayout() : false;
  53         Context newcontext(true, context.textclass);
  54         if (forcePlainLayout)
  55                 newcontext.layout = &context.textclass.plainLayout();
  56         else
  57                 newcontext.font = context.font;
  58         parse_text(p, os, flags, outer, newcontext);
  59         newcontext.check_end_layout(os);
  60 }
  61
  62
  63 namespace {
  64
  65 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  66                 Context const & context, string const & name)
  67 {
  68         InsetLayout const * layout = 0;
  69         DocumentClass::InsetLayouts::const_iterator it =
  70                 context.textclass.insetLayouts().find(from_ascii(name));
  71         if (it != context.textclass.insetLayouts().end())
  72                 layout = &(it->second);
  73         parse_text_in_inset(p, os, flags, outer, context, layout);
  74 }
  75
  76 /// parses a paragraph snippet, useful for example for \\emph{...}
  77 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  78                 Context & context)
  79 {
  80         Context newcontext(context);
  81         // Don't inherit the paragraph-level extra stuff
  82         newcontext.par_extra_stuff.clear();
  83         parse_text(p, os, flags, outer, newcontext);
  84         // Make sure that we don't create invalid .lyx files
  85         context.need_layout = newcontext.need_layout;
  86         context.need_end_layout = newcontext.need_end_layout;
  87 }
  88
  89
  90 /*!
  91  * Thin wrapper around parse_text_snippet() using a string.
  92  *
  93  * We completely ignore \c context.need_layout and \c context.need_end_layout,
  94  * because our return value is not used directly (otherwise the stream version
  95  * of parse_text_snippet() could be used). That means that the caller needs
  96  * to do layout management manually.
  97  * This is intended to parse text that does not create any layout changes.
  98  */
  99 string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 100                   Context & context)
 101 {
 102         Context newcontext(context);
 103         newcontext.need_layout = false;
 104         newcontext.need_end_layout = false;
 105         newcontext.new_layout_allowed = false;
 106         // Avoid warning by Context::~Context()
 107         newcontext.par_extra_stuff.clear();
 108         ostringstream os;
 109         parse_text_snippet(p, os, flags, outer, newcontext);
 110         return os.str();
 111 }
 112
 113
 114 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 115  "vpageref", "prettyref", "eqref", 0 };
 116
 117 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
 118  "vpageref", "formatted", "eqref", 0 };
 119
 120 /**
 121  * known polyglossia language names (including variants)
 122  */
 123 const char * const polyglossia_languages[] = {
 124 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 125 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 126 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 127 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 128 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 129 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 130 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 131 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 132 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 133 "marathi", "spanish",
 134 "american", "ancient", "australian", "british", "monotonic", "newzealand",
 135 "polytonic", 0};
 136
 137 /**
 138  * the same as polyglossia_languages with .lyx names
 139  * please keep this in sync with polyglossia_languages line by line!
 140  */
 141 const char * const coded_polyglossia_languages[] = {
 142 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 143 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 144 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 145 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 146 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 147 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 148 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 149 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 150 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 151 "marathi", "spanish",
 152 "american", "ancientgreek", "australian", "british", "greek", "newzealand",
 153 "polutonikogreek", 0};
 154
 155 /**
 156  * supported CJK encodings
 157  */
 158 const char * const supported_CJK_encodings[] = {
 159 "EUC-JP", "KS", "GB", "UTF8", 0};
 160
 161 /**
 162  * the same as supported_CJK_encodings with their corresponding LyX language name
 163  * please keep this in sync with supported_CJK_encodings line by line!
 164  */
 165 const char * const coded_supported_CJK_encodings[] = {
 166 "japanese-cjk", "korean", "chinese-simplified", "chinese-traditional", 0};
 167
 168 string CJK2lyx(string const & encoding)
 169 {
 170         char const * const * where = is_known(encoding, supported_CJK_encodings);
 171         if (where)
 172                 return coded_supported_CJK_encodings[where - supported_CJK_encodings];
 173         return encoding;
 174 }
 175
 176 /*!
 177  * natbib commands.
 178  * The starred forms are also known except for "citefullauthor",
 179  * "citeyear" and "citeyearpar".
 180  */
 181 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 182 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 183 "citefullauthor", "Citet", "Citep", "Citealt", "Citealp", "Citeauthor", 0 };
 184
 185 /*!
 186  * jurabib commands.
 187  * No starred form other than "cite*" known.
 188  */
 189 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 190 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 191 // jurabib commands not (yet) supported by LyX:
 192 // "fullcite",
 193 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 194 // "footciteauthor", "footciteyear", "footciteyearpar",
 195 "citefield", "citetitle", 0 };
 196
 197 /// LaTeX names for quotes
 198 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
 199 "guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
 200 "quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
 201 "guilsinglleft", "frq", "guilsinglright", 0};
 202
 203 /// the same as known_quotes with .lyx names
 204 char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 205 "ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
 206 "gld", "grd", "gls", "ers", "fls",
 207 "fls", "frs", "frs", 0};
 208
 209 /// LaTeX names for font sizes
 210 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 211 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 212
 213 /// the same as known_sizes with .lyx names
 214 char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 215 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 216
 217 /// LaTeX 2.09 names for font families
 218 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 219
 220 /// LaTeX names for font families
 221 char const * const known_font_families[] = { "rmfamily", "sffamily",
 222 "ttfamily", 0};
 223
 224 /// LaTeX names for font family changing commands
 225 char const * const known_text_font_families[] = { "textrm", "textsf",
 226 "texttt", 0};
 227
 228 /// The same as known_old_font_families, known_font_families and
 229 /// known_text_font_families with .lyx names
 230 char const * const known_coded_font_families[] = { "roman", "sans",
 231 "typewriter", 0};
 232
 233 /// LaTeX 2.09 names for font series
 234 char const * const known_old_font_series[] = { "bf", 0};
 235
 236 /// LaTeX names for font series
 237 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 238
 239 /// LaTeX names for font series changing commands
 240 char const * const known_text_font_series[] = { "textbf", "textmd", 0};
 241
 242 /// The same as known_old_font_series, known_font_series and
 243 /// known_text_font_series with .lyx names
 244 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 245
 246 /// LaTeX 2.09 names for font shapes
 247 char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 248
 249 /// LaTeX names for font shapes
 250 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 251 "upshape", 0};
 252
 253 /// LaTeX names for font shape changing commands
 254 char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
 255 "textup", 0};
 256
 257 /// The same as known_old_font_shapes, known_font_shapes and
 258 /// known_text_font_shapes with .lyx names
 259 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 260 "smallcaps", "up", 0};
 261
 262 /// Known special characters which need skip_spaces_braces() afterwards
 263 char const * const known_special_chars[] = {"ldots", "lyxarrow",
 264 "textcompwordmark", "slash", 0};
 265
 266 /// the same as known_special_chars with .lyx names
 267 char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
 268 "textcompwordmark{}", "slash{}", 0};
 269
 270 /*!
 271  * Graphics file extensions known by the dvips driver of the graphics package.
 272  * These extensions are used to complete the filename of an included
 273  * graphics file if it does not contain an extension.
 274  * The order must be the same that latex uses to find a file, because we
 275  * will use the first extension that matches.
 276  * This is only an approximation for the common cases. If we would want to
 277  * do it right in all cases, we would need to know which graphics driver is
 278  * used and know the extensions of every driver of the graphics package.
 279  */
 280 char const * const known_dvips_graphics_formats[] = {"eps", "ps", "eps.gz",
 281 "ps.gz", "eps.Z", "ps.Z", 0};
 282
 283 /*!
 284  * Graphics file extensions known by the pdftex driver of the graphics package.
 285  * \sa known_dvips_graphics_formats
 286  */
 287 char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 288 "mps", "tif", 0};
 289
 290 /*!
 291  * Known file extensions for TeX files as used by \\include.
 292  */
 293 char const * const known_tex_extensions[] = {"tex", 0};
 294
 295 /// spaces known by InsetSpace
 296 char const * const known_spaces[] = { " ", "space", ",",
 297 "thinspace", "quad", "qquad", "enspace", "enskip",
 298 "negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
 299 "hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
 300 "upbracefill", "downbracefill", 0};
 301
 302 /// the same as known_spaces with .lyx names
 303 char const * const known_coded_spaces[] = { "space{}", "space{}",
 304 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
 305 "negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
 306 "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
 307 "upbracefill{}", "downbracefill{}", 0};
 308
 309 /// These are translated by LyX to commands like "\\LyX{}", so we have to put
 310 /// them in ERT. "LaTeXe" must come before "LaTeX"!
 311 char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 312 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 313 int const known_phrase_lengths[] = {3, 5, 7, 0};
 314
 315 // string to store the float type to be able to determine the type of subfloats
 316 string float_type = "";
 317
 318
 319 /// splits "x=z, y=b" into a map and an ordered keyword vector
 320 void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 321 {
 322         vector<string> v;
 323         split(s, v);
 324         res.clear();
 325         keys.resize(v.size());
 326         for (size_t i = 0; i < v.size(); ++i) {
 327                 size_t const pos   = v[i].find('=');
 328                 string const index = trimSpaceAndEol(v[i].substr(0, pos));
 329                 string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
 330                 res[index] = value;
 331                 keys[i] = index;
 332         }
 333 }
 334
 335
 336 /*!
 337  * Split a LaTeX length into value and unit.
 338  * The latter can be a real unit like "pt", or a latex length variable
 339  * like "\textwidth". The unit may contain additional stuff like glue
 340  * lengths, but we don't care, because such lengths are ERT anyway.
 341  * \returns true if \p value and \p unit are valid.
 342  */
 343 bool splitLatexLength(string const & len, string & value, string & unit)
 344 {
 345         if (len.empty())
 346                 return false;
 347         const string::size_type i = len.find_first_not_of(" -+0123456789.,");
 348         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 349         string const length = subst(len, ',', '.');
 350         if (i == string::npos)
 351                 return false;
 352         if (i == 0) {
 353                 if (len[0] == '\\') {
 354                         // We had something like \textwidth without a factor
 355                         value = "1.0";
 356                 } else {
 357                         return false;
 358                 }
 359         } else {
 360                 value = trimSpaceAndEol(string(length, 0, i));
 361         }
 362         if (value == "-")
 363                 value = "-1.0";
 364         // 'cM' is a valid LaTeX length unit. Change it to 'cm'
 365         if (contains(len, '\\'))
 366                 unit = trimSpaceAndEol(string(len, i));
 367         else
 368                 unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
 369         return true;
 370 }
 371
 372
 373 /// A simple function to translate a latex length to something LyX can
 374 /// understand. Not perfect, but rather best-effort.
 375 bool translate_len(string const & length, string & valstring, string & unit)
 376 {
 377         if (!splitLatexLength(length, valstring, unit))
 378                 return false;
 379         // LyX uses percent values
 380         double value;
 381         istringstream iss(valstring);
 382         iss >> value;
 383         value *= 100;
 384         ostringstream oss;
 385         oss << value;
 386         string const percentval = oss.str();
 387         // a normal length
 388         if (unit.empty() || unit[0] != '\\')
 389                 return true;
 390         string::size_type const i = unit.find(' ');
 391         string const endlen = (i == string::npos) ? string() : string(unit, i);
 392         if (unit == "\\textwidth") {
 393                 valstring = percentval;
 394                 unit = "text%" + endlen;
 395         } else if (unit == "\\columnwidth") {
 396                 valstring = percentval;
 397                 unit = "col%" + endlen;
 398         } else if (unit == "\\paperwidth") {
 399                 valstring = percentval;
 400                 unit = "page%" + endlen;
 401         } else if (unit == "\\linewidth") {
 402                 valstring = percentval;
 403                 unit = "line%" + endlen;
 404         } else if (unit == "\\paperheight") {
 405                 valstring = percentval;
 406                 unit = "pheight%" + endlen;
 407         } else if (unit == "\\textheight") {
 408                 valstring = percentval;
 409                 unit = "theight%" + endlen;
 410         }
 411         return true;
 412 }
 413
 414 }
 415
 416
 417 string translate_len(string const & length)
 418 {
 419         string unit;
 420         string value;
 421         if (translate_len(length, value, unit))
 422                 return value + unit;
 423         // If the input is invalid, return what we have.
 424         return length;
 425 }
 426
 427
 428 namespace {
 429
 430 /*!
 431  * Translates a LaTeX length into \p value, \p unit and
 432  * \p special parts suitable for a box inset.
 433  * The difference from translate_len() is that a box inset knows about
 434  * some special "units" that are stored in \p special.
 435  */
 436 void translate_box_len(string const & length, string & value, string & unit, string & special)
 437 {
 438         if (translate_len(length, value, unit)) {
 439                 if (unit == "\\height" || unit == "\\depth" ||
 440                     unit == "\\totalheight" || unit == "\\width") {
 441                         special = unit.substr(1);
 442                         // The unit is not used, but LyX requires a dummy setting
 443                         unit = "in";
 444                 } else
 445                         special = "none";
 446         } else {
 447                 value.clear();
 448                 unit = length;
 449                 special = "none";
 450         }
 451 }
 452
 453
 454 /*!
 455  * Find a file with basename \p name in path \p path and an extension
 456  * in \p extensions.
 457  */
 458 string find_file(string const & name, string const & path,
 459                  char const * const * extensions)
 460 {
 461         for (char const * const * what = extensions; *what; ++what) {
 462                 string const trial = addExtension(name, *what);
 463                 if (makeAbsPath(trial, path).exists())
 464                         return trial;
 465         }
 466         return string();
 467 }
 468
 469
 470 void begin_inset(ostream & os, string const & name)
 471 {
 472         os << "\n\\begin_inset " << name;
 473 }
 474
 475
 476 void begin_command_inset(ostream & os, string const & name,
 477                          string const & latexname)
 478 {
 479         begin_inset(os, "CommandInset ");
 480         os << name << "\nLatexCommand " << latexname << '\n';
 481 }
 482
 483
 484 void end_inset(ostream & os)
 485 {
 486         os << "\n\\end_inset\n\n";
 487 }
 488
 489
 490 bool skip_braces(Parser & p)
 491 {
 492         if (p.next_token().cat() != catBegin)
 493                 return false;
 494         p.get_token();
 495         if (p.next_token().cat() == catEnd) {
 496                 p.get_token();
 497                 return true;
 498         }
 499         p.putback();
 500         return false;
 501 }
 502
 503
 504 /// replace LaTeX commands in \p s from the unicodesymbols file with their
 505 /// unicode points
 506 docstring convert_unicodesymbols(docstring s)
 507 {
 508         odocstringstream os;
 509         for (size_t i = 0; i < s.size();) {
 510                 if (s[i] != '\\') {
 511                         os.put(s[i++]);
 512                         continue;
 513                 }
 514                 s = s.substr(i);
 515                 bool termination;
 516                 docstring rem;
 517                 set<string> req;
 518                 docstring parsed = encodings.fromLaTeXCommand(s,
 519                                 Encodings::TEXT_CMD, termination, rem, &req);
 520                 set<string>::const_iterator it = req.begin();
 521                 set<string>::const_iterator en = req.end();
 522                 for (; it != en; ++it)
 523                         preamble.registerAutomaticallyLoadedPackage(*it);
 524                 os << parsed;
 525                 s = rem;
 526                 if (s.empty() || s[0] != '\\')
 527                         i = 0;
 528                 else
 529                         i = 1;
 530         }
 531         return os.str();
 532 }
 533
 534
 535 /// try to convert \p s to a valid InsetCommand argument
 536 string convert_command_inset_arg(string s)
 537 {
 538         if (isAscii(s))
 539                 // since we don't know the input encoding we can't use from_utf8
 540                 s = to_utf8(convert_unicodesymbols(from_ascii(s)));
 541         // LyX cannot handle newlines in a latex command
 542         return subst(s, "\n", " ");
 543 }
 544
 545
 546 void handle_backslash(ostream & os, string const & s)
 547 {
 548         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 549                 if (*it == '\\')
 550                         os << "\n\\backslash\n";
 551                 else
 552                         os << *it;
 553         }
 554 }
 555
 556
 557 void handle_ert(ostream & os, string const & s, Context & context)
 558 {
 559         // We must have a valid layout before outputting the ERT inset.
 560         context.check_layout(os);
 561         Context newcontext(true, context.textclass);
 562         begin_inset(os, "ERT");
 563         os << "\nstatus collapsed\n";
 564         newcontext.check_layout(os);
 565         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 566                 if (*it == '\\')
 567                         os << "\n\\backslash\n";
 568                 else if (*it == '\n') {
 569                         newcontext.new_paragraph(os);
 570                         newcontext.check_layout(os);
 571                 } else
 572                         os << *it;
 573         }
 574         newcontext.check_end_layout(os);
 575         end_inset(os);
 576 }
 577
 578
 579 void handle_comment(ostream & os, string const & s, Context & context)
 580 {
 581         // TODO: Handle this better
 582         Context newcontext(true, context.textclass);
 583         begin_inset(os, "ERT");
 584         os << "\nstatus collapsed\n";
 585         newcontext.check_layout(os);
 586         handle_backslash(os, s);
 587         // make sure that our comment is the last thing on the line
 588         newcontext.new_paragraph(os);
 589         newcontext.check_layout(os);
 590         newcontext.check_end_layout(os);
 591         end_inset(os);
 592 }
 593
 594
 595 Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
 596 {
 597         Layout const * layout = findLayoutWithoutModule(textclass, name, command);
 598         if (layout)
 599                 return layout;
 600         if (checkModule(name, command))
 601                 return findLayoutWithoutModule(textclass, name, command);
 602         return layout;
 603 }
 604
 605
 606 InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 607 {
 608         InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
 609         if (insetlayout)
 610                 return insetlayout;
 611         if (checkModule(name, command))
 612                 return findInsetLayoutWithoutModule(textclass, name, command);
 613         return insetlayout;
 614 }
 615
 616
 617 void eat_whitespace(Parser &, ostream &, Context &, bool);
 618
 619
 620 /*!
 621  * Skips whitespace and braces.
 622  * This should be called after a command has been parsed that is not put into
 623  * ERT, and where LyX adds "{}" if needed.
 624  */
 625 void skip_spaces_braces(Parser & p, bool keepws = false)
 626 {
 627         /* The following four examples produce the same typeset output and
 628            should be handled by this function:
 629            - abc \j{} xyz
 630            - abc \j {} xyz
 631            - abc \j
 632              {} xyz
 633            - abc \j %comment
 634              {} xyz
 635          */
 636         // Unfortunately we need to skip comments, too.
 637         // We can't use eat_whitespace since writing them after the {}
 638         // results in different output in some cases.
 639         bool const skipped_spaces = p.skip_spaces(true);
 640         bool const skipped_braces = skip_braces(p);
 641         if (keepws && skipped_spaces && !skipped_braces)
 642                 // put back the space (it is better handled by check_space)
 643                 p.unskip_spaces(true);
 644 }
 645
 646
 647 void output_command_layout(ostream & os, Parser & p, bool outer,
 648                            Context & parent_context,
 649                            Layout const * newlayout)
 650 {
 651         TeXFont const oldFont = parent_context.font;
 652         // save the current font size
 653         string const size = oldFont.size;
 654         // reset the font size to default, because the font size switches
 655         // don't affect section headings and the like
 656         parent_context.font.size = Context::normalfont.size;
 657         // we only need to write the font change if we have an open layout
 658         if (!parent_context.atParagraphStart())
 659                 output_font_change(os, oldFont, parent_context.font);
 660         parent_context.check_end_layout(os);
 661         Context context(true, parent_context.textclass, newlayout,
 662                         parent_context.layout, parent_context.font);
 663         if (parent_context.deeper_paragraph) {
 664                 // We are beginning a nested environment after a
 665                 // deeper paragraph inside the outer list environment.
 666                 // Therefore we don't need to output a "begin deeper".
 667                 context.need_end_deeper = true;
 668         }
 669         context.check_deeper(os);
 670         context.check_layout(os);
 671         unsigned int optargs = 0;
 672         while (optargs < context.layout->optargs) {
 673                 eat_whitespace(p, os, context, false);
 674                 if (p.next_token().cat() == catEscape ||
 675                     p.next_token().character() != '[')
 676                         break;
 677                 p.get_token(); // eat '['
 678                 begin_inset(os, "Argument\n");
 679                 os << "status collapsed\n\n";
 680                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 681                 end_inset(os);
 682                 eat_whitespace(p, os, context, false);
 683                 ++optargs;
 684         }
 685         unsigned int reqargs = 0;
 686         while (reqargs < context.layout->reqargs) {
 687                 eat_whitespace(p, os, context, false);
 688                 if (p.next_token().cat() != catBegin)
 689                         break;
 690                 p.get_token(); // eat '{'
 691                 begin_inset(os, "Argument\n");
 692                 os << "status collapsed\n\n";
 693                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
 694                 end_inset(os);
 695                 eat_whitespace(p, os, context, false);
 696                 ++reqargs;
 697         }
 698         parse_text(p, os, FLAG_ITEM, outer, context);
 699         context.check_end_layout(os);
 700         if (parent_context.deeper_paragraph) {
 701                 // We must suppress the "end deeper" because we
 702                 // suppressed the "begin deeper" above.
 703                 context.need_end_deeper = false;
 704         }
 705         context.check_end_deeper(os);
 706         // We don't need really a new paragraph, but
 707         // we must make sure that the next item gets a \begin_layout.
 708         parent_context.new_paragraph(os);
 709         // Set the font size to the original value. No need to output it here
 710         // (Context::begin_layout() will do that if needed)
 711         parent_context.font.size = size;
 712 }
 713
 714
 715 /*!
 716  * Output a space if necessary.
 717  * This function gets called for every whitespace token.
 718  *
 719  * We have three cases here:
 720  * 1. A space must be suppressed. Example: The lyxcode case below
 721  * 2. A space may be suppressed. Example: Spaces before "\par"
 722  * 3. A space must not be suppressed. Example: A space between two words
 723  *
 724  * We currently handle only 1. and 3 and from 2. only the case of
 725  * spaces before newlines as a side effect.
 726  *
 727  * 2. could be used to suppress as many spaces as possible. This has two effects:
 728  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 729  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 730  * The drawback is that the logic inside the function becomes
 731  * complicated, and that is the reason why it is not implemented.
 732  */
 733 void check_space(Parser & p, ostream & os, Context & context)
 734 {
 735         Token const next = p.next_token();
 736         Token const curr = p.curr_token();
 737         // A space before a single newline and vice versa must be ignored
 738         // LyX emits a newline before \end{lyxcode}.
 739         // This newline must be ignored,
 740         // otherwise LyX will add an additional protected space.
 741         if (next.cat() == catSpace ||
 742             next.cat() == catNewline ||
 743             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 744                 return;
 745         }
 746         context.check_layout(os);
 747         os << ' ';
 748 }
 749
 750
 751 /*!
 752  * Parse all arguments of \p command
 753  */
 754 void parse_arguments(string const & command,
 755                      vector<ArgumentType> const & template_arguments,
 756                      Parser & p, ostream & os, bool outer, Context & context)
 757 {
 758         string ert = command;
 759         size_t no_arguments = template_arguments.size();
 760         for (size_t i = 0; i < no_arguments; ++i) {
 761                 switch (template_arguments[i]) {
 762                 case required:
 763                 case req_group:
 764                         // This argument contains regular LaTeX
 765                         handle_ert(os, ert + '{', context);
 766                         eat_whitespace(p, os, context, false);
 767                         if (template_arguments[i] == required)
 768                                 parse_text(p, os, FLAG_ITEM, outer, context);
 769                         else
 770                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 771                         ert = "}";
 772                         break;
 773                 case item:
 774                         // This argument consists only of a single item.
 775                         // The presence of '{' or not must be preserved.
 776                         p.skip_spaces();
 777                         if (p.next_token().cat() == catBegin)
 778                                 ert += '{' + p.verbatim_item() + '}';
 779                         else
 780                                 ert += p.verbatim_item();
 781                         break;
 782                 case displaymath:
 783                 case verbatim:
 784                         // This argument may contain special characters
 785                         ert += '{' + p.verbatim_item() + '}';
 786                         break;
 787                 case optional:
 788                 case opt_group:
 789                         // true because we must not eat whitespace
 790                         // if an optional arg follows we must not strip the
 791                         // brackets from this one
 792                         if (i < no_arguments - 1 &&
 793                             template_arguments[i+1] == optional)
 794                                 ert += p.getFullOpt(true);
 795                         else
 796                                 ert += p.getOpt(true);
 797                         break;
 798                 }
 799         }
 800         handle_ert(os, ert, context);
 801 }
 802
 803
 804 /*!
 805  * Check whether \p command is a known command. If yes,
 806  * handle the command with all arguments.
 807  * \return true if the command was parsed, false otherwise.
 808  */
 809 bool parse_command(string const & command, Parser & p, ostream & os,
 810                    bool outer, Context & context)
 811 {
 812         if (known_commands.find(command) != known_commands.end()) {
 813                 parse_arguments(command, known_commands[command], p, os,
 814                                 outer, context);
 815                 return true;
 816         }
 817         return false;
 818 }
 819
 820
 821 /// Parses a minipage or parbox
 822 void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 823                unsigned inner_flags, bool outer, Context & parent_context,
 824                string const & outer_type, string const & special,
 825                string const & inner_type)
 826 {
 827         string position;
 828         string inner_pos;
 829         string hor_pos = "c";
 830         // We need to set the height to the LaTeX default of 1\\totalheight
 831         // for the case when no height argument is given
 832         string height_value = "1";
 833         string height_unit = "in";
 834         string height_special = "totalheight";
 835         string latex_height;
 836         string width_value;
 837         string width_unit;
 838         string latex_width;
 839         string width_special = "none";
 840         if (!inner_type.empty() && p.hasOpt()) {
 841                 if (inner_type != "makebox")
 842                         position = p.getArg('[', ']');
 843                 else {
 844                         latex_width = p.getArg('[', ']');
 845                         translate_box_len(latex_width, width_value, width_unit, width_special);
 846                         position = "t";
 847                 }
 848                 if (position != "t" && position != "c" && position != "b") {
 849                         cerr << "invalid position " << position << " for "
 850                              << inner_type << endl;
 851                         position = "c";
 852                 }
 853                 if (p.hasOpt()) {
 854                         if (inner_type != "makebox") {
 855                                 latex_height = p.getArg('[', ']');
 856                                 translate_box_len(latex_height, height_value, height_unit, height_special);
 857                         } else
 858                                 hor_pos = p.getArg('[', ']');
 859
 860                         if (p.hasOpt()) {
 861                                 inner_pos = p.getArg('[', ']');
 862                                 if (inner_pos != "c" && inner_pos != "t" &&
 863                                     inner_pos != "b" && inner_pos != "s") {
 864                                         cerr << "invalid inner_pos "
 865                                              << inner_pos << " for "
 866                                              << inner_type << endl;
 867                                         inner_pos = position;
 868                                 }
 869                         }
 870                 }
 871         }
 872         if (inner_type.empty()) {
 873                 if (special.empty() && outer_type != "framebox")
 874                         latex_width = "1\\columnwidth";
 875                 else {
 876                         Parser p2(special);
 877                         latex_width = p2.getArg('[', ']');
 878                         string const opt = p2.getArg('[', ']');
 879                         if (!opt.empty()) {
 880                                 hor_pos = opt;
 881                                 if (hor_pos != "l" && hor_pos != "c" &&
 882                                     hor_pos != "r") {
 883                                         cerr << "invalid hor_pos " << hor_pos
 884                                              << " for " << outer_type << endl;
 885                                         hor_pos = "c";
 886                                 }
 887                         }
 888                 }
 889         } else if (inner_type != "makebox")
 890                 latex_width = p.verbatim_item();
 891         // if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 892         // as this is LyX's standard for such cases (except for makebox)
 893         // \framebox is more special and handled below
 894         if (latex_width.empty() && inner_type != "makebox"
 895                 && outer_type != "framebox")
 896                 latex_width = "1\\columnwidth";
 897
 898         translate_len(latex_width, width_value, width_unit);
 899
 900         bool shadedparbox = false;
 901         if (inner_type == "shaded") {
 902                 eat_whitespace(p, os, parent_context, false);
 903                 if (outer_type == "parbox") {
 904                         // Eat '{'
 905                         if (p.next_token().cat() == catBegin)
 906                                 p.get_token();
 907                         eat_whitespace(p, os, parent_context, false);
 908                         shadedparbox = true;
 909                 }
 910                 p.get_token();
 911                 p.getArg('{', '}');
 912         }
 913         // If we already read the inner box we have to push the inner env
 914         if (!outer_type.empty() && !inner_type.empty() &&
 915             (inner_flags & FLAG_END))
 916                 active_environments.push_back(inner_type);
 917         // LyX can't handle length variables
 918         bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
 919         if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
 920                 // Look whether there is some content after the end of the
 921                 // inner box, but before the end of the outer box.
 922                 // If yes, we need to output ERT.
 923                 p.pushPosition();
 924                 if (inner_flags & FLAG_END)
 925                         p.verbatimEnvironment(inner_type);
 926                 else
 927                         p.verbatim_item();
 928                 p.skip_spaces(true);
 929                 bool const outer_env(outer_type == "framed" || outer_type == "minipage");
 930                 if ((outer_env && p.next_token().asInput() != "\\end") ||
 931                     (!outer_env && p.next_token().cat() != catEnd)) {
 932                         // something is between the end of the inner box and
 933                         // the end of the outer box, so we need to use ERT.
 934                         use_ert = true;
 935                 }
 936                 p.popPosition();
 937         }
 938         // if only \makebox{content} was used we can set its width to 1\width
 939         // because this identic and also identic to \mbox
 940         // this doesn't work for \framebox{content}, thus we have to use ERT for this
 941         if (latex_width.empty() && inner_type == "makebox") {
 942                 width_value = "1";
 943                 width_unit = "in";
 944                 width_special = "width";
 945         } else if (latex_width.empty() && outer_type == "framebox") {
 946                 use_ert = true;
 947         }
 948         if (use_ert) {
 949                 ostringstream ss;
 950                 if (!outer_type.empty()) {
 951                         if (outer_flags & FLAG_END)
 952                                 ss << "\\begin{" << outer_type << '}';
 953                         else {
 954                                 ss << '\\' << outer_type << '{';
 955                                 if (!special.empty())
 956                                         ss << special;
 957                         }
 958                 }
 959                 if (!inner_type.empty()) {
 960                         if (inner_type != "shaded") {
 961                                 if (inner_flags & FLAG_END)
 962                                         ss << "\\begin{" << inner_type << '}';
 963                                 else
 964                                         ss << '\\' << inner_type;
 965                         }
 966                         if (!position.empty())
 967                                 ss << '[' << position << ']';
 968                         if (!latex_height.empty())
 969                                 ss << '[' << latex_height << ']';
 970                         if (!inner_pos.empty())
 971                                 ss << '[' << inner_pos << ']';
 972                         ss << '{' << latex_width << '}';
 973                         if (!(inner_flags & FLAG_END))
 974                                 ss << '{';
 975                 }
 976                 if (inner_type == "shaded")
 977                         ss << "\\begin{shaded}";
 978                 handle_ert(os, ss.str(), parent_context);
 979                 if (!inner_type.empty()) {
 980                         parse_text(p, os, inner_flags, outer, parent_context);
 981                         if (inner_flags & FLAG_END)
 982                                 handle_ert(os, "\\end{" + inner_type + '}',
 983                                            parent_context);
 984                         else
 985                                 handle_ert(os, "}", parent_context);
 986                 }
 987                 if (!outer_type.empty()) {
 988                         // If we already read the inner box we have to pop
 989                         // the inner env
 990                         if (!inner_type.empty() && (inner_flags & FLAG_END))
 991                                 active_environments.pop_back();
 992
 993                         // Ensure that the end of the outer box is parsed correctly:
 994                         // The opening brace has been eaten by parse_outer_box()
 995                         if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
 996                                 outer_flags &= ~FLAG_ITEM;
 997                                 outer_flags |= FLAG_BRACE_LAST;
 998                         }
 999                         parse_text(p, os, outer_flags, outer, parent_context);
1000                         if (outer_flags & FLAG_END)
1001                                 handle_ert(os, "\\end{" + outer_type + '}',
1002                                            parent_context);
1003                         else if (inner_type.empty() && outer_type == "framebox")
1004                                 // in this case it is already closed later
1005                                 ;
1006                         else
1007                                 handle_ert(os, "}", parent_context);
1008                 }
1009         } else {
1010                 // LyX does not like empty positions, so we have
1011                 // to set them to the LaTeX default values here.
1012                 if (position.empty())
1013                         position = "c";
1014                 if (inner_pos.empty())
1015                         inner_pos = position;
1016                 parent_context.check_layout(os);
1017                 begin_inset(os, "Box ");
1018                 if (outer_type == "framed")
1019                         os << "Framed\n";
1020                 else if (outer_type == "framebox")
1021                         os << "Boxed\n";
1022                 else if (outer_type == "shadowbox")
1023                         os << "Shadowbox\n";
1024                 else if ((outer_type == "shaded" && inner_type.empty()) ||
1025                              (outer_type == "minipage" && inner_type == "shaded") ||
1026                              (outer_type == "parbox" && inner_type == "shaded")) {
1027                         os << "Shaded\n";
1028                         preamble.registerAutomaticallyLoadedPackage("color");
1029                 } else if (outer_type == "doublebox")
1030                         os << "Doublebox\n";
1031                 else if (outer_type.empty())
1032                         os << "Frameless\n";
1033                 else
1034                         os << outer_type << '\n';
1035                 os << "position \"" << position << "\"\n";
1036                 os << "hor_pos \"" << hor_pos << "\"\n";
1037                 os << "has_inner_box " << !inner_type.empty() << "\n";
1038                 os << "inner_pos \"" << inner_pos << "\"\n";
1039                 os << "use_parbox " << (inner_type == "parbox" || shadedparbox)
1040                    << '\n';
1041                 os << "use_makebox " << (inner_type == "makebox") << '\n';
1042                 os << "width \"" << width_value << width_unit << "\"\n";
1043                 os << "special \"" << width_special << "\"\n";
1044                 os << "height \"" << height_value << height_unit << "\"\n";
1045                 os << "height_special \"" << height_special << "\"\n";
1046                 os << "status open\n\n";
1047
1048                 // Unfortunately we can't use parse_text_in_inset:
1049                 // InsetBox::forcePlainLayout() is hard coded and does not
1050                 // use the inset layout. Apart from that do we call parse_text
1051                 // up to two times, but need only one check_end_layout.
1052                 bool const forcePlainLayout =
1053                         (!inner_type.empty() || inner_type == "makebox") &&
1054                         outer_type != "shaded" && outer_type != "framed";
1055                 Context context(true, parent_context.textclass);
1056                 if (forcePlainLayout)
1057                         context.layout = &context.textclass.plainLayout();
1058                 else
1059                         context.font = parent_context.font;
1060
1061                 // If we have no inner box the contents will be read with the outer box
1062                 if (!inner_type.empty())
1063                         parse_text(p, os, inner_flags, outer, context);
1064
1065                 // Ensure that the end of the outer box is parsed correctly:
1066                 // The opening brace has been eaten by parse_outer_box()
1067                 if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
1068                         outer_flags &= ~FLAG_ITEM;
1069                         outer_flags |= FLAG_BRACE_LAST;
1070                 }
1071
1072                 // Find end of outer box, output contents if inner_type is
1073                 // empty and output possible comments
1074                 if (!outer_type.empty()) {
1075                         // If we already read the inner box we have to pop
1076                         // the inner env
1077                         if (!inner_type.empty() && (inner_flags & FLAG_END))
1078                                 active_environments.pop_back();
1079                         // This does not output anything but comments if
1080                         // inner_type is not empty (see use_ert)
1081                         parse_text(p, os, outer_flags, outer, context);
1082                 }
1083
1084                 context.check_end_layout(os);
1085                 end_inset(os);
1086 #ifdef PRESERVE_LAYOUT
1087                 // LyX puts a % after the end of the minipage
1088                 if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
1089                         // new paragraph
1090                         //handle_comment(os, "%dummy", parent_context);
1091                         p.get_token();
1092                         p.skip_spaces();
1093                         parent_context.new_paragraph(os);
1094                 }
1095                 else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
1096                         //handle_comment(os, "%dummy", parent_context);
1097                         p.get_token();
1098                         p.skip_spaces();
1099                         // We add a protected space if something real follows
1100                         if (p.good() && p.next_token().cat() != catComment) {
1101                                 begin_inset(os, "space ~\n");
1102                                 end_inset(os);
1103                         }
1104                 }
1105 #endif
1106         }
1107 }
1108
1109
1110 void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
1111                      Context & parent_context, string const & outer_type,
1112                      string const & special)
1113 {
1114         eat_whitespace(p, os, parent_context, false);
1115         if (flags & FLAG_ITEM) {
1116                 // Eat '{'
1117                 if (p.next_token().cat() == catBegin)
1118                         p.get_token();
1119                 else
1120                         cerr << "Warning: Ignoring missing '{' after \\"
1121                              << outer_type << '.' << endl;
1122                 eat_whitespace(p, os, parent_context, false);
1123         }
1124         string inner;
1125         unsigned int inner_flags = 0;
1126         p.pushPosition();
1127         if (outer_type == "minipage" || outer_type == "parbox") {
1128                 p.skip_spaces(true);
1129                 while (p.hasOpt()) {
1130                         p.getArg('[', ']');
1131                         p.skip_spaces(true);
1132                 }
1133                 p.getArg('{', '}');
1134                 p.skip_spaces(true);
1135                 if (outer_type == "parbox") {
1136                         // Eat '{'
1137                         if (p.next_token().cat() == catBegin)
1138                                 p.get_token();
1139                         p.skip_spaces(true);
1140                 }
1141         }
1142         if (outer_type == "shaded") {
1143                 // These boxes never have an inner box
1144                 ;
1145         } else if (p.next_token().asInput() == "\\parbox") {
1146                 inner = p.get_token().cs();
1147                 inner_flags = FLAG_ITEM;
1148         } else if (p.next_token().asInput() == "\\begin") {
1149                 // Is this a minipage or shaded box?
1150                 p.pushPosition();
1151                 p.get_token();
1152                 inner = p.getArg('{', '}');
1153                 p.popPosition();
1154                 if (inner == "minipage" || inner == "shaded")
1155                         inner_flags = FLAG_END;
1156                 else
1157                         inner = "";
1158         }
1159         p.popPosition();
1160         if (inner_flags == FLAG_END) {
1161                 if (inner != "shaded")
1162                 {
1163                         p.get_token();
1164                         p.getArg('{', '}');
1165                         eat_whitespace(p, os, parent_context, false);
1166                 }
1167                 parse_box(p, os, flags, FLAG_END, outer, parent_context,
1168                           outer_type, special, inner);
1169         } else {
1170                 if (inner_flags == FLAG_ITEM) {
1171                         p.get_token();
1172                         eat_whitespace(p, os, parent_context, false);
1173                 }
1174                 parse_box(p, os, flags, inner_flags, outer, parent_context,
1175                           outer_type, special, inner);
1176         }
1177 }
1178
1179
1180 void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
1181 {
1182         parent_context.check_layout(os);
1183         begin_inset(os, "listings\n");
1184         if (p.hasOpt()) {
1185                 string arg = p.verbatimOption();
1186                 os << "lstparams " << '"' << arg << '"' << '\n';
1187         }
1188         if (in_line)
1189                 os << "inline true\n";
1190         else
1191                 os << "inline false\n";
1192         os << "status collapsed\n";
1193         Context context(true, parent_context.textclass);
1194         context.layout = &parent_context.textclass.plainLayout();
1195         string s;
1196         if (in_line) {
1197                 s = p.plainCommand('!', '!', "lstinline");
1198                 context.new_paragraph(os);
1199                 context.check_layout(os);
1200         } else
1201                 s = p.plainEnvironment("lstlisting");
1202         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1203                 if (*it == '\\')
1204                         os << "\n\\backslash\n";
1205                 else if (*it == '\n') {
1206                         // avoid adding an empty paragraph at the end
1207                         if (it + 1 != et) {
1208                                 context.new_paragraph(os);
1209                                 context.check_layout(os);
1210                         }
1211                 } else
1212                         os << *it;
1213         }
1214         context.check_end_layout(os);
1215         end_inset(os);
1216 }
1217
1218
1219 /// parse an unknown environment
1220 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
1221                                unsigned flags, bool outer,
1222                                Context & parent_context)
1223 {
1224         if (name == "tabbing")
1225                 // We need to remember that we have to handle '\=' specially
1226                 flags |= FLAG_TABBING;
1227
1228         // We need to translate font changes and paragraphs inside the
1229         // environment to ERT if we have a non standard font.
1230         // Otherwise things like
1231         // \large\begin{foo}\huge bar\end{foo}
1232         // will not work.
1233         bool const specialfont =
1234                 (parent_context.font != parent_context.normalfont);
1235         bool const new_layout_allowed = parent_context.new_layout_allowed;
1236         if (specialfont)
1237                 parent_context.new_layout_allowed = false;
1238         handle_ert(os, "\\begin{" + name + "}", parent_context);
1239         parse_text_snippet(p, os, flags, outer, parent_context);
1240         handle_ert(os, "\\end{" + name + "}", parent_context);
1241         if (specialfont)
1242                 parent_context.new_layout_allowed = new_layout_allowed;
1243 }
1244
1245
1246 void parse_environment(Parser & p, ostream & os, bool outer,
1247                        string & last_env, Context & parent_context)
1248 {
1249         Layout const * newlayout;
1250         InsetLayout const * newinsetlayout = 0;
1251         string const name = p.getArg('{', '}');
1252         const bool is_starred = suffixIs(name, '*');
1253         string const unstarred_name = rtrim(name, "*");
1254         active_environments.push_back(name);
1255
1256         if (is_math_env(name)) {
1257                 parent_context.check_layout(os);
1258                 begin_inset(os, "Formula ");
1259                 os << "\\begin{" << name << "}";
1260                 parse_math(p, os, FLAG_END, MATH_MODE);
1261                 os << "\\end{" << name << "}";
1262                 end_inset(os);
1263                 if (is_display_math_env(name)) {
1264                         // Prevent the conversion of a line break to a space
1265                         // (bug 7668). This does not change the output, but
1266                         // looks ugly in LyX.
1267                         eat_whitespace(p, os, parent_context, false);
1268                 }
1269         }
1270
1271         else if (is_known(name, polyglossia_languages)) {
1272                 // We must begin a new paragraph if not already done
1273                 if (! parent_context.atParagraphStart()) {
1274                         parent_context.check_end_layout(os);
1275                         parent_context.new_paragraph(os);
1276                 }
1277                 // save the language in the context so that it is
1278                 // handled by parse_text
1279                 parent_context.font.language = polyglossia2lyx(name);
1280                 parse_text(p, os, FLAG_END, outer, parent_context);
1281                 // Just in case the environment is empty
1282                 parent_context.extra_stuff.erase();
1283                 // We must begin a new paragraph to reset the language
1284                 parent_context.new_paragraph(os);
1285                 p.skip_spaces();
1286         }
1287
1288         else if (unstarred_name == "tabular" || name == "longtable") {
1289                 eat_whitespace(p, os, parent_context, false);
1290                 string width = "0pt";
1291                 if (name == "tabular*") {
1292                         width = lyx::translate_len(p.getArg('{', '}'));
1293                         eat_whitespace(p, os, parent_context, false);
1294                 }
1295                 parent_context.check_layout(os);
1296                 begin_inset(os, "Tabular ");
1297                 handle_tabular(p, os, name, width, parent_context);
1298                 end_inset(os);
1299                 p.skip_spaces();
1300         }
1301
1302         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
1303                 eat_whitespace(p, os, parent_context, false);
1304                 string const opt = p.hasOpt() ? p.getArg('[', ']') : string();
1305                 eat_whitespace(p, os, parent_context, false);
1306                 parent_context.check_layout(os);
1307                 begin_inset(os, "Float " + unstarred_name + "\n");
1308                 // store the float type for subfloats
1309                 // subfloats only work with figures and tables
1310                 if (unstarred_name == "figure")
1311                         float_type = unstarred_name;
1312                 else if (unstarred_name == "table")
1313                         float_type = unstarred_name;
1314                 else
1315                         float_type = "";
1316                 if (!opt.empty())
1317                         os << "placement " << opt << '\n';
1318                 if (contains(opt, "H"))
1319                         preamble.registerAutomaticallyLoadedPackage("float");
1320                 else {
1321                         Floating const & fl = parent_context.textclass.floats()
1322                                 .getType(unstarred_name);
1323                         if (!fl.floattype().empty() && fl.usesFloatPkg())
1324                                 preamble.registerAutomaticallyLoadedPackage("float");
1325                 }
1326
1327                 os << "wide " << convert<string>(is_starred)
1328                    << "\nsideways false"
1329                    << "\nstatus open\n\n";
1330                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1331                 end_inset(os);
1332                 // We don't need really a new paragraph, but
1333                 // we must make sure that the next item gets a \begin_layout.
1334                 parent_context.new_paragraph(os);
1335                 p.skip_spaces();
1336                 // the float is parsed thus delete the type
1337                 float_type = "";
1338         }
1339
1340         else if (unstarred_name == "sidewaysfigure"
1341                 || unstarred_name == "sidewaystable") {
1342                 eat_whitespace(p, os, parent_context, false);
1343                 parent_context.check_layout(os);
1344                 if (unstarred_name == "sidewaysfigure")
1345                         begin_inset(os, "Float figure\n");
1346                 else
1347                         begin_inset(os, "Float table\n");
1348                 os << "wide " << convert<string>(is_starred)
1349                    << "\nsideways true"
1350                    << "\nstatus open\n\n";
1351                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1352                 end_inset(os);
1353                 // We don't need really a new paragraph, but
1354                 // we must make sure that the next item gets a \begin_layout.
1355                 parent_context.new_paragraph(os);
1356                 p.skip_spaces();
1357                 preamble.registerAutomaticallyLoadedPackage("rotfloat");
1358         }
1359
1360         else if (name == "wrapfigure" || name == "wraptable") {
1361                 // syntax is \begin{wrapfigure}[lines]{placement}[overhang]{width}
1362                 eat_whitespace(p, os, parent_context, false);
1363                 parent_context.check_layout(os);
1364                 // default values
1365                 string lines = "0";
1366                 string overhang = "0col%";
1367                 // parse
1368                 if (p.hasOpt())
1369                         lines = p.getArg('[', ']');
1370                 string const placement = p.getArg('{', '}');
1371                 if (p.hasOpt())
1372                         overhang = p.getArg('[', ']');
1373                 string const width = p.getArg('{', '}');
1374                 // write
1375                 if (name == "wrapfigure")
1376                         begin_inset(os, "Wrap figure\n");
1377                 else
1378                         begin_inset(os, "Wrap table\n");
1379                 os << "lines " << lines
1380                    << "\nplacement " << placement
1381                    << "\noverhang " << lyx::translate_len(overhang)
1382                    << "\nwidth " << lyx::translate_len(width)
1383                    << "\nstatus open\n\n";
1384                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1385                 end_inset(os);
1386                 // We don't need really a new paragraph, but
1387                 // we must make sure that the next item gets a \begin_layout.
1388                 parent_context.new_paragraph(os);
1389                 p.skip_spaces();
1390                 preamble.registerAutomaticallyLoadedPackage("wrapfig");
1391         }
1392
1393         else if (name == "minipage") {
1394                 eat_whitespace(p, os, parent_context, false);
1395                 // Test whether this is an outer box of a shaded box
1396                 p.pushPosition();
1397                 // swallow arguments
1398                 while (p.hasOpt()) {
1399                         p.getArg('[', ']');
1400                         p.skip_spaces(true);
1401                 }
1402                 p.getArg('{', '}');
1403                 p.skip_spaces(true);
1404                 Token t = p.get_token();
1405                 bool shaded = false;
1406                 if (t.asInput() == "\\begin") {
1407                         p.skip_spaces(true);
1408                         if (p.getArg('{', '}') == "shaded")
1409                                 shaded = true;
1410                 }
1411                 p.popPosition();
1412                 if (shaded)
1413                         parse_outer_box(p, os, FLAG_END, outer,
1414                                         parent_context, name, "shaded");
1415                 else
1416                         parse_box(p, os, 0, FLAG_END, outer, parent_context,
1417                                   "", "", name);
1418                 p.skip_spaces();
1419         }
1420
1421         else if (name == "comment") {
1422                 eat_whitespace(p, os, parent_context, false);
1423                 parent_context.check_layout(os);
1424                 begin_inset(os, "Note Comment\n");
1425                 os << "status open\n";
1426                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1427                 end_inset(os);
1428                 p.skip_spaces();
1429                 skip_braces(p); // eat {} that might by set by LyX behind comments
1430                 preamble.registerAutomaticallyLoadedPackage("verbatim");
1431         }
1432
1433         else if (name == "verbatim") {
1434                 os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
1435                 string const s = p.plainEnvironment("verbatim");
1436                 string::const_iterator it2 = s.begin();
1437                 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1438                         if (*it == '\\')
1439                                 os << "\\backslash ";
1440                         else if (*it == '\n') {
1441                                 it2 = it + 1;
1442                                 // avoid adding an empty paragraph at the end
1443                                 // FIXME: if there are 2 consecutive spaces at the end ignore it
1444                                 // because LyX will re-add a \n
1445                                 // This hack must be removed once bug 8049 is fixed!
1446                                 if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
1447                                         os << "\n\\end_layout\n\\begin_layout Verbatim\n";
1448                         } else
1449                                 os << *it;
1450                 }
1451                 os << "\n\\end_layout\n\n";
1452                 p.skip_spaces();
1453                 // reset to Standard layout
1454                 os << "\n\\begin_layout Standard\n";
1455         }
1456
1457         else if (name == "CJK") {
1458                 // the scheme is \begin{CJK}{encoding}{mapping}{text}
1459                 // It is impossible to decide if a CJK environment was in its own paragraph or within
1460                 // a line. We therefore always assume a paragraph since the latter is a rare case.
1461                 eat_whitespace(p, os, parent_context, false);
1462                 parent_context.check_end_layout(os);
1463                 // store the encoding to be able to reset it
1464                 string const encoding_old = p.getEncoding();
1465                 string const encoding = p.getArg('{', '}');
1466                 // SJIS and BIG5 don't work with LaTeX according to the comment in unicode.cpp
1467                 // JIS does not work with LyX's encoding conversion
1468                 if (encoding != "SJIS" && encoding != "BIG5" && encoding != "JIS")
1469                         p.setEncoding(encoding);
1470                 else
1471                         p.setEncoding("utf8");
1472                 // LyX doesn't support the second argument so if
1473                 // this is used we need to output everything as ERT
1474                 string const mapping = p.getArg('{', '}');
1475                 if ((!mapping.empty() && mapping != " ")
1476                         || (!is_known(encoding, supported_CJK_encodings))) {
1477                         parent_context.check_layout(os);
1478                         handle_ert(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}",
1479                                        parent_context);
1480                         // we must parse the content as verbatim because e.g. SJIS can contain
1481                         // normally invalid characters
1482                         string const s = p.plainEnvironment("CJK");
1483                         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1484                                 if (*it == '\\')
1485                                         handle_ert(os, "\\", parent_context);
1486                                 else if (*it == '$')
1487                                         handle_ert(os, "$", parent_context);
1488                                 else
1489                                         os << *it;
1490                         }
1491                         p.skip_spaces();
1492                         handle_ert(os, "\\end{" + name + "}",
1493                                        parent_context);
1494                 } else {
1495                         string const lang = CJK2lyx(encoding);
1496                         // store the language because we must reset it at the end
1497                         string const lang_old = parent_context.font.language;
1498                         parent_context.font.language = lang;
1499                         parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1500                         parent_context.font.language = lang_old;
1501                         parent_context.new_paragraph(os);
1502                 }
1503                 p.setEncoding(encoding_old);
1504                 p.skip_spaces();
1505         }
1506
1507         else if (name == "lyxgreyedout") {
1508                 eat_whitespace(p, os, parent_context, false);
1509                 parent_context.check_layout(os);
1510                 begin_inset(os, "Note Greyedout\n");
1511                 os << "status open\n";
1512                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1513                 end_inset(os);
1514                 p.skip_spaces();
1515                 if (!preamble.notefontcolor().empty())
1516                         preamble.registerAutomaticallyLoadedPackage("color");
1517         }
1518
1519         else if (name == "framed" || name == "shaded") {
1520                 eat_whitespace(p, os, parent_context, false);
1521                 parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
1522                 p.skip_spaces();
1523         }
1524
1525         else if (name == "lstlisting") {
1526                 eat_whitespace(p, os, parent_context, false);
1527                 // FIXME handle the automatic color package loading
1528                 // uwestoehr asks: In what case color is loaded?
1529                 parse_listings(p, os, parent_context, false);
1530                 p.skip_spaces();
1531         }
1532
1533         else if (!parent_context.new_layout_allowed)
1534                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1535                                           parent_context);
1536
1537         // Alignment and spacing settings
1538         // FIXME (bug xxxx): These settings can span multiple paragraphs and
1539         //                                       therefore are totally broken!
1540         // Note that \centering, raggedright, and raggedleft cannot be handled, as
1541         // they are commands not environments. They are furthermore switches that
1542         // can be ended by another switches, but also by commands like \footnote or
1543         // \parbox. So the only safe way is to leave them untouched.
1544         else if (name == "center" || name == "centering" ||
1545                  name == "flushleft" || name == "flushright" ||
1546                  name == "singlespace" || name == "onehalfspace" ||
1547                  name == "doublespace" || name == "spacing") {
1548                 eat_whitespace(p, os, parent_context, false);
1549                 // We must begin a new paragraph if not already done
1550                 if (! parent_context.atParagraphStart()) {
1551                         parent_context.check_end_layout(os);
1552                         parent_context.new_paragraph(os);
1553                 }
1554                 if (name == "flushleft")
1555                         parent_context.add_extra_stuff("\\align left\n");
1556                 else if (name == "flushright")
1557                         parent_context.add_extra_stuff("\\align right\n");
1558                 else if (name == "center" || name == "centering")
1559                         parent_context.add_extra_stuff("\\align center\n");
1560                 else if (name == "singlespace")
1561                         parent_context.add_extra_stuff("\\paragraph_spacing single\n");
1562                 else if (name == "onehalfspace") {
1563                         parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
1564                         preamble.registerAutomaticallyLoadedPackage("setspace");
1565                 } else if (name == "doublespace") {
1566                         parent_context.add_extra_stuff("\\paragraph_spacing double\n");
1567                         preamble.registerAutomaticallyLoadedPackage("setspace");
1568                 } else if (name == "spacing") {
1569                         parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
1570                         preamble.registerAutomaticallyLoadedPackage("setspace");
1571                 }
1572                 parse_text(p, os, FLAG_END, outer, parent_context);
1573                 // Just in case the environment is empty
1574                 parent_context.extra_stuff.erase();
1575                 // We must begin a new paragraph to reset the alignment
1576                 parent_context.new_paragraph(os);
1577                 p.skip_spaces();
1578         }
1579
1580         // The single '=' is meant here.
1581         else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
1582                 eat_whitespace(p, os, parent_context, false);
1583                 Context context(true, parent_context.textclass, newlayout,
1584                                 parent_context.layout, parent_context.font);
1585                 if (parent_context.deeper_paragraph) {
1586                         // We are beginning a nested environment after a
1587                         // deeper paragraph inside the outer list environment.
1588                         // Therefore we don't need to output a "begin deeper".
1589                         context.need_end_deeper = true;
1590                 }
1591                 parent_context.check_end_layout(os);
1592                 if (last_env == name) {
1593                         // we need to output a separator since LyX would export
1594                         // the two environments as one otherwise (bug 5716)
1595                         docstring const sep = from_ascii("--Separator--");
1596                         TeX2LyXDocClass const & textclass(parent_context.textclass);
1597                         if (textclass.hasLayout(sep)) {
1598                                 Context newcontext(parent_context);
1599                                 newcontext.layout = &(textclass[sep]);
1600                                 newcontext.check_layout(os);
1601                                 newcontext.check_end_layout(os);
1602                         } else {
1603                                 parent_context.check_layout(os);
1604                                 begin_inset(os, "Note Note\n");
1605                                 os << "status closed\n";
1606                                 Context newcontext(true, textclass,
1607                                                 &(textclass.defaultLayout()));
1608                                 newcontext.check_layout(os);
1609                                 newcontext.check_end_layout(os);
1610                                 end_inset(os);
1611                                 parent_context.check_end_layout(os);
1612                         }
1613                 }
1614                 switch (context.layout->latextype) {
1615                 case  LATEX_LIST_ENVIRONMENT:
1616                         context.add_par_extra_stuff("\\labelwidthstring "
1617                                                     + p.verbatim_item() + '\n');
1618                         p.skip_spaces();
1619                         break;
1620                 case  LATEX_BIB_ENVIRONMENT:
1621                         p.verbatim_item(); // swallow next arg
1622                         p.skip_spaces();
1623                         break;
1624                 default:
1625                         break;
1626                 }
1627                 context.check_deeper(os);
1628                 // handle known optional and required arguments
1629                 // layouts require all optional arguments before the required ones
1630                 // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
1631                 // It is impossible to place anything after the environment name,
1632                 // but before the first \\item.
1633                 if (context.layout->latextype == LATEX_ENVIRONMENT) {
1634                         bool need_layout = true;
1635                         unsigned int optargs = 0;
1636                         while (optargs < context.layout->optargs) {
1637                                 eat_whitespace(p, os, context, false);
1638                                 if (p.next_token().cat() == catEscape ||
1639                                     p.next_token().character() != '[')
1640                                         break;
1641                                 p.get_token(); // eat '['
1642                                 if (need_layout) {
1643                                         context.check_layout(os);
1644                                         need_layout = false;
1645                                 }
1646                                 begin_inset(os, "Argument\n");
1647                                 os << "status collapsed\n\n";
1648                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
1649                                 end_inset(os);
1650                                 eat_whitespace(p, os, context, false);
1651                                 ++optargs;
1652                         }
1653                         unsigned int reqargs = 0;
1654                         while (reqargs < context.layout->reqargs) {
1655                                 eat_whitespace(p, os, context, false);
1656                                 if (p.next_token().cat() != catBegin)
1657                                         break;
1658                                 p.get_token(); // eat '{'
1659                                 if (need_layout) {
1660                                         context.check_layout(os);
1661                                         need_layout = false;
1662                                 }
1663                                 begin_inset(os, "Argument\n");
1664                                 os << "status collapsed\n\n";
1665                                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
1666                                 end_inset(os);
1667                                 eat_whitespace(p, os, context, false);
1668                                 ++reqargs;
1669                         }
1670                 }
1671                 parse_text(p, os, FLAG_END, outer, context);
1672                 context.check_end_layout(os);
1673                 if (parent_context.deeper_paragraph) {
1674                         // We must suppress the "end deeper" because we
1675                         // suppressed the "begin deeper" above.
1676                         context.need_end_deeper = false;
1677                 }
1678                 context.check_end_deeper(os);
1679                 parent_context.new_paragraph(os);
1680                 p.skip_spaces();
1681                 if (!preamble.titleLayoutFound())
1682                         preamble.titleLayoutFound(newlayout->intitle);
1683                 set<string> const & req = newlayout->requires();
1684                 set<string>::const_iterator it = req.begin();
1685                 set<string>::const_iterator en = req.end();
1686                 for (; it != en; ++it)
1687                         preamble.registerAutomaticallyLoadedPackage(*it);
1688         }
1689
1690         // The single '=' is meant here.
1691         else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
1692                 eat_whitespace(p, os, parent_context, false);
1693                 parent_context.check_layout(os);
1694                 begin_inset(os, "Flex ");
1695                 os << to_utf8(newinsetlayout->name()) << '\n'
1696                    << "status collapsed\n";
1697                 parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
1698                 end_inset(os);
1699         }
1700
1701         else if (name == "appendix") {
1702                 // This is no good latex style, but it works and is used in some documents...
1703                 eat_whitespace(p, os, parent_context, false);
1704                 parent_context.check_end_layout(os);
1705                 Context context(true, parent_context.textclass, parent_context.layout,
1706                                 parent_context.layout, parent_context.font);
1707                 context.check_layout(os);
1708                 os << "\\start_of_appendix\n";
1709                 parse_text(p, os, FLAG_END, outer, context);
1710                 context.check_end_layout(os);
1711                 p.skip_spaces();
1712         }
1713
1714         else if (known_environments.find(name) != known_environments.end()) {
1715                 vector<ArgumentType> arguments = known_environments[name];
1716                 // The last "argument" denotes wether we may translate the
1717                 // environment contents to LyX
1718                 // The default required if no argument is given makes us
1719                 // compatible with the reLyXre environment.
1720                 ArgumentType contents = arguments.empty() ?
1721                         required :
1722                         arguments.back();
1723                 if (!arguments.empty())
1724                         arguments.pop_back();
1725                 // See comment in parse_unknown_environment()
1726                 bool const specialfont =
1727                         (parent_context.font != parent_context.normalfont);
1728                 bool const new_layout_allowed =
1729                         parent_context.new_layout_allowed;
1730                 if (specialfont)
1731                         parent_context.new_layout_allowed = false;
1732                 parse_arguments("\\begin{" + name + "}", arguments, p, os,
1733                                 outer, parent_context);
1734                 if (contents == verbatim)
1735                         handle_ert(os, p.verbatimEnvironment(name),
1736                                    parent_context);
1737                 else
1738                         parse_text_snippet(p, os, FLAG_END, outer,
1739                                            parent_context);
1740                 handle_ert(os, "\\end{" + name + "}", parent_context);
1741                 if (specialfont)
1742                         parent_context.new_layout_allowed = new_layout_allowed;
1743         }
1744
1745         else
1746                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1747                                           parent_context);
1748
1749         last_env = name;
1750         active_environments.pop_back();
1751 }
1752
1753
1754 /// parses a comment and outputs it to \p os.
1755 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
1756 {
1757         LASSERT(t.cat() == catComment, return);
1758         if (!t.cs().empty()) {
1759                 context.check_layout(os);
1760                 handle_comment(os, '%' + t.cs(), context);
1761                 if (p.next_token().cat() == catNewline) {
1762                         // A newline after a comment line starts a new
1763                         // paragraph
1764                         if (context.new_layout_allowed) {
1765                                 if(!context.atParagraphStart())
1766                                         // Only start a new paragraph if not already
1767                                         // done (we might get called recursively)
1768                                         context.new_paragraph(os);
1769                         } else
1770                                 handle_ert(os, "\n", context);
1771                         eat_whitespace(p, os, context, true);
1772                 }
1773         } else {
1774                 // "%\n" combination
1775                 p.skip_spaces();
1776         }
1777 }
1778
1779
1780 /*!
1781  * Reads spaces and comments until the first non-space, non-comment token.
1782  * New paragraphs (double newlines or \\par) are handled like simple spaces
1783  * if \p eatParagraph is true.
1784  * Spaces are skipped, but comments are written to \p os.
1785  */
1786 void eat_whitespace(Parser & p, ostream & os, Context & context,
1787                     bool eatParagraph)
1788 {
1789         while (p.good()) {
1790                 Token const & t = p.get_token();
1791                 if (t.cat() == catComment)
1792                         parse_comment(p, os, t, context);
1793                 else if ((! eatParagraph && p.isParagraph()) ||
1794                          (t.cat() != catSpace && t.cat() != catNewline)) {
1795                         p.putback();
1796                         return;
1797                 }
1798         }
1799 }
1800
1801
1802 /*!
1803  * Set a font attribute, parse text and reset the font attribute.
1804  * \param attribute Attribute name (e.g. \\family, \\shape etc.)
1805  * \param currentvalue Current value of the attribute. Is set to the new
1806  * value during parsing.
1807  * \param newvalue New value of the attribute
1808  */
1809 void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
1810                            Context & context, string const & attribute,
1811                            string & currentvalue, string const & newvalue)
1812 {
1813         context.check_layout(os);
1814         string const oldvalue = currentvalue;
1815         currentvalue = newvalue;
1816         os << '\n' << attribute << ' ' << newvalue << "\n";
1817         parse_text_snippet(p, os, flags, outer, context);
1818         context.check_layout(os);
1819         os << '\n' << attribute << ' ' << oldvalue << "\n";
1820         currentvalue = oldvalue;
1821 }
1822
1823
1824 /// get the arguments of a natbib or jurabib citation command
1825 void get_cite_arguments(Parser & p, bool natbibOrder,
1826         string & before, string & after)
1827 {
1828         // We need to distinguish "" and "[]", so we can't use p.getOpt().
1829
1830         // text before the citation
1831         before.clear();
1832         // text after the citation
1833         after = p.getFullOpt();
1834
1835         if (!after.empty()) {
1836                 before = p.getFullOpt();
1837                 if (natbibOrder && !before.empty())
1838                         swap(before, after);
1839         }
1840 }
1841
1842
1843 /// Convert filenames with TeX macros and/or quotes to something LyX
1844 /// can understand
1845 string const normalize_filename(string const & name)
1846 {
1847         Parser p(trim(name, "\""));
1848         ostringstream os;
1849         while (p.good()) {
1850                 Token const & t = p.get_token();
1851                 if (t.cat() != catEscape)
1852                         os << t.asInput();
1853                 else if (t.cs() == "lyxdot") {
1854                         // This is used by LyX for simple dots in relative
1855                         // names
1856                         os << '.';
1857                         p.skip_spaces();
1858                 } else if (t.cs() == "space") {
1859                         os << ' ';
1860                         p.skip_spaces();
1861                 } else
1862                         os << t.asInput();
1863         }
1864         return os.str();
1865 }
1866
1867
1868 /// Convert \p name from TeX convention (relative to master file) to LyX
1869 /// convention (relative to .lyx file) if it is relative
1870 void fix_relative_filename(string & name)
1871 {
1872         if (FileName::isAbsolute(name))
1873                 return;
1874
1875         name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
1876                                    from_utf8(getParentFilePath())));
1877 }
1878
1879
1880 /// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
1881 void parse_noweb(Parser & p, ostream & os, Context & context)
1882 {
1883         // assemble the rest of the keyword
1884         string name("<<");
1885         bool scrap = false;
1886         while (p.good()) {
1887                 Token const & t = p.get_token();
1888                 if (t.asInput() == ">" && p.next_token().asInput() == ">") {
1889                         name += ">>";
1890                         p.get_token();
1891                         scrap = (p.good() && p.next_token().asInput() == "=");
1892                         if (scrap)
1893                                 name += p.get_token().asInput();
1894                         break;
1895                 }
1896                 name += t.asInput();
1897         }
1898
1899         if (!scrap || !context.new_layout_allowed ||
1900             !context.textclass.hasLayout(from_ascii("Scrap"))) {
1901                 cerr << "Warning: Could not interpret '" << name
1902                      << "'. Ignoring it." << endl;
1903                 return;
1904         }
1905
1906         // We use new_paragraph instead of check_end_layout because the stuff
1907         // following the noweb chunk needs to start with a \begin_layout.
1908         // This may create a new paragraph even if there was none in the
1909         // noweb file, but the alternative is an invalid LyX file. Since
1910         // noweb code chunks are implemented with a layout style in LyX they
1911         // always must be in an own paragraph.
1912         context.new_paragraph(os);
1913         Context newcontext(true, context.textclass,
1914                 &context.textclass[from_ascii("Scrap")]);
1915         newcontext.check_layout(os);
1916         os << name;
1917         while (p.good()) {
1918                 Token const & t = p.get_token();
1919                 // We abuse the parser a bit, because this is no TeX syntax
1920                 // at all.
1921                 if (t.cat() == catEscape)
1922                         os << subst(t.asInput(), "\\", "\n\\backslash\n");
1923                 else {
1924                         ostringstream oss;
1925                         Context tmp(false, context.textclass,
1926                                     &context.textclass[from_ascii("Scrap")]);
1927                         tmp.need_end_layout = true;
1928                         tmp.check_layout(oss);
1929                         os << subst(t.asInput(), "\n", oss.str());
1930                 }
1931                 // The scrap chunk is ended by an @ at the beginning of a line.
1932                 // After the @ the line may contain a comment and/or
1933                 // whitespace, but nothing else.
1934                 if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
1935                     (p.next_token().cat() == catSpace ||
1936                      p.next_token().cat() == catNewline ||
1937                      p.next_token().cat() == catComment)) {
1938                         while (p.good() && p.next_token().cat() == catSpace)
1939                                 os << p.get_token().asInput();
1940                         if (p.next_token().cat() == catComment)
1941                                 // The comment includes a final '\n'
1942                                 os << p.get_token().asInput();
1943                         else {
1944                                 if (p.next_token().cat() == catNewline)
1945                                         p.get_token();
1946                                 os << '\n';
1947                         }
1948                         break;
1949                 }
1950         }
1951         newcontext.check_end_layout(os);
1952 }
1953
1954
1955 /// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
1956 bool is_macro(Parser & p)
1957 {
1958         Token first = p.curr_token();
1959         if (first.cat() != catEscape || !p.good())
1960                 return false;
1961         if (first.cs() == "def")
1962                 return true;
1963         if (first.cs() != "global" && first.cs() != "long")
1964                 return false;
1965         Token second = p.get_token();
1966         int pos = 1;
1967         while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
1968                second.cat() == catNewline || second.cat() == catComment)) {
1969                 second = p.get_token();
1970                 pos++;
1971         }
1972         bool secondvalid = second.cat() == catEscape;
1973         Token third;
1974         bool thirdvalid = false;
1975         if (p.good() && first.cs() == "global" && secondvalid &&
1976             second.cs() == "long") {
1977                 third = p.get_token();
1978                 pos++;
1979                 while (p.good() && !p.isParagraph() &&
1980                        (third.cat() == catSpace ||
1981                         third.cat() == catNewline ||
1982                         third.cat() == catComment)) {
1983                         third = p.get_token();
1984                         pos++;
1985                 }
1986                 thirdvalid = third.cat() == catEscape;
1987         }
1988         for (int i = 0; i < pos; ++i)
1989                 p.putback();
1990         if (!secondvalid)
1991                 return false;
1992         if (!thirdvalid)
1993                 return (first.cs() == "global" || first.cs() == "long") &&
1994                        second.cs() == "def";
1995         return first.cs() == "global" && second.cs() == "long" &&
1996                third.cs() == "def";
1997 }
1998
1999
2000 /// Parse a macro definition (assumes that is_macro() returned true)
2001 void parse_macro(Parser & p, ostream & os, Context & context)
2002 {
2003         context.check_layout(os);
2004         Token first = p.curr_token();
2005         Token second;
2006         Token third;
2007         string command = first.asInput();
2008         if (first.cs() != "def") {
2009                 p.get_token();
2010                 eat_whitespace(p, os, context, false);
2011                 second = p.curr_token();
2012                 command += second.asInput();
2013                 if (second.cs() != "def") {
2014                         p.get_token();
2015                         eat_whitespace(p, os, context, false);
2016                         third = p.curr_token();
2017                         command += third.asInput();
2018                 }
2019         }
2020         eat_whitespace(p, os, context, false);
2021         string const name = p.get_token().cs();
2022         eat_whitespace(p, os, context, false);
2023
2024         // parameter text
2025         bool simple = true;
2026         string paramtext;
2027         int arity = 0;
2028         while (p.next_token().cat() != catBegin) {
2029                 if (p.next_token().cat() == catParameter) {
2030                         // # found
2031                         p.get_token();
2032                         paramtext += "#";
2033
2034                         // followed by number?
2035                         if (p.next_token().cat() == catOther) {
2036                                 char c = p.getChar();
2037                                 paramtext += c;
2038                                 // number = current arity + 1?
2039                                 if (c == arity + '0' + 1)
2040                                         ++arity;
2041                                 else
2042                                         simple = false;
2043                         } else
2044                                 paramtext += p.get_token().cs();
2045                 } else {
2046                         paramtext += p.get_token().cs();
2047                         simple = false;
2048                 }
2049         }
2050
2051         // only output simple (i.e. compatible) macro as FormulaMacros
2052         string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
2053         if (simple) {
2054                 context.check_layout(os);
2055                 begin_inset(os, "FormulaMacro");
2056                 os << "\n\\def" << ert;
2057                 end_inset(os);
2058         } else
2059                 handle_ert(os, command + ert, context);
2060 }
2061
2062
2063 void registerExternalTemplatePackages(string const & name)
2064 {
2065         external::TemplateManager const & etm = external::TemplateManager::get();
2066         external::Template const * const et = etm.getTemplateByName(name);
2067         if (!et)
2068                 return;
2069         external::Template::Formats::const_iterator cit = et->formats.end();
2070         if (pdflatex)
2071                 cit = et->formats.find("PDFLaTeX");
2072         if (cit == et->formats.end())
2073                 // If the template has not specified a PDFLaTeX output,
2074                 // we try the LaTeX format.
2075                 cit = et->formats.find("LaTeX");
2076         if (cit == et->formats.end())
2077                 return;
2078         vector<string>::const_iterator qit = cit->second.requirements.begin();
2079         vector<string>::const_iterator qend = cit->second.requirements.end();
2080         for (; qit != qend; ++qit)
2081                 preamble.registerAutomaticallyLoadedPackage(*qit);
2082 }
2083
2084 } // anonymous namespace
2085
2086
2087 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
2088                 Context & context)
2089 {
2090         Layout const * newlayout = 0;
2091         InsetLayout const * newinsetlayout = 0;
2092         char const * const * where = 0;
2093         // Store the latest bibliographystyle and nocite{*} option
2094         // (needed for bibtex inset)
2095         string btprint;
2096         string bibliographystyle = "default";
2097         bool const use_natbib = preamble.isPackageUsed("natbib");
2098         bool const use_jurabib = preamble.isPackageUsed("jurabib");
2099         string last_env;
2100         while (p.good()) {
2101                 Token const & t = p.get_token();
2102
2103         // it is impossible to determine the correct document language if CJK is used.
2104         // Therefore write a note at the beginning of the document
2105         if (have_CJK) {
2106                 context.check_layout(os);
2107                 begin_inset(os, "Note Note\n");
2108                 os << "status open\n\\begin_layout Plain Layout\n"
2109                    << "\\series bold\n"
2110                    << "Important information:\n"
2111                    << "\\end_layout\n\n"
2112                    << "\\begin_layout Plain Layout\n"
2113                    << "This document contains text in Chinese, Japanese or Korean.\n"
2114                    << " It was therefore impossible for tex2lyx to set the correct document langue for your document."
2115                    << " Please set the language manually in the document settings.\n"
2116                    << "\\end_layout\n";
2117                 end_inset(os);
2118                 have_CJK = false;
2119         }
2120
2121         // it is impossible to determine the correct encoding for non-CJK Japanese.
2122         // Therefore write a note at the beginning of the document
2123         if (is_nonCJKJapanese) {
2124                 context.check_layout(os);
2125                 begin_inset(os, "Note Note\n");
2126                 os << "status open\n\\begin_layout Plain Layout\n"
2127                    << "\\series bold\n"
2128                    << "Important information:\n"
2129                    << "\\end_layout\n\n"
2130                    << "\\begin_layout Plain Layout\n"
2131                    << "This document is in Japanese (non-CJK).\n"
2132                    << " It was therefore impossible for tex2lyx to determine the correct encoding."
2133                    << " The encoding EUC-JP was assumed. If this is incorrect, please set the correct"
2134                    << " encoding in the document settings.\n"
2135                    << "\\end_layout\n";
2136                 end_inset(os);
2137                 is_nonCJKJapanese = false;
2138         }
2139
2140 #ifdef FILEDEBUG
2141                 debugToken(cerr, t, flags);
2142 #endif
2143
2144                 if (flags & FLAG_ITEM) {
2145                         if (t.cat() == catSpace)
2146                                 continue;
2147
2148                         flags &= ~FLAG_ITEM;
2149                         if (t.cat() == catBegin) {
2150                                 // skip the brace and collect everything to the next matching
2151                                 // closing brace
2152                                 flags |= FLAG_BRACE_LAST;
2153                                 continue;
2154                         }
2155
2156                         // handle only this single token, leave the loop if done
2157                         flags |= FLAG_LEAVE;
2158                 }
2159
2160                 if (t.cat() != catEscape && t.character() == ']' &&
2161                     (flags & FLAG_BRACK_LAST))
2162                         return;
2163                 if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
2164                         return;
2165
2166                 // If there is anything between \end{env} and \begin{env} we
2167                 // don't need to output a separator.
2168                 if (t.cat() != catSpace && t.cat() != catNewline &&
2169                     t.asInput() != "\\begin")
2170                         last_env = "";
2171
2172                 //
2173                 // cat codes
2174                 //
2175                 if (t.cat() == catMath) {
2176                         // we are inside some text mode thingy, so opening new math is allowed
2177                         context.check_layout(os);
2178                         begin_inset(os, "Formula ");
2179                         Token const & n = p.get_token();
2180                         bool const display(n.cat() == catMath && outer);
2181                         if (display) {
2182                                 // TeX's $$...$$ syntax for displayed math
2183                                 os << "\\[";
2184                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2185                                 os << "\\]";
2186                                 p.get_token(); // skip the second '$' token
2187                         } else {
2188                                 // simple $...$  stuff
2189                                 p.putback();
2190                                 os << '$';
2191                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2192                                 os << '$';
2193                         }
2194                         end_inset(os);
2195                         if (display) {
2196                                 // Prevent the conversion of a line break to a
2197                                 // space (bug 7668). This does not change the
2198                                 // output, but looks ugly in LyX.
2199                                 eat_whitespace(p, os, context, false);
2200                         }
2201                 }
2202
2203                 else if (t.cat() == catSuper || t.cat() == catSub)
2204                         cerr << "catcode " << t << " illegal in text mode\n";
2205
2206                 // Basic support for english quotes. This should be
2207                 // extended to other quotes, but is not so easy (a
2208                 // left english quote is the same as a right german
2209                 // quote...)
2210                 else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
2211                         context.check_layout(os);
2212                         begin_inset(os, "Quotes ");
2213                         os << "eld";
2214                         end_inset(os);
2215                         p.get_token();
2216                         skip_braces(p);
2217                 }
2218                 else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
2219                         context.check_layout(os);
2220                         begin_inset(os, "Quotes ");
2221                         os << "erd";
2222                         end_inset(os);
2223                         p.get_token();
2224                         skip_braces(p);
2225                 }
2226
2227                 else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
2228                         context.check_layout(os);
2229                         begin_inset(os, "Quotes ");
2230                         os << "ald";
2231                         end_inset(os);
2232                         p.get_token();
2233                         skip_braces(p);
2234                 }
2235
2236                 else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
2237                         context.check_layout(os);
2238                         begin_inset(os, "Quotes ");
2239                         os << "ard";
2240                         end_inset(os);
2241                         p.get_token();
2242                         skip_braces(p);
2243                 }
2244
2245                 else if (t.asInput() == "<"
2246                          && p.next_token().asInput() == "<" && noweb_mode) {
2247                         p.get_token();
2248                         parse_noweb(p, os, context);
2249                 }
2250
2251                 else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
2252                         check_space(p, os, context);
2253
2254                 else if (t.character() == '[' && noweb_mode &&
2255                          p.next_token().character() == '[') {
2256                         // These can contain underscores
2257                         p.putback();
2258                         string const s = p.getFullOpt() + ']';
2259                         if (p.next_token().character() == ']')
2260                                 p.get_token();
2261                         else
2262                                 cerr << "Warning: Inserting missing ']' in '"
2263                                      << s << "'." << endl;
2264                         handle_ert(os, s, context);
2265                 }
2266
2267                 else if (t.cat() == catLetter) {
2268                         context.check_layout(os);
2269                         // Workaround for bug 4752.
2270                         // FIXME: This whole code block needs to be removed
2271                         //        when the bug is fixed and tex2lyx produces
2272                         //        the updated file format.
2273                         // The replacement algorithm in LyX is so stupid that
2274                         // it even translates a phrase if it is part of a word.
2275                         bool handled = false;
2276                         for (int const * l = known_phrase_lengths; *l; ++l) {
2277                                 string phrase = t.cs();
2278                                 for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
2279                                         phrase += p.get_token().cs();
2280                                 if (is_known(phrase, known_coded_phrases)) {
2281                                         handle_ert(os, phrase, context);
2282                                         handled = true;
2283                                         break;
2284                                 } else {
2285                                         for (size_t i = 1; i < phrase.length(); ++i)
2286                                                 p.putback();
2287                                 }
2288                         }
2289                         if (!handled)
2290                                 os << t.cs();
2291                 }
2292
2293                 else if (t.cat() == catOther ||
2294                                t.cat() == catAlign ||
2295                                t.cat() == catParameter) {
2296                         // This translates "&" to "\\&" which may be wrong...
2297                         context.check_layout(os);
2298                         os << t.cs();
2299                 }
2300
2301                 else if (p.isParagraph()) {
2302                         if (context.new_layout_allowed)
2303                                 context.new_paragraph(os);
2304                         else
2305                                 handle_ert(os, "\\par ", context);
2306                         eat_whitespace(p, os, context, true);
2307                 }
2308
2309                 else if (t.cat() == catActive) {
2310                         context.check_layout(os);
2311                         if (t.character() == '~') {
2312                                 if (context.layout->free_spacing)
2313                                         os << ' ';
2314                                 else {
2315                                         begin_inset(os, "space ~\n");
2316                                         end_inset(os);
2317                                 }
2318                         } else
2319                                 os << t.cs();
2320                 }
2321
2322                 else if (t.cat() == catBegin) {
2323                         Token const next = p.next_token();
2324                         Token const end = p.next_next_token();
2325                         if (next.cat() == catEnd) {
2326                         // {}
2327                         Token const prev = p.prev_token();
2328                         p.get_token();
2329                         if (p.next_token().character() == '`' ||
2330                             (prev.character() == '-' &&
2331                              p.next_token().character() == '-'))
2332                                 ; // ignore it in {}`` or -{}-
2333                         else
2334                                 handle_ert(os, "{}", context);
2335                         } else if (next.cat() == catEscape &&
2336                                    is_known(next.cs(), known_quotes) &&
2337                                    end.cat() == catEnd) {
2338                                 // Something like {\textquoteright} (e.g.
2339                                 // from writer2latex). LyX writes
2340                                 // \textquoteright{}, so we may skip the
2341                                 // braces here for better readability.
2342                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2343                                                    outer, context);
2344                         } else {
2345                         context.check_layout(os);
2346                         // special handling of font attribute changes
2347                         Token const prev = p.prev_token();
2348                         TeXFont const oldFont = context.font;
2349                         if (next.character() == '[' ||
2350                             next.character() == ']' ||
2351                             next.character() == '*') {
2352                                 p.get_token();
2353                                 if (p.next_token().cat() == catEnd) {
2354                                         os << next.cs();
2355                                         p.get_token();
2356                                 } else {
2357                                         p.putback();
2358                                         handle_ert(os, "{", context);
2359                                         parse_text_snippet(p, os,
2360                                                         FLAG_BRACE_LAST,
2361                                                         outer, context);
2362                                         handle_ert(os, "}", context);
2363                                 }
2364                         } else if (! context.new_layout_allowed) {
2365                                 handle_ert(os, "{", context);
2366                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2367                                                    outer, context);
2368                                 handle_ert(os, "}", context);
2369                         } else if (is_known(next.cs(), known_sizes)) {
2370                                 // next will change the size, so we must
2371                                 // reset it here
2372                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2373                                                    outer, context);
2374                                 if (!context.atParagraphStart())
2375                                         os << "\n\\size "
2376                                            << context.font.size << "\n";
2377                         } else if (is_known(next.cs(), known_font_families)) {
2378                                 // next will change the font family, so we
2379                                 // must reset it here
2380                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2381                                                    outer, context);
2382                                 if (!context.atParagraphStart())
2383                                         os << "\n\\family "
2384                                            << context.font.family << "\n";
2385                         } else if (is_known(next.cs(), known_font_series)) {
2386                                 // next will change the font series, so we
2387                                 // must reset it here
2388                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2389                                                    outer, context);
2390                                 if (!context.atParagraphStart())
2391                                         os << "\n\\series "
2392                                            << context.font.series << "\n";
2393                         } else if (is_known(next.cs(), known_font_shapes)) {
2394                                 // next will change the font shape, so we
2395                                 // must reset it here
2396                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2397                                                    outer, context);
2398                                 if (!context.atParagraphStart())
2399                                         os << "\n\\shape "
2400                                            << context.font.shape << "\n";
2401                         } else if (is_known(next.cs(), known_old_font_families) ||
2402                                    is_known(next.cs(), known_old_font_series) ||
2403                                    is_known(next.cs(), known_old_font_shapes)) {
2404                                 // next will change the font family, series
2405                                 // and shape, so we must reset it here
2406                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2407                                                    outer, context);
2408                                 if (!context.atParagraphStart())
2409                                         os <<  "\n\\family "
2410                                            << context.font.family
2411                                            << "\n\\series "
2412                                            << context.font.series
2413                                            << "\n\\shape "
2414                                            << context.font.shape << "\n";
2415                         } else {
2416                                 handle_ert(os, "{", context);
2417                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2418                                                    outer, context);
2419                                 handle_ert(os, "}", context);
2420                                 }
2421                         }
2422                 }
2423
2424                 else if (t.cat() == catEnd) {
2425                         if (flags & FLAG_BRACE_LAST) {
2426                                 return;
2427                         }
2428                         cerr << "stray '}' in text\n";
2429                         handle_ert(os, "}", context);
2430                 }
2431
2432                 else if (t.cat() == catComment)
2433                         parse_comment(p, os, t, context);
2434
2435                 //
2436                 // control sequences
2437                 //
2438
2439                 else if (t.cs() == "(") {
2440                         context.check_layout(os);
2441                         begin_inset(os, "Formula");
2442                         os << " \\(";
2443                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
2444                         os << "\\)";
2445                         end_inset(os);
2446                 }
2447
2448                 else if (t.cs() == "[") {
2449                         context.check_layout(os);
2450                         begin_inset(os, "Formula");
2451                         os << " \\[";
2452                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
2453                         os << "\\]";
2454                         end_inset(os);
2455                         // Prevent the conversion of a line break to a space
2456                         // (bug 7668). This does not change the output, but
2457                         // looks ugly in LyX.
2458                         eat_whitespace(p, os, context, false);
2459                 }
2460
2461                 else if (t.cs() == "begin")
2462                         parse_environment(p, os, outer, last_env,
2463                                           context);
2464
2465                 else if (t.cs() == "end") {
2466                         if (flags & FLAG_END) {
2467                                 // eat environment name
2468                                 string const name = p.getArg('{', '}');
2469                                 if (name != active_environment())
2470                                         cerr << "\\end{" + name + "} does not match \\begin{"
2471                                                 + active_environment() + "}\n";
2472                                 return;
2473                         }
2474                         p.error("found 'end' unexpectedly");
2475                 }
2476
2477                 else if (t.cs() == "item") {
2478                         string s;
2479                         bool const optarg = p.hasOpt();
2480                         if (optarg) {
2481                                 // FIXME: This swallows comments, but we cannot use
2482                                 //        eat_whitespace() since we must not output
2483                                 //        anything before the item.
2484                                 p.skip_spaces(true);
2485                                 s = p.verbatimOption();
2486                         } else
2487                                 p.skip_spaces(false);
2488                         context.set_item();
2489                         context.check_layout(os);
2490                         if (context.has_item) {
2491                                 // An item in an unknown list-like environment
2492                                 // FIXME: Do this in check_layout()!
2493                                 context.has_item = false;
2494                                 if (optarg)
2495                                         handle_ert(os, "\\item", context);
2496                                 else
2497                                         handle_ert(os, "\\item ", context);
2498                         }
2499                         if (optarg) {
2500                                 if (context.layout->labeltype != LABEL_MANUAL) {
2501                                         // LyX does not support \item[\mybullet]
2502                                         // in itemize environments
2503                                         Parser p2(s + ']');
2504                                         os << parse_text_snippet(p2,
2505                                                 FLAG_BRACK_LAST, outer, context);
2506                                 } else if (!s.empty()) {
2507                                         // LyX adds braces around the argument,
2508                                         // so we need to remove them here.
2509                                         if (s.size() > 2 && s[0] == '{' &&
2510                                             s[s.size()-1] == '}')
2511                                                 s = s.substr(1, s.size()-2);
2512                                         // If the argument contains a space we
2513                                         // must put it into ERT: Otherwise LyX
2514                                         // would misinterpret the space as
2515                                         // item delimiter (bug 7663)
2516                                         if (contains(s, ' ')) {
2517                                                 handle_ert(os, s, context);
2518                                         } else {
2519                                                 Parser p2(s + ']');
2520                                                 os << parse_text_snippet(p2,
2521                                                         FLAG_BRACK_LAST,
2522                                                         outer, context);
2523                                         }
2524                                         // The space is needed to separate the
2525                                         // item from the rest of the sentence.
2526                                         os << ' ';
2527                                         eat_whitespace(p, os, context, false);
2528                                 }
2529                         }
2530                 }
2531
2532                 else if (t.cs() == "bibitem") {
2533                         context.set_item();
2534                         context.check_layout(os);
2535                         eat_whitespace(p, os, context, false);
2536                         string label = convert_command_inset_arg(p.verbatimOption());
2537                         string key = convert_command_inset_arg(p.verbatim_item());
2538                         if (contains(label, '\\') || contains(key, '\\')) {
2539                                 // LyX can't handle LaTeX commands in labels or keys
2540                                 handle_ert(os, t.asInput() + '[' + label +
2541                                                "]{" + p.verbatim_item() + '}',
2542                                            context);
2543                         } else {
2544                                 begin_command_inset(os, "bibitem", "bibitem");
2545                                 os << "label \"" << label << "\"\n"
2546                                       "key \"" << key << "\"\n";
2547                                 end_inset(os);
2548                         }
2549                 }
2550
2551                 else if (is_macro(p)) {
2552                         // catch the case of \def\inputGnumericTable
2553                         bool macro = true;
2554                         if (t.cs() == "def") {
2555                                 Token second = p.next_token();
2556                                 if (second.cs() == "inputGnumericTable") {
2557                                         p.pushPosition();
2558                                         p.get_token();
2559                                         skip_braces(p);
2560                                         Token third = p.get_token();
2561                                         p.popPosition();
2562                                         if (third.cs() == "input") {
2563                                                 p.get_token();
2564                                                 skip_braces(p);
2565                                                 p.get_token();
2566                                                 string name = normalize_filename(p.verbatim_item());
2567                                                 string const path = getMasterFilePath();
2568                                                 // We want to preserve relative / absolute filenames,
2569                                                 // therefore path is only used for testing
2570                                                 // The file extension is in every case ".tex".
2571                                                 // So we need to remove this extension and check for
2572                                                 // the original one.
2573                                                 name = removeExtension(name);
2574                                                 if (!makeAbsPath(name, path).exists()) {
2575                                                         char const * const Gnumeric_formats[] = {"gnumeric",
2576                                                                 "ods", "xls", 0};
2577                                                         string const Gnumeric_name =
2578                                                                 find_file(name, path, Gnumeric_formats);
2579                                                         if (!Gnumeric_name.empty())
2580                                                                 name = Gnumeric_name;
2581                                                 }
2582                                                 if (makeAbsPath(name, path).exists())
2583                                                         fix_relative_filename(name);
2584                                                 else
2585                                                         cerr << "Warning: Could not find file '"
2586                                                              << name << "'." << endl;
2587                                                 context.check_layout(os);
2588                                                 begin_inset(os, "External\n\ttemplate ");
2589                                                 os << "GnumericSpreadsheet\n\tfilename "
2590                                                    << name << "\n";
2591                                                 end_inset(os);
2592                                                 context.check_layout(os);
2593                                                 macro = false;
2594                                                 // register the packages that are automatically reloaded
2595                                                 // by the Gnumeric template
2596                                                 registerExternalTemplatePackages("GnumericSpreadsheet");
2597                                         }
2598                                 }
2599                         }
2600                         if (macro)
2601                                 parse_macro(p, os, context);
2602                 }
2603
2604                 else if (t.cs() == "noindent") {
2605                         p.skip_spaces();
2606                         context.add_par_extra_stuff("\\noindent\n");
2607                 }
2608
2609                 else if (t.cs() == "appendix") {
2610                         context.add_par_extra_stuff("\\start_of_appendix\n");
2611                         // We need to start a new paragraph. Otherwise the
2612                         // appendix in 'bla\appendix\chapter{' would start
2613                         // too late.
2614                         context.new_paragraph(os);
2615                         // We need to make sure that the paragraph is
2616                         // generated even if it is empty. Otherwise the
2617                         // appendix in '\par\appendix\par\chapter{' would
2618                         // start too late.
2619                         context.check_layout(os);
2620                         // FIXME: This is a hack to prevent paragraph
2621                         // deletion if it is empty. Handle this better!
2622                         handle_comment(os,
2623                                 "%dummy comment inserted by tex2lyx to "
2624                                 "ensure that this paragraph is not empty",
2625                                 context);
2626                         // Both measures above may generate an additional
2627                         // empty paragraph, but that does not hurt, because
2628                         // whitespace does not matter here.
2629                         eat_whitespace(p, os, context, true);
2630                 }
2631
2632                 // Must catch empty dates before findLayout is called below
2633                 else if (t.cs() == "date") {
2634                         eat_whitespace(p, os, context, false);
2635                         p.pushPosition();
2636                         string const date = p.verbatim_item();
2637                         p.popPosition();
2638                         if (date.empty()) {
2639                                 preamble.suppressDate(true);
2640                                 p.verbatim_item();
2641                         } else {
2642                                 preamble.suppressDate(false);
2643                                 if (context.new_layout_allowed &&
2644                                     (newlayout = findLayout(context.textclass,
2645                                                             t.cs(), true))) {
2646                                         // write the layout
2647                                         output_command_layout(os, p, outer,
2648                                                         context, newlayout);
2649                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2650                                         if (!preamble.titleLayoutFound())
2651                                                 preamble.titleLayoutFound(newlayout->intitle);
2652                                         set<string> const & req = newlayout->requires();
2653                                         set<string>::const_iterator it = req.begin();
2654                                         set<string>::const_iterator en = req.end();
2655                                         for (; it != en; ++it)
2656                                                 preamble.registerAutomaticallyLoadedPackage(*it);
2657                                 } else
2658                                         handle_ert(os,
2659                                                 "\\date{" + p.verbatim_item() + '}',
2660                                                 context);
2661                         }
2662                 }
2663
2664                 // Starred section headings
2665                 // Must attempt to parse "Section*" before "Section".
2666                 else if ((p.next_token().asInput() == "*") &&
2667                          context.new_layout_allowed &&
2668                          (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
2669                         // write the layout
2670                         p.get_token();
2671                         output_command_layout(os, p, outer, context, newlayout);
2672                         p.skip_spaces();
2673                         if (!preamble.titleLayoutFound())
2674                                 preamble.titleLayoutFound(newlayout->intitle);
2675                         set<string> const & req = newlayout->requires();
2676                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2677                                 preamble.registerAutomaticallyLoadedPackage(*it);
2678                 }
2679
2680                 // Section headings and the like
2681                 else if (context.new_layout_allowed &&
2682                          (newlayout = findLayout(context.textclass, t.cs(), true))) {
2683                         // write the layout
2684                         output_command_layout(os, p, outer, context, newlayout);
2685                         p.skip_spaces();
2686                         if (!preamble.titleLayoutFound())
2687                                 preamble.titleLayoutFound(newlayout->intitle);
2688                         set<string> const & req = newlayout->requires();
2689                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2690                                 preamble.registerAutomaticallyLoadedPackage(*it);
2691                 }
2692
2693                 else if (t.cs() == "caption") {
2694                         p.skip_spaces();
2695                         context.check_layout(os);
2696                         p.skip_spaces();
2697                         begin_inset(os, "Caption\n");
2698                         Context newcontext(true, context.textclass);
2699                         newcontext.font = context.font;
2700                         newcontext.check_layout(os);
2701                         if (p.next_token().cat() != catEscape &&
2702                             p.next_token().character() == '[') {
2703                                 p.get_token(); // eat '['
2704                                 begin_inset(os, "Argument\n");
2705                                 os << "status collapsed\n";
2706                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
2707                                 end_inset(os);
2708                                 eat_whitespace(p, os, context, false);
2709                         }
2710                         parse_text(p, os, FLAG_ITEM, outer, context);
2711                         context.check_end_layout(os);
2712                         // We don't need really a new paragraph, but
2713                         // we must make sure that the next item gets a \begin_layout.
2714                         context.new_paragraph(os);
2715                         end_inset(os);
2716                         p.skip_spaces();
2717                         newcontext.check_end_layout(os);
2718                 }
2719
2720                 else if (t.cs() == "subfloat") {
2721                         // the syntax is \subfloat[caption]{content}
2722                         // if it is a table of figure depends on the surrounding float
2723                         bool has_caption = false;
2724                         p.skip_spaces();
2725                         // do nothing if there is no outer float
2726                         if (!float_type.empty()) {
2727                                 context.check_layout(os);
2728                                 p.skip_spaces();
2729                                 begin_inset(os, "Float " + float_type + "\n");
2730                                 os << "wide false"
2731                                    << "\nsideways false"
2732                                    << "\nstatus collapsed\n\n";
2733                                 // test for caption
2734                                 string caption;
2735                                 if (p.next_token().cat() != catEscape &&
2736                                                 p.next_token().character() == '[') {
2737                                                         p.get_token(); // eat '['
2738                                                         caption = parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
2739                                                         has_caption = true;
2740                                 }
2741                                 // the content
2742                                 parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
2743                                 // the caption comes always as the last
2744                                 if (has_caption) {
2745                                         // we must make sure that the caption gets a \begin_layout
2746                                         os << "\n\\begin_layout Plain Layout";
2747                                         p.skip_spaces();
2748                                         begin_inset(os, "Caption\n");
2749                                         Context newcontext(true, context.textclass);
2750                                         newcontext.font = context.font;
2751                                         newcontext.check_layout(os);
2752                                         os << caption << "\n";
2753                                         newcontext.check_end_layout(os);
2754                                         // We don't need really a new paragraph, but
2755                                         // we must make sure that the next item gets a \begin_layout.
2756                                         //newcontext.new_paragraph(os);
2757                                         end_inset(os);
2758                                         p.skip_spaces();
2759                                 }
2760                                 // We don't need really a new paragraph, but
2761                                 // we must make sure that the next item gets a \begin_layout.
2762                                 if (has_caption)
2763                                         context.new_paragraph(os);
2764                                 end_inset(os);
2765                                 p.skip_spaces();
2766                                 context.check_end_layout(os);
2767                                 // close the layout we opened
2768                                 if (has_caption)
2769                                         os << "\n\\end_layout\n";
2770                         } else {
2771                                 // if the float type is not supported or there is no surrounding float
2772                                 // output it as ERT
2773                                 if (p.hasOpt()) {
2774                                         string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
2775                                         handle_ert(os, t.asInput() + '[' + opt_arg +
2776                                                "]{" + p.verbatim_item() + '}', context);
2777                                 } else
2778                                         handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
2779                         }
2780                 }
2781
2782                 else if (t.cs() == "includegraphics") {
2783                         bool const clip = p.next_token().asInput() == "*";
2784                         if (clip)
2785                                 p.get_token();
2786                         string const arg = p.getArg('[', ']');
2787                         map<string, string> opts;
2788                         vector<string> keys;
2789                         split_map(arg, opts, keys);
2790                         if (clip)
2791                                 opts["clip"] = string();
2792                         string name = normalize_filename(p.verbatim_item());
2793
2794                         string const path = getMasterFilePath();
2795                         // We want to preserve relative / absolute filenames,
2796                         // therefore path is only used for testing
2797                         if (!makeAbsPath(name, path).exists()) {
2798                                 // The file extension is probably missing.
2799                                 // Now try to find it out.
2800                                 string const dvips_name =
2801                                         find_file(name, path,
2802                                                   known_dvips_graphics_formats);
2803                                 string const pdftex_name =
2804                                         find_file(name, path,
2805                                                   known_pdftex_graphics_formats);
2806                                 if (!dvips_name.empty()) {
2807                                         if (!pdftex_name.empty()) {
2808                                                 cerr << "This file contains the "
2809                                                         "latex snippet\n"
2810                                                         "\"\\includegraphics{"
2811                                                      << name << "}\".\n"
2812                                                         "However, files\n\""
2813                                                      << dvips_name << "\" and\n\""
2814                                                      << pdftex_name << "\"\n"
2815                                                         "both exist, so I had to make a "
2816                                                         "choice and took the first one.\n"
2817                                                         "Please move the unwanted one "
2818                                                         "someplace else and try again\n"
2819                                                         "if my choice was wrong."
2820                                                      << endl;
2821                                         }
2822                                         name = dvips_name;
2823                                 } else if (!pdftex_name.empty()) {
2824                                         name = pdftex_name;
2825                                         pdflatex = true;
2826                                 }
2827                         }
2828
2829                         if (makeAbsPath(name, path).exists())
2830                                 fix_relative_filename(name);
2831                         else
2832                                 cerr << "Warning: Could not find graphics file '"
2833                                      << name << "'." << endl;
2834
2835                         context.check_layout(os);
2836                         begin_inset(os, "Graphics ");
2837                         os << "\n\tfilename " << name << '\n';
2838                         if (opts.find("width") != opts.end())
2839                                 os << "\twidth "
2840                                    << translate_len(opts["width"]) << '\n';
2841                         if (opts.find("height") != opts.end())
2842                                 os << "\theight "
2843                                    << translate_len(opts["height"]) << '\n';
2844                         if (opts.find("scale") != opts.end()) {
2845                                 istringstream iss(opts["scale"]);
2846                                 double val;
2847                                 iss >> val;
2848                                 val = val*100;
2849                                 os << "\tscale " << val << '\n';
2850                         }
2851                         if (opts.find("angle") != opts.end()) {
2852                                 os << "\trotateAngle "
2853                                    << opts["angle"] << '\n';
2854                                 vector<string>::const_iterator a =
2855                                         find(keys.begin(), keys.end(), "angle");
2856                                 vector<string>::const_iterator s =
2857                                         find(keys.begin(), keys.end(), "width");
2858                                 if (s == keys.end())
2859                                         s = find(keys.begin(), keys.end(), "height");
2860                                 if (s == keys.end())
2861                                         s = find(keys.begin(), keys.end(), "scale");
2862                                 if (s != keys.end() && distance(s, a) > 0)
2863                                         os << "\tscaleBeforeRotation\n";
2864                         }
2865                         if (opts.find("origin") != opts.end()) {
2866                                 ostringstream ss;
2867                                 string const opt = opts["origin"];
2868                                 if (opt.find('l') != string::npos) ss << "left";
2869                                 if (opt.find('r') != string::npos) ss << "right";
2870                                 if (opt.find('c') != string::npos) ss << "center";
2871                                 if (opt.find('t') != string::npos) ss << "Top";
2872                                 if (opt.find('b') != string::npos) ss << "Bottom";
2873                                 if (opt.find('B') != string::npos) ss << "Baseline";
2874                                 if (!ss.str().empty())
2875                                         os << "\trotateOrigin " << ss.str() << '\n';
2876                                 else
2877                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
2878                         }
2879                         if (opts.find("keepaspectratio") != opts.end())
2880                                 os << "\tkeepAspectRatio\n";
2881                         if (opts.find("clip") != opts.end())
2882                                 os << "\tclip\n";
2883                         if (opts.find("draft") != opts.end())
2884                                 os << "\tdraft\n";
2885                         if (opts.find("bb") != opts.end())
2886                                 os << "\tBoundingBox "
2887                                    << opts["bb"] << '\n';
2888                         int numberOfbbOptions = 0;
2889                         if (opts.find("bbllx") != opts.end())
2890                                 numberOfbbOptions++;
2891                         if (opts.find("bblly") != opts.end())
2892                                 numberOfbbOptions++;
2893                         if (opts.find("bburx") != opts.end())
2894                                 numberOfbbOptions++;
2895                         if (opts.find("bbury") != opts.end())
2896                                 numberOfbbOptions++;
2897                         if (numberOfbbOptions == 4)
2898                                 os << "\tBoundingBox "
2899                                    << opts["bbllx"] << " " << opts["bblly"] << " "
2900                                    << opts["bburx"] << " " << opts["bbury"] << '\n';
2901                         else if (numberOfbbOptions > 0)
2902                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2903                         numberOfbbOptions = 0;
2904                         if (opts.find("natwidth") != opts.end())
2905                                 numberOfbbOptions++;
2906                         if (opts.find("natheight") != opts.end())
2907                                 numberOfbbOptions++;
2908                         if (numberOfbbOptions == 2)
2909                                 os << "\tBoundingBox 0bp 0bp "
2910                                    << opts["natwidth"] << " " << opts["natheight"] << '\n';
2911                         else if (numberOfbbOptions > 0)
2912                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2913                         ostringstream special;
2914                         if (opts.find("hiresbb") != opts.end())
2915                                 special << "hiresbb,";
2916                         if (opts.find("trim") != opts.end())
2917                                 special << "trim,";
2918                         if (opts.find("viewport") != opts.end())
2919                                 special << "viewport=" << opts["viewport"] << ',';
2920                         if (opts.find("totalheight") != opts.end())
2921                                 special << "totalheight=" << opts["totalheight"] << ',';
2922                         if (opts.find("type") != opts.end())
2923                                 special << "type=" << opts["type"] << ',';
2924                         if (opts.find("ext") != opts.end())
2925                                 special << "ext=" << opts["ext"] << ',';
2926                         if (opts.find("read") != opts.end())
2927                                 special << "read=" << opts["read"] << ',';
2928                         if (opts.find("command") != opts.end())
2929                                 special << "command=" << opts["command"] << ',';
2930                         string s_special = special.str();
2931                         if (!s_special.empty()) {
2932                                 // We had special arguments. Remove the trailing ','.
2933                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
2934                         }
2935                         // TODO: Handle the unknown settings better.
2936                         // Warn about invalid options.
2937                         // Check whether some option was given twice.
2938                         end_inset(os);
2939                         preamble.registerAutomaticallyLoadedPackage("graphicx");
2940                 }
2941
2942                 else if (t.cs() == "footnote" ||
2943                          (t.cs() == "thanks" && context.layout->intitle)) {
2944                         p.skip_spaces();
2945                         context.check_layout(os);
2946                         begin_inset(os, "Foot\n");
2947                         os << "status collapsed\n\n";
2948                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2949                         end_inset(os);
2950                 }
2951
2952                 else if (t.cs() == "marginpar") {
2953                         p.skip_spaces();
2954                         context.check_layout(os);
2955                         begin_inset(os, "Marginal\n");
2956                         os << "status collapsed\n\n";
2957                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2958                         end_inset(os);
2959                 }
2960
2961                 else if (t.cs() == "lstinline") {
2962                         p.skip_spaces();
2963                         parse_listings(p, os, context, true);
2964                 }
2965
2966                 else if (t.cs() == "ensuremath") {
2967                         p.skip_spaces();
2968                         context.check_layout(os);
2969                         string const s = p.verbatim_item();
2970                         //FIXME: this never triggers in UTF8
2971                         if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
2972                                 os << s;
2973                         else
2974                                 handle_ert(os, "\\ensuremath{" + s + "}",
2975                                            context);
2976                 }
2977
2978                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
2979                         if (preamble.titleLayoutFound()) {
2980                                 // swallow this
2981                                 skip_spaces_braces(p);
2982                         } else
2983                                 handle_ert(os, t.asInput(), context);
2984                 }
2985
2986                 else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
2987                         context.check_layout(os);
2988                         begin_command_inset(os, "toc", t.cs());
2989                         end_inset(os);
2990                         skip_spaces_braces(p);
2991                         if (t.cs() == "lstlistoflistings")
2992                                 preamble.registerAutomaticallyLoadedPackage("listings");
2993                 }
2994
2995                 else if (t.cs() == "listoffigures") {
2996                         context.check_layout(os);
2997                         begin_inset(os, "FloatList figure\n");
2998                         end_inset(os);
2999                         skip_spaces_braces(p);
3000                 }
3001
3002                 else if (t.cs() == "listoftables") {
3003                         context.check_layout(os);
3004                         begin_inset(os, "FloatList table\n");
3005                         end_inset(os);
3006                         skip_spaces_braces(p);
3007                 }
3008
3009                 else if (t.cs() == "listof") {
3010                         p.skip_spaces(true);
3011                         string const name = p.get_token().cs();
3012                         if (context.textclass.floats().typeExist(name)) {
3013                                 context.check_layout(os);
3014                                 begin_inset(os, "FloatList ");
3015                                 os << name << "\n";
3016                                 end_inset(os);
3017                                 p.get_token(); // swallow second arg
3018                         } else
3019                                 handle_ert(os, "\\listof{" + name + "}", context);
3020                 }
3021
3022                 else if ((where = is_known(t.cs(), known_text_font_families)))
3023                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3024                                 context, "\\family", context.font.family,
3025                                 known_coded_font_families[where - known_text_font_families]);
3026
3027                 else if ((where = is_known(t.cs(), known_text_font_series)))
3028                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3029                                 context, "\\series", context.font.series,
3030                                 known_coded_font_series[where - known_text_font_series]);
3031
3032                 else if ((where = is_known(t.cs(), known_text_font_shapes)))
3033                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3034                                 context, "\\shape", context.font.shape,
3035                                 known_coded_font_shapes[where - known_text_font_shapes]);
3036
3037                 else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
3038                         context.check_layout(os);
3039                         TeXFont oldFont = context.font;
3040                         context.font.init();
3041                         context.font.size = oldFont.size;
3042                         os << "\n\\family " << context.font.family << "\n";
3043                         os << "\n\\series " << context.font.series << "\n";
3044                         os << "\n\\shape " << context.font.shape << "\n";
3045                         if (t.cs() == "textnormal") {
3046                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3047                                 output_font_change(os, context.font, oldFont);
3048                                 context.font = oldFont;
3049                         } else
3050                                 eat_whitespace(p, os, context, false);
3051                 }
3052
3053                 else if (t.cs() == "textcolor") {
3054                         // scheme is \textcolor{color name}{text}
3055                         string const color = p.verbatim_item();
3056                         // we only support the predefined colors of the color package
3057                         if (color == "black" || color == "blue" || color == "cyan"
3058                                 || color == "green" || color == "magenta" || color == "red"
3059                                 || color == "white" || color == "yellow") {
3060                                         context.check_layout(os);
3061                                         os << "\n\\color " << color << "\n";
3062                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3063                                         context.check_layout(os);
3064                                         os << "\n\\color inherit\n";
3065                                         preamble.registerAutomaticallyLoadedPackage("color");
3066                         } else
3067                                 // for custom defined colors
3068                                 handle_ert(os, t.asInput() + "{" + color + "}", context);
3069                 }
3070
3071                 else if (t.cs() == "underbar" || t.cs() == "uline") {
3072                         // \underbar is not 100% correct (LyX outputs \uline
3073                         // of ulem.sty). The difference is that \ulem allows
3074                         // line breaks, and \underbar does not.
3075                         // Do NOT handle \underline.
3076                         // \underbar cuts through y, g, q, p etc.,
3077                         // \underline does not.
3078                         context.check_layout(os);
3079                         os << "\n\\bar under\n";
3080                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3081                         context.check_layout(os);
3082                         os << "\n\\bar default\n";
3083                         preamble.registerAutomaticallyLoadedPackage("ulem");
3084                 }
3085
3086                 else if (t.cs() == "sout") {
3087                         context.check_layout(os);
3088                         os << "\n\\strikeout on\n";
3089                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3090                         context.check_layout(os);
3091                         os << "\n\\strikeout default\n";
3092                         preamble.registerAutomaticallyLoadedPackage("ulem");
3093                 }
3094
3095                 else if (t.cs() == "uuline" || t.cs() == "uwave" ||
3096                          t.cs() == "emph" || t.cs() == "noun") {
3097                         context.check_layout(os);
3098                         os << "\n\\" << t.cs() << " on\n";
3099                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3100                         context.check_layout(os);
3101                         os << "\n\\" << t.cs() << " default\n";
3102                         if (t.cs() == "uuline" || t.cs() == "uwave")
3103                                 preamble.registerAutomaticallyLoadedPackage("ulem");
3104                 }
3105
3106                 else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
3107                         context.check_layout(os);
3108                         string name = p.getArg('{', '}');
3109                         string localtime = p.getArg('{', '}');
3110                         preamble.registerAuthor(name);
3111                         Author const & author = preamble.getAuthor(name);
3112                         // from_ctime() will fail if LyX decides to output the
3113                         // time in the text language. It might also use a wrong
3114                         // time zone (if the original LyX document was exported
3115                         // with a different time zone).
3116                         time_t ptime = from_ctime(localtime);
3117                         if (ptime == static_cast<time_t>(-1)) {
3118                                 cerr << "Warning: Could not parse time `" << localtime
3119                                      << "´ for change tracking, using current time instead.\n";
3120                                 ptime = current_time();
3121                         }
3122                         if (t.cs() == "lyxadded")
3123                                 os << "\n\\change_inserted ";
3124                         else
3125                                 os << "\n\\change_deleted ";
3126                         os << author.bufferId() << ' ' << ptime << '\n';
3127                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3128                         bool dvipost    = LaTeXPackages::isAvailable("dvipost");
3129                         bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
3130                                           LaTeXPackages::isAvailable("xcolor");
3131                         // No need to test for luatex, since luatex comes in
3132                         // two flavours (dvi and pdf), like latex, and those
3133                         // are detected by pdflatex.
3134                         if (pdflatex || xetex) {
3135                                 if (xcolorulem) {
3136                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3137                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3138                                         preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
3139                                 }
3140                         } else {
3141                                 if (dvipost) {
3142                                         preamble.registerAutomaticallyLoadedPackage("dvipost");
3143                                 } else if (xcolorulem) {
3144                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3145                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3146                                 }
3147                         }
3148                 }
3149
3150                 else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
3151                              t.cs() == "vphantom") {
3152                         context.check_layout(os);
3153                         if (t.cs() == "phantom")
3154                                 begin_inset(os, "Phantom Phantom\n");
3155                         if (t.cs() == "hphantom")
3156                                 begin_inset(os, "Phantom HPhantom\n");
3157                         if (t.cs() == "vphantom")
3158                                 begin_inset(os, "Phantom VPhantom\n");
3159                         os << "status open\n";
3160                         parse_text_in_inset(p, os, FLAG_ITEM, outer, context,
3161                                             "Phantom");
3162                         end_inset(os);
3163                 }
3164
3165                 else if (t.cs() == "href") {
3166                         context.check_layout(os);
3167                         string target = p.getArg('{', '}');
3168                         string name = p.getArg('{', '}');
3169                         string type;
3170                         size_t i = target.find(':');
3171                         if (i != string::npos) {
3172                                 type = target.substr(0, i + 1);
3173                                 if (type == "mailto:" || type == "file:")
3174                                         target = target.substr(i + 1);
3175                                 // handle the case that name is equal to target, except of "http://"
3176                                 else if (target.substr(i + 3) == name && type == "http:")
3177                                         target = name;
3178                         }
3179                         begin_command_inset(os, "href", "href");
3180                         if (name != target)
3181                                 os << "name \"" << name << "\"\n";
3182                         os << "target \"" << target << "\"\n";
3183                         if (type == "mailto:" || type == "file:")
3184                                 os << "type \"" << type << "\"\n";
3185                         end_inset(os);
3186                         skip_spaces_braces(p);
3187                 }
3188
3189                 else if (t.cs() == "lyxline") {
3190                         // swallow size argument (it is not used anyway)
3191                         p.getArg('{', '}');
3192                         if (!context.atParagraphStart()) {
3193                                 // so our line is in the middle of a paragraph
3194                                 // we need to add a new line, lest this line
3195                                 // follow the other content on that line and
3196                                 // run off the side of the page
3197                                 // FIXME: This may create an empty paragraph,
3198                                 //        but without that it would not be
3199                                 //        possible to set noindent below.
3200                                 //        Fortunately LaTeX does not care
3201                                 //        about the empty paragraph.
3202                                 context.new_paragraph(os);
3203                         }
3204                         if (preamble.indentParagraphs()) {
3205                                 // we need to unindent, lest the line be too long
3206                                 context.add_par_extra_stuff("\\noindent\n");
3207                         }
3208                         context.check_layout(os);
3209                         begin_command_inset(os, "line", "rule");
3210                         os << "offset \"0.5ex\"\n"
3211                               "width \"100line%\"\n"
3212                               "height \"1pt\"\n";
3213                         end_inset(os);
3214                 }
3215
3216                 else if (t.cs() == "rule") {
3217                         string const offset = (p.hasOpt() ? p.getArg('[', ']') : string());
3218                         string const width = p.getArg('{', '}');
3219                         string const thickness = p.getArg('{', '}');
3220                         context.check_layout(os);
3221                         begin_command_inset(os, "line", "rule");
3222                         if (!offset.empty())
3223                                 os << "offset \"" << translate_len(offset) << "\"\n";
3224                         os << "width \"" << translate_len(width) << "\"\n"
3225                                   "height \"" << translate_len(thickness) << "\"\n";
3226                         end_inset(os);
3227                 }
3228
3229                 else if (is_known(t.cs(), known_phrases) ||
3230                          (t.cs() == "protect" &&
3231                           p.next_token().cat() == catEscape &&
3232                           is_known(p.next_token().cs(), known_phrases))) {
3233                         // LyX sometimes puts a \protect in front, so we have to ignore it
3234                         // FIXME: This needs to be changed when bug 4752 is fixed.
3235                         where = is_known(
3236                                 t.cs() == "protect" ? p.get_token().cs() : t.cs(),
3237                                 known_phrases);
3238                         context.check_layout(os);
3239                         os << known_coded_phrases[where - known_phrases];
3240                         skip_spaces_braces(p);
3241                 }
3242
3243                 else if ((where = is_known(t.cs(), known_ref_commands))) {
3244                         string const opt = p.getOpt();
3245                         if (opt.empty()) {
3246                                 context.check_layout(os);
3247                                 begin_command_inset(os, "ref",
3248                                         known_coded_ref_commands[where - known_ref_commands]);
3249                                 os << "reference \""
3250                                    << convert_command_inset_arg(p.verbatim_item())
3251                                    << "\"\n";
3252                                 end_inset(os);
3253                                 if (t.cs() == "vref" || t.cs() == "vpageref")
3254                                         preamble.registerAutomaticallyLoadedPackage("varioref");
3255
3256                         } else {
3257                                 // LyX does not support optional arguments of ref commands
3258                                 handle_ert(os, t.asInput() + '[' + opt + "]{" +
3259                                                p.verbatim_item() + "}", context);
3260                         }
3261                 }
3262
3263                 else if (use_natbib &&
3264                          is_known(t.cs(), known_natbib_commands) &&
3265                          ((t.cs() != "citefullauthor" &&
3266                            t.cs() != "citeyear" &&
3267                            t.cs() != "citeyearpar") ||
3268                           p.next_token().asInput() != "*")) {
3269                         context.check_layout(os);
3270                         string command = t.cs();
3271                         if (p.next_token().asInput() == "*") {
3272                                 command += '*';
3273                                 p.get_token();
3274                         }
3275                         if (command == "citefullauthor")
3276                                 // alternative name for "\\citeauthor*"
3277                                 command = "citeauthor*";
3278
3279                         // text before the citation
3280                         string before;
3281                         // text after the citation
3282                         string after;
3283                         get_cite_arguments(p, true, before, after);
3284
3285                         if (command == "cite") {
3286                                 // \cite without optional argument means
3287                                 // \citet, \cite with at least one optional
3288                                 // argument means \citep.
3289                                 if (before.empty() && after.empty())
3290                                         command = "citet";
3291                                 else
3292                                         command = "citep";
3293                         }
3294                         if (before.empty() && after == "[]")
3295                                 // avoid \citet[]{a}
3296                                 after.erase();
3297                         else if (before == "[]" && after == "[]") {
3298                                 // avoid \citet[][]{a}
3299                                 before.erase();
3300                                 after.erase();
3301                         }
3302                         // remove the brackets around after and before
3303                         if (!after.empty()) {
3304                                 after.erase(0, 1);
3305                                 after.erase(after.length() - 1, 1);
3306                                 after = convert_command_inset_arg(after);
3307                         }
3308                         if (!before.empty()) {
3309                                 before.erase(0, 1);
3310                                 before.erase(before.length() - 1, 1);
3311                                 before = convert_command_inset_arg(before);
3312                         }
3313                         begin_command_inset(os, "citation", command);
3314                         os << "after " << '"' << after << '"' << "\n";
3315                         os << "before " << '"' << before << '"' << "\n";
3316                         os << "key \""
3317                            << convert_command_inset_arg(p.verbatim_item())
3318                            << "\"\n";
3319                         end_inset(os);
3320                 }
3321
3322                 else if (use_jurabib &&
3323                          is_known(t.cs(), known_jurabib_commands) &&
3324                          (t.cs() == "cite" || p.next_token().asInput() != "*")) {
3325                         context.check_layout(os);
3326                         string command = t.cs();
3327                         if (p.next_token().asInput() == "*") {
3328                                 command += '*';
3329                                 p.get_token();
3330                         }
3331                         char argumentOrder = '\0';
3332                         vector<string> const options =
3333                                 preamble.getPackageOptions("jurabib");
3334                         if (find(options.begin(), options.end(),
3335                                       "natbiborder") != options.end())
3336                                 argumentOrder = 'n';
3337                         else if (find(options.begin(), options.end(),
3338                                            "jurabiborder") != options.end())
3339                                 argumentOrder = 'j';
3340
3341                         // text before the citation
3342                         string before;
3343                         // text after the citation
3344                         string after;
3345                         get_cite_arguments(p, argumentOrder != 'j', before, after);
3346
3347                         string const citation = p.verbatim_item();
3348                         if (!before.empty() && argumentOrder == '\0') {
3349                                 cerr << "Warning: Assuming argument order "
3350                                         "of jurabib version 0.6 for\n'"
3351                                      << command << before << after << '{'
3352                                      << citation << "}'.\n"
3353                                         "Add 'jurabiborder' to the jurabib "
3354                                         "package options if you used an\n"
3355                                         "earlier jurabib version." << endl;
3356                         }
3357                         if (!after.empty()) {
3358                                 after.erase(0, 1);
3359                                 after.erase(after.length() - 1, 1);
3360                         }
3361                         if (!before.empty()) {
3362                                 before.erase(0, 1);
3363                                 before.erase(before.length() - 1, 1);
3364                         }
3365                         begin_command_inset(os, "citation", command);
3366                         os << "after " << '"' << after << '"' << "\n";
3367                         os << "before " << '"' << before << '"' << "\n";
3368                         os << "key " << '"' << citation << '"' << "\n";
3369                         end_inset(os);
3370                 }
3371
3372                 else if (t.cs() == "cite"
3373                         || t.cs() == "nocite") {
3374                         context.check_layout(os);
3375                         string after = convert_command_inset_arg(p.getArg('[', ']'));
3376                         string key = convert_command_inset_arg(p.verbatim_item());
3377                         // store the case that it is "\nocite{*}" to use it later for
3378                         // the BibTeX inset
3379                         if (key != "*") {
3380                                 begin_command_inset(os, "citation", t.cs());
3381                                 os << "after " << '"' << after << '"' << "\n";
3382                                 os << "key " << '"' << key << '"' << "\n";
3383                                 end_inset(os);
3384                         } else if (t.cs() == "nocite")
3385                                 btprint = key;
3386                 }
3387
3388                 else if (t.cs() == "index" ||
3389                          (t.cs() == "sindex" && preamble.use_indices() == "true")) {
3390                         context.check_layout(os);
3391                         string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
3392                                 p.getArg('[', ']') : "";
3393                         string const kind = arg.empty() ? "idx" : arg;
3394                         begin_inset(os, "Index ");
3395                         os << kind << "\nstatus collapsed\n";
3396                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
3397                         end_inset(os);
3398                         if (kind != "idx")
3399                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3400                 }
3401
3402                 else if (t.cs() == "nomenclature") {
3403                         context.check_layout(os);
3404                         begin_command_inset(os, "nomenclature", "nomenclature");
3405                         string prefix = convert_command_inset_arg(p.getArg('[', ']'));
3406                         if (!prefix.empty())
3407                                 os << "prefix " << '"' << prefix << '"' << "\n";
3408                         os << "symbol " << '"'
3409                            << convert_command_inset_arg(p.verbatim_item());
3410                         os << "\"\ndescription \""
3411                            << convert_command_inset_arg(p.verbatim_item())
3412                            << "\"\n";
3413                         end_inset(os);
3414                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3415                 }
3416
3417                 else if (t.cs() == "label") {
3418                         context.check_layout(os);
3419                         begin_command_inset(os, "label", "label");
3420                         os << "name \""
3421                            << convert_command_inset_arg(p.verbatim_item())
3422                            << "\"\n";
3423                         end_inset(os);
3424                 }
3425
3426                 else if (t.cs() == "printindex") {
3427                         context.check_layout(os);
3428                         begin_command_inset(os, "index_print", "printindex");
3429                         os << "type \"idx\"\n";
3430                         end_inset(os);
3431                         skip_spaces_braces(p);
3432                         preamble.registerAutomaticallyLoadedPackage("makeidx");
3433                         if (preamble.use_indices() == "true")
3434                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3435                 }
3436
3437                 else if (t.cs() == "printnomenclature") {
3438                         string width = "";
3439                         string width_type = "";
3440                         context.check_layout(os);
3441                         begin_command_inset(os, "nomencl_print", "printnomenclature");
3442                         // case of a custom width
3443                         if (p.hasOpt()) {
3444                                 width = p.getArg('[', ']');
3445                                 width = translate_len(width);
3446                                 width_type = "custom";
3447                         }
3448                         // case of no custom width
3449                         // the case of no custom width but the width set
3450                         // via \settowidth{\nomlabelwidth}{***} cannot be supported
3451                         // because the user could have set anything, not only the width
3452                         // of the longest label (which would be width_type = "auto")
3453                         string label = convert_command_inset_arg(p.getArg('{', '}'));
3454                         if (label.empty() && width_type.empty())
3455                                 width_type = "none";
3456                         os << "set_width \"" << width_type << "\"\n";
3457                         if (width_type == "custom")
3458                                 os << "width \"" << width << '\"';
3459                         end_inset(os);
3460                         skip_spaces_braces(p);
3461                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3462                 }
3463
3464                 else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
3465                         context.check_layout(os);
3466                         begin_inset(os, "script ");
3467                         os << t.cs().substr(4) << '\n';
3468                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
3469                         end_inset(os);
3470                         if (t.cs() == "textsubscript")
3471                                 preamble.registerAutomaticallyLoadedPackage("subscript");
3472                 }
3473
3474                 else if ((where = is_known(t.cs(), known_quotes))) {
3475                         context.check_layout(os);
3476                         begin_inset(os, "Quotes ");
3477                         os << known_coded_quotes[where - known_quotes];
3478                         end_inset(os);
3479                         // LyX adds {} after the quote, so we have to eat
3480                         // spaces here if there are any before a possible
3481                         // {} pair.
3482                         eat_whitespace(p, os, context, false);
3483                         skip_braces(p);
3484                 }
3485
3486                 else if ((where = is_known(t.cs(), known_sizes)) &&
3487                          context.new_layout_allowed) {
3488                         context.check_layout(os);
3489                         TeXFont const oldFont = context.font;
3490                         context.font.size = known_coded_sizes[where - known_sizes];
3491                         output_font_change(os, oldFont, context.font);
3492                         eat_whitespace(p, os, context, false);
3493                 }
3494
3495                 else if ((where = is_known(t.cs(), known_font_families)) &&
3496                          context.new_layout_allowed) {
3497                         context.check_layout(os);
3498                         TeXFont const oldFont = context.font;
3499                         context.font.family =
3500                                 known_coded_font_families[where - known_font_families];
3501                         output_font_change(os, oldFont, context.font);
3502                         eat_whitespace(p, os, context, false);
3503                 }
3504
3505                 else if ((where = is_known(t.cs(), known_font_series)) &&
3506                          context.new_layout_allowed) {
3507                         context.check_layout(os);
3508                         TeXFont const oldFont = context.font;
3509                         context.font.series =
3510                                 known_coded_font_series[where - known_font_series];
3511                         output_font_change(os, oldFont, context.font);
3512                         eat_whitespace(p, os, context, false);
3513                 }
3514
3515                 else if ((where = is_known(t.cs(), known_font_shapes)) &&
3516                          context.new_layout_allowed) {
3517                         context.check_layout(os);
3518                         TeXFont const oldFont = context.font;
3519                         context.font.shape =
3520                                 known_coded_font_shapes[where - known_font_shapes];
3521                         output_font_change(os, oldFont, context.font);
3522                         eat_whitespace(p, os, context, false);
3523                 }
3524                 else if ((where = is_known(t.cs(), known_old_font_families)) &&
3525                          context.new_layout_allowed) {
3526                         context.check_layout(os);
3527                         TeXFont const oldFont = context.font;
3528                         context.font.init();
3529                         context.font.size = oldFont.size;
3530                         context.font.family =
3531                                 known_coded_font_families[where - known_old_font_families];
3532                         output_font_change(os, oldFont, context.font);
3533                         eat_whitespace(p, os, context, false);
3534                 }
3535
3536                 else if ((where = is_known(t.cs(), known_old_font_series)) &&
3537                          context.new_layout_allowed) {
3538                         context.check_layout(os);
3539                         TeXFont const oldFont = context.font;
3540                         context.font.init();
3541                         context.font.size = oldFont.size;
3542                         context.font.series =
3543                                 known_coded_font_series[where - known_old_font_series];
3544                         output_font_change(os, oldFont, context.font);
3545                         eat_whitespace(p, os, context, false);
3546                 }
3547
3548                 else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
3549                          context.new_layout_allowed) {
3550                         context.check_layout(os);
3551                         TeXFont const oldFont = context.font;
3552                         context.font.init();
3553                         context.font.size = oldFont.size;
3554                         context.font.shape =
3555                                 known_coded_font_shapes[where - known_old_font_shapes];
3556                         output_font_change(os, oldFont, context.font);
3557                         eat_whitespace(p, os, context, false);
3558                 }
3559
3560                 else if (t.cs() == "selectlanguage") {
3561                         context.check_layout(os);
3562                         // save the language for the case that a
3563                         // \foreignlanguage is used
3564                         context.font.language = babel2lyx(p.verbatim_item());
3565                         os << "\n\\lang " << context.font.language << "\n";
3566                 }
3567
3568                 else if (t.cs() == "foreignlanguage") {
3569                         string const lang = babel2lyx(p.verbatim_item());
3570                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3571                                               context, "\\lang",
3572                                               context.font.language, lang);
3573                 }
3574
3575                 else if (prefixIs(t.cs(), "text")
3576                          && is_known(t.cs().substr(4), polyglossia_languages)) {
3577                         // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[]
3578                         string lang;
3579                         // We have to output the whole command if it has an option
3580                         // because LyX doesn't support this yet, see bug #8214,
3581                         // only if there is a single option specifying a variant, we can handle it.
3582                         if (p.hasOpt()) {
3583                                 string langopts = p.getOpt();
3584                                 // check if the option contains a variant, if yes, extract it
3585                                 string::size_type pos_var = langopts.find("variant");
3586                                 string::size_type i = langopts.find(',');
3587                                 string::size_type k = langopts.find('=', pos_var);
3588                                 if (pos_var != string::npos && i == string::npos) {
3589                                         string variant;
3590                                         variant = langopts.substr(k + 1, langopts.length() - k - 2);
3591                                         lang = polyglossia2lyx(variant);
3592                                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3593                                                                   context, "\\lang",
3594                                                                   context.font.language, lang);
3595                                 } else
3596                                         handle_ert(os, t.asInput() + langopts, context);
3597                         } else {
3598                                 lang = polyglossia2lyx(t.cs().substr(4, string::npos));
3599                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3600                                                           context, "\\lang",
3601                                                           context.font.language, lang);
3602                         }
3603                 }
3604
3605                 else if (t.cs() == "inputencoding") {
3606                         // nothing to write here
3607                         string const enc = subst(p.verbatim_item(), "\n", " ");
3608                         p.setEncoding(enc);
3609                 }
3610
3611                 else if ((where = is_known(t.cs(), known_special_chars))) {
3612                         context.check_layout(os);
3613                         os << "\\SpecialChar \\"
3614                            << known_coded_special_chars[where - known_special_chars]
3615                            << '\n';
3616                         skip_spaces_braces(p);
3617                 }
3618
3619                 else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
3620                         context.check_layout(os);
3621                         os << "\\SpecialChar \\nobreakdash-\n";
3622                         p.get_token();
3623                 }
3624
3625                 else if (t.cs() == "textquotedbl") {
3626                         context.check_layout(os);
3627                         os << "\"";
3628                         skip_braces(p);
3629                 }
3630
3631                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
3632                         context.check_layout(os);
3633                         os << "\\SpecialChar \\@.\n";
3634                         p.get_token();
3635                 }
3636
3637                 else if (t.cs() == "-") {
3638                         context.check_layout(os);
3639                         os << "\\SpecialChar \\-\n";
3640                 }
3641
3642                 else if (t.cs() == "textasciitilde") {
3643                         context.check_layout(os);
3644                         os << '~';
3645                         skip_spaces_braces(p);
3646                 }
3647
3648                 else if (t.cs() == "textasciicircum") {
3649                         context.check_layout(os);
3650                         os << '^';
3651                         skip_spaces_braces(p);
3652                 }
3653
3654                 else if (t.cs() == "textbackslash") {
3655                         context.check_layout(os);
3656                         os << "\n\\backslash\n";
3657                         skip_spaces_braces(p);
3658                 }
3659
3660                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
3661                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
3662                             || t.cs() == "%") {
3663                         context.check_layout(os);
3664                         os << t.cs();
3665                 }
3666
3667                 else if (t.cs() == "char") {
3668                         context.check_layout(os);
3669                         if (p.next_token().character() == '`') {
3670                                 p.get_token();
3671                                 if (p.next_token().cs() == "\"") {
3672                                         p.get_token();
3673                                         os << '"';
3674                                         skip_braces(p);
3675                                 } else {
3676                                         handle_ert(os, "\\char`", context);
3677                                 }
3678                         } else {
3679                                 handle_ert(os, "\\char", context);
3680                         }
3681                 }
3682
3683                 else if (t.cs() == "verb") {
3684                         context.check_layout(os);
3685                         char const delimiter = p.next_token().character();
3686                         string const arg = p.getArg(delimiter, delimiter);
3687                         ostringstream oss;
3688                         oss << "\\verb" << delimiter << arg << delimiter;
3689                         handle_ert(os, oss.str(), context);
3690                 }
3691
3692                 // Problem: \= creates a tabstop inside the tabbing environment
3693                 // and else an accent. In the latter case we really would want
3694                 // \={o} instead of \= o.
3695                 else if (t.cs() == "=" && (flags & FLAG_TABBING))
3696                         handle_ert(os, t.asInput(), context);
3697
3698                 // accents (see Table 6 in Comprehensive LaTeX Symbol List)
3699                 else if (t.cs().size() == 1
3700                          && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
3701                         context.check_layout(os);
3702                         // try to see whether the string is in unicodesymbols
3703                         bool termination;
3704                         docstring rem;
3705                         string command = t.asInput() + "{"
3706                                 + trimSpaceAndEol(p.verbatim_item())
3707                                 + "}";
3708                         set<string> req;
3709                         docstring s = encodings.fromLaTeXCommand(from_utf8(command),
3710                                 Encodings::TEXT_CMD | Encodings::MATH_CMD,
3711                                 termination, rem, &req);
3712                         if (!s.empty()) {
3713                                 if (!rem.empty())
3714                                         cerr << "When parsing " << command
3715                                              << ", result is " << to_utf8(s)
3716                                              << "+" << to_utf8(rem) << endl;
3717                                 os << to_utf8(s);
3718                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
3719                                         preamble.registerAutomaticallyLoadedPackage(*it);
3720                         } else
3721                                 // we did not find a non-ert version
3722                                 handle_ert(os, command, context);
3723                 }
3724
3725                 else if (t.cs() == "\\") {
3726                         context.check_layout(os);
3727                         if (p.hasOpt())
3728                                 handle_ert(os, "\\\\" + p.getOpt(), context);
3729                         else if (p.next_token().asInput() == "*") {
3730                                 p.get_token();
3731                                 // getOpt() eats the following space if there
3732                                 // is no optional argument, but that is OK
3733                                 // here since it has no effect in the output.
3734                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
3735                         }
3736                         else {
3737                                 begin_inset(os, "Newline newline");
3738                                 end_inset(os);
3739                         }
3740                 }
3741
3742                 else if (t.cs() == "newline" ||
3743                          (t.cs() == "linebreak" && !p.hasOpt())) {
3744                         context.check_layout(os);
3745                         begin_inset(os, "Newline ");
3746                         os << t.cs();
3747                         end_inset(os);
3748                         skip_spaces_braces(p);
3749                 }
3750
3751                 else if (t.cs() == "input" || t.cs() == "include"
3752                          || t.cs() == "verbatiminput") {
3753                         string name = t.cs();
3754                         if (t.cs() == "verbatiminput"
3755                             && p.next_token().asInput() == "*")
3756                                 name += p.get_token().asInput();
3757                         context.check_layout(os);
3758                         string filename(normalize_filename(p.getArg('{', '}')));
3759                         string const path = getMasterFilePath();
3760                         // We want to preserve relative / absolute filenames,
3761                         // therefore path is only used for testing
3762                         if ((t.cs() == "include" || t.cs() == "input") &&
3763                             !makeAbsPath(filename, path).exists()) {
3764                                 // The file extension is probably missing.
3765                                 // Now try to find it out.
3766                                 string const tex_name =
3767                                         find_file(filename, path,
3768                                                   known_tex_extensions);
3769                                 if (!tex_name.empty())
3770                                         filename = tex_name;
3771                         }
3772                         bool external = false;
3773                         string outname;
3774                         if (makeAbsPath(filename, path).exists()) {
3775                                 string const abstexname =
3776                                         makeAbsPath(filename, path).absFileName();
3777                                 string const abslyxname =
3778                                         changeExtension(abstexname, ".lyx");
3779                                 string const absfigname =
3780                                         changeExtension(abstexname, ".fig");
3781                                 fix_relative_filename(filename);
3782                                 string const lyxname =
3783                                         changeExtension(filename, ".lyx");
3784                                 bool xfig = false;
3785                                 external = FileName(absfigname).exists();
3786                                 if (t.cs() == "input") {
3787                                         string const ext = getExtension(abstexname);
3788
3789                                         // Combined PS/LaTeX:
3790                                         // x.eps, x.pstex_t (old xfig)
3791                                         // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
3792                                         FileName const absepsname(
3793                                                 changeExtension(abstexname, ".eps"));
3794                                         FileName const abspstexname(
3795                                                 changeExtension(abstexname, ".pstex"));
3796                                         bool const xfigeps =
3797                                                 (absepsname.exists() ||
3798                                                  abspstexname.exists()) &&
3799                                                 ext == "pstex_t";
3800
3801                                         // Combined PDF/LaTeX:
3802                                         // x.pdf, x.pdftex_t (old xfig)
3803                                         // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
3804                                         FileName const abspdfname(
3805                                                 changeExtension(abstexname, ".pdf"));
3806                                         bool const xfigpdf =
3807                                                 abspdfname.exists() &&
3808                                                 (ext == "pdftex_t" || ext == "pdf_t");
3809                                         if (xfigpdf)
3810                                                 pdflatex = true;
3811
3812                                         // Combined PS/PDF/LaTeX:
3813                                         // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
3814                                         string const absbase2(
3815                                                 removeExtension(abstexname) + "_pspdftex");
3816                                         FileName const abseps2name(
3817                                                 addExtension(absbase2, ".eps"));
3818                                         FileName const abspdf2name(
3819                                                 addExtension(absbase2, ".pdf"));
3820                                         bool const xfigboth =
3821                                                 abspdf2name.exists() &&
3822                                                 abseps2name.exists() && ext == "pspdftex";
3823
3824                                         xfig = xfigpdf || xfigeps || xfigboth;
3825                                         external = external && xfig;
3826                                 }
3827                                 if (external) {
3828                                         outname = changeExtension(filename, ".fig");
3829                                 } else if (xfig) {
3830                                         // Don't try to convert, the result
3831                                         // would be full of ERT.
3832                                         outname = filename;
3833                                 } else if (t.cs() != "verbatiminput" &&
3834                                     tex2lyx(abstexname, FileName(abslyxname),
3835                                             p.getEncoding())) {
3836                                         outname = lyxname;
3837                                 } else {
3838                                         outname = filename;
3839                                 }
3840                         } else {
3841                                 cerr << "Warning: Could not find included file '"
3842                                      << filename << "'." << endl;
3843                                 outname = filename;
3844                         }
3845                         if (external) {
3846                                 begin_inset(os, "External\n");
3847                                 os << "\ttemplate XFig\n"
3848                                    << "\tfilename " << outname << '\n';
3849                                 registerExternalTemplatePackages("XFig");
3850                         } else {
3851                                 begin_command_inset(os, "include", name);
3852                                 os << "preview false\n"
3853                                       "filename \"" << outname << "\"\n";
3854                                 if (t.cs() == "verbatiminput")
3855                                         preamble.registerAutomaticallyLoadedPackage("verbatim");
3856                         }
3857                         end_inset(os);
3858                 }
3859
3860                 else if (t.cs() == "bibliographystyle") {
3861                         // store new bibliographystyle
3862                         bibliographystyle = p.verbatim_item();
3863                         // If any other command than \bibliography and
3864                         // \nocite{*} follows, we need to output the style
3865                         // (because it might be used by that command).
3866                         // Otherwise, it will automatically be output by LyX.
3867                         p.pushPosition();
3868                         bool output = true;
3869                         for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
3870                                 if (t2.cat() == catBegin)
3871                                         break;
3872                                 if (t2.cat() != catEscape)
3873                                         continue;
3874                                 if (t2.cs() == "nocite") {
3875                                         if (p.getArg('{', '}') == "*")
3876                                                 continue;
3877                                 } else if (t2.cs() == "bibliography")
3878                                         output = false;
3879                                 break;
3880                         }
3881                         p.popPosition();
3882                         if (output) {
3883                                 handle_ert(os,
3884                                         "\\bibliographystyle{" + bibliographystyle + '}',
3885                                         context);
3886                         }
3887                 }
3888
3889                 else if (t.cs() == "bibliography") {
3890                         context.check_layout(os);
3891                         begin_command_inset(os, "bibtex", "bibtex");
3892                         if (!btprint.empty()) {
3893                                 os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
3894                                 // clear the string because the next BibTeX inset can be without the
3895                                 // \nocite{*} option
3896                                 btprint.clear();
3897                         }
3898                         os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
3899                         // Do we have a bibliographystyle set?
3900                         if (!bibliographystyle.empty())
3901                                 os << "options " << '"' << bibliographystyle << '"' << "\n";
3902                         end_inset(os);
3903                 }
3904
3905                 else if (t.cs() == "parbox") {
3906                         // Test whether this is an outer box of a shaded box
3907                         p.pushPosition();
3908                         // swallow arguments
3909                         while (p.hasOpt()) {
3910                                 p.getArg('[', ']');
3911                                 p.skip_spaces(true);
3912                         }
3913                         p.getArg('{', '}');
3914                         p.skip_spaces(true);
3915                         // eat the '{'
3916                         if (p.next_token().cat() == catBegin)
3917                                 p.get_token();
3918                         p.skip_spaces(true);
3919                         Token to = p.get_token();
3920                         bool shaded = false;
3921                         if (to.asInput() == "\\begin") {
3922                                 p.skip_spaces(true);
3923                                 if (p.getArg('{', '}') == "shaded")
3924                                         shaded = true;
3925                         }
3926                         p.popPosition();
3927                         if (shaded) {
3928                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3929                                                 context, "parbox", "shaded");
3930                         } else
3931                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3932                                           "", "", t.cs());
3933                 }
3934
3935                 else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
3936                          t.cs() == "shadowbox" || t.cs() == "doublebox")
3937                         parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
3938
3939                 else if (t.cs() == "framebox") {
3940                         if (p.next_token().character() == '(') {
3941                                 //the syntax is: \framebox(x,y)[position]{content}
3942                                 string arg = t.asInput();
3943                                 arg += p.getFullParentheseArg();
3944                                 arg += p.getFullOpt();
3945                                 eat_whitespace(p, os, context, false);
3946                                 handle_ert(os, arg + '{', context);
3947                                 eat_whitespace(p, os, context, false);
3948                                 parse_text(p, os, FLAG_ITEM, outer, context);
3949                                 handle_ert(os, "}", context);
3950                         } else {
3951                                 string special = p.getFullOpt();
3952                                 special += p.getOpt();
3953                                 // LyX does not yet support \framebox without any option
3954                                 if (!special.empty())
3955                                         parse_outer_box(p, os, FLAG_ITEM, outer,
3956                                                         context, t.cs(), special);
3957                                 else {
3958                                         eat_whitespace(p, os, context, false);
3959                                         handle_ert(os, "\\framebox{", context);
3960                                         eat_whitespace(p, os, context, false);
3961                                         parse_text(p, os, FLAG_ITEM, outer, context);
3962                                         handle_ert(os, "}", context);
3963                                 }
3964                         }
3965                 }
3966
3967                 //\makebox() is part of the picture environment and different from \makebox{}
3968                 //\makebox{} will be parsed by parse_box
3969                 else if (t.cs() == "makebox") {
3970                         if (p.next_token().character() == '(') {
3971                                 //the syntax is: \makebox(x,y)[position]{content}
3972                                 string arg = t.asInput();
3973                                 arg += p.getFullParentheseArg();
3974                                 arg += p.getFullOpt();
3975                                 eat_whitespace(p, os, context, false);
3976                                 handle_ert(os, arg + '{', context);
3977                                 eat_whitespace(p, os, context, false);
3978                                 parse_text(p, os, FLAG_ITEM, outer, context);
3979                                 handle_ert(os, "}", context);
3980                         } else
3981                                 //the syntax is: \makebox[width][position]{content}
3982                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3983                                           "", "", t.cs());
3984                 }
3985
3986                 else if (t.cs() == "smallskip" ||
3987                          t.cs() == "medskip" ||
3988                          t.cs() == "bigskip" ||
3989                          t.cs() == "vfill") {
3990                         context.check_layout(os);
3991                         begin_inset(os, "VSpace ");
3992                         os << t.cs();
3993                         end_inset(os);
3994                         skip_spaces_braces(p);
3995                 }
3996
3997                 else if ((where = is_known(t.cs(), known_spaces))) {
3998                         context.check_layout(os);
3999                         begin_inset(os, "space ");
4000                         os << '\\' << known_coded_spaces[where - known_spaces]
4001                            << '\n';
4002                         end_inset(os);
4003                         // LaTeX swallows whitespace after all spaces except
4004                         // "\\,". We have to do that here, too, because LyX
4005                         // adds "{}" which would make the spaces significant.
4006                         if (t.cs() !=  ",")
4007                                 eat_whitespace(p, os, context, false);
4008                         // LyX adds "{}" after all spaces except "\\ " and
4009                         // "\\,", so we have to remove "{}".
4010                         // "\\,{}" is equivalent to "\\," in LaTeX, so we
4011                         // remove the braces after "\\,", too.
4012                         if (t.cs() != " ")
4013                                 skip_braces(p);
4014                 }
4015
4016                 else if (t.cs() == "newpage" ||
4017                          (t.cs() == "pagebreak" && !p.hasOpt()) ||
4018                          t.cs() == "clearpage" ||
4019                          t.cs() == "cleardoublepage") {
4020                         context.check_layout(os);
4021                         begin_inset(os, "Newpage ");
4022                         os << t.cs();
4023                         end_inset(os);
4024                         skip_spaces_braces(p);
4025                 }
4026
4027                 else if (t.cs() == "DeclareRobustCommand" ||
4028                          t.cs() == "DeclareRobustCommandx" ||
4029                          t.cs() == "newcommand" ||
4030                          t.cs() == "newcommandx" ||
4031                          t.cs() == "providecommand" ||
4032                          t.cs() == "providecommandx" ||
4033                          t.cs() == "renewcommand" ||
4034                          t.cs() == "renewcommandx") {
4035                         // DeclareRobustCommand, DeclareRobustCommandx,
4036                         // providecommand and providecommandx could be handled
4037                         // by parse_command(), but we need to call
4038                         // add_known_command() here.
4039                         string name = t.asInput();
4040                         if (p.next_token().asInput() == "*") {
4041                                 // Starred form. Eat '*'
4042                                 p.get_token();
4043                                 name += '*';
4044                         }
4045                         string const command = p.verbatim_item();
4046                         string const opt1 = p.getFullOpt();
4047                         string const opt2 = p.getFullOpt();
4048                         add_known_command(command, opt1, !opt2.empty());
4049                         string const ert = name + '{' + command + '}' +
4050                                            opt1 + opt2 +
4051                                            '{' + p.verbatim_item() + '}';
4052
4053                         if (t.cs() == "DeclareRobustCommand" ||
4054                             t.cs() == "DeclareRobustCommandx" ||
4055                             t.cs() == "providecommand" ||
4056                             t.cs() == "providecommandx" ||
4057                             name[name.length()-1] == '*')
4058                                 handle_ert(os, ert, context);
4059                         else {
4060                                 context.check_layout(os);
4061                                 begin_inset(os, "FormulaMacro");
4062                                 os << "\n" << ert;
4063                                 end_inset(os);
4064                         }
4065                 }
4066
4067                 else if (t.cs() == "let" && p.next_token().asInput() != "*") {
4068                         // let could be handled by parse_command(),
4069                         // but we need to call add_known_command() here.
4070                         string ert = t.asInput();
4071                         string name;
4072                         p.skip_spaces();
4073                         if (p.next_token().cat() == catBegin) {
4074                                 name = p.verbatim_item();
4075                                 ert += '{' + name + '}';
4076                         } else {
4077                                 name = p.verbatim_item();
4078                                 ert += name;
4079                         }
4080                         string command;
4081                         p.skip_spaces();
4082                         if (p.next_token().cat() == catBegin) {
4083                                 command = p.verbatim_item();
4084                                 ert += '{' + command + '}';
4085                         } else {
4086                                 command = p.verbatim_item();
4087                                 ert += command;
4088                         }
4089                         // If command is known, make name known too, to parse
4090                         // its arguments correctly. For this reason we also
4091                         // have commands in syntax.default that are hardcoded.
4092                         CommandMap::iterator it = known_commands.find(command);
4093                         if (it != known_commands.end())
4094                                 known_commands[t.asInput()] = it->second;
4095                         handle_ert(os, ert, context);
4096                 }
4097
4098                 else if (t.cs() == "hspace" || t.cs() == "vspace") {
4099                         bool starred = false;
4100                         if (p.next_token().asInput() == "*") {
4101                                 p.get_token();
4102                                 starred = true;
4103                         }
4104                         string name = t.asInput();
4105                         string const length = p.verbatim_item();
4106                         string unit;
4107                         string valstring;
4108                         bool valid = splitLatexLength(length, valstring, unit);
4109                         bool known_hspace = false;
4110                         bool known_vspace = false;
4111                         bool known_unit = false;
4112                         double value;
4113                         if (valid) {
4114                                 istringstream iss(valstring);
4115                                 iss >> value;
4116                                 if (value == 1.0) {
4117                                         if (t.cs()[0] == 'h') {
4118                                                 if (unit == "\\fill") {
4119                                                         if (!starred) {
4120                                                                 unit = "";
4121                                                                 name = "\\hfill";
4122                                                         }
4123                                                         known_hspace = true;
4124                                                 }
4125                                         } else {
4126                                                 if (unit == "\\smallskipamount") {
4127                                                         unit = "smallskip";
4128                                                         known_vspace = true;
4129                                                 } else if (unit == "\\medskipamount") {
4130                                                         unit = "medskip";
4131                                                         known_vspace = true;
4132                                                 } else if (unit == "\\bigskipamount") {
4133                                                         unit = "bigskip";
4134                                                         known_vspace = true;
4135                                                 } else if (unit == "\\fill") {
4136                                                         unit = "vfill";
4137                                                         known_vspace = true;
4138                                                 }
4139                                         }
4140                                 }
4141                                 if (!known_hspace && !known_vspace) {
4142                                         switch (unitFromString(unit)) {
4143                                         case Length::SP:
4144                                         case Length::PT:
4145                                         case Length::BP:
4146                                         case Length::DD:
4147                                         case Length::MM:
4148                                         case Length::PC:
4149                                         case Length::CC:
4150                                         case Length::CM:
4151                                         case Length::IN:
4152                                         case Length::EX:
4153                                         case Length::EM:
4154                                         case Length::MU:
4155                                                 known_unit = true;
4156                                                 break;
4157                                         default:
4158                                                 break;
4159                                         }
4160                                 }
4161                         }
4162
4163                         if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
4164                                 // Literal horizontal length or known variable
4165                                 context.check_layout(os);
4166                                 begin_inset(os, "space ");
4167                                 os << name;
4168                                 if (starred)
4169                                         os << '*';
4170                                 os << '{';
4171                                 if (known_hspace)
4172                                         os << unit;
4173                                 os << "}";
4174                                 if (known_unit && !known_hspace)
4175                                         os << "\n\\length "
4176                                            << translate_len(length);
4177                                 end_inset(os);
4178                         } else if (known_unit || known_vspace) {
4179                                 // Literal vertical length or known variable
4180                                 context.check_layout(os);
4181                                 begin_inset(os, "VSpace ");
4182                                 if (known_unit)
4183                                         os << value;
4184                                 os << unit;
4185                                 if (starred)
4186                                         os << '*';
4187                                 end_inset(os);
4188                         } else {
4189                                 // LyX can't handle other length variables in Inset VSpace/space
4190                                 if (starred)
4191                                         name += '*';
4192                                 if (valid) {
4193                                         if (value == 1.0)
4194                                                 handle_ert(os, name + '{' + unit + '}', context);
4195                                         else if (value == -1.0)
4196                                                 handle_ert(os, name + "{-" + unit + '}', context);
4197                                         else
4198                                                 handle_ert(os, name + '{' + valstring + unit + '}', context);
4199                                 } else
4200                                         handle_ert(os, name + '{' + length + '}', context);
4201                         }
4202                 }
4203
4204                 // The single '=' is meant here.
4205                 else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
4206                         p.skip_spaces();
4207                         context.check_layout(os);
4208                         begin_inset(os, "Flex ");
4209                         os << to_utf8(newinsetlayout->name()) << '\n'
4210                            << "status collapsed\n";
4211                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
4212                         end_inset(os);
4213                 }
4214
4215                 else if (t.cs() == "includepdf") {
4216                         p.skip_spaces();
4217                         string const arg = p.getArg('[', ']');
4218                         map<string, string> opts;
4219                         vector<string> keys;
4220                         split_map(arg, opts, keys);
4221                         string name = normalize_filename(p.verbatim_item());
4222                         string const path = getMasterFilePath();
4223                         // We want to preserve relative / absolute filenames,
4224                         // therefore path is only used for testing
4225                         if (!makeAbsPath(name, path).exists()) {
4226                                 // The file extension is probably missing.
4227                                 // Now try to find it out.
4228                                 char const * const pdfpages_format[] = {"pdf", 0};
4229                                 string const pdftex_name =
4230                                         find_file(name, path, pdfpages_format);
4231                                 if (!pdftex_name.empty()) {
4232                                         name = pdftex_name;
4233                                         pdflatex = true;
4234                                 }
4235                         }
4236                         if (makeAbsPath(name, path).exists())
4237                                 fix_relative_filename(name);
4238                         else
4239                                 cerr << "Warning: Could not find file '"
4240                                      << name << "'." << endl;
4241                         // write output
4242                         context.check_layout(os);
4243                         begin_inset(os, "External\n\ttemplate ");
4244                         os << "PDFPages\n\tfilename "
4245                            << name << "\n";
4246                         // parse the options
4247                         if (opts.find("pages") != opts.end())
4248                                 os << "\textra LaTeX \"pages="
4249                                    << opts["pages"] << "\"\n";
4250                         if (opts.find("angle") != opts.end())
4251                                 os << "\trotateAngle "
4252                                    << opts["angle"] << '\n';
4253                         if (opts.find("origin") != opts.end()) {
4254                                 ostringstream ss;
4255                                 string const opt = opts["origin"];
4256                                 if (opt == "tl") ss << "topleft";
4257                                 if (opt == "bl") ss << "bottomleft";
4258                                 if (opt == "Bl") ss << "baselineleft";
4259                                 if (opt == "c") ss << "center";
4260                                 if (opt == "tc") ss << "topcenter";
4261                                 if (opt == "bc") ss << "bottomcenter";
4262                                 if (opt == "Bc") ss << "baselinecenter";
4263                                 if (opt == "tr") ss << "topright";
4264                                 if (opt == "br") ss << "bottomright";
4265                                 if (opt == "Br") ss << "baselineright";
4266                                 if (!ss.str().empty())
4267                                         os << "\trotateOrigin " << ss.str() << '\n';
4268                                 else
4269                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
4270                         }
4271                         if (opts.find("width") != opts.end())
4272                                 os << "\twidth "
4273                                    << translate_len(opts["width"]) << '\n';
4274                         if (opts.find("height") != opts.end())
4275                                 os << "\theight "
4276                                    << translate_len(opts["height"]) << '\n';
4277                         if (opts.find("keepaspectratio") != opts.end())
4278                                 os << "\tkeepAspectRatio\n";
4279                         end_inset(os);
4280                         context.check_layout(os);
4281                         registerExternalTemplatePackages("PDFPages");
4282                 }
4283
4284                 else if (t.cs() == "loadgame") {
4285                         p.skip_spaces();
4286                         string name = normalize_filename(p.verbatim_item());
4287                         string const path = getMasterFilePath();
4288                         // We want to preserve relative / absolute filenames,
4289                         // therefore path is only used for testing
4290                         if (!makeAbsPath(name, path).exists()) {
4291                                 // The file extension is probably missing.
4292                                 // Now try to find it out.
4293                                 char const * const lyxskak_format[] = {"fen", 0};
4294                                 string const lyxskak_name =
4295                                         find_file(name, path, lyxskak_format);
4296                                 if (!lyxskak_name.empty())
4297                                         name = lyxskak_name;
4298                         }
4299                         if (makeAbsPath(name, path).exists())
4300                                 fix_relative_filename(name);
4301                         else
4302                                 cerr << "Warning: Could not find file '"
4303                                      << name << "'." << endl;
4304                         context.check_layout(os);
4305                         begin_inset(os, "External\n\ttemplate ");
4306                         os << "ChessDiagram\n\tfilename "
4307                            << name << "\n";
4308                         end_inset(os);
4309                         context.check_layout(os);
4310                         // after a \loadgame follows a \showboard
4311                         if (p.get_token().asInput() == "showboard")
4312                                 p.get_token();
4313                         registerExternalTemplatePackages("ChessDiagram");
4314                 }
4315
4316                 else {
4317                         // try to see whether the string is in unicodesymbols
4318                         // Only use text mode commands, since we are in text mode here,
4319                         // and math commands may be invalid (bug 6797)
4320                         bool termination;
4321                         docstring rem;
4322                         set<string> req;
4323                         docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
4324                                         Encodings::TEXT_CMD, termination, rem, &req);
4325                         if (!s.empty()) {
4326                                 if (!rem.empty())
4327                                         cerr << "When parsing " << t.cs()
4328                                              << ", result is " << to_utf8(s)
4329                                              << "+" << to_utf8(rem) << endl;
4330                                 context.check_layout(os);
4331                                 os << to_utf8(s);
4332                                 if (termination)
4333                                         skip_spaces_braces(p);
4334                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
4335                                         preamble.registerAutomaticallyLoadedPackage(*it);
4336                         }
4337                         //cerr << "#: " << t << " mode: " << mode << endl;
4338                         // heuristic: read up to next non-nested space
4339                         /*
4340                         string s = t.asInput();
4341                         string z = p.verbatim_item();
4342                         while (p.good() && z != " " && z.size()) {
4343                                 //cerr << "read: " << z << endl;
4344                                 s += z;
4345                                 z = p.verbatim_item();
4346                         }
4347                         cerr << "found ERT: " << s << endl;
4348                         handle_ert(os, s + ' ', context);
4349                         */
4350                         else {
4351                                 string name = t.asInput();
4352                                 if (p.next_token().asInput() == "*") {
4353                                         // Starred commands like \vspace*{}
4354                                         p.get_token();  // Eat '*'
4355                                         name += '*';
4356                                 }
4357                                 if (!parse_command(name, p, os, outer, context))
4358                                         handle_ert(os, name, context);
4359                         }
4360                 }
4361
4362                 if (flags & FLAG_LEAVE) {
4363                         flags &= ~FLAG_LEAVE;
4364                         break;
4365                 }
4366         }
4367 }
4368
4369 // }])
4370
4371
4372 } // namespace lyx