src/tex2lyx/text.cpp

   1 /**
   2  * \file tex2lyx/text.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  * \author Uwe Stöhr
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 // {[(
  14
  15 #include <config.h>
  16
  17 #include "tex2lyx.h"
  18
  19 #include "Context.h"
  20 #include "Encoding.h"
  21 #include "FloatList.h"
  22 #include "LaTeXPackages.h"
  23 #include "Layout.h"
  24 #include "Length.h"
  25 #include "Preamble.h"
  26
  27 #include "insets/ExternalTemplate.h"
  28
  29 #include "support/lassert.h"
  30 #include "support/convert.h"
  31 #include "support/FileName.h"
  32 #include "support/filetools.h"
  33 #include "support/lstrings.h"
  34 #include "support/lyxtime.h"
  35
  36 #include <algorithm>
  37 #include <iostream>
  38 #include <map>
  39 #include <sstream>
  40 #include <vector>
  41
  42 using namespace std;
  43 using namespace lyx::support;
  44
  45 namespace lyx {
  46
  47
  48 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  49                 Context const & context, InsetLayout const * layout)
  50 {
  51         bool const forcePlainLayout =
  52                 layout ? layout->forcePlainLayout() : false;
  53         Context newcontext(true, context.textclass);
  54         if (forcePlainLayout)
  55                 newcontext.layout = &context.textclass.plainLayout();
  56         else
  57                 newcontext.font = context.font;
  58         parse_text(p, os, flags, outer, newcontext);
  59         newcontext.check_end_layout(os);
  60 }
  61
  62
  63 namespace {
  64
  65 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  66                 Context const & context, string const & name)
  67 {
  68         InsetLayout const * layout = 0;
  69         DocumentClass::InsetLayouts::const_iterator it =
  70                 context.textclass.insetLayouts().find(from_ascii(name));
  71         if (it != context.textclass.insetLayouts().end())
  72                 layout = &(it->second);
  73         parse_text_in_inset(p, os, flags, outer, context, layout);
  74 }
  75
  76 /// parses a paragraph snippet, useful for example for \\emph{...}
  77 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  78                 Context & context)
  79 {
  80         Context newcontext(context);
  81         // Don't inherit the paragraph-level extra stuff
  82         newcontext.par_extra_stuff.clear();
  83         parse_text(p, os, flags, outer, newcontext);
  84         // Make sure that we don't create invalid .lyx files
  85         context.need_layout = newcontext.need_layout;
  86         context.need_end_layout = newcontext.need_end_layout;
  87 }
  88
  89
  90 /*!
  91  * Thin wrapper around parse_text_snippet() using a string.
  92  *
  93  * We completely ignore \c context.need_layout and \c context.need_end_layout,
  94  * because our return value is not used directly (otherwise the stream version
  95  * of parse_text_snippet() could be used). That means that the caller needs
  96  * to do layout management manually.
  97  * This is intended to parse text that does not create any layout changes.
  98  */
  99 string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 100                   Context & context)
 101 {
 102         Context newcontext(context);
 103         newcontext.need_layout = false;
 104         newcontext.need_end_layout = false;
 105         newcontext.new_layout_allowed = false;
 106         // Avoid warning by Context::~Context()
 107         newcontext.par_extra_stuff.clear();
 108         ostringstream os;
 109         parse_text_snippet(p, os, flags, outer, newcontext);
 110         return os.str();
 111 }
 112
 113
 114 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 115  "vpageref", "prettyref", "eqref", 0 };
 116
 117 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
 118  "vpageref", "formatted", "eqref", 0 };
 119
 120 /**
 121  * known polyglossia language names (inluding synomyms)
 122  */
 123 const char * const polyglossia_languages[] = {
 124 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 125 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 126 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 127 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 128 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 129 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 130 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 131 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 132 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 133 "marathi", "spanish", 0};
 134
 135 /**
 136  * the same as polyglossia_languages with .lyx names
 137  * please keep this in sync with polyglossia_languages line by line!
 138  */
 139 const char * const coded_polyglossia_languages[] = {
 140 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 141 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 142 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 143 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 144 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 145 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 146 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 147 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 148 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 149 "marathi", "spanish", 0};
 150
 151 string polyglossia2lyx(string const & language)
 152 {
 153         char const * const * where = is_known(language, polyglossia_languages);
 154         if (where)
 155                 return coded_polyglossia_languages[where - polyglossia_languages];
 156         return language;
 157 }
 158
 159 /*!
 160  * natbib commands.
 161  * The starred forms are also known except for "citefullauthor",
 162  * "citeyear" and "citeyearpar".
 163  */
 164 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 165 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 166 "citefullauthor", "Citet", "Citep", "Citealt", "Citealp", "Citeauthor", 0 };
 167
 168 /*!
 169  * jurabib commands.
 170  * No starred form other than "cite*" known.
 171  */
 172 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 173 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 174 // jurabib commands not (yet) supported by LyX:
 175 // "fullcite",
 176 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 177 // "footciteauthor", "footciteyear", "footciteyearpar",
 178 "citefield", "citetitle", 0 };
 179
 180 /// LaTeX names for quotes
 181 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
 182 "guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
 183 "quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
 184 "guilsinglleft", "frq", "guilsinglright", 0};
 185
 186 /// the same as known_quotes with .lyx names
 187 char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 188 "ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
 189 "gld", "grd", "gls", "ers", "fls",
 190 "fls", "frs", "frs", 0};
 191
 192 /// LaTeX names for font sizes
 193 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 194 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 195
 196 /// the same as known_sizes with .lyx names
 197 char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 198 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 199
 200 /// LaTeX 2.09 names for font families
 201 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 202
 203 /// LaTeX names for font families
 204 char const * const known_font_families[] = { "rmfamily", "sffamily",
 205 "ttfamily", 0};
 206
 207 /// LaTeX names for font family changing commands
 208 char const * const known_text_font_families[] = { "textrm", "textsf",
 209 "texttt", 0};
 210
 211 /// The same as known_old_font_families, known_font_families and
 212 /// known_text_font_families with .lyx names
 213 char const * const known_coded_font_families[] = { "roman", "sans",
 214 "typewriter", 0};
 215
 216 /// LaTeX 2.09 names for font series
 217 char const * const known_old_font_series[] = { "bf", 0};
 218
 219 /// LaTeX names for font series
 220 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 221
 222 /// LaTeX names for font series changing commands
 223 char const * const known_text_font_series[] = { "textbf", "textmd", 0};
 224
 225 /// The same as known_old_font_series, known_font_series and
 226 /// known_text_font_series with .lyx names
 227 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 228
 229 /// LaTeX 2.09 names for font shapes
 230 char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 231
 232 /// LaTeX names for font shapes
 233 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 234 "upshape", 0};
 235
 236 /// LaTeX names for font shape changing commands
 237 char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
 238 "textup", 0};
 239
 240 /// The same as known_old_font_shapes, known_font_shapes and
 241 /// known_text_font_shapes with .lyx names
 242 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 243 "smallcaps", "up", 0};
 244
 245 /// Known special characters which need skip_spaces_braces() afterwards
 246 char const * const known_special_chars[] = {"ldots", "lyxarrow",
 247 "textcompwordmark", "slash", 0};
 248
 249 /// the same as known_special_chars with .lyx names
 250 char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
 251 "textcompwordmark{}", "slash{}", 0};
 252
 253 /*!
 254  * Graphics file extensions known by the dvips driver of the graphics package.
 255  * These extensions are used to complete the filename of an included
 256  * graphics file if it does not contain an extension.
 257  * The order must be the same that latex uses to find a file, because we
 258  * will use the first extension that matches.
 259  * This is only an approximation for the common cases. If we would want to
 260  * do it right in all cases, we would need to know which graphics driver is
 261  * used and know the extensions of every driver of the graphics package.
 262  */
 263 char const * const known_dvips_graphics_formats[] = {"eps", "ps", "eps.gz",
 264 "ps.gz", "eps.Z", "ps.Z", 0};
 265
 266 /*!
 267  * Graphics file extensions known by the pdftex driver of the graphics package.
 268  * \sa known_dvips_graphics_formats
 269  */
 270 char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 271 "mps", "tif", 0};
 272
 273 /*!
 274  * Known file extensions for TeX files as used by \\include.
 275  */
 276 char const * const known_tex_extensions[] = {"tex", 0};
 277
 278 /// spaces known by InsetSpace
 279 char const * const known_spaces[] = { " ", "space", ",",
 280 "thinspace", "quad", "qquad", "enspace", "enskip",
 281 "negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
 282 "hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
 283 "upbracefill", "downbracefill", 0};
 284
 285 /// the same as known_spaces with .lyx names
 286 char const * const known_coded_spaces[] = { "space{}", "space{}",
 287 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
 288 "negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
 289 "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
 290 "upbracefill{}", "downbracefill{}", 0};
 291
 292 /// These are translated by LyX to commands like "\\LyX{}", so we have to put
 293 /// them in ERT. "LaTeXe" must come before "LaTeX"!
 294 char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 295 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 296 int const known_phrase_lengths[] = {3, 5, 7, 0};
 297
 298 // string to store the float type to be able to determine the type of subfloats
 299 string float_type = "";
 300
 301
 302 /// splits "x=z, y=b" into a map and an ordered keyword vector
 303 void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 304 {
 305         vector<string> v;
 306         split(s, v);
 307         res.clear();
 308         keys.resize(v.size());
 309         for (size_t i = 0; i < v.size(); ++i) {
 310                 size_t const pos   = v[i].find('=');
 311                 string const index = trimSpaceAndEol(v[i].substr(0, pos));
 312                 string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
 313                 res[index] = value;
 314                 keys[i] = index;
 315         }
 316 }
 317
 318
 319 /*!
 320  * Split a LaTeX length into value and unit.
 321  * The latter can be a real unit like "pt", or a latex length variable
 322  * like "\textwidth". The unit may contain additional stuff like glue
 323  * lengths, but we don't care, because such lengths are ERT anyway.
 324  * \returns true if \p value and \p unit are valid.
 325  */
 326 bool splitLatexLength(string const & len, string & value, string & unit)
 327 {
 328         if (len.empty())
 329                 return false;
 330         const string::size_type i = len.find_first_not_of(" -+0123456789.,");
 331         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 332         string const length = subst(len, ',', '.');
 333         if (i == string::npos)
 334                 return false;
 335         if (i == 0) {
 336                 if (len[0] == '\\') {
 337                         // We had something like \textwidth without a factor
 338                         value = "1.0";
 339                 } else {
 340                         return false;
 341                 }
 342         } else {
 343                 value = trimSpaceAndEol(string(length, 0, i));
 344         }
 345         if (value == "-")
 346                 value = "-1.0";
 347         // 'cM' is a valid LaTeX length unit. Change it to 'cm'
 348         if (contains(len, '\\'))
 349                 unit = trimSpaceAndEol(string(len, i));
 350         else
 351                 unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
 352         return true;
 353 }
 354
 355
 356 /// A simple function to translate a latex length to something LyX can
 357 /// understand. Not perfect, but rather best-effort.
 358 bool translate_len(string const & length, string & valstring, string & unit)
 359 {
 360         if (!splitLatexLength(length, valstring, unit))
 361                 return false;
 362         // LyX uses percent values
 363         double value;
 364         istringstream iss(valstring);
 365         iss >> value;
 366         value *= 100;
 367         ostringstream oss;
 368         oss << value;
 369         string const percentval = oss.str();
 370         // a normal length
 371         if (unit.empty() || unit[0] != '\\')
 372                 return true;
 373         string::size_type const i = unit.find(' ');
 374         string const endlen = (i == string::npos) ? string() : string(unit, i);
 375         if (unit == "\\textwidth") {
 376                 valstring = percentval;
 377                 unit = "text%" + endlen;
 378         } else if (unit == "\\columnwidth") {
 379                 valstring = percentval;
 380                 unit = "col%" + endlen;
 381         } else if (unit == "\\paperwidth") {
 382                 valstring = percentval;
 383                 unit = "page%" + endlen;
 384         } else if (unit == "\\linewidth") {
 385                 valstring = percentval;
 386                 unit = "line%" + endlen;
 387         } else if (unit == "\\paperheight") {
 388                 valstring = percentval;
 389                 unit = "pheight%" + endlen;
 390         } else if (unit == "\\textheight") {
 391                 valstring = percentval;
 392                 unit = "theight%" + endlen;
 393         }
 394         return true;
 395 }
 396
 397 }
 398
 399
 400 string translate_len(string const & length)
 401 {
 402         string unit;
 403         string value;
 404         if (translate_len(length, value, unit))
 405                 return value + unit;
 406         // If the input is invalid, return what we have.
 407         return length;
 408 }
 409
 410
 411 namespace {
 412
 413 /*!
 414  * Translates a LaTeX length into \p value, \p unit and
 415  * \p special parts suitable for a box inset.
 416  * The difference from translate_len() is that a box inset knows about
 417  * some special "units" that are stored in \p special.
 418  */
 419 void translate_box_len(string const & length, string & value, string & unit, string & special)
 420 {
 421         if (translate_len(length, value, unit)) {
 422                 if (unit == "\\height" || unit == "\\depth" ||
 423                     unit == "\\totalheight" || unit == "\\width") {
 424                         special = unit.substr(1);
 425                         // The unit is not used, but LyX requires a dummy setting
 426                         unit = "in";
 427                 } else
 428                         special = "none";
 429         } else {
 430                 value.clear();
 431                 unit = length;
 432                 special = "none";
 433         }
 434 }
 435
 436
 437 /*!
 438  * Find a file with basename \p name in path \p path and an extension
 439  * in \p extensions.
 440  */
 441 string find_file(string const & name, string const & path,
 442                  char const * const * extensions)
 443 {
 444         for (char const * const * what = extensions; *what; ++what) {
 445                 string const trial = addExtension(name, *what);
 446                 if (makeAbsPath(trial, path).exists())
 447                         return trial;
 448         }
 449         return string();
 450 }
 451
 452
 453 void begin_inset(ostream & os, string const & name)
 454 {
 455         os << "\n\\begin_inset " << name;
 456 }
 457
 458
 459 void begin_command_inset(ostream & os, string const & name,
 460                          string const & latexname)
 461 {
 462         begin_inset(os, "CommandInset ");
 463         os << name << "\nLatexCommand " << latexname << '\n';
 464 }
 465
 466
 467 void end_inset(ostream & os)
 468 {
 469         os << "\n\\end_inset\n\n";
 470 }
 471
 472
 473 bool skip_braces(Parser & p)
 474 {
 475         if (p.next_token().cat() != catBegin)
 476                 return false;
 477         p.get_token();
 478         if (p.next_token().cat() == catEnd) {
 479                 p.get_token();
 480                 return true;
 481         }
 482         p.putback();
 483         return false;
 484 }
 485
 486
 487 /// replace LaTeX commands in \p s from the unicodesymbols file with their
 488 /// unicode points
 489 docstring convert_unicodesymbols(docstring s)
 490 {
 491         odocstringstream os;
 492         for (size_t i = 0; i < s.size();) {
 493                 if (s[i] != '\\') {
 494                         os.put(s[i++]);
 495                         continue;
 496                 }
 497                 s = s.substr(i);
 498                 bool termination;
 499                 docstring rem;
 500                 set<string> req;
 501                 docstring parsed = encodings.fromLaTeXCommand(s,
 502                                 Encodings::TEXT_CMD, termination, rem, &req);
 503                 set<string>::const_iterator it = req.begin();
 504                 set<string>::const_iterator en = req.end();
 505                 for (; it != en; ++it)
 506                         preamble.registerAutomaticallyLoadedPackage(*it);
 507                 os << parsed;
 508                 s = rem;
 509                 if (s.empty() || s[0] != '\\')
 510                         i = 0;
 511                 else
 512                         i = 1;
 513         }
 514         return os.str();
 515 }
 516
 517
 518 /// try to convert \p s to a valid InsetCommand argument
 519 string convert_command_inset_arg(string s)
 520 {
 521         if (isAscii(s))
 522                 // since we don't know the input encoding we can't use from_utf8
 523                 s = to_utf8(convert_unicodesymbols(from_ascii(s)));
 524         // LyX cannot handle newlines in a latex command
 525         return subst(s, "\n", " ");
 526 }
 527
 528
 529 void handle_backslash(ostream & os, string const & s)
 530 {
 531         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 532                 if (*it == '\\')
 533                         os << "\n\\backslash\n";
 534                 else
 535                         os << *it;
 536         }
 537 }
 538
 539
 540 void handle_ert(ostream & os, string const & s, Context & context)
 541 {
 542         // We must have a valid layout before outputting the ERT inset.
 543         context.check_layout(os);
 544         Context newcontext(true, context.textclass);
 545         begin_inset(os, "ERT");
 546         os << "\nstatus collapsed\n";
 547         newcontext.check_layout(os);
 548         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 549                 if (*it == '\\')
 550                         os << "\n\\backslash\n";
 551                 else if (*it == '\n') {
 552                         newcontext.new_paragraph(os);
 553                         newcontext.check_layout(os);
 554                 } else
 555                         os << *it;
 556         }
 557         newcontext.check_end_layout(os);
 558         end_inset(os);
 559 }
 560
 561
 562 void handle_comment(ostream & os, string const & s, Context & context)
 563 {
 564         // TODO: Handle this better
 565         Context newcontext(true, context.textclass);
 566         begin_inset(os, "ERT");
 567         os << "\nstatus collapsed\n";
 568         newcontext.check_layout(os);
 569         handle_backslash(os, s);
 570         // make sure that our comment is the last thing on the line
 571         newcontext.new_paragraph(os);
 572         newcontext.check_layout(os);
 573         newcontext.check_end_layout(os);
 574         end_inset(os);
 575 }
 576
 577
 578 Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
 579 {
 580         Layout const * layout = findLayoutWithoutModule(textclass, name, command);
 581         if (layout)
 582                 return layout;
 583         if (checkModule(name, command))
 584                 return findLayoutWithoutModule(textclass, name, command);
 585         return layout;
 586 }
 587
 588
 589 InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 590 {
 591         InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
 592         if (insetlayout)
 593                 return insetlayout;
 594         if (checkModule(name, command))
 595                 return findInsetLayoutWithoutModule(textclass, name, command);
 596         return insetlayout;
 597 }
 598
 599
 600 void eat_whitespace(Parser &, ostream &, Context &, bool);
 601
 602
 603 /*!
 604  * Skips whitespace and braces.
 605  * This should be called after a command has been parsed that is not put into
 606  * ERT, and where LyX adds "{}" if needed.
 607  */
 608 void skip_spaces_braces(Parser & p, bool keepws = false)
 609 {
 610         /* The following four examples produce the same typeset output and
 611            should be handled by this function:
 612            - abc \j{} xyz
 613            - abc \j {} xyz
 614            - abc \j
 615              {} xyz
 616            - abc \j %comment
 617              {} xyz
 618          */
 619         // Unfortunately we need to skip comments, too.
 620         // We can't use eat_whitespace since writing them after the {}
 621         // results in different output in some cases.
 622         bool const skipped_spaces = p.skip_spaces(true);
 623         bool const skipped_braces = skip_braces(p);
 624         if (keepws && skipped_spaces && !skipped_braces)
 625                 // put back the space (it is better handled by check_space)
 626                 p.unskip_spaces(true);
 627 }
 628
 629
 630 void output_command_layout(ostream & os, Parser & p, bool outer,
 631                            Context & parent_context,
 632                            Layout const * newlayout)
 633 {
 634         TeXFont const oldFont = parent_context.font;
 635         // save the current font size
 636         string const size = oldFont.size;
 637         // reset the font size to default, because the font size switches
 638         // don't affect section headings and the like
 639         parent_context.font.size = Context::normalfont.size;
 640         // we only need to write the font change if we have an open layout
 641         if (!parent_context.atParagraphStart())
 642                 output_font_change(os, oldFont, parent_context.font);
 643         parent_context.check_end_layout(os);
 644         Context context(true, parent_context.textclass, newlayout,
 645                         parent_context.layout, parent_context.font);
 646         if (parent_context.deeper_paragraph) {
 647                 // We are beginning a nested environment after a
 648                 // deeper paragraph inside the outer list environment.
 649                 // Therefore we don't need to output a "begin deeper".
 650                 context.need_end_deeper = true;
 651         }
 652         context.check_deeper(os);
 653         context.check_layout(os);
 654         unsigned int optargs = 0;
 655         while (optargs < context.layout->optargs) {
 656                 eat_whitespace(p, os, context, false);
 657                 if (p.next_token().cat() == catEscape ||
 658                     p.next_token().character() != '[')
 659                         break;
 660                 p.get_token(); // eat '['
 661                 begin_inset(os, "Argument\n");
 662                 os << "status collapsed\n\n";
 663                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 664                 end_inset(os);
 665                 eat_whitespace(p, os, context, false);
 666                 ++optargs;
 667         }
 668         unsigned int reqargs = 0;
 669         while (reqargs < context.layout->reqargs) {
 670                 eat_whitespace(p, os, context, false);
 671                 if (p.next_token().cat() != catBegin)
 672                         break;
 673                 p.get_token(); // eat '{'
 674                 begin_inset(os, "Argument\n");
 675                 os << "status collapsed\n\n";
 676                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
 677                 end_inset(os);
 678                 eat_whitespace(p, os, context, false);
 679                 ++reqargs;
 680         }
 681         parse_text(p, os, FLAG_ITEM, outer, context);
 682         context.check_end_layout(os);
 683         if (parent_context.deeper_paragraph) {
 684                 // We must suppress the "end deeper" because we
 685                 // suppressed the "begin deeper" above.
 686                 context.need_end_deeper = false;
 687         }
 688         context.check_end_deeper(os);
 689         // We don't need really a new paragraph, but
 690         // we must make sure that the next item gets a \begin_layout.
 691         parent_context.new_paragraph(os);
 692         // Set the font size to the original value. No need to output it here
 693         // (Context::begin_layout() will do that if needed)
 694         parent_context.font.size = size;
 695 }
 696
 697
 698 /*!
 699  * Output a space if necessary.
 700  * This function gets called for every whitespace token.
 701  *
 702  * We have three cases here:
 703  * 1. A space must be suppressed. Example: The lyxcode case below
 704  * 2. A space may be suppressed. Example: Spaces before "\par"
 705  * 3. A space must not be suppressed. Example: A space between two words
 706  *
 707  * We currently handle only 1. and 3 and from 2. only the case of
 708  * spaces before newlines as a side effect.
 709  *
 710  * 2. could be used to suppress as many spaces as possible. This has two effects:
 711  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 712  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 713  * The drawback is that the logic inside the function becomes
 714  * complicated, and that is the reason why it is not implemented.
 715  */
 716 void check_space(Parser & p, ostream & os, Context & context)
 717 {
 718         Token const next = p.next_token();
 719         Token const curr = p.curr_token();
 720         // A space before a single newline and vice versa must be ignored
 721         // LyX emits a newline before \end{lyxcode}.
 722         // This newline must be ignored,
 723         // otherwise LyX will add an additional protected space.
 724         if (next.cat() == catSpace ||
 725             next.cat() == catNewline ||
 726             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 727                 return;
 728         }
 729         context.check_layout(os);
 730         os << ' ';
 731 }
 732
 733
 734 /*!
 735  * Parse all arguments of \p command
 736  */
 737 void parse_arguments(string const & command,
 738                      vector<ArgumentType> const & template_arguments,
 739                      Parser & p, ostream & os, bool outer, Context & context)
 740 {
 741         string ert = command;
 742         size_t no_arguments = template_arguments.size();
 743         for (size_t i = 0; i < no_arguments; ++i) {
 744                 switch (template_arguments[i]) {
 745                 case required:
 746                 case req_group:
 747                         // This argument contains regular LaTeX
 748                         handle_ert(os, ert + '{', context);
 749                         eat_whitespace(p, os, context, false);
 750                         if (template_arguments[i] == required)
 751                                 parse_text(p, os, FLAG_ITEM, outer, context);
 752                         else
 753                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 754                         ert = "}";
 755                         break;
 756                 case item:
 757                         // This argument consists only of a single item.
 758                         // The presence of '{' or not must be preserved.
 759                         p.skip_spaces();
 760                         if (p.next_token().cat() == catBegin)
 761                                 ert += '{' + p.verbatim_item() + '}';
 762                         else
 763                                 ert += p.verbatim_item();
 764                         break;
 765                 case displaymath:
 766                 case verbatim:
 767                         // This argument may contain special characters
 768                         ert += '{' + p.verbatim_item() + '}';
 769                         break;
 770                 case optional:
 771                 case opt_group:
 772                         // true because we must not eat whitespace
 773                         // if an optional arg follows we must not strip the
 774                         // brackets from this one
 775                         if (i < no_arguments - 1 &&
 776                             template_arguments[i+1] == optional)
 777                                 ert += p.getFullOpt(true);
 778                         else
 779                                 ert += p.getOpt(true);
 780                         break;
 781                 }
 782         }
 783         handle_ert(os, ert, context);
 784 }
 785
 786
 787 /*!
 788  * Check whether \p command is a known command. If yes,
 789  * handle the command with all arguments.
 790  * \return true if the command was parsed, false otherwise.
 791  */
 792 bool parse_command(string const & command, Parser & p, ostream & os,
 793                    bool outer, Context & context)
 794 {
 795         if (known_commands.find(command) != known_commands.end()) {
 796                 parse_arguments(command, known_commands[command], p, os,
 797                                 outer, context);
 798                 return true;
 799         }
 800         return false;
 801 }
 802
 803
 804 /// Parses a minipage or parbox
 805 void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 806                unsigned inner_flags, bool outer, Context & parent_context,
 807                string const & outer_type, string const & special,
 808                string const & inner_type)
 809 {
 810         string position;
 811         string inner_pos;
 812         string hor_pos = "c";
 813         // We need to set the height to the LaTeX default of 1\\totalheight
 814         // for the case when no height argument is given
 815         string height_value = "1";
 816         string height_unit = "in";
 817         string height_special = "totalheight";
 818         string latex_height;
 819         string width_value;
 820         string width_unit;
 821         string latex_width;
 822         string width_special = "none";
 823         if (!inner_type.empty() && p.hasOpt()) {
 824                 if (inner_type != "makebox")
 825                         position = p.getArg('[', ']');
 826                 else {
 827                         latex_width = p.getArg('[', ']');
 828                         translate_box_len(latex_width, width_value, width_unit, width_special);
 829                         position = "t";
 830                 }
 831                 if (position != "t" && position != "c" && position != "b") {
 832                         cerr << "invalid position " << position << " for "
 833                              << inner_type << endl;
 834                         position = "c";
 835                 }
 836                 if (p.hasOpt()) {
 837                         if (inner_type != "makebox") {
 838                                 latex_height = p.getArg('[', ']');
 839                                 translate_box_len(latex_height, height_value, height_unit, height_special);
 840                         } else
 841                                 hor_pos = p.getArg('[', ']');
 842
 843                         if (p.hasOpt()) {
 844                                 inner_pos = p.getArg('[', ']');
 845                                 if (inner_pos != "c" && inner_pos != "t" &&
 846                                     inner_pos != "b" && inner_pos != "s") {
 847                                         cerr << "invalid inner_pos "
 848                                              << inner_pos << " for "
 849                                              << inner_type << endl;
 850                                         inner_pos = position;
 851                                 }
 852                         }
 853                 }
 854         }
 855         if (inner_type.empty()) {
 856                 if (special.empty() && outer_type != "framebox")
 857                         latex_width = "1\\columnwidth";
 858                 else {
 859                         Parser p2(special);
 860                         latex_width = p2.getArg('[', ']');
 861                         string const opt = p2.getArg('[', ']');
 862                         if (!opt.empty()) {
 863                                 hor_pos = opt;
 864                                 if (hor_pos != "l" && hor_pos != "c" &&
 865                                     hor_pos != "r") {
 866                                         cerr << "invalid hor_pos " << hor_pos
 867                                              << " for " << outer_type << endl;
 868                                         hor_pos = "c";
 869                                 }
 870                         }
 871                 }
 872         } else if (inner_type != "makebox")
 873                 latex_width = p.verbatim_item();
 874         // if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 875         // as this is LyX's standard for such cases (except for makebox)
 876         // \framebox is more special and handled below
 877         if (latex_width.empty() && inner_type != "makebox"
 878                 && outer_type != "framebox")
 879                 latex_width = "1\\columnwidth";
 880
 881         translate_len(latex_width, width_value, width_unit);
 882
 883         bool shadedparbox = false;
 884         if (inner_type == "shaded") {
 885                 eat_whitespace(p, os, parent_context, false);
 886                 if (outer_type == "parbox") {
 887                         // Eat '{'
 888                         if (p.next_token().cat() == catBegin)
 889                                 p.get_token();
 890                         eat_whitespace(p, os, parent_context, false);
 891                         shadedparbox = true;
 892                 }
 893                 p.get_token();
 894                 p.getArg('{', '}');
 895         }
 896         // If we already read the inner box we have to push the inner env
 897         if (!outer_type.empty() && !inner_type.empty() &&
 898             (inner_flags & FLAG_END))
 899                 active_environments.push_back(inner_type);
 900         // LyX can't handle length variables
 901         bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
 902         if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
 903                 // Look whether there is some content after the end of the
 904                 // inner box, but before the end of the outer box.
 905                 // If yes, we need to output ERT.
 906                 p.pushPosition();
 907                 if (inner_flags & FLAG_END)
 908                         p.verbatimEnvironment(inner_type);
 909                 else
 910                         p.verbatim_item();
 911                 p.skip_spaces(true);
 912                 bool const outer_env(outer_type == "framed" || outer_type == "minipage");
 913                 if ((outer_env && p.next_token().asInput() != "\\end") ||
 914                     (!outer_env && p.next_token().cat() != catEnd)) {
 915                         // something is between the end of the inner box and
 916                         // the end of the outer box, so we need to use ERT.
 917                         use_ert = true;
 918                 }
 919                 p.popPosition();
 920         }
 921         // if only \makebox{content} was used we can set its width to 1\width
 922         // because this identic and also identic to \mbox
 923         // this doesn't work for \framebox{content}, thus we have to use ERT for this
 924         if (latex_width.empty() && inner_type == "makebox") {
 925                 width_value = "1";
 926                 width_unit = "in";
 927                 width_special = "width";
 928         } else if (latex_width.empty() && outer_type == "framebox") {
 929                 use_ert = true;
 930         }
 931         if (use_ert) {
 932                 ostringstream ss;
 933                 if (!outer_type.empty()) {
 934                         if (outer_flags & FLAG_END)
 935                                 ss << "\\begin{" << outer_type << '}';
 936                         else {
 937                                 ss << '\\' << outer_type << '{';
 938                                 if (!special.empty())
 939                                         ss << special;
 940                         }
 941                 }
 942                 if (!inner_type.empty()) {
 943                         if (inner_type != "shaded") {
 944                                 if (inner_flags & FLAG_END)
 945                                         ss << "\\begin{" << inner_type << '}';
 946                                 else
 947                                         ss << '\\' << inner_type;
 948                         }
 949                         if (!position.empty())
 950                                 ss << '[' << position << ']';
 951                         if (!latex_height.empty())
 952                                 ss << '[' << latex_height << ']';
 953                         if (!inner_pos.empty())
 954                                 ss << '[' << inner_pos << ']';
 955                         ss << '{' << latex_width << '}';
 956                         if (!(inner_flags & FLAG_END))
 957                                 ss << '{';
 958                 }
 959                 if (inner_type == "shaded")
 960                         ss << "\\begin{shaded}";
 961                 handle_ert(os, ss.str(), parent_context);
 962                 if (!inner_type.empty()) {
 963                         parse_text(p, os, inner_flags, outer, parent_context);
 964                         if (inner_flags & FLAG_END)
 965                                 handle_ert(os, "\\end{" + inner_type + '}',
 966                                            parent_context);
 967                         else
 968                                 handle_ert(os, "}", parent_context);
 969                 }
 970                 if (!outer_type.empty()) {
 971                         // If we already read the inner box we have to pop
 972                         // the inner env
 973                         if (!inner_type.empty() && (inner_flags & FLAG_END))
 974                                 active_environments.pop_back();
 975
 976                         // Ensure that the end of the outer box is parsed correctly:
 977                         // The opening brace has been eaten by parse_outer_box()
 978                         if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
 979                                 outer_flags &= ~FLAG_ITEM;
 980                                 outer_flags |= FLAG_BRACE_LAST;
 981                         }
 982                         parse_text(p, os, outer_flags, outer, parent_context);
 983                         if (outer_flags & FLAG_END)
 984                                 handle_ert(os, "\\end{" + outer_type + '}',
 985                                            parent_context);
 986                         else if (inner_type.empty() && outer_type == "framebox")
 987                                 // in this case it is already closed later
 988                                 ;
 989                         else
 990                                 handle_ert(os, "}", parent_context);
 991                 }
 992         } else {
 993                 // LyX does not like empty positions, so we have
 994                 // to set them to the LaTeX default values here.
 995                 if (position.empty())
 996                         position = "c";
 997                 if (inner_pos.empty())
 998                         inner_pos = position;
 999                 parent_context.check_layout(os);
1000                 begin_inset(os, "Box ");
1001                 if (outer_type == "framed")
1002                         os << "Framed\n";
1003                 else if (outer_type == "framebox")
1004                         os << "Boxed\n";
1005                 else if (outer_type == "shadowbox")
1006                         os << "Shadowbox\n";
1007                 else if ((outer_type == "shaded" && inner_type.empty()) ||
1008                              (outer_type == "minipage" && inner_type == "shaded") ||
1009                              (outer_type == "parbox" && inner_type == "shaded")) {
1010                         os << "Shaded\n";
1011                         preamble.registerAutomaticallyLoadedPackage("color");
1012                 } else if (outer_type == "doublebox")
1013                         os << "Doublebox\n";
1014                 else if (outer_type.empty())
1015                         os << "Frameless\n";
1016                 else
1017                         os << outer_type << '\n';
1018                 os << "position \"" << position << "\"\n";
1019                 os << "hor_pos \"" << hor_pos << "\"\n";
1020                 os << "has_inner_box " << !inner_type.empty() << "\n";
1021                 os << "inner_pos \"" << inner_pos << "\"\n";
1022                 os << "use_parbox " << (inner_type == "parbox" || shadedparbox)
1023                    << '\n';
1024                 os << "use_makebox " << (inner_type == "makebox") << '\n';
1025                 os << "width \"" << width_value << width_unit << "\"\n";
1026                 os << "special \"" << width_special << "\"\n";
1027                 os << "height \"" << height_value << height_unit << "\"\n";
1028                 os << "height_special \"" << height_special << "\"\n";
1029                 os << "status open\n\n";
1030
1031                 // Unfortunately we can't use parse_text_in_inset:
1032                 // InsetBox::forcePlainLayout() is hard coded and does not
1033                 // use the inset layout. Apart from that do we call parse_text
1034                 // up to two times, but need only one check_end_layout.
1035                 bool const forcePlainLayout =
1036                         (!inner_type.empty() || inner_type == "makebox") &&
1037                         outer_type != "shaded" && outer_type != "framed";
1038                 Context context(true, parent_context.textclass);
1039                 if (forcePlainLayout)
1040                         context.layout = &context.textclass.plainLayout();
1041                 else
1042                         context.font = parent_context.font;
1043
1044                 // If we have no inner box the contents will be read with the outer box
1045                 if (!inner_type.empty())
1046                         parse_text(p, os, inner_flags, outer, context);
1047
1048                 // Ensure that the end of the outer box is parsed correctly:
1049                 // The opening brace has been eaten by parse_outer_box()
1050                 if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
1051                         outer_flags &= ~FLAG_ITEM;
1052                         outer_flags |= FLAG_BRACE_LAST;
1053                 }
1054
1055                 // Find end of outer box, output contents if inner_type is
1056                 // empty and output possible comments
1057                 if (!outer_type.empty()) {
1058                         // If we already read the inner box we have to pop
1059                         // the inner env
1060                         if (!inner_type.empty() && (inner_flags & FLAG_END))
1061                                 active_environments.pop_back();
1062                         // This does not output anything but comments if
1063                         // inner_type is not empty (see use_ert)
1064                         parse_text(p, os, outer_flags, outer, context);
1065                 }
1066
1067                 context.check_end_layout(os);
1068                 end_inset(os);
1069 #ifdef PRESERVE_LAYOUT
1070                 // LyX puts a % after the end of the minipage
1071                 if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
1072                         // new paragraph
1073                         //handle_comment(os, "%dummy", parent_context);
1074                         p.get_token();
1075                         p.skip_spaces();
1076                         parent_context.new_paragraph(os);
1077                 }
1078                 else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
1079                         //handle_comment(os, "%dummy", parent_context);
1080                         p.get_token();
1081                         p.skip_spaces();
1082                         // We add a protected space if something real follows
1083                         if (p.good() && p.next_token().cat() != catComment) {
1084                                 begin_inset(os, "space ~\n");
1085                                 end_inset(os);
1086                         }
1087                 }
1088 #endif
1089         }
1090 }
1091
1092
1093 void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
1094                      Context & parent_context, string const & outer_type,
1095                      string const & special)
1096 {
1097         eat_whitespace(p, os, parent_context, false);
1098         if (flags & FLAG_ITEM) {
1099                 // Eat '{'
1100                 if (p.next_token().cat() == catBegin)
1101                         p.get_token();
1102                 else
1103                         cerr << "Warning: Ignoring missing '{' after \\"
1104                              << outer_type << '.' << endl;
1105                 eat_whitespace(p, os, parent_context, false);
1106         }
1107         string inner;
1108         unsigned int inner_flags = 0;
1109         p.pushPosition();
1110         if (outer_type == "minipage" || outer_type == "parbox") {
1111                 p.skip_spaces(true);
1112                 while (p.hasOpt()) {
1113                         p.getArg('[', ']');
1114                         p.skip_spaces(true);
1115                 }
1116                 p.getArg('{', '}');
1117                 p.skip_spaces(true);
1118                 if (outer_type == "parbox") {
1119                         // Eat '{'
1120                         if (p.next_token().cat() == catBegin)
1121                                 p.get_token();
1122                         p.skip_spaces(true);
1123                 }
1124         }
1125         if (outer_type == "shaded") {
1126                 // These boxes never have an inner box
1127                 ;
1128         } else if (p.next_token().asInput() == "\\parbox") {
1129                 inner = p.get_token().cs();
1130                 inner_flags = FLAG_ITEM;
1131         } else if (p.next_token().asInput() == "\\begin") {
1132                 // Is this a minipage or shaded box?
1133                 p.pushPosition();
1134                 p.get_token();
1135                 inner = p.getArg('{', '}');
1136                 p.popPosition();
1137                 if (inner == "minipage" || inner == "shaded")
1138                         inner_flags = FLAG_END;
1139                 else
1140                         inner = "";
1141         }
1142         p.popPosition();
1143         if (inner_flags == FLAG_END) {
1144                 if (inner != "shaded")
1145                 {
1146                         p.get_token();
1147                         p.getArg('{', '}');
1148                         eat_whitespace(p, os, parent_context, false);
1149                 }
1150                 parse_box(p, os, flags, FLAG_END, outer, parent_context,
1151                           outer_type, special, inner);
1152         } else {
1153                 if (inner_flags == FLAG_ITEM) {
1154                         p.get_token();
1155                         eat_whitespace(p, os, parent_context, false);
1156                 }
1157                 parse_box(p, os, flags, inner_flags, outer, parent_context,
1158                           outer_type, special, inner);
1159         }
1160 }
1161
1162
1163 void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
1164 {
1165         parent_context.check_layout(os);
1166         begin_inset(os, "listings\n");
1167         if (p.hasOpt()) {
1168                 string arg = p.verbatimOption();
1169                 os << "lstparams " << '"' << arg << '"' << '\n';
1170         }
1171         if (in_line)
1172                 os << "inline true\n";
1173         else
1174                 os << "inline false\n";
1175         os << "status collapsed\n";
1176         Context context(true, parent_context.textclass);
1177         context.layout = &parent_context.textclass.plainLayout();
1178         string s;
1179         if (in_line) {
1180                 s = p.plainCommand('!', '!', "lstinline");
1181                 context.new_paragraph(os);
1182                 context.check_layout(os);
1183         } else
1184                 s = p.plainEnvironment("lstlisting");
1185         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1186                 if (*it == '\\')
1187                         os << "\n\\backslash\n";
1188                 else if (*it == '\n') {
1189                         // avoid adding an empty paragraph at the end
1190                         if (it + 1 != et) {
1191                                 context.new_paragraph(os);
1192                                 context.check_layout(os);
1193                         }
1194                 } else
1195                         os << *it;
1196         }
1197         context.check_end_layout(os);
1198         end_inset(os);
1199 }
1200
1201
1202 /// parse an unknown environment
1203 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
1204                                unsigned flags, bool outer,
1205                                Context & parent_context)
1206 {
1207         if (name == "tabbing")
1208                 // We need to remember that we have to handle '\=' specially
1209                 flags |= FLAG_TABBING;
1210
1211         // We need to translate font changes and paragraphs inside the
1212         // environment to ERT if we have a non standard font.
1213         // Otherwise things like
1214         // \large\begin{foo}\huge bar\end{foo}
1215         // will not work.
1216         bool const specialfont =
1217                 (parent_context.font != parent_context.normalfont);
1218         bool const new_layout_allowed = parent_context.new_layout_allowed;
1219         if (specialfont)
1220                 parent_context.new_layout_allowed = false;
1221         handle_ert(os, "\\begin{" + name + "}", parent_context);
1222         parse_text_snippet(p, os, flags, outer, parent_context);
1223         handle_ert(os, "\\end{" + name + "}", parent_context);
1224         if (specialfont)
1225                 parent_context.new_layout_allowed = new_layout_allowed;
1226 }
1227
1228
1229 void parse_environment(Parser & p, ostream & os, bool outer,
1230                        string & last_env, Context & parent_context)
1231 {
1232         Layout const * newlayout;
1233         InsetLayout const * newinsetlayout = 0;
1234         string const name = p.getArg('{', '}');
1235         const bool is_starred = suffixIs(name, '*');
1236         string const unstarred_name = rtrim(name, "*");
1237         active_environments.push_back(name);
1238
1239         if (is_math_env(name)) {
1240                 parent_context.check_layout(os);
1241                 begin_inset(os, "Formula ");
1242                 os << "\\begin{" << name << "}";
1243                 parse_math(p, os, FLAG_END, MATH_MODE);
1244                 os << "\\end{" << name << "}";
1245                 end_inset(os);
1246                 if (is_display_math_env(name)) {
1247                         // Prevent the conversion of a line break to a space
1248                         // (bug 7668). This does not change the output, but
1249                         // looks ugly in LyX.
1250                         eat_whitespace(p, os, parent_context, false);
1251                 }
1252         }
1253
1254         else if (is_known(name, polyglossia_languages)) {
1255                 // We must begin a new paragraph if not already done
1256                 if (! parent_context.atParagraphStart()) {
1257                         parent_context.check_end_layout(os);
1258                         parent_context.new_paragraph(os);
1259                 }
1260                 // save the language in the context so that it is
1261                 // handled by parse_text
1262                 parent_context.font.language = polyglossia2lyx(name);
1263                 parse_text(p, os, FLAG_END, outer, parent_context);
1264                 // Just in case the environment is empty
1265                 parent_context.extra_stuff.erase();
1266                 // We must begin a new paragraph to reset the language
1267                 parent_context.new_paragraph(os);
1268                 p.skip_spaces();
1269         }
1270
1271         else if (unstarred_name == "tabular" || name == "longtable") {
1272                 eat_whitespace(p, os, parent_context, false);
1273                 string width = "0pt";
1274                 if (name == "tabular*") {
1275                         width = lyx::translate_len(p.getArg('{', '}'));
1276                         eat_whitespace(p, os, parent_context, false);
1277                 }
1278                 parent_context.check_layout(os);
1279                 begin_inset(os, "Tabular ");
1280                 handle_tabular(p, os, name, width, parent_context);
1281                 end_inset(os);
1282                 p.skip_spaces();
1283         }
1284
1285         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
1286                 eat_whitespace(p, os, parent_context, false);
1287                 string const opt = p.hasOpt() ? p.getArg('[', ']') : string();
1288                 eat_whitespace(p, os, parent_context, false);
1289                 parent_context.check_layout(os);
1290                 begin_inset(os, "Float " + unstarred_name + "\n");
1291                 // store the float type for subfloats
1292                 // subfloats only work with figures and tables
1293                 if (unstarred_name == "figure")
1294                         float_type = unstarred_name;
1295                 else if (unstarred_name == "table")
1296                         float_type = unstarred_name;
1297                 else
1298                         float_type = "";
1299                 if (!opt.empty())
1300                         os << "placement " << opt << '\n';
1301                 if (contains(opt, "H"))
1302                         preamble.registerAutomaticallyLoadedPackage("float");
1303                 else {
1304                         Floating const & fl = parent_context.textclass.floats()
1305                                 .getType(unstarred_name);
1306                         if (!fl.floattype().empty() && fl.usesFloatPkg())
1307                                 preamble.registerAutomaticallyLoadedPackage("float");
1308                 }
1309
1310                 os << "wide " << convert<string>(is_starred)
1311                    << "\nsideways false"
1312                    << "\nstatus open\n\n";
1313                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1314                 end_inset(os);
1315                 // We don't need really a new paragraph, but
1316                 // we must make sure that the next item gets a \begin_layout.
1317                 parent_context.new_paragraph(os);
1318                 p.skip_spaces();
1319                 // the float is parsed thus delete the type
1320                 float_type = "";
1321         }
1322
1323         else if (unstarred_name == "sidewaysfigure"
1324                 || unstarred_name == "sidewaystable") {
1325                 eat_whitespace(p, os, parent_context, false);
1326                 parent_context.check_layout(os);
1327                 if (unstarred_name == "sidewaysfigure")
1328                         begin_inset(os, "Float figure\n");
1329                 else
1330                         begin_inset(os, "Float table\n");
1331                 os << "wide " << convert<string>(is_starred)
1332                    << "\nsideways true"
1333                    << "\nstatus open\n\n";
1334                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1335                 end_inset(os);
1336                 // We don't need really a new paragraph, but
1337                 // we must make sure that the next item gets a \begin_layout.
1338                 parent_context.new_paragraph(os);
1339                 p.skip_spaces();
1340                 preamble.registerAutomaticallyLoadedPackage("rotfloat");
1341         }
1342
1343         else if (name == "wrapfigure" || name == "wraptable") {
1344                 // syntax is \begin{wrapfigure}[lines]{placement}[overhang]{width}
1345                 eat_whitespace(p, os, parent_context, false);
1346                 parent_context.check_layout(os);
1347                 // default values
1348                 string lines = "0";
1349                 string overhang = "0col%";
1350                 // parse
1351                 if (p.hasOpt())
1352                         lines = p.getArg('[', ']');
1353                 string const placement = p.getArg('{', '}');
1354                 if (p.hasOpt())
1355                         overhang = p.getArg('[', ']');
1356                 string const width = p.getArg('{', '}');
1357                 // write
1358                 if (name == "wrapfigure")
1359                         begin_inset(os, "Wrap figure\n");
1360                 else
1361                         begin_inset(os, "Wrap table\n");
1362                 os << "lines " << lines
1363                    << "\nplacement " << placement
1364                    << "\noverhang " << lyx::translate_len(overhang)
1365                    << "\nwidth " << lyx::translate_len(width)
1366                    << "\nstatus open\n\n";
1367                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1368                 end_inset(os);
1369                 // We don't need really a new paragraph, but
1370                 // we must make sure that the next item gets a \begin_layout.
1371                 parent_context.new_paragraph(os);
1372                 p.skip_spaces();
1373                 preamble.registerAutomaticallyLoadedPackage("wrapfig");
1374         }
1375
1376         else if (name == "minipage") {
1377                 eat_whitespace(p, os, parent_context, false);
1378                 // Test whether this is an outer box of a shaded box
1379                 p.pushPosition();
1380                 // swallow arguments
1381                 while (p.hasOpt()) {
1382                         p.getArg('[', ']');
1383                         p.skip_spaces(true);
1384                 }
1385                 p.getArg('{', '}');
1386                 p.skip_spaces(true);
1387                 Token t = p.get_token();
1388                 bool shaded = false;
1389                 if (t.asInput() == "\\begin") {
1390                         p.skip_spaces(true);
1391                         if (p.getArg('{', '}') == "shaded")
1392                                 shaded = true;
1393                 }
1394                 p.popPosition();
1395                 if (shaded)
1396                         parse_outer_box(p, os, FLAG_END, outer,
1397                                         parent_context, name, "shaded");
1398                 else
1399                         parse_box(p, os, 0, FLAG_END, outer, parent_context,
1400                                   "", "", name);
1401                 p.skip_spaces();
1402         }
1403
1404         else if (name == "comment") {
1405                 eat_whitespace(p, os, parent_context, false);
1406                 parent_context.check_layout(os);
1407                 begin_inset(os, "Note Comment\n");
1408                 os << "status open\n";
1409                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1410                 end_inset(os);
1411                 p.skip_spaces();
1412                 skip_braces(p); // eat {} that might by set by LyX behind comments
1413                 preamble.registerAutomaticallyLoadedPackage("verbatim");
1414         }
1415
1416         else if (name == "verbatim") {
1417                 os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
1418                 string const s = p.plainEnvironment("verbatim");
1419                 string::const_iterator it2 = s.begin();
1420                 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1421                         if (*it == '\\')
1422                                 os << "\\backslash ";
1423                         else if (*it == '\n') {
1424                                 it2 = it + 1;
1425                                 // avoid adding an empty paragraph at the end
1426                                 // FIXME: if there are 2 consecutive spaces at the end ignore it
1427                                 // because LyX will re-add a \n
1428                                 // This hack must be removed once bug 8049 is fixed!
1429                                 if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
1430                                         os << "\n\\end_layout\n\\begin_layout Verbatim\n";
1431                         } else
1432                                 os << *it;
1433                 }
1434                 os << "\n\\end_layout\n\n";
1435                 p.skip_spaces();
1436                 // reset to Standard layout
1437                 os << "\n\\begin_layout Standard\n";
1438         }
1439
1440         else if (name == "lyxgreyedout") {
1441                 eat_whitespace(p, os, parent_context, false);
1442                 parent_context.check_layout(os);
1443                 begin_inset(os, "Note Greyedout\n");
1444                 os << "status open\n";
1445                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1446                 end_inset(os);
1447                 p.skip_spaces();
1448                 if (!preamble.notefontcolor().empty())
1449                         preamble.registerAutomaticallyLoadedPackage("color");
1450         }
1451
1452         else if (name == "framed" || name == "shaded") {
1453                 eat_whitespace(p, os, parent_context, false);
1454                 parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
1455                 p.skip_spaces();
1456         }
1457
1458         else if (name == "lstlisting") {
1459                 eat_whitespace(p, os, parent_context, false);
1460                 // FIXME handle the automatic color package loading
1461                 // uwestoehr asks: In what case color is loaded?
1462                 parse_listings(p, os, parent_context, false);
1463                 p.skip_spaces();
1464         }
1465
1466         else if (!parent_context.new_layout_allowed)
1467                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1468                                           parent_context);
1469
1470         // Alignment and spacing settings
1471         // FIXME (bug xxxx): These settings can span multiple paragraphs and
1472         //                                       therefore are totally broken!
1473         // Note that \centering, raggedright, and raggedleft cannot be handled, as
1474         // they are commands not environments. They are furthermore switches that
1475         // can be ended by another switches, but also by commands like \footnote or
1476         // \parbox. So the only safe way is to leave them untouched.
1477         else if (name == "center" || name == "centering" ||
1478                  name == "flushleft" || name == "flushright" ||
1479                  name == "singlespace" || name == "onehalfspace" ||
1480                  name == "doublespace" || name == "spacing") {
1481                 eat_whitespace(p, os, parent_context, false);
1482                 // We must begin a new paragraph if not already done
1483                 if (! parent_context.atParagraphStart()) {
1484                         parent_context.check_end_layout(os);
1485                         parent_context.new_paragraph(os);
1486                 }
1487                 if (name == "flushleft")
1488                         parent_context.add_extra_stuff("\\align left\n");
1489                 else if (name == "flushright")
1490                         parent_context.add_extra_stuff("\\align right\n");
1491                 else if (name == "center" || name == "centering")
1492                         parent_context.add_extra_stuff("\\align center\n");
1493                 else if (name == "singlespace")
1494                         parent_context.add_extra_stuff("\\paragraph_spacing single\n");
1495                 else if (name == "onehalfspace") {
1496                         parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
1497                         preamble.registerAutomaticallyLoadedPackage("setspace");
1498                 } else if (name == "doublespace") {
1499                         parent_context.add_extra_stuff("\\paragraph_spacing double\n");
1500                         preamble.registerAutomaticallyLoadedPackage("setspace");
1501                 } else if (name == "spacing") {
1502                         parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
1503                         preamble.registerAutomaticallyLoadedPackage("setspace");
1504                 }
1505                 parse_text(p, os, FLAG_END, outer, parent_context);
1506                 // Just in case the environment is empty
1507                 parent_context.extra_stuff.erase();
1508                 // We must begin a new paragraph to reset the alignment
1509                 parent_context.new_paragraph(os);
1510                 p.skip_spaces();
1511         }
1512
1513         // The single '=' is meant here.
1514         else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
1515                 eat_whitespace(p, os, parent_context, false);
1516                 Context context(true, parent_context.textclass, newlayout,
1517                                 parent_context.layout, parent_context.font);
1518                 if (parent_context.deeper_paragraph) {
1519                         // We are beginning a nested environment after a
1520                         // deeper paragraph inside the outer list environment.
1521                         // Therefore we don't need to output a "begin deeper".
1522                         context.need_end_deeper = true;
1523                 }
1524                 parent_context.check_end_layout(os);
1525                 if (last_env == name) {
1526                         // we need to output a separator since LyX would export
1527                         // the two environments as one otherwise (bug 5716)
1528                         docstring const sep = from_ascii("--Separator--");
1529                         TeX2LyXDocClass const & textclass(parent_context.textclass);
1530                         if (textclass.hasLayout(sep)) {
1531                                 Context newcontext(parent_context);
1532                                 newcontext.layout = &(textclass[sep]);
1533                                 newcontext.check_layout(os);
1534                                 newcontext.check_end_layout(os);
1535                         } else {
1536                                 parent_context.check_layout(os);
1537                                 begin_inset(os, "Note Note\n");
1538                                 os << "status closed\n";
1539                                 Context newcontext(true, textclass,
1540                                                 &(textclass.defaultLayout()));
1541                                 newcontext.check_layout(os);
1542                                 newcontext.check_end_layout(os);
1543                                 end_inset(os);
1544                                 parent_context.check_end_layout(os);
1545                         }
1546                 }
1547                 switch (context.layout->latextype) {
1548                 case  LATEX_LIST_ENVIRONMENT:
1549                         context.add_par_extra_stuff("\\labelwidthstring "
1550                                                     + p.verbatim_item() + '\n');
1551                         p.skip_spaces();
1552                         break;
1553                 case  LATEX_BIB_ENVIRONMENT:
1554                         p.verbatim_item(); // swallow next arg
1555                         p.skip_spaces();
1556                         break;
1557                 default:
1558                         break;
1559                 }
1560                 context.check_deeper(os);
1561                 // handle known optional and required arguments
1562                 // layouts require all optional arguments before the required ones
1563                 // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
1564                 // It is impossible to place anything after the environment name,
1565                 // but before the first \\item.
1566                 if (context.layout->latextype == LATEX_ENVIRONMENT) {
1567                         bool need_layout = true;
1568                         unsigned int optargs = 0;
1569                         while (optargs < context.layout->optargs) {
1570                                 eat_whitespace(p, os, context, false);
1571                                 if (p.next_token().cat() == catEscape ||
1572                                     p.next_token().character() != '[')
1573                                         break;
1574                                 p.get_token(); // eat '['
1575                                 if (need_layout) {
1576                                         context.check_layout(os);
1577                                         need_layout = false;
1578                                 }
1579                                 begin_inset(os, "Argument\n");
1580                                 os << "status collapsed\n\n";
1581                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
1582                                 end_inset(os);
1583                                 eat_whitespace(p, os, context, false);
1584                                 ++optargs;
1585                         }
1586                         unsigned int reqargs = 0;
1587                         while (reqargs < context.layout->reqargs) {
1588                                 eat_whitespace(p, os, context, false);
1589                                 if (p.next_token().cat() != catBegin)
1590                                         break;
1591                                 p.get_token(); // eat '{'
1592                                 if (need_layout) {
1593                                         context.check_layout(os);
1594                                         need_layout = false;
1595                                 }
1596                                 begin_inset(os, "Argument\n");
1597                                 os << "status collapsed\n\n";
1598                                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
1599                                 end_inset(os);
1600                                 eat_whitespace(p, os, context, false);
1601                                 ++reqargs;
1602                         }
1603                 }
1604                 parse_text(p, os, FLAG_END, outer, context);
1605                 context.check_end_layout(os);
1606                 if (parent_context.deeper_paragraph) {
1607                         // We must suppress the "end deeper" because we
1608                         // suppressed the "begin deeper" above.
1609                         context.need_end_deeper = false;
1610                 }
1611                 context.check_end_deeper(os);
1612                 parent_context.new_paragraph(os);
1613                 p.skip_spaces();
1614                 if (!preamble.titleLayoutFound())
1615                         preamble.titleLayoutFound(newlayout->intitle);
1616                 set<string> const & req = newlayout->requires();
1617                 set<string>::const_iterator it = req.begin();
1618                 set<string>::const_iterator en = req.end();
1619                 for (; it != en; ++it)
1620                         preamble.registerAutomaticallyLoadedPackage(*it);
1621         }
1622
1623         // The single '=' is meant here.
1624         else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
1625                 eat_whitespace(p, os, parent_context, false);
1626                 parent_context.check_layout(os);
1627                 begin_inset(os, "Flex ");
1628                 os << to_utf8(newinsetlayout->name()) << '\n'
1629                    << "status collapsed\n";
1630                 parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
1631                 end_inset(os);
1632         }
1633
1634         else if (name == "appendix") {
1635                 // This is no good latex style, but it works and is used in some documents...
1636                 eat_whitespace(p, os, parent_context, false);
1637                 parent_context.check_end_layout(os);
1638                 Context context(true, parent_context.textclass, parent_context.layout,
1639                                 parent_context.layout, parent_context.font);
1640                 context.check_layout(os);
1641                 os << "\\start_of_appendix\n";
1642                 parse_text(p, os, FLAG_END, outer, context);
1643                 context.check_end_layout(os);
1644                 p.skip_spaces();
1645         }
1646
1647         else if (known_environments.find(name) != known_environments.end()) {
1648                 vector<ArgumentType> arguments = known_environments[name];
1649                 // The last "argument" denotes wether we may translate the
1650                 // environment contents to LyX
1651                 // The default required if no argument is given makes us
1652                 // compatible with the reLyXre environment.
1653                 ArgumentType contents = arguments.empty() ?
1654                         required :
1655                         arguments.back();
1656                 if (!arguments.empty())
1657                         arguments.pop_back();
1658                 // See comment in parse_unknown_environment()
1659                 bool const specialfont =
1660                         (parent_context.font != parent_context.normalfont);
1661                 bool const new_layout_allowed =
1662                         parent_context.new_layout_allowed;
1663                 if (specialfont)
1664                         parent_context.new_layout_allowed = false;
1665                 parse_arguments("\\begin{" + name + "}", arguments, p, os,
1666                                 outer, parent_context);
1667                 if (contents == verbatim)
1668                         handle_ert(os, p.verbatimEnvironment(name),
1669                                    parent_context);
1670                 else
1671                         parse_text_snippet(p, os, FLAG_END, outer,
1672                                            parent_context);
1673                 handle_ert(os, "\\end{" + name + "}", parent_context);
1674                 if (specialfont)
1675                         parent_context.new_layout_allowed = new_layout_allowed;
1676         }
1677
1678         else
1679                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1680                                           parent_context);
1681
1682         last_env = name;
1683         active_environments.pop_back();
1684 }
1685
1686
1687 /// parses a comment and outputs it to \p os.
1688 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
1689 {
1690         LASSERT(t.cat() == catComment, return);
1691         if (!t.cs().empty()) {
1692                 context.check_layout(os);
1693                 handle_comment(os, '%' + t.cs(), context);
1694                 if (p.next_token().cat() == catNewline) {
1695                         // A newline after a comment line starts a new
1696                         // paragraph
1697                         if (context.new_layout_allowed) {
1698                                 if(!context.atParagraphStart())
1699                                         // Only start a new paragraph if not already
1700                                         // done (we might get called recursively)
1701                                         context.new_paragraph(os);
1702                         } else
1703                                 handle_ert(os, "\n", context);
1704                         eat_whitespace(p, os, context, true);
1705                 }
1706         } else {
1707                 // "%\n" combination
1708                 p.skip_spaces();
1709         }
1710 }
1711
1712
1713 /*!
1714  * Reads spaces and comments until the first non-space, non-comment token.
1715  * New paragraphs (double newlines or \\par) are handled like simple spaces
1716  * if \p eatParagraph is true.
1717  * Spaces are skipped, but comments are written to \p os.
1718  */
1719 void eat_whitespace(Parser & p, ostream & os, Context & context,
1720                     bool eatParagraph)
1721 {
1722         while (p.good()) {
1723                 Token const & t = p.get_token();
1724                 if (t.cat() == catComment)
1725                         parse_comment(p, os, t, context);
1726                 else if ((! eatParagraph && p.isParagraph()) ||
1727                          (t.cat() != catSpace && t.cat() != catNewline)) {
1728                         p.putback();
1729                         return;
1730                 }
1731         }
1732 }
1733
1734
1735 /*!
1736  * Set a font attribute, parse text and reset the font attribute.
1737  * \param attribute Attribute name (e.g. \\family, \\shape etc.)
1738  * \param currentvalue Current value of the attribute. Is set to the new
1739  * value during parsing.
1740  * \param newvalue New value of the attribute
1741  */
1742 void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
1743                            Context & context, string const & attribute,
1744                            string & currentvalue, string const & newvalue)
1745 {
1746         context.check_layout(os);
1747         string const oldvalue = currentvalue;
1748         currentvalue = newvalue;
1749         os << '\n' << attribute << ' ' << newvalue << "\n";
1750         parse_text_snippet(p, os, flags, outer, context);
1751         context.check_layout(os);
1752         os << '\n' << attribute << ' ' << oldvalue << "\n";
1753         currentvalue = oldvalue;
1754 }
1755
1756
1757 /// get the arguments of a natbib or jurabib citation command
1758 void get_cite_arguments(Parser & p, bool natbibOrder,
1759         string & before, string & after)
1760 {
1761         // We need to distinguish "" and "[]", so we can't use p.getOpt().
1762
1763         // text before the citation
1764         before.clear();
1765         // text after the citation
1766         after = p.getFullOpt();
1767
1768         if (!after.empty()) {
1769                 before = p.getFullOpt();
1770                 if (natbibOrder && !before.empty())
1771                         swap(before, after);
1772         }
1773 }
1774
1775
1776 /// Convert filenames with TeX macros and/or quotes to something LyX
1777 /// can understand
1778 string const normalize_filename(string const & name)
1779 {
1780         Parser p(trim(name, "\""));
1781         ostringstream os;
1782         while (p.good()) {
1783                 Token const & t = p.get_token();
1784                 if (t.cat() != catEscape)
1785                         os << t.asInput();
1786                 else if (t.cs() == "lyxdot") {
1787                         // This is used by LyX for simple dots in relative
1788                         // names
1789                         os << '.';
1790                         p.skip_spaces();
1791                 } else if (t.cs() == "space") {
1792                         os << ' ';
1793                         p.skip_spaces();
1794                 } else
1795                         os << t.asInput();
1796         }
1797         return os.str();
1798 }
1799
1800
1801 /// Convert \p name from TeX convention (relative to master file) to LyX
1802 /// convention (relative to .lyx file) if it is relative
1803 void fix_relative_filename(string & name)
1804 {
1805         if (FileName::isAbsolute(name))
1806                 return;
1807
1808         name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
1809                                    from_utf8(getParentFilePath())));
1810 }
1811
1812
1813 /// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
1814 void parse_noweb(Parser & p, ostream & os, Context & context)
1815 {
1816         // assemble the rest of the keyword
1817         string name("<<");
1818         bool scrap = false;
1819         while (p.good()) {
1820                 Token const & t = p.get_token();
1821                 if (t.asInput() == ">" && p.next_token().asInput() == ">") {
1822                         name += ">>";
1823                         p.get_token();
1824                         scrap = (p.good() && p.next_token().asInput() == "=");
1825                         if (scrap)
1826                                 name += p.get_token().asInput();
1827                         break;
1828                 }
1829                 name += t.asInput();
1830         }
1831
1832         if (!scrap || !context.new_layout_allowed ||
1833             !context.textclass.hasLayout(from_ascii("Scrap"))) {
1834                 cerr << "Warning: Could not interpret '" << name
1835                      << "'. Ignoring it." << endl;
1836                 return;
1837         }
1838
1839         // We use new_paragraph instead of check_end_layout because the stuff
1840         // following the noweb chunk needs to start with a \begin_layout.
1841         // This may create a new paragraph even if there was none in the
1842         // noweb file, but the alternative is an invalid LyX file. Since
1843         // noweb code chunks are implemented with a layout style in LyX they
1844         // always must be in an own paragraph.
1845         context.new_paragraph(os);
1846         Context newcontext(true, context.textclass,
1847                 &context.textclass[from_ascii("Scrap")]);
1848         newcontext.check_layout(os);
1849         os << name;
1850         while (p.good()) {
1851                 Token const & t = p.get_token();
1852                 // We abuse the parser a bit, because this is no TeX syntax
1853                 // at all.
1854                 if (t.cat() == catEscape)
1855                         os << subst(t.asInput(), "\\", "\n\\backslash\n");
1856                 else {
1857                         ostringstream oss;
1858                         Context tmp(false, context.textclass,
1859                                     &context.textclass[from_ascii("Scrap")]);
1860                         tmp.need_end_layout = true;
1861                         tmp.check_layout(oss);
1862                         os << subst(t.asInput(), "\n", oss.str());
1863                 }
1864                 // The scrap chunk is ended by an @ at the beginning of a line.
1865                 // After the @ the line may contain a comment and/or
1866                 // whitespace, but nothing else.
1867                 if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
1868                     (p.next_token().cat() == catSpace ||
1869                      p.next_token().cat() == catNewline ||
1870                      p.next_token().cat() == catComment)) {
1871                         while (p.good() && p.next_token().cat() == catSpace)
1872                                 os << p.get_token().asInput();
1873                         if (p.next_token().cat() == catComment)
1874                                 // The comment includes a final '\n'
1875                                 os << p.get_token().asInput();
1876                         else {
1877                                 if (p.next_token().cat() == catNewline)
1878                                         p.get_token();
1879                                 os << '\n';
1880                         }
1881                         break;
1882                 }
1883         }
1884         newcontext.check_end_layout(os);
1885 }
1886
1887
1888 /// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
1889 bool is_macro(Parser & p)
1890 {
1891         Token first = p.curr_token();
1892         if (first.cat() != catEscape || !p.good())
1893                 return false;
1894         if (first.cs() == "def")
1895                 return true;
1896         if (first.cs() != "global" && first.cs() != "long")
1897                 return false;
1898         Token second = p.get_token();
1899         int pos = 1;
1900         while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
1901                second.cat() == catNewline || second.cat() == catComment)) {
1902                 second = p.get_token();
1903                 pos++;
1904         }
1905         bool secondvalid = second.cat() == catEscape;
1906         Token third;
1907         bool thirdvalid = false;
1908         if (p.good() && first.cs() == "global" && secondvalid &&
1909             second.cs() == "long") {
1910                 third = p.get_token();
1911                 pos++;
1912                 while (p.good() && !p.isParagraph() &&
1913                        (third.cat() == catSpace ||
1914                         third.cat() == catNewline ||
1915                         third.cat() == catComment)) {
1916                         third = p.get_token();
1917                         pos++;
1918                 }
1919                 thirdvalid = third.cat() == catEscape;
1920         }
1921         for (int i = 0; i < pos; ++i)
1922                 p.putback();
1923         if (!secondvalid)
1924                 return false;
1925         if (!thirdvalid)
1926                 return (first.cs() == "global" || first.cs() == "long") &&
1927                        second.cs() == "def";
1928         return first.cs() == "global" && second.cs() == "long" &&
1929                third.cs() == "def";
1930 }
1931
1932
1933 /// Parse a macro definition (assumes that is_macro() returned true)
1934 void parse_macro(Parser & p, ostream & os, Context & context)
1935 {
1936         context.check_layout(os);
1937         Token first = p.curr_token();
1938         Token second;
1939         Token third;
1940         string command = first.asInput();
1941         if (first.cs() != "def") {
1942                 p.get_token();
1943                 eat_whitespace(p, os, context, false);
1944                 second = p.curr_token();
1945                 command += second.asInput();
1946                 if (second.cs() != "def") {
1947                         p.get_token();
1948                         eat_whitespace(p, os, context, false);
1949                         third = p.curr_token();
1950                         command += third.asInput();
1951                 }
1952         }
1953         eat_whitespace(p, os, context, false);
1954         string const name = p.get_token().cs();
1955         eat_whitespace(p, os, context, false);
1956
1957         // parameter text
1958         bool simple = true;
1959         string paramtext;
1960         int arity = 0;
1961         while (p.next_token().cat() != catBegin) {
1962                 if (p.next_token().cat() == catParameter) {
1963                         // # found
1964                         p.get_token();
1965                         paramtext += "#";
1966
1967                         // followed by number?
1968                         if (p.next_token().cat() == catOther) {
1969                                 char c = p.getChar();
1970                                 paramtext += c;
1971                                 // number = current arity + 1?
1972                                 if (c == arity + '0' + 1)
1973                                         ++arity;
1974                                 else
1975                                         simple = false;
1976                         } else
1977                                 paramtext += p.get_token().cs();
1978                 } else {
1979                         paramtext += p.get_token().cs();
1980                         simple = false;
1981                 }
1982         }
1983
1984         // only output simple (i.e. compatible) macro as FormulaMacros
1985         string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
1986         if (simple) {
1987                 context.check_layout(os);
1988                 begin_inset(os, "FormulaMacro");
1989                 os << "\n\\def" << ert;
1990                 end_inset(os);
1991         } else
1992                 handle_ert(os, command + ert, context);
1993 }
1994
1995
1996 void registerExternalTemplatePackages(string const & name)
1997 {
1998         external::TemplateManager const & etm = external::TemplateManager::get();
1999         external::Template const * const et = etm.getTemplateByName(name);
2000         if (!et)
2001                 return;
2002         external::Template::Formats::const_iterator cit = et->formats.end();
2003         if (pdflatex)
2004                 cit = et->formats.find("PDFLaTeX");
2005         if (cit == et->formats.end())
2006                 // If the template has not specified a PDFLaTeX output,
2007                 // we try the LaTeX format.
2008                 cit = et->formats.find("LaTeX");
2009         if (cit == et->formats.end())
2010                 return;
2011         vector<string>::const_iterator qit = cit->second.requirements.begin();
2012         vector<string>::const_iterator qend = cit->second.requirements.end();
2013         for (; qit != qend; ++qit)
2014                 preamble.registerAutomaticallyLoadedPackage(*qit);
2015 }
2016
2017 } // anonymous namespace
2018
2019
2020 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
2021                 Context & context)
2022 {
2023         Layout const * newlayout = 0;
2024         InsetLayout const * newinsetlayout = 0;
2025         char const * const * where = 0;
2026         // Store the latest bibliographystyle and nocite{*} option
2027         // (needed for bibtex inset)
2028         string btprint;
2029         string bibliographystyle = "default";
2030         bool const use_natbib = preamble.isPackageUsed("natbib");
2031         bool const use_jurabib = preamble.isPackageUsed("jurabib");
2032         string last_env;
2033         while (p.good()) {
2034                 Token const & t = p.get_token();
2035
2036 #ifdef FILEDEBUG
2037                 debugToken(cerr, t, flags);
2038 #endif
2039
2040                 if (flags & FLAG_ITEM) {
2041                         if (t.cat() == catSpace)
2042                                 continue;
2043
2044                         flags &= ~FLAG_ITEM;
2045                         if (t.cat() == catBegin) {
2046                                 // skip the brace and collect everything to the next matching
2047                                 // closing brace
2048                                 flags |= FLAG_BRACE_LAST;
2049                                 continue;
2050                         }
2051
2052                         // handle only this single token, leave the loop if done
2053                         flags |= FLAG_LEAVE;
2054                 }
2055
2056                 if (t.cat() != catEscape && t.character() == ']' &&
2057                     (flags & FLAG_BRACK_LAST))
2058                         return;
2059                 if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
2060                         return;
2061
2062                 // If there is anything between \end{env} and \begin{env} we
2063                 // don't need to output a separator.
2064                 if (t.cat() != catSpace && t.cat() != catNewline &&
2065                     t.asInput() != "\\begin")
2066                         last_env = "";
2067
2068                 //
2069                 // cat codes
2070                 //
2071                 if (t.cat() == catMath) {
2072                         // we are inside some text mode thingy, so opening new math is allowed
2073                         context.check_layout(os);
2074                         begin_inset(os, "Formula ");
2075                         Token const & n = p.get_token();
2076                         bool const display(n.cat() == catMath && outer);
2077                         if (display) {
2078                                 // TeX's $$...$$ syntax for displayed math
2079                                 os << "\\[";
2080                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2081                                 os << "\\]";
2082                                 p.get_token(); // skip the second '$' token
2083                         } else {
2084                                 // simple $...$  stuff
2085                                 p.putback();
2086                                 os << '$';
2087                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2088                                 os << '$';
2089                         }
2090                         end_inset(os);
2091                         if (display) {
2092                                 // Prevent the conversion of a line break to a
2093                                 // space (bug 7668). This does not change the
2094                                 // output, but looks ugly in LyX.
2095                                 eat_whitespace(p, os, context, false);
2096                         }
2097                 }
2098
2099                 else if (t.cat() == catSuper || t.cat() == catSub)
2100                         cerr << "catcode " << t << " illegal in text mode\n";
2101
2102                 // Basic support for english quotes. This should be
2103                 // extended to other quotes, but is not so easy (a
2104                 // left english quote is the same as a right german
2105                 // quote...)
2106                 else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
2107                         context.check_layout(os);
2108                         begin_inset(os, "Quotes ");
2109                         os << "eld";
2110                         end_inset(os);
2111                         p.get_token();
2112                         skip_braces(p);
2113                 }
2114                 else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
2115                         context.check_layout(os);
2116                         begin_inset(os, "Quotes ");
2117                         os << "erd";
2118                         end_inset(os);
2119                         p.get_token();
2120                         skip_braces(p);
2121                 }
2122
2123                 else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
2124                         context.check_layout(os);
2125                         begin_inset(os, "Quotes ");
2126                         os << "ald";
2127                         end_inset(os);
2128                         p.get_token();
2129                         skip_braces(p);
2130                 }
2131
2132                 else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
2133                         context.check_layout(os);
2134                         begin_inset(os, "Quotes ");
2135                         os << "ard";
2136                         end_inset(os);
2137                         p.get_token();
2138                         skip_braces(p);
2139                 }
2140
2141                 else if (t.asInput() == "<"
2142                          && p.next_token().asInput() == "<" && noweb_mode) {
2143                         p.get_token();
2144                         parse_noweb(p, os, context);
2145                 }
2146
2147                 else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
2148                         check_space(p, os, context);
2149
2150                 else if (t.character() == '[' && noweb_mode &&
2151                          p.next_token().character() == '[') {
2152                         // These can contain underscores
2153                         p.putback();
2154                         string const s = p.getFullOpt() + ']';
2155                         if (p.next_token().character() == ']')
2156                                 p.get_token();
2157                         else
2158                                 cerr << "Warning: Inserting missing ']' in '"
2159                                      << s << "'." << endl;
2160                         handle_ert(os, s, context);
2161                 }
2162
2163                 else if (t.cat() == catLetter) {
2164                         context.check_layout(os);
2165                         // Workaround for bug 4752.
2166                         // FIXME: This whole code block needs to be removed
2167                         //        when the bug is fixed and tex2lyx produces
2168                         //        the updated file format.
2169                         // The replacement algorithm in LyX is so stupid that
2170                         // it even translates a phrase if it is part of a word.
2171                         bool handled = false;
2172                         for (int const * l = known_phrase_lengths; *l; ++l) {
2173                                 string phrase = t.cs();
2174                                 for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
2175                                         phrase += p.get_token().cs();
2176                                 if (is_known(phrase, known_coded_phrases)) {
2177                                         handle_ert(os, phrase, context);
2178                                         handled = true;
2179                                         break;
2180                                 } else {
2181                                         for (size_t i = 1; i < phrase.length(); ++i)
2182                                                 p.putback();
2183                                 }
2184                         }
2185                         if (!handled)
2186                                 os << t.cs();
2187                 }
2188
2189                 else if (t.cat() == catOther ||
2190                                t.cat() == catAlign ||
2191                                t.cat() == catParameter) {
2192                         // This translates "&" to "\\&" which may be wrong...
2193                         context.check_layout(os);
2194                         os << t.cs();
2195                 }
2196
2197                 else if (p.isParagraph()) {
2198                         if (context.new_layout_allowed)
2199                                 context.new_paragraph(os);
2200                         else
2201                                 handle_ert(os, "\\par ", context);
2202                         eat_whitespace(p, os, context, true);
2203                 }
2204
2205                 else if (t.cat() == catActive) {
2206                         context.check_layout(os);
2207                         if (t.character() == '~') {
2208                                 if (context.layout->free_spacing)
2209                                         os << ' ';
2210                                 else {
2211                                         begin_inset(os, "space ~\n");
2212                                         end_inset(os);
2213                                 }
2214                         } else
2215                                 os << t.cs();
2216                 }
2217
2218                 else if (t.cat() == catBegin) {
2219                         Token const next = p.next_token();
2220                         Token const end = p.next_next_token();
2221                         if (next.cat() == catEnd) {
2222                         // {}
2223                         Token const prev = p.prev_token();
2224                         p.get_token();
2225                         if (p.next_token().character() == '`' ||
2226                             (prev.character() == '-' &&
2227                              p.next_token().character() == '-'))
2228                                 ; // ignore it in {}`` or -{}-
2229                         else
2230                                 handle_ert(os, "{}", context);
2231                         } else if (next.cat() == catEscape &&
2232                                    is_known(next.cs(), known_quotes) &&
2233                                    end.cat() == catEnd) {
2234                                 // Something like {\textquoteright} (e.g.
2235                                 // from writer2latex). LyX writes
2236                                 // \textquoteright{}, so we may skip the
2237                                 // braces here for better readability.
2238                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2239                                                    outer, context);
2240                         } else {
2241                         context.check_layout(os);
2242                         // special handling of font attribute changes
2243                         Token const prev = p.prev_token();
2244                         TeXFont const oldFont = context.font;
2245                         if (next.character() == '[' ||
2246                             next.character() == ']' ||
2247                             next.character() == '*') {
2248                                 p.get_token();
2249                                 if (p.next_token().cat() == catEnd) {
2250                                         os << next.cs();
2251                                         p.get_token();
2252                                 } else {
2253                                         p.putback();
2254                                         handle_ert(os, "{", context);
2255                                         parse_text_snippet(p, os,
2256                                                         FLAG_BRACE_LAST,
2257                                                         outer, context);
2258                                         handle_ert(os, "}", context);
2259                                 }
2260                         } else if (! context.new_layout_allowed) {
2261                                 handle_ert(os, "{", context);
2262                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2263                                                    outer, context);
2264                                 handle_ert(os, "}", context);
2265                         } else if (is_known(next.cs(), known_sizes)) {
2266                                 // next will change the size, so we must
2267                                 // reset it here
2268                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2269                                                    outer, context);
2270                                 if (!context.atParagraphStart())
2271                                         os << "\n\\size "
2272                                            << context.font.size << "\n";
2273                         } else if (is_known(next.cs(), known_font_families)) {
2274                                 // next will change the font family, so we
2275                                 // must reset it here
2276                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2277                                                    outer, context);
2278                                 if (!context.atParagraphStart())
2279                                         os << "\n\\family "
2280                                            << context.font.family << "\n";
2281                         } else if (is_known(next.cs(), known_font_series)) {
2282                                 // next will change the font series, so we
2283                                 // must reset it here
2284                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2285                                                    outer, context);
2286                                 if (!context.atParagraphStart())
2287                                         os << "\n\\series "
2288                                            << context.font.series << "\n";
2289                         } else if (is_known(next.cs(), known_font_shapes)) {
2290                                 // next will change the font shape, so we
2291                                 // must reset it here
2292                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2293                                                    outer, context);
2294                                 if (!context.atParagraphStart())
2295                                         os << "\n\\shape "
2296                                            << context.font.shape << "\n";
2297                         } else if (is_known(next.cs(), known_old_font_families) ||
2298                                    is_known(next.cs(), known_old_font_series) ||
2299                                    is_known(next.cs(), known_old_font_shapes)) {
2300                                 // next will change the font family, series
2301                                 // and shape, so we must reset it here
2302                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2303                                                    outer, context);
2304                                 if (!context.atParagraphStart())
2305                                         os <<  "\n\\family "
2306                                            << context.font.family
2307                                            << "\n\\series "
2308                                            << context.font.series
2309                                            << "\n\\shape "
2310                                            << context.font.shape << "\n";
2311                         } else {
2312                                 handle_ert(os, "{", context);
2313                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2314                                                    outer, context);
2315                                 handle_ert(os, "}", context);
2316                                 }
2317                         }
2318                 }
2319
2320                 else if (t.cat() == catEnd) {
2321                         if (flags & FLAG_BRACE_LAST) {
2322                                 return;
2323                         }
2324                         cerr << "stray '}' in text\n";
2325                         handle_ert(os, "}", context);
2326                 }
2327
2328                 else if (t.cat() == catComment)
2329                         parse_comment(p, os, t, context);
2330
2331                 //
2332                 // control sequences
2333                 //
2334
2335                 else if (t.cs() == "(") {
2336                         context.check_layout(os);
2337                         begin_inset(os, "Formula");
2338                         os << " \\(";
2339                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
2340                         os << "\\)";
2341                         end_inset(os);
2342                 }
2343
2344                 else if (t.cs() == "[") {
2345                         context.check_layout(os);
2346                         begin_inset(os, "Formula");
2347                         os << " \\[";
2348                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
2349                         os << "\\]";
2350                         end_inset(os);
2351                         // Prevent the conversion of a line break to a space
2352                         // (bug 7668). This does not change the output, but
2353                         // looks ugly in LyX.
2354                         eat_whitespace(p, os, context, false);
2355                 }
2356
2357                 else if (t.cs() == "begin")
2358                         parse_environment(p, os, outer, last_env,
2359                                           context);
2360
2361                 else if (t.cs() == "end") {
2362                         if (flags & FLAG_END) {
2363                                 // eat environment name
2364                                 string const name = p.getArg('{', '}');
2365                                 if (name != active_environment())
2366                                         cerr << "\\end{" + name + "} does not match \\begin{"
2367                                                 + active_environment() + "}\n";
2368                                 return;
2369                         }
2370                         p.error("found 'end' unexpectedly");
2371                 }
2372
2373                 else if (t.cs() == "item") {
2374                         string s;
2375                         bool const optarg = p.hasOpt();
2376                         if (optarg) {
2377                                 // FIXME: This swallows comments, but we cannot use
2378                                 //        eat_whitespace() since we must not output
2379                                 //        anything before the item.
2380                                 p.skip_spaces(true);
2381                                 s = p.verbatimOption();
2382                         } else
2383                                 p.skip_spaces(false);
2384                         context.set_item();
2385                         context.check_layout(os);
2386                         if (context.has_item) {
2387                                 // An item in an unknown list-like environment
2388                                 // FIXME: Do this in check_layout()!
2389                                 context.has_item = false;
2390                                 if (optarg)
2391                                         handle_ert(os, "\\item", context);
2392                                 else
2393                                         handle_ert(os, "\\item ", context);
2394                         }
2395                         if (optarg) {
2396                                 if (context.layout->labeltype != LABEL_MANUAL) {
2397                                         // LyX does not support \item[\mybullet]
2398                                         // in itemize environments
2399                                         Parser p2(s + ']');
2400                                         os << parse_text_snippet(p2,
2401                                                 FLAG_BRACK_LAST, outer, context);
2402                                 } else if (!s.empty()) {
2403                                         // LyX adds braces around the argument,
2404                                         // so we need to remove them here.
2405                                         if (s.size() > 2 && s[0] == '{' &&
2406                                             s[s.size()-1] == '}')
2407                                                 s = s.substr(1, s.size()-2);
2408                                         // If the argument contains a space we
2409                                         // must put it into ERT: Otherwise LyX
2410                                         // would misinterpret the space as
2411                                         // item delimiter (bug 7663)
2412                                         if (contains(s, ' ')) {
2413                                                 handle_ert(os, s, context);
2414                                         } else {
2415                                                 Parser p2(s + ']');
2416                                                 os << parse_text_snippet(p2,
2417                                                         FLAG_BRACK_LAST,
2418                                                         outer, context);
2419                                         }
2420                                         // The space is needed to separate the
2421                                         // item from the rest of the sentence.
2422                                         os << ' ';
2423                                         eat_whitespace(p, os, context, false);
2424                                 }
2425                         }
2426                 }
2427
2428                 else if (t.cs() == "bibitem") {
2429                         context.set_item();
2430                         context.check_layout(os);
2431                         eat_whitespace(p, os, context, false);
2432                         string label = convert_command_inset_arg(p.verbatimOption());
2433                         string key = convert_command_inset_arg(p.verbatim_item());
2434                         if (contains(label, '\\') || contains(key, '\\')) {
2435                                 // LyX can't handle LaTeX commands in labels or keys
2436                                 handle_ert(os, t.asInput() + '[' + label +
2437                                                "]{" + p.verbatim_item() + '}',
2438                                            context);
2439                         } else {
2440                                 begin_command_inset(os, "bibitem", "bibitem");
2441                                 os << "label \"" << label << "\"\n"
2442                                       "key \"" << key << "\"\n";
2443                                 end_inset(os);
2444                         }
2445                 }
2446
2447                 else if (is_macro(p)) {
2448                         // catch the case of \def\inputGnumericTable
2449                         bool macro = true;
2450                         if (t.cs() == "def") {
2451                                 Token second = p.next_token();
2452                                 if (second.cs() == "inputGnumericTable") {
2453                                         p.pushPosition();
2454                                         p.get_token();
2455                                         skip_braces(p);
2456                                         Token third = p.get_token();
2457                                         p.popPosition();
2458                                         if (third.cs() == "input") {
2459                                                 p.get_token();
2460                                                 skip_braces(p);
2461                                                 p.get_token();
2462                                                 string name = normalize_filename(p.verbatim_item());
2463                                                 string const path = getMasterFilePath();
2464                                                 // We want to preserve relative / absolute filenames,
2465                                                 // therefore path is only used for testing
2466                                                 // The file extension is in every case ".tex".
2467                                                 // So we need to remove this extension and check for
2468                                                 // the original one.
2469                                                 name = removeExtension(name);
2470                                                 if (!makeAbsPath(name, path).exists()) {
2471                                                         char const * const Gnumeric_formats[] = {"gnumeric",
2472                                                                 "ods", "xls", 0};
2473                                                         string const Gnumeric_name =
2474                                                                 find_file(name, path, Gnumeric_formats);
2475                                                         if (!Gnumeric_name.empty())
2476                                                                 name = Gnumeric_name;
2477                                                 }
2478                                                 if (makeAbsPath(name, path).exists())
2479                                                         fix_relative_filename(name);
2480                                                 else
2481                                                         cerr << "Warning: Could not find file '"
2482                                                              << name << "'." << endl;
2483                                                 context.check_layout(os);
2484                                                 begin_inset(os, "External\n\ttemplate ");
2485                                                 os << "GnumericSpreadsheet\n\tfilename "
2486                                                    << name << "\n";
2487                                                 end_inset(os);
2488                                                 context.check_layout(os);
2489                                                 macro = false;
2490                                                 // register the packages that are automatically reloaded
2491                                                 // by the Gnumeric template
2492                                                 registerExternalTemplatePackages("GnumericSpreadsheet");
2493                                         }
2494                                 }
2495                         }
2496                         if (macro)
2497                                 parse_macro(p, os, context);
2498                 }
2499
2500                 else if (t.cs() == "noindent") {
2501                         p.skip_spaces();
2502                         context.add_par_extra_stuff("\\noindent\n");
2503                 }
2504
2505                 else if (t.cs() == "appendix") {
2506                         context.add_par_extra_stuff("\\start_of_appendix\n");
2507                         // We need to start a new paragraph. Otherwise the
2508                         // appendix in 'bla\appendix\chapter{' would start
2509                         // too late.
2510                         context.new_paragraph(os);
2511                         // We need to make sure that the paragraph is
2512                         // generated even if it is empty. Otherwise the
2513                         // appendix in '\par\appendix\par\chapter{' would
2514                         // start too late.
2515                         context.check_layout(os);
2516                         // FIXME: This is a hack to prevent paragraph
2517                         // deletion if it is empty. Handle this better!
2518                         handle_comment(os,
2519                                 "%dummy comment inserted by tex2lyx to "
2520                                 "ensure that this paragraph is not empty",
2521                                 context);
2522                         // Both measures above may generate an additional
2523                         // empty paragraph, but that does not hurt, because
2524                         // whitespace does not matter here.
2525                         eat_whitespace(p, os, context, true);
2526                 }
2527
2528                 // Must catch empty dates before findLayout is called below
2529                 else if (t.cs() == "date") {
2530                         eat_whitespace(p, os, context, false);
2531                         p.pushPosition();
2532                         string const date = p.verbatim_item();
2533                         p.popPosition();
2534                         if (date.empty()) {
2535                                 preamble.suppressDate(true);
2536                                 p.verbatim_item();
2537                         } else {
2538                                 preamble.suppressDate(false);
2539                                 if (context.new_layout_allowed &&
2540                                     (newlayout = findLayout(context.textclass,
2541                                                             t.cs(), true))) {
2542                                         // write the layout
2543                                         output_command_layout(os, p, outer,
2544                                                         context, newlayout);
2545                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2546                                         if (!preamble.titleLayoutFound())
2547                                                 preamble.titleLayoutFound(newlayout->intitle);
2548                                         set<string> const & req = newlayout->requires();
2549                                         set<string>::const_iterator it = req.begin();
2550                                         set<string>::const_iterator en = req.end();
2551                                         for (; it != en; ++it)
2552                                                 preamble.registerAutomaticallyLoadedPackage(*it);
2553                                 } else
2554                                         handle_ert(os,
2555                                                 "\\date{" + p.verbatim_item() + '}',
2556                                                 context);
2557                         }
2558                 }
2559
2560                 // Starred section headings
2561                 // Must attempt to parse "Section*" before "Section".
2562                 else if ((p.next_token().asInput() == "*") &&
2563                          context.new_layout_allowed &&
2564                          (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
2565                         // write the layout
2566                         p.get_token();
2567                         output_command_layout(os, p, outer, context, newlayout);
2568                         p.skip_spaces();
2569                         if (!preamble.titleLayoutFound())
2570                                 preamble.titleLayoutFound(newlayout->intitle);
2571                         set<string> const & req = newlayout->requires();
2572                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2573                                 preamble.registerAutomaticallyLoadedPackage(*it);
2574                 }
2575
2576                 // Section headings and the like
2577                 else if (context.new_layout_allowed &&
2578                          (newlayout = findLayout(context.textclass, t.cs(), true))) {
2579                         // write the layout
2580                         output_command_layout(os, p, outer, context, newlayout);
2581                         p.skip_spaces();
2582                         if (!preamble.titleLayoutFound())
2583                                 preamble.titleLayoutFound(newlayout->intitle);
2584                         set<string> const & req = newlayout->requires();
2585                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2586                                 preamble.registerAutomaticallyLoadedPackage(*it);
2587                 }
2588
2589                 else if (t.cs() == "caption") {
2590                         p.skip_spaces();
2591                         context.check_layout(os);
2592                         p.skip_spaces();
2593                         begin_inset(os, "Caption\n");
2594                         Context newcontext(true, context.textclass);
2595                         newcontext.font = context.font;
2596                         newcontext.check_layout(os);
2597                         if (p.next_token().cat() != catEscape &&
2598                             p.next_token().character() == '[') {
2599                                 p.get_token(); // eat '['
2600                                 begin_inset(os, "Argument\n");
2601                                 os << "status collapsed\n";
2602                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
2603                                 end_inset(os);
2604                                 eat_whitespace(p, os, context, false);
2605                         }
2606                         parse_text(p, os, FLAG_ITEM, outer, context);
2607                         context.check_end_layout(os);
2608                         // We don't need really a new paragraph, but
2609                         // we must make sure that the next item gets a \begin_layout.
2610                         context.new_paragraph(os);
2611                         end_inset(os);
2612                         p.skip_spaces();
2613                         newcontext.check_end_layout(os);
2614                 }
2615
2616                 else if (t.cs() == "subfloat") {
2617                         // the syntax is \subfloat[caption]{content}
2618                         // if it is a table of figure depends on the surrounding float
2619                         bool has_caption = false;
2620                         p.skip_spaces();
2621                         // do nothing if there is no outer float
2622                         if (!float_type.empty()) {
2623                                 context.check_layout(os);
2624                                 p.skip_spaces();
2625                                 begin_inset(os, "Float " + float_type + "\n");
2626                                 os << "wide false"
2627                                    << "\nsideways false"
2628                                    << "\nstatus collapsed\n\n";
2629                                 // test for caption
2630                                 string caption;
2631                                 if (p.next_token().cat() != catEscape &&
2632                                                 p.next_token().character() == '[') {
2633                                                         p.get_token(); // eat '['
2634                                                         caption = parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
2635                                                         has_caption = true;
2636                                 }
2637                                 // the content
2638                                 parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
2639                                 // the caption comes always as the last
2640                                 if (has_caption) {
2641                                         // we must make sure that the caption gets a \begin_layout
2642                                         os << "\n\\begin_layout Plain Layout";
2643                                         p.skip_spaces();
2644                                         begin_inset(os, "Caption\n");
2645                                         Context newcontext(true, context.textclass);
2646                                         newcontext.font = context.font;
2647                                         newcontext.check_layout(os);
2648                                         os << caption << "\n";
2649                                         newcontext.check_end_layout(os);
2650                                         // We don't need really a new paragraph, but
2651                                         // we must make sure that the next item gets a \begin_layout.
2652                                         //newcontext.new_paragraph(os);
2653                                         end_inset(os);
2654                                         p.skip_spaces();
2655                                 }
2656                                 // We don't need really a new paragraph, but
2657                                 // we must make sure that the next item gets a \begin_layout.
2658                                 if (has_caption)
2659                                         context.new_paragraph(os);
2660                                 end_inset(os);
2661                                 p.skip_spaces();
2662                                 context.check_end_layout(os);
2663                                 // close the layout we opened
2664                                 if (has_caption)
2665                                         os << "\n\\end_layout\n";
2666                         } else {
2667                                 // if the float type is not supported or there is no surrounding float
2668                                 // output it as ERT
2669                                 if (p.hasOpt()) {
2670                                         string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
2671                                         handle_ert(os, t.asInput() + '[' + opt_arg +
2672                                                "]{" + p.verbatim_item() + '}', context);
2673                                 } else
2674                                         handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
2675                         }
2676                 }
2677
2678                 else if (t.cs() == "includegraphics") {
2679                         bool const clip = p.next_token().asInput() == "*";
2680                         if (clip)
2681                                 p.get_token();
2682                         string const arg = p.getArg('[', ']');
2683                         map<string, string> opts;
2684                         vector<string> keys;
2685                         split_map(arg, opts, keys);
2686                         if (clip)
2687                                 opts["clip"] = string();
2688                         string name = normalize_filename(p.verbatim_item());
2689
2690                         string const path = getMasterFilePath();
2691                         // We want to preserve relative / absolute filenames,
2692                         // therefore path is only used for testing
2693                         if (!makeAbsPath(name, path).exists()) {
2694                                 // The file extension is probably missing.
2695                                 // Now try to find it out.
2696                                 string const dvips_name =
2697                                         find_file(name, path,
2698                                                   known_dvips_graphics_formats);
2699                                 string const pdftex_name =
2700                                         find_file(name, path,
2701                                                   known_pdftex_graphics_formats);
2702                                 if (!dvips_name.empty()) {
2703                                         if (!pdftex_name.empty()) {
2704                                                 cerr << "This file contains the "
2705                                                         "latex snippet\n"
2706                                                         "\"\\includegraphics{"
2707                                                      << name << "}\".\n"
2708                                                         "However, files\n\""
2709                                                      << dvips_name << "\" and\n\""
2710                                                      << pdftex_name << "\"\n"
2711                                                         "both exist, so I had to make a "
2712                                                         "choice and took the first one.\n"
2713                                                         "Please move the unwanted one "
2714                                                         "someplace else and try again\n"
2715                                                         "if my choice was wrong."
2716                                                      << endl;
2717                                         }
2718                                         name = dvips_name;
2719                                 } else if (!pdftex_name.empty()) {
2720                                         name = pdftex_name;
2721                                         pdflatex = true;
2722                                 }
2723                         }
2724
2725                         if (makeAbsPath(name, path).exists())
2726                                 fix_relative_filename(name);
2727                         else
2728                                 cerr << "Warning: Could not find graphics file '"
2729                                      << name << "'." << endl;
2730
2731                         context.check_layout(os);
2732                         begin_inset(os, "Graphics ");
2733                         os << "\n\tfilename " << name << '\n';
2734                         if (opts.find("width") != opts.end())
2735                                 os << "\twidth "
2736                                    << translate_len(opts["width"]) << '\n';
2737                         if (opts.find("height") != opts.end())
2738                                 os << "\theight "
2739                                    << translate_len(opts["height"]) << '\n';
2740                         if (opts.find("scale") != opts.end()) {
2741                                 istringstream iss(opts["scale"]);
2742                                 double val;
2743                                 iss >> val;
2744                                 val = val*100;
2745                                 os << "\tscale " << val << '\n';
2746                         }
2747                         if (opts.find("angle") != opts.end()) {
2748                                 os << "\trotateAngle "
2749                                    << opts["angle"] << '\n';
2750                                 vector<string>::const_iterator a =
2751                                         find(keys.begin(), keys.end(), "angle");
2752                                 vector<string>::const_iterator s =
2753                                         find(keys.begin(), keys.end(), "width");
2754                                 if (s == keys.end())
2755                                         s = find(keys.begin(), keys.end(), "height");
2756                                 if (s == keys.end())
2757                                         s = find(keys.begin(), keys.end(), "scale");
2758                                 if (s != keys.end() && distance(s, a) > 0)
2759                                         os << "\tscaleBeforeRotation\n";
2760                         }
2761                         if (opts.find("origin") != opts.end()) {
2762                                 ostringstream ss;
2763                                 string const opt = opts["origin"];
2764                                 if (opt.find('l') != string::npos) ss << "left";
2765                                 if (opt.find('r') != string::npos) ss << "right";
2766                                 if (opt.find('c') != string::npos) ss << "center";
2767                                 if (opt.find('t') != string::npos) ss << "Top";
2768                                 if (opt.find('b') != string::npos) ss << "Bottom";
2769                                 if (opt.find('B') != string::npos) ss << "Baseline";
2770                                 if (!ss.str().empty())
2771                                         os << "\trotateOrigin " << ss.str() << '\n';
2772                                 else
2773                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
2774                         }
2775                         if (opts.find("keepaspectratio") != opts.end())
2776                                 os << "\tkeepAspectRatio\n";
2777                         if (opts.find("clip") != opts.end())
2778                                 os << "\tclip\n";
2779                         if (opts.find("draft") != opts.end())
2780                                 os << "\tdraft\n";
2781                         if (opts.find("bb") != opts.end())
2782                                 os << "\tBoundingBox "
2783                                    << opts["bb"] << '\n';
2784                         int numberOfbbOptions = 0;
2785                         if (opts.find("bbllx") != opts.end())
2786                                 numberOfbbOptions++;
2787                         if (opts.find("bblly") != opts.end())
2788                                 numberOfbbOptions++;
2789                         if (opts.find("bburx") != opts.end())
2790                                 numberOfbbOptions++;
2791                         if (opts.find("bbury") != opts.end())
2792                                 numberOfbbOptions++;
2793                         if (numberOfbbOptions == 4)
2794                                 os << "\tBoundingBox "
2795                                    << opts["bbllx"] << " " << opts["bblly"] << " "
2796                                    << opts["bburx"] << " " << opts["bbury"] << '\n';
2797                         else if (numberOfbbOptions > 0)
2798                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2799                         numberOfbbOptions = 0;
2800                         if (opts.find("natwidth") != opts.end())
2801                                 numberOfbbOptions++;
2802                         if (opts.find("natheight") != opts.end())
2803                                 numberOfbbOptions++;
2804                         if (numberOfbbOptions == 2)
2805                                 os << "\tBoundingBox 0bp 0bp "
2806                                    << opts["natwidth"] << " " << opts["natheight"] << '\n';
2807                         else if (numberOfbbOptions > 0)
2808                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2809                         ostringstream special;
2810                         if (opts.find("hiresbb") != opts.end())
2811                                 special << "hiresbb,";
2812                         if (opts.find("trim") != opts.end())
2813                                 special << "trim,";
2814                         if (opts.find("viewport") != opts.end())
2815                                 special << "viewport=" << opts["viewport"] << ',';
2816                         if (opts.find("totalheight") != opts.end())
2817                                 special << "totalheight=" << opts["totalheight"] << ',';
2818                         if (opts.find("type") != opts.end())
2819                                 special << "type=" << opts["type"] << ',';
2820                         if (opts.find("ext") != opts.end())
2821                                 special << "ext=" << opts["ext"] << ',';
2822                         if (opts.find("read") != opts.end())
2823                                 special << "read=" << opts["read"] << ',';
2824                         if (opts.find("command") != opts.end())
2825                                 special << "command=" << opts["command"] << ',';
2826                         string s_special = special.str();
2827                         if (!s_special.empty()) {
2828                                 // We had special arguments. Remove the trailing ','.
2829                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
2830                         }
2831                         // TODO: Handle the unknown settings better.
2832                         // Warn about invalid options.
2833                         // Check whether some option was given twice.
2834                         end_inset(os);
2835                         preamble.registerAutomaticallyLoadedPackage("graphicx");
2836                 }
2837
2838                 else if (t.cs() == "footnote" ||
2839                          (t.cs() == "thanks" && context.layout->intitle)) {
2840                         p.skip_spaces();
2841                         context.check_layout(os);
2842                         begin_inset(os, "Foot\n");
2843                         os << "status collapsed\n\n";
2844                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2845                         end_inset(os);
2846                 }
2847
2848                 else if (t.cs() == "marginpar") {
2849                         p.skip_spaces();
2850                         context.check_layout(os);
2851                         begin_inset(os, "Marginal\n");
2852                         os << "status collapsed\n\n";
2853                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2854                         end_inset(os);
2855                 }
2856
2857                 else if (t.cs() == "lstinline") {
2858                         p.skip_spaces();
2859                         parse_listings(p, os, context, true);
2860                 }
2861
2862                 else if (t.cs() == "ensuremath") {
2863                         p.skip_spaces();
2864                         context.check_layout(os);
2865                         string const s = p.verbatim_item();
2866                         //FIXME: this never triggers in UTF8
2867                         if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
2868                                 os << s;
2869                         else
2870                                 handle_ert(os, "\\ensuremath{" + s + "}",
2871                                            context);
2872                 }
2873
2874                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
2875                         if (preamble.titleLayoutFound()) {
2876                                 // swallow this
2877                                 skip_spaces_braces(p);
2878                         } else
2879                                 handle_ert(os, t.asInput(), context);
2880                 }
2881
2882                 else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
2883                         context.check_layout(os);
2884                         begin_command_inset(os, "toc", t.cs());
2885                         end_inset(os);
2886                         skip_spaces_braces(p);
2887                         if (t.cs() == "lstlistoflistings")
2888                                 preamble.registerAutomaticallyLoadedPackage("listings");
2889                 }
2890
2891                 else if (t.cs() == "listoffigures") {
2892                         context.check_layout(os);
2893                         begin_inset(os, "FloatList figure\n");
2894                         end_inset(os);
2895                         skip_spaces_braces(p);
2896                 }
2897
2898                 else if (t.cs() == "listoftables") {
2899                         context.check_layout(os);
2900                         begin_inset(os, "FloatList table\n");
2901                         end_inset(os);
2902                         skip_spaces_braces(p);
2903                 }
2904
2905                 else if (t.cs() == "listof") {
2906                         p.skip_spaces(true);
2907                         string const name = p.get_token().cs();
2908                         if (context.textclass.floats().typeExist(name)) {
2909                                 context.check_layout(os);
2910                                 begin_inset(os, "FloatList ");
2911                                 os << name << "\n";
2912                                 end_inset(os);
2913                                 p.get_token(); // swallow second arg
2914                         } else
2915                                 handle_ert(os, "\\listof{" + name + "}", context);
2916                 }
2917
2918                 else if ((where = is_known(t.cs(), known_text_font_families)))
2919                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2920                                 context, "\\family", context.font.family,
2921                                 known_coded_font_families[where - known_text_font_families]);
2922
2923                 else if ((where = is_known(t.cs(), known_text_font_series)))
2924                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2925                                 context, "\\series", context.font.series,
2926                                 known_coded_font_series[where - known_text_font_series]);
2927
2928                 else if ((where = is_known(t.cs(), known_text_font_shapes)))
2929                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2930                                 context, "\\shape", context.font.shape,
2931                                 known_coded_font_shapes[where - known_text_font_shapes]);
2932
2933                 else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
2934                         context.check_layout(os);
2935                         TeXFont oldFont = context.font;
2936                         context.font.init();
2937                         context.font.size = oldFont.size;
2938                         os << "\n\\family " << context.font.family << "\n";
2939                         os << "\n\\series " << context.font.series << "\n";
2940                         os << "\n\\shape " << context.font.shape << "\n";
2941                         if (t.cs() == "textnormal") {
2942                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2943                                 output_font_change(os, context.font, oldFont);
2944                                 context.font = oldFont;
2945                         } else
2946                                 eat_whitespace(p, os, context, false);
2947                 }
2948
2949                 else if (t.cs() == "textcolor") {
2950                         // scheme is \textcolor{color name}{text}
2951                         string const color = p.verbatim_item();
2952                         // we only support the predefined colors of the color package
2953                         if (color == "black" || color == "blue" || color == "cyan"
2954                                 || color == "green" || color == "magenta" || color == "red"
2955                                 || color == "white" || color == "yellow") {
2956                                         context.check_layout(os);
2957                                         os << "\n\\color " << color << "\n";
2958                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2959                                         context.check_layout(os);
2960                                         os << "\n\\color inherit\n";
2961                                         preamble.registerAutomaticallyLoadedPackage("color");
2962                         } else
2963                                 // for custom defined colors
2964                                 handle_ert(os, t.asInput() + "{" + color + "}", context);
2965                 }
2966
2967                 else if (t.cs() == "underbar" || t.cs() == "uline") {
2968                         // \underbar is not 100% correct (LyX outputs \uline
2969                         // of ulem.sty). The difference is that \ulem allows
2970                         // line breaks, and \underbar does not.
2971                         // Do NOT handle \underline.
2972                         // \underbar cuts through y, g, q, p etc.,
2973                         // \underline does not.
2974                         context.check_layout(os);
2975                         os << "\n\\bar under\n";
2976                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2977                         context.check_layout(os);
2978                         os << "\n\\bar default\n";
2979                         preamble.registerAutomaticallyLoadedPackage("ulem");
2980                 }
2981
2982                 else if (t.cs() == "sout") {
2983                         context.check_layout(os);
2984                         os << "\n\\strikeout on\n";
2985                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2986                         context.check_layout(os);
2987                         os << "\n\\strikeout default\n";
2988                         preamble.registerAutomaticallyLoadedPackage("ulem");
2989                 }
2990
2991                 else if (t.cs() == "uuline" || t.cs() == "uwave" ||
2992                          t.cs() == "emph" || t.cs() == "noun") {
2993                         context.check_layout(os);
2994                         os << "\n\\" << t.cs() << " on\n";
2995                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2996                         context.check_layout(os);
2997                         os << "\n\\" << t.cs() << " default\n";
2998                         if (t.cs() == "uuline" || t.cs() == "uwave")
2999                                 preamble.registerAutomaticallyLoadedPackage("ulem");
3000                 }
3001
3002                 else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
3003                         context.check_layout(os);
3004                         string name = p.getArg('{', '}');
3005                         string localtime = p.getArg('{', '}');
3006                         preamble.registerAuthor(name);
3007                         Author const & author = preamble.getAuthor(name);
3008                         // from_ctime() will fail if LyX decides to output the
3009                         // time in the text language. It might also use a wrong
3010                         // time zone (if the original LyX document was exported
3011                         // with a different time zone).
3012                         time_t ptime = from_ctime(localtime);
3013                         if (ptime == static_cast<time_t>(-1)) {
3014                                 cerr << "Warning: Could not parse time `" << localtime
3015                                      << "´ for change tracking, using current time instead.\n";
3016                                 ptime = current_time();
3017                         }
3018                         if (t.cs() == "lyxadded")
3019                                 os << "\n\\change_inserted ";
3020                         else
3021                                 os << "\n\\change_deleted ";
3022                         os << author.bufferId() << ' ' << ptime << '\n';
3023                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3024                         bool dvipost    = LaTeXPackages::isAvailable("dvipost");
3025                         bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
3026                                           LaTeXPackages::isAvailable("xcolor");
3027                         // No need to test for luatex, since luatex comes in
3028                         // two flavours (dvi and pdf), like latex, and those
3029                         // are detected by pdflatex.
3030                         if (pdflatex || xetex) {
3031                                 if (xcolorulem) {
3032                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3033                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3034                                         preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
3035                                 }
3036                         } else {
3037                                 if (dvipost) {
3038                                         preamble.registerAutomaticallyLoadedPackage("dvipost");
3039                                 } else if (xcolorulem) {
3040                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3041                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3042                                 }
3043                         }
3044                 }
3045
3046                 else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
3047                              t.cs() == "vphantom") {
3048                         context.check_layout(os);
3049                         if (t.cs() == "phantom")
3050                                 begin_inset(os, "Phantom Phantom\n");
3051                         if (t.cs() == "hphantom")
3052                                 begin_inset(os, "Phantom HPhantom\n");
3053                         if (t.cs() == "vphantom")
3054                                 begin_inset(os, "Phantom VPhantom\n");
3055                         os << "status open\n";
3056                         parse_text_in_inset(p, os, FLAG_ITEM, outer, context,
3057                                             "Phantom");
3058                         end_inset(os);
3059                 }
3060
3061                 else if (t.cs() == "href") {
3062                         context.check_layout(os);
3063                         string target = p.getArg('{', '}');
3064                         string name = p.getArg('{', '}');
3065                         string type;
3066                         size_t i = target.find(':');
3067                         if (i != string::npos) {
3068                                 type = target.substr(0, i + 1);
3069                                 if (type == "mailto:" || type == "file:")
3070                                         target = target.substr(i + 1);
3071                                 // handle the case that name is equal to target, except of "http://"
3072                                 else if (target.substr(i + 3) == name && type == "http:")
3073                                         target = name;
3074                         }
3075                         begin_command_inset(os, "href", "href");
3076                         if (name != target)
3077                                 os << "name \"" << name << "\"\n";
3078                         os << "target \"" << target << "\"\n";
3079                         if (type == "mailto:" || type == "file:")
3080                                 os << "type \"" << type << "\"\n";
3081                         end_inset(os);
3082                         skip_spaces_braces(p);
3083                 }
3084
3085                 else if (t.cs() == "lyxline") {
3086                         // swallow size argument (it is not used anyway)
3087                         p.getArg('{', '}');
3088                         if (!context.atParagraphStart()) {
3089                                 // so our line is in the middle of a paragraph
3090                                 // we need to add a new line, lest this line
3091                                 // follow the other content on that line and
3092                                 // run off the side of the page
3093                                 // FIXME: This may create an empty paragraph,
3094                                 //        but without that it would not be
3095                                 //        possible to set noindent below.
3096                                 //        Fortunately LaTeX does not care
3097                                 //        about the empty paragraph.
3098                                 context.new_paragraph(os);
3099                         }
3100                         if (preamble.indentParagraphs()) {
3101                                 // we need to unindent, lest the line be too long
3102                                 context.add_par_extra_stuff("\\noindent\n");
3103                         }
3104                         context.check_layout(os);
3105                         begin_command_inset(os, "line", "rule");
3106                         os << "offset \"0.5ex\"\n"
3107                               "width \"100line%\"\n"
3108                               "height \"1pt\"\n";
3109                         end_inset(os);
3110                 }
3111
3112                 else if (t.cs() == "rule") {
3113                         string const offset = (p.hasOpt() ? p.getArg('[', ']') : string());
3114                         string const width = p.getArg('{', '}');
3115                         string const thickness = p.getArg('{', '}');
3116                         context.check_layout(os);
3117                         begin_command_inset(os, "line", "rule");
3118                         if (!offset.empty())
3119                                 os << "offset \"" << translate_len(offset) << "\"\n";
3120                         os << "width \"" << translate_len(width) << "\"\n"
3121                                   "height \"" << translate_len(thickness) << "\"\n";
3122                         end_inset(os);
3123                 }
3124
3125                 else if (is_known(t.cs(), known_phrases) ||
3126                          (t.cs() == "protect" &&
3127                           p.next_token().cat() == catEscape &&
3128                           is_known(p.next_token().cs(), known_phrases))) {
3129                         // LyX sometimes puts a \protect in front, so we have to ignore it
3130                         // FIXME: This needs to be changed when bug 4752 is fixed.
3131                         where = is_known(
3132                                 t.cs() == "protect" ? p.get_token().cs() : t.cs(),
3133                                 known_phrases);
3134                         context.check_layout(os);
3135                         os << known_coded_phrases[where - known_phrases];
3136                         skip_spaces_braces(p);
3137                 }
3138
3139                 else if ((where = is_known(t.cs(), known_ref_commands))) {
3140                         string const opt = p.getOpt();
3141                         if (opt.empty()) {
3142                                 context.check_layout(os);
3143                                 begin_command_inset(os, "ref",
3144                                         known_coded_ref_commands[where - known_ref_commands]);
3145                                 os << "reference \""
3146                                    << convert_command_inset_arg(p.verbatim_item())
3147                                    << "\"\n";
3148                                 end_inset(os);
3149                                 if (t.cs() == "vref" || t.cs() == "vpageref")
3150                                         preamble.registerAutomaticallyLoadedPackage("varioref");
3151
3152                         } else {
3153                                 // LyX does not support optional arguments of ref commands
3154                                 handle_ert(os, t.asInput() + '[' + opt + "]{" +
3155                                                p.verbatim_item() + "}", context);
3156                         }
3157                 }
3158
3159                 else if (use_natbib &&
3160                          is_known(t.cs(), known_natbib_commands) &&
3161                          ((t.cs() != "citefullauthor" &&
3162                            t.cs() != "citeyear" &&
3163                            t.cs() != "citeyearpar") ||
3164                           p.next_token().asInput() != "*")) {
3165                         context.check_layout(os);
3166                         string command = t.cs();
3167                         if (p.next_token().asInput() == "*") {
3168                                 command += '*';
3169                                 p.get_token();
3170                         }
3171                         if (command == "citefullauthor")
3172                                 // alternative name for "\\citeauthor*"
3173                                 command = "citeauthor*";
3174
3175                         // text before the citation
3176                         string before;
3177                         // text after the citation
3178                         string after;
3179                         get_cite_arguments(p, true, before, after);
3180
3181                         if (command == "cite") {
3182                                 // \cite without optional argument means
3183                                 // \citet, \cite with at least one optional
3184                                 // argument means \citep.
3185                                 if (before.empty() && after.empty())
3186                                         command = "citet";
3187                                 else
3188                                         command = "citep";
3189                         }
3190                         if (before.empty() && after == "[]")
3191                                 // avoid \citet[]{a}
3192                                 after.erase();
3193                         else if (before == "[]" && after == "[]") {
3194                                 // avoid \citet[][]{a}
3195                                 before.erase();
3196                                 after.erase();
3197                         }
3198                         // remove the brackets around after and before
3199                         if (!after.empty()) {
3200                                 after.erase(0, 1);
3201                                 after.erase(after.length() - 1, 1);
3202                                 after = convert_command_inset_arg(after);
3203                         }
3204                         if (!before.empty()) {
3205                                 before.erase(0, 1);
3206                                 before.erase(before.length() - 1, 1);
3207                                 before = convert_command_inset_arg(before);
3208                         }
3209                         begin_command_inset(os, "citation", command);
3210                         os << "after " << '"' << after << '"' << "\n";
3211                         os << "before " << '"' << before << '"' << "\n";
3212                         os << "key \""
3213                            << convert_command_inset_arg(p.verbatim_item())
3214                            << "\"\n";
3215                         end_inset(os);
3216                 }
3217
3218                 else if (use_jurabib &&
3219                          is_known(t.cs(), known_jurabib_commands) &&
3220                          (t.cs() == "cite" || p.next_token().asInput() != "*")) {
3221                         context.check_layout(os);
3222                         string command = t.cs();
3223                         if (p.next_token().asInput() == "*") {
3224                                 command += '*';
3225                                 p.get_token();
3226                         }
3227                         char argumentOrder = '\0';
3228                         vector<string> const options =
3229                                 preamble.getPackageOptions("jurabib");
3230                         if (find(options.begin(), options.end(),
3231                                       "natbiborder") != options.end())
3232                                 argumentOrder = 'n';
3233                         else if (find(options.begin(), options.end(),
3234                                            "jurabiborder") != options.end())
3235                                 argumentOrder = 'j';
3236
3237                         // text before the citation
3238                         string before;
3239                         // text after the citation
3240                         string after;
3241                         get_cite_arguments(p, argumentOrder != 'j', before, after);
3242
3243                         string const citation = p.verbatim_item();
3244                         if (!before.empty() && argumentOrder == '\0') {
3245                                 cerr << "Warning: Assuming argument order "
3246                                         "of jurabib version 0.6 for\n'"
3247                                      << command << before << after << '{'
3248                                      << citation << "}'.\n"
3249                                         "Add 'jurabiborder' to the jurabib "
3250                                         "package options if you used an\n"
3251                                         "earlier jurabib version." << endl;
3252                         }
3253                         if (!after.empty()) {
3254                                 after.erase(0, 1);
3255                                 after.erase(after.length() - 1, 1);
3256                         }
3257                         if (!before.empty()) {
3258                                 before.erase(0, 1);
3259                                 before.erase(before.length() - 1, 1);
3260                         }
3261                         begin_command_inset(os, "citation", command);
3262                         os << "after " << '"' << after << '"' << "\n";
3263                         os << "before " << '"' << before << '"' << "\n";
3264                         os << "key " << '"' << citation << '"' << "\n";
3265                         end_inset(os);
3266                 }
3267
3268                 else if (t.cs() == "cite"
3269                         || t.cs() == "nocite") {
3270                         context.check_layout(os);
3271                         string after = convert_command_inset_arg(p.getArg('[', ']'));
3272                         string key = convert_command_inset_arg(p.verbatim_item());
3273                         // store the case that it is "\nocite{*}" to use it later for
3274                         // the BibTeX inset
3275                         if (key != "*") {
3276                                 begin_command_inset(os, "citation", t.cs());
3277                                 os << "after " << '"' << after << '"' << "\n";
3278                                 os << "key " << '"' << key << '"' << "\n";
3279                                 end_inset(os);
3280                         } else if (t.cs() == "nocite")
3281                                 btprint = key;
3282                 }
3283
3284                 else if (t.cs() == "index" ||
3285                          (t.cs() == "sindex" && preamble.use_indices() == "true")) {
3286                         context.check_layout(os);
3287                         string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
3288                                 p.getArg('[', ']') : "";
3289                         string const kind = arg.empty() ? "idx" : arg;
3290                         begin_inset(os, "Index ");
3291                         os << kind << "\nstatus collapsed\n";
3292                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
3293                         end_inset(os);
3294                         if (kind != "idx")
3295                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3296                 }
3297
3298                 else if (t.cs() == "nomenclature") {
3299                         context.check_layout(os);
3300                         begin_command_inset(os, "nomenclature", "nomenclature");
3301                         string prefix = convert_command_inset_arg(p.getArg('[', ']'));
3302                         if (!prefix.empty())
3303                                 os << "prefix " << '"' << prefix << '"' << "\n";
3304                         os << "symbol " << '"'
3305                            << convert_command_inset_arg(p.verbatim_item());
3306                         os << "\"\ndescription \""
3307                            << convert_command_inset_arg(p.verbatim_item())
3308                            << "\"\n";
3309                         end_inset(os);
3310                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3311                 }
3312
3313                 else if (t.cs() == "label") {
3314                         context.check_layout(os);
3315                         begin_command_inset(os, "label", "label");
3316                         os << "name \""
3317                            << convert_command_inset_arg(p.verbatim_item())
3318                            << "\"\n";
3319                         end_inset(os);
3320                 }
3321
3322                 else if (t.cs() == "printindex") {
3323                         context.check_layout(os);
3324                         begin_command_inset(os, "index_print", "printindex");
3325                         os << "type \"idx\"\n";
3326                         end_inset(os);
3327                         skip_spaces_braces(p);
3328                         preamble.registerAutomaticallyLoadedPackage("makeidx");
3329                         if (preamble.use_indices() == "true")
3330                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3331                 }
3332
3333                 else if (t.cs() == "printnomenclature") {
3334                         string width = "";
3335                         string width_type = "";
3336                         context.check_layout(os);
3337                         begin_command_inset(os, "nomencl_print", "printnomenclature");
3338                         // case of a custom width
3339                         if (p.hasOpt()) {
3340                                 width = p.getArg('[', ']');
3341                                 width = translate_len(width);
3342                                 width_type = "custom";
3343                         }
3344                         // case of no custom width
3345                         // the case of no custom width but the width set
3346                         // via \settowidth{\nomlabelwidth}{***} cannot be supported
3347                         // because the user could have set anything, not only the width
3348                         // of the longest label (which would be width_type = "auto")
3349                         string label = convert_command_inset_arg(p.getArg('{', '}'));
3350                         if (label.empty() && width_type.empty())
3351                                 width_type = "none";
3352                         os << "set_width \"" << width_type << "\"\n";
3353                         if (width_type == "custom")
3354                                 os << "width \"" << width << '\"';
3355                         end_inset(os);
3356                         skip_spaces_braces(p);
3357                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3358                 }
3359
3360                 else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
3361                         context.check_layout(os);
3362                         begin_inset(os, "script ");
3363                         os << t.cs().substr(4) << '\n';
3364                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
3365                         end_inset(os);
3366                         if (t.cs() == "textsubscript")
3367                                 preamble.registerAutomaticallyLoadedPackage("subscript");
3368                 }
3369
3370                 else if ((where = is_known(t.cs(), known_quotes))) {
3371                         context.check_layout(os);
3372                         begin_inset(os, "Quotes ");
3373                         os << known_coded_quotes[where - known_quotes];
3374                         end_inset(os);
3375                         // LyX adds {} after the quote, so we have to eat
3376                         // spaces here if there are any before a possible
3377                         // {} pair.
3378                         eat_whitespace(p, os, context, false);
3379                         skip_braces(p);
3380                 }
3381
3382                 else if ((where = is_known(t.cs(), known_sizes)) &&
3383                          context.new_layout_allowed) {
3384                         context.check_layout(os);
3385                         TeXFont const oldFont = context.font;
3386                         context.font.size = known_coded_sizes[where - known_sizes];
3387                         output_font_change(os, oldFont, context.font);
3388                         eat_whitespace(p, os, context, false);
3389                 }
3390
3391                 else if ((where = is_known(t.cs(), known_font_families)) &&
3392                          context.new_layout_allowed) {
3393                         context.check_layout(os);
3394                         TeXFont const oldFont = context.font;
3395                         context.font.family =
3396                                 known_coded_font_families[where - known_font_families];
3397                         output_font_change(os, oldFont, context.font);
3398                         eat_whitespace(p, os, context, false);
3399                 }
3400
3401                 else if ((where = is_known(t.cs(), known_font_series)) &&
3402                          context.new_layout_allowed) {
3403                         context.check_layout(os);
3404                         TeXFont const oldFont = context.font;
3405                         context.font.series =
3406                                 known_coded_font_series[where - known_font_series];
3407                         output_font_change(os, oldFont, context.font);
3408                         eat_whitespace(p, os, context, false);
3409                 }
3410
3411                 else if ((where = is_known(t.cs(), known_font_shapes)) &&
3412                          context.new_layout_allowed) {
3413                         context.check_layout(os);
3414                         TeXFont const oldFont = context.font;
3415                         context.font.shape =
3416                                 known_coded_font_shapes[where - known_font_shapes];
3417                         output_font_change(os, oldFont, context.font);
3418                         eat_whitespace(p, os, context, false);
3419                 }
3420                 else if ((where = is_known(t.cs(), known_old_font_families)) &&
3421                          context.new_layout_allowed) {
3422                         context.check_layout(os);
3423                         TeXFont const oldFont = context.font;
3424                         context.font.init();
3425                         context.font.size = oldFont.size;
3426                         context.font.family =
3427                                 known_coded_font_families[where - known_old_font_families];
3428                         output_font_change(os, oldFont, context.font);
3429                         eat_whitespace(p, os, context, false);
3430                 }
3431
3432                 else if ((where = is_known(t.cs(), known_old_font_series)) &&
3433                          context.new_layout_allowed) {
3434                         context.check_layout(os);
3435                         TeXFont const oldFont = context.font;
3436                         context.font.init();
3437                         context.font.size = oldFont.size;
3438                         context.font.series =
3439                                 known_coded_font_series[where - known_old_font_series];
3440                         output_font_change(os, oldFont, context.font);
3441                         eat_whitespace(p, os, context, false);
3442                 }
3443
3444                 else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
3445                          context.new_layout_allowed) {
3446                         context.check_layout(os);
3447                         TeXFont const oldFont = context.font;
3448                         context.font.init();
3449                         context.font.size = oldFont.size;
3450                         context.font.shape =
3451                                 known_coded_font_shapes[where - known_old_font_shapes];
3452                         output_font_change(os, oldFont, context.font);
3453                         eat_whitespace(p, os, context, false);
3454                 }
3455
3456                 else if (t.cs() == "selectlanguage") {
3457                         context.check_layout(os);
3458                         // save the language for the case that a
3459                         // \foreignlanguage is used
3460                         context.font.language = babel2lyx(p.verbatim_item());
3461                         os << "\n\\lang " << context.font.language << "\n";
3462                 }
3463
3464                 else if (t.cs() == "foreignlanguage") {
3465                         string const lang = babel2lyx(p.verbatim_item());
3466                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3467                                               context, "\\lang",
3468                                               context.font.language, lang);
3469                 }
3470
3471                 else if (is_known(t.cs().substr(4, string::npos), polyglossia_languages)) {
3472                         // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[]
3473                         string const lang = polyglossia2lyx(t.cs().substr(4, string::npos));
3474                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3475                                               context, "\\lang",
3476                                               context.font.language, lang);
3477                 }
3478
3479                 else if (t.cs() == "inputencoding") {
3480                         // nothing to write here
3481                         string const enc = subst(p.verbatim_item(), "\n", " ");
3482                         p.setEncoding(enc);
3483                 }
3484
3485                 else if ((where = is_known(t.cs(), known_special_chars))) {
3486                         context.check_layout(os);
3487                         os << "\\SpecialChar \\"
3488                            << known_coded_special_chars[where - known_special_chars]
3489                            << '\n';
3490                         skip_spaces_braces(p);
3491                 }
3492
3493                 else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
3494                         context.check_layout(os);
3495                         os << "\\SpecialChar \\nobreakdash-\n";
3496                         p.get_token();
3497                 }
3498
3499                 else if (t.cs() == "textquotedbl") {
3500                         context.check_layout(os);
3501                         os << "\"";
3502                         skip_braces(p);
3503                 }
3504
3505                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
3506                         context.check_layout(os);
3507                         os << "\\SpecialChar \\@.\n";
3508                         p.get_token();
3509                 }
3510
3511                 else if (t.cs() == "-") {
3512                         context.check_layout(os);
3513                         os << "\\SpecialChar \\-\n";
3514                 }
3515
3516                 else if (t.cs() == "textasciitilde") {
3517                         context.check_layout(os);
3518                         os << '~';
3519                         skip_spaces_braces(p);
3520                 }
3521
3522                 else if (t.cs() == "textasciicircum") {
3523                         context.check_layout(os);
3524                         os << '^';
3525                         skip_spaces_braces(p);
3526                 }
3527
3528                 else if (t.cs() == "textbackslash") {
3529                         context.check_layout(os);
3530                         os << "\n\\backslash\n";
3531                         skip_spaces_braces(p);
3532                 }
3533
3534                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
3535                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
3536                             || t.cs() == "%") {
3537                         context.check_layout(os);
3538                         os << t.cs();
3539                 }
3540
3541                 else if (t.cs() == "char") {
3542                         context.check_layout(os);
3543                         if (p.next_token().character() == '`') {
3544                                 p.get_token();
3545                                 if (p.next_token().cs() == "\"") {
3546                                         p.get_token();
3547                                         os << '"';
3548                                         skip_braces(p);
3549                                 } else {
3550                                         handle_ert(os, "\\char`", context);
3551                                 }
3552                         } else {
3553                                 handle_ert(os, "\\char", context);
3554                         }
3555                 }
3556
3557                 else if (t.cs() == "verb") {
3558                         context.check_layout(os);
3559                         char const delimiter = p.next_token().character();
3560                         string const arg = p.getArg(delimiter, delimiter);
3561                         ostringstream oss;
3562                         oss << "\\verb" << delimiter << arg << delimiter;
3563                         handle_ert(os, oss.str(), context);
3564                 }
3565
3566                 // Problem: \= creates a tabstop inside the tabbing environment
3567                 // and else an accent. In the latter case we really would want
3568                 // \={o} instead of \= o.
3569                 else if (t.cs() == "=" && (flags & FLAG_TABBING))
3570                         handle_ert(os, t.asInput(), context);
3571
3572                 // accents (see Table 6 in Comprehensive LaTeX Symbol List)
3573                 else if (t.cs().size() == 1
3574                          && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
3575                         context.check_layout(os);
3576                         // try to see whether the string is in unicodesymbols
3577                         bool termination;
3578                         docstring rem;
3579                         string command = t.asInput() + "{"
3580                                 + trimSpaceAndEol(p.verbatim_item())
3581                                 + "}";
3582                         set<string> req;
3583                         docstring s = encodings.fromLaTeXCommand(from_utf8(command),
3584                                 Encodings::TEXT_CMD | Encodings::MATH_CMD,
3585                                 termination, rem, &req);
3586                         if (!s.empty()) {
3587                                 if (!rem.empty())
3588                                         cerr << "When parsing " << command
3589                                              << ", result is " << to_utf8(s)
3590                                              << "+" << to_utf8(rem) << endl;
3591                                 os << to_utf8(s);
3592                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
3593                                         preamble.registerAutomaticallyLoadedPackage(*it);
3594                         } else
3595                                 // we did not find a non-ert version
3596                                 handle_ert(os, command, context);
3597                 }
3598
3599                 else if (t.cs() == "\\") {
3600                         context.check_layout(os);
3601                         if (p.hasOpt())
3602                                 handle_ert(os, "\\\\" + p.getOpt(), context);
3603                         else if (p.next_token().asInput() == "*") {
3604                                 p.get_token();
3605                                 // getOpt() eats the following space if there
3606                                 // is no optional argument, but that is OK
3607                                 // here since it has no effect in the output.
3608                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
3609                         }
3610                         else {
3611                                 begin_inset(os, "Newline newline");
3612                                 end_inset(os);
3613                         }
3614                 }
3615
3616                 else if (t.cs() == "newline" ||
3617                          (t.cs() == "linebreak" && !p.hasOpt())) {
3618                         context.check_layout(os);
3619                         begin_inset(os, "Newline ");
3620                         os << t.cs();
3621                         end_inset(os);
3622                         skip_spaces_braces(p);
3623                 }
3624
3625                 else if (t.cs() == "input" || t.cs() == "include"
3626                          || t.cs() == "verbatiminput") {
3627                         string name = t.cs();
3628                         if (t.cs() == "verbatiminput"
3629                             && p.next_token().asInput() == "*")
3630                                 name += p.get_token().asInput();
3631                         context.check_layout(os);
3632                         string filename(normalize_filename(p.getArg('{', '}')));
3633                         string const path = getMasterFilePath();
3634                         // We want to preserve relative / absolute filenames,
3635                         // therefore path is only used for testing
3636                         if ((t.cs() == "include" || t.cs() == "input") &&
3637                             !makeAbsPath(filename, path).exists()) {
3638                                 // The file extension is probably missing.
3639                                 // Now try to find it out.
3640                                 string const tex_name =
3641                                         find_file(filename, path,
3642                                                   known_tex_extensions);
3643                                 if (!tex_name.empty())
3644                                         filename = tex_name;
3645                         }
3646                         bool external = false;
3647                         string outname;
3648                         if (makeAbsPath(filename, path).exists()) {
3649                                 string const abstexname =
3650                                         makeAbsPath(filename, path).absFileName();
3651                                 string const abslyxname =
3652                                         changeExtension(abstexname, ".lyx");
3653                                 string const absfigname =
3654                                         changeExtension(abstexname, ".fig");
3655                                 fix_relative_filename(filename);
3656                                 string const lyxname =
3657                                         changeExtension(filename, ".lyx");
3658                                 bool xfig = false;
3659                                 external = FileName(absfigname).exists();
3660                                 if (t.cs() == "input") {
3661                                         string const ext = getExtension(abstexname);
3662
3663                                         // Combined PS/LaTeX:
3664                                         // x.eps, x.pstex_t (old xfig)
3665                                         // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
3666                                         FileName const absepsname(
3667                                                 changeExtension(abstexname, ".eps"));
3668                                         FileName const abspstexname(
3669                                                 changeExtension(abstexname, ".pstex"));
3670                                         bool const xfigeps =
3671                                                 (absepsname.exists() ||
3672                                                  abspstexname.exists()) &&
3673                                                 ext == "pstex_t";
3674
3675                                         // Combined PDF/LaTeX:
3676                                         // x.pdf, x.pdftex_t (old xfig)
3677                                         // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
3678                                         FileName const abspdfname(
3679                                                 changeExtension(abstexname, ".pdf"));
3680                                         bool const xfigpdf =
3681                                                 abspdfname.exists() &&
3682                                                 (ext == "pdftex_t" || ext == "pdf_t");
3683                                         if (xfigpdf)
3684                                                 pdflatex = true;
3685
3686                                         // Combined PS/PDF/LaTeX:
3687                                         // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
3688                                         string const absbase2(
3689                                                 removeExtension(abstexname) + "_pspdftex");
3690                                         FileName const abseps2name(
3691                                                 addExtension(absbase2, ".eps"));
3692                                         FileName const abspdf2name(
3693                                                 addExtension(absbase2, ".pdf"));
3694                                         bool const xfigboth =
3695                                                 abspdf2name.exists() &&
3696                                                 abseps2name.exists() && ext == "pspdftex";
3697
3698                                         xfig = xfigpdf || xfigeps || xfigboth;
3699                                         external = external && xfig;
3700                                 }
3701                                 if (external) {
3702                                         outname = changeExtension(filename, ".fig");
3703                                 } else if (xfig) {
3704                                         // Don't try to convert, the result
3705                                         // would be full of ERT.
3706                                         outname = filename;
3707                                 } else if (t.cs() != "verbatiminput" &&
3708                                     tex2lyx(abstexname, FileName(abslyxname),
3709                                             p.getEncoding())) {
3710                                         outname = lyxname;
3711                                 } else {
3712                                         outname = filename;
3713                                 }
3714                         } else {
3715                                 cerr << "Warning: Could not find included file '"
3716                                      << filename << "'." << endl;
3717                                 outname = filename;
3718                         }
3719                         if (external) {
3720                                 begin_inset(os, "External\n");
3721                                 os << "\ttemplate XFig\n"
3722                                    << "\tfilename " << outname << '\n';
3723                                 registerExternalTemplatePackages("XFig");
3724                         } else {
3725                                 begin_command_inset(os, "include", name);
3726                                 os << "preview false\n"
3727                                       "filename \"" << outname << "\"\n";
3728                                 if (t.cs() == "verbatiminput")
3729                                         preamble.registerAutomaticallyLoadedPackage("verbatim");
3730                         }
3731                         end_inset(os);
3732                 }
3733
3734                 else if (t.cs() == "bibliographystyle") {
3735                         // store new bibliographystyle
3736                         bibliographystyle = p.verbatim_item();
3737                         // If any other command than \bibliography and
3738                         // \nocite{*} follows, we need to output the style
3739                         // (because it might be used by that command).
3740                         // Otherwise, it will automatically be output by LyX.
3741                         p.pushPosition();
3742                         bool output = true;
3743                         for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
3744                                 if (t2.cat() == catBegin)
3745                                         break;
3746                                 if (t2.cat() != catEscape)
3747                                         continue;
3748                                 if (t2.cs() == "nocite") {
3749                                         if (p.getArg('{', '}') == "*")
3750                                                 continue;
3751                                 } else if (t2.cs() == "bibliography")
3752                                         output = false;
3753                                 break;
3754                         }
3755                         p.popPosition();
3756                         if (output) {
3757                                 handle_ert(os,
3758                                         "\\bibliographystyle{" + bibliographystyle + '}',
3759                                         context);
3760                         }
3761                 }
3762
3763                 else if (t.cs() == "bibliography") {
3764                         context.check_layout(os);
3765                         begin_command_inset(os, "bibtex", "bibtex");
3766                         if (!btprint.empty()) {
3767                                 os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
3768                                 // clear the string because the next BibTeX inset can be without the
3769                                 // \nocite{*} option
3770                                 btprint.clear();
3771                         }
3772                         os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
3773                         // Do we have a bibliographystyle set?
3774                         if (!bibliographystyle.empty())
3775                                 os << "options " << '"' << bibliographystyle << '"' << "\n";
3776                         end_inset(os);
3777                 }
3778
3779                 else if (t.cs() == "parbox") {
3780                         // Test whether this is an outer box of a shaded box
3781                         p.pushPosition();
3782                         // swallow arguments
3783                         while (p.hasOpt()) {
3784                                 p.getArg('[', ']');
3785                                 p.skip_spaces(true);
3786                         }
3787                         p.getArg('{', '}');
3788                         p.skip_spaces(true);
3789                         // eat the '{'
3790                         if (p.next_token().cat() == catBegin)
3791                                 p.get_token();
3792                         p.skip_spaces(true);
3793                         Token to = p.get_token();
3794                         bool shaded = false;
3795                         if (to.asInput() == "\\begin") {
3796                                 p.skip_spaces(true);
3797                                 if (p.getArg('{', '}') == "shaded")
3798                                         shaded = true;
3799                         }
3800                         p.popPosition();
3801                         if (shaded) {
3802                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3803                                                 context, "parbox", "shaded");
3804                         } else
3805                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3806                                           "", "", t.cs());
3807                 }
3808
3809                 else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
3810                          t.cs() == "shadowbox" || t.cs() == "doublebox")
3811                         parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
3812
3813                 else if (t.cs() == "framebox") {
3814                         if (p.next_token().character() == '(') {
3815                                 //the syntax is: \framebox(x,y)[position]{content}
3816                                 string arg = t.asInput();
3817                                 arg += p.getFullParentheseArg();
3818                                 arg += p.getFullOpt();
3819                                 eat_whitespace(p, os, context, false);
3820                                 handle_ert(os, arg + '{', context);
3821                                 eat_whitespace(p, os, context, false);
3822                                 parse_text(p, os, FLAG_ITEM, outer, context);
3823                                 handle_ert(os, "}", context);
3824                         } else {
3825                                 string special = p.getFullOpt();
3826                                 special += p.getOpt();
3827                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3828                                                 context, t.cs(), special);
3829                         }
3830                 }
3831
3832                 //\makebox() is part of the picture environment and different from \makebox{}
3833                 //\makebox{} will be parsed by parse_box
3834                 else if (t.cs() == "makebox") {
3835                         if (p.next_token().character() == '(') {
3836                                 //the syntax is: \makebox(x,y)[position]{content}
3837                                 string arg = t.asInput();
3838                                 arg += p.getFullParentheseArg();
3839                                 arg += p.getFullOpt();
3840                                 eat_whitespace(p, os, context, false);
3841                                 handle_ert(os, arg + '{', context);
3842                                 eat_whitespace(p, os, context, false);
3843                                 parse_text(p, os, FLAG_ITEM, outer, context);
3844                                 handle_ert(os, "}", context);
3845                         } else
3846                                 //the syntax is: \makebox[width][position]{content}
3847                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3848                                           "", "", t.cs());
3849                 }
3850
3851                 else if (t.cs() == "smallskip" ||
3852                          t.cs() == "medskip" ||
3853                          t.cs() == "bigskip" ||
3854                          t.cs() == "vfill") {
3855                         context.check_layout(os);
3856                         begin_inset(os, "VSpace ");
3857                         os << t.cs();
3858                         end_inset(os);
3859                         skip_spaces_braces(p);
3860                 }
3861
3862                 else if ((where = is_known(t.cs(), known_spaces))) {
3863                         context.check_layout(os);
3864                         begin_inset(os, "space ");
3865                         os << '\\' << known_coded_spaces[where - known_spaces]
3866                            << '\n';
3867                         end_inset(os);
3868                         // LaTeX swallows whitespace after all spaces except
3869                         // "\\,". We have to do that here, too, because LyX
3870                         // adds "{}" which would make the spaces significant.
3871                         if (t.cs() !=  ",")
3872                                 eat_whitespace(p, os, context, false);
3873                         // LyX adds "{}" after all spaces except "\\ " and
3874                         // "\\,", so we have to remove "{}".
3875                         // "\\,{}" is equivalent to "\\," in LaTeX, so we
3876                         // remove the braces after "\\,", too.
3877                         if (t.cs() != " ")
3878                                 skip_braces(p);
3879                 }
3880
3881                 else if (t.cs() == "newpage" ||
3882                          (t.cs() == "pagebreak" && !p.hasOpt()) ||
3883                          t.cs() == "clearpage" ||
3884                          t.cs() == "cleardoublepage") {
3885                         context.check_layout(os);
3886                         begin_inset(os, "Newpage ");
3887                         os << t.cs();
3888                         end_inset(os);
3889                         skip_spaces_braces(p);
3890                 }
3891
3892                 else if (t.cs() == "DeclareRobustCommand" ||
3893                          t.cs() == "DeclareRobustCommandx" ||
3894                          t.cs() == "newcommand" ||
3895                          t.cs() == "newcommandx" ||
3896                          t.cs() == "providecommand" ||
3897                          t.cs() == "providecommandx" ||
3898                          t.cs() == "renewcommand" ||
3899                          t.cs() == "renewcommandx") {
3900                         // DeclareRobustCommand, DeclareRobustCommandx,
3901                         // providecommand and providecommandx could be handled
3902                         // by parse_command(), but we need to call
3903                         // add_known_command() here.
3904                         string name = t.asInput();
3905                         if (p.next_token().asInput() == "*") {
3906                                 // Starred form. Eat '*'
3907                                 p.get_token();
3908                                 name += '*';
3909                         }
3910                         string const command = p.verbatim_item();
3911                         string const opt1 = p.getFullOpt();
3912                         string const opt2 = p.getFullOpt();
3913                         add_known_command(command, opt1, !opt2.empty());
3914                         string const ert = name + '{' + command + '}' +
3915                                            opt1 + opt2 +
3916                                            '{' + p.verbatim_item() + '}';
3917
3918                         if (t.cs() == "DeclareRobustCommand" ||
3919                             t.cs() == "DeclareRobustCommandx" ||
3920                             t.cs() == "providecommand" ||
3921                             t.cs() == "providecommandx" ||
3922                             name[name.length()-1] == '*')
3923                                 handle_ert(os, ert, context);
3924                         else {
3925                                 context.check_layout(os);
3926                                 begin_inset(os, "FormulaMacro");
3927                                 os << "\n" << ert;
3928                                 end_inset(os);
3929                         }
3930                 }
3931
3932                 else if (t.cs() == "let" && p.next_token().asInput() != "*") {
3933                         // let could be handled by parse_command(),
3934                         // but we need to call add_known_command() here.
3935                         string ert = t.asInput();
3936                         string name;
3937                         p.skip_spaces();
3938                         if (p.next_token().cat() == catBegin) {
3939                                 name = p.verbatim_item();
3940                                 ert += '{' + name + '}';
3941                         } else {
3942                                 name = p.verbatim_item();
3943                                 ert += name;
3944                         }
3945                         string command;
3946                         p.skip_spaces();
3947                         if (p.next_token().cat() == catBegin) {
3948                                 command = p.verbatim_item();
3949                                 ert += '{' + command + '}';
3950                         } else {
3951                                 command = p.verbatim_item();
3952                                 ert += command;
3953                         }
3954                         // If command is known, make name known too, to parse
3955                         // its arguments correctly. For this reason we also
3956                         // have commands in syntax.default that are hardcoded.
3957                         CommandMap::iterator it = known_commands.find(command);
3958                         if (it != known_commands.end())
3959                                 known_commands[t.asInput()] = it->second;
3960                         handle_ert(os, ert, context);
3961                 }
3962
3963                 else if (t.cs() == "hspace" || t.cs() == "vspace") {
3964                         bool starred = false;
3965                         if (p.next_token().asInput() == "*") {
3966                                 p.get_token();
3967                                 starred = true;
3968                         }
3969                         string name = t.asInput();
3970                         string const length = p.verbatim_item();
3971                         string unit;
3972                         string valstring;
3973                         bool valid = splitLatexLength(length, valstring, unit);
3974                         bool known_hspace = false;
3975                         bool known_vspace = false;
3976                         bool known_unit = false;
3977                         double value;
3978                         if (valid) {
3979                                 istringstream iss(valstring);
3980                                 iss >> value;
3981                                 if (value == 1.0) {
3982                                         if (t.cs()[0] == 'h') {
3983                                                 if (unit == "\\fill") {
3984                                                         if (!starred) {
3985                                                                 unit = "";
3986                                                                 name = "\\hfill";
3987                                                         }
3988                                                         known_hspace = true;
3989                                                 }
3990                                         } else {
3991                                                 if (unit == "\\smallskipamount") {
3992                                                         unit = "smallskip";
3993                                                         known_vspace = true;
3994                                                 } else if (unit == "\\medskipamount") {
3995                                                         unit = "medskip";
3996                                                         known_vspace = true;
3997                                                 } else if (unit == "\\bigskipamount") {
3998                                                         unit = "bigskip";
3999                                                         known_vspace = true;
4000                                                 } else if (unit == "\\fill") {
4001                                                         unit = "vfill";
4002                                                         known_vspace = true;
4003                                                 }
4004                                         }
4005                                 }
4006                                 if (!known_hspace && !known_vspace) {
4007                                         switch (unitFromString(unit)) {
4008                                         case Length::SP:
4009                                         case Length::PT:
4010                                         case Length::BP:
4011                                         case Length::DD:
4012                                         case Length::MM:
4013                                         case Length::PC:
4014                                         case Length::CC:
4015                                         case Length::CM:
4016                                         case Length::IN:
4017                                         case Length::EX:
4018                                         case Length::EM:
4019                                         case Length::MU:
4020                                                 known_unit = true;
4021                                                 break;
4022                                         default:
4023                                                 break;
4024                                         }
4025                                 }
4026                         }
4027
4028                         if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
4029                                 // Literal horizontal length or known variable
4030                                 context.check_layout(os);
4031                                 begin_inset(os, "space ");
4032                                 os << name;
4033                                 if (starred)
4034                                         os << '*';
4035                                 os << '{';
4036                                 if (known_hspace)
4037                                         os << unit;
4038                                 os << "}";
4039                                 if (known_unit && !known_hspace)
4040                                         os << "\n\\length "
4041                                            << translate_len(length);
4042                                 end_inset(os);
4043                         } else if (known_unit || known_vspace) {
4044                                 // Literal vertical length or known variable
4045                                 context.check_layout(os);
4046                                 begin_inset(os, "VSpace ");
4047                                 if (known_unit)
4048                                         os << value;
4049                                 os << unit;
4050                                 if (starred)
4051                                         os << '*';
4052                                 end_inset(os);
4053                         } else {
4054                                 // LyX can't handle other length variables in Inset VSpace/space
4055                                 if (starred)
4056                                         name += '*';
4057                                 if (valid) {
4058                                         if (value == 1.0)
4059                                                 handle_ert(os, name + '{' + unit + '}', context);
4060                                         else if (value == -1.0)
4061                                                 handle_ert(os, name + "{-" + unit + '}', context);
4062                                         else
4063                                                 handle_ert(os, name + '{' + valstring + unit + '}', context);
4064                                 } else
4065                                         handle_ert(os, name + '{' + length + '}', context);
4066                         }
4067                 }
4068
4069                 // The single '=' is meant here.
4070                 else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
4071                         p.skip_spaces();
4072                         context.check_layout(os);
4073                         begin_inset(os, "Flex ");
4074                         os << to_utf8(newinsetlayout->name()) << '\n'
4075                            << "status collapsed\n";
4076                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
4077                         end_inset(os);
4078                 }
4079
4080                 else if (t.cs() == "includepdf") {
4081                         p.skip_spaces();
4082                         string const arg = p.getArg('[', ']');
4083                         map<string, string> opts;
4084                         vector<string> keys;
4085                         split_map(arg, opts, keys);
4086                         string name = normalize_filename(p.verbatim_item());
4087                         string const path = getMasterFilePath();
4088                         // We want to preserve relative / absolute filenames,
4089                         // therefore path is only used for testing
4090                         if (!makeAbsPath(name, path).exists()) {
4091                                 // The file extension is probably missing.
4092                                 // Now try to find it out.
4093                                 char const * const pdfpages_format[] = {"pdf", 0};
4094                                 string const pdftex_name =
4095                                         find_file(name, path, pdfpages_format);
4096                                 if (!pdftex_name.empty()) {
4097                                         name = pdftex_name;
4098                                         pdflatex = true;
4099                                 }
4100                         }
4101                         if (makeAbsPath(name, path).exists())
4102                                 fix_relative_filename(name);
4103                         else
4104                                 cerr << "Warning: Could not find file '"
4105                                      << name << "'." << endl;
4106                         // write output
4107                         context.check_layout(os);
4108                         begin_inset(os, "External\n\ttemplate ");
4109                         os << "PDFPages\n\tfilename "
4110                            << name << "\n";
4111                         // parse the options
4112                         if (opts.find("pages") != opts.end())
4113                                 os << "\textra LaTeX \"pages="
4114                                    << opts["pages"] << "\"\n";
4115                         if (opts.find("angle") != opts.end())
4116                                 os << "\trotateAngle "
4117                                    << opts["angle"] << '\n';
4118                         if (opts.find("origin") != opts.end()) {
4119                                 ostringstream ss;
4120                                 string const opt = opts["origin"];
4121                                 if (opt == "tl") ss << "topleft";
4122                                 if (opt == "bl") ss << "bottomleft";
4123                                 if (opt == "Bl") ss << "baselineleft";
4124                                 if (opt == "c") ss << "center";
4125                                 if (opt == "tc") ss << "topcenter";
4126                                 if (opt == "bc") ss << "bottomcenter";
4127                                 if (opt == "Bc") ss << "baselinecenter";
4128                                 if (opt == "tr") ss << "topright";
4129                                 if (opt == "br") ss << "bottomright";
4130                                 if (opt == "Br") ss << "baselineright";
4131                                 if (!ss.str().empty())
4132                                         os << "\trotateOrigin " << ss.str() << '\n';
4133                                 else
4134                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
4135                         }
4136                         if (opts.find("width") != opts.end())
4137                                 os << "\twidth "
4138                                    << translate_len(opts["width"]) << '\n';
4139                         if (opts.find("height") != opts.end())
4140                                 os << "\theight "
4141                                    << translate_len(opts["height"]) << '\n';
4142                         if (opts.find("keepaspectratio") != opts.end())
4143                                 os << "\tkeepAspectRatio\n";
4144                         end_inset(os);
4145                         context.check_layout(os);
4146                         registerExternalTemplatePackages("PDFPages");
4147                 }
4148
4149                 else if (t.cs() == "loadgame") {
4150                         p.skip_spaces();
4151                         string name = normalize_filename(p.verbatim_item());
4152                         string const path = getMasterFilePath();
4153                         // We want to preserve relative / absolute filenames,
4154                         // therefore path is only used for testing
4155                         if (!makeAbsPath(name, path).exists()) {
4156                                 // The file extension is probably missing.
4157                                 // Now try to find it out.
4158                                 char const * const lyxskak_format[] = {"fen", 0};
4159                                 string const lyxskak_name =
4160                                         find_file(name, path, lyxskak_format);
4161                                 if (!lyxskak_name.empty())
4162                                         name = lyxskak_name;
4163                         }
4164                         if (makeAbsPath(name, path).exists())
4165                                 fix_relative_filename(name);
4166                         else
4167                                 cerr << "Warning: Could not find file '"
4168                                      << name << "'." << endl;
4169                         context.check_layout(os);
4170                         begin_inset(os, "External\n\ttemplate ");
4171                         os << "ChessDiagram\n\tfilename "
4172                            << name << "\n";
4173                         end_inset(os);
4174                         context.check_layout(os);
4175                         // after a \loadgame follows a \showboard
4176                         if (p.get_token().asInput() == "showboard")
4177                                 p.get_token();
4178                         registerExternalTemplatePackages("ChessDiagram");
4179                 }
4180
4181                 else {
4182                         // try to see whether the string is in unicodesymbols
4183                         // Only use text mode commands, since we are in text mode here,
4184                         // and math commands may be invalid (bug 6797)
4185                         bool termination;
4186                         docstring rem;
4187                         set<string> req;
4188                         docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
4189                                         Encodings::TEXT_CMD, termination, rem, &req);
4190                         if (!s.empty()) {
4191                                 if (!rem.empty())
4192                                         cerr << "When parsing " << t.cs()
4193                                              << ", result is " << to_utf8(s)
4194                                              << "+" << to_utf8(rem) << endl;
4195                                 context.check_layout(os);
4196                                 os << to_utf8(s);
4197                                 if (termination)
4198                                         skip_spaces_braces(p);
4199                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
4200                                         preamble.registerAutomaticallyLoadedPackage(*it);
4201                         }
4202                         //cerr << "#: " << t << " mode: " << mode << endl;
4203                         // heuristic: read up to next non-nested space
4204                         /*
4205                         string s = t.asInput();
4206                         string z = p.verbatim_item();
4207                         while (p.good() && z != " " && z.size()) {
4208                                 //cerr << "read: " << z << endl;
4209                                 s += z;
4210                                 z = p.verbatim_item();
4211                         }
4212                         cerr << "found ERT: " << s << endl;
4213                         handle_ert(os, s + ' ', context);
4214                         */
4215                         else {
4216                                 string name = t.asInput();
4217                                 if (p.next_token().asInput() == "*") {
4218                                         // Starred commands like \vspace*{}
4219                                         p.get_token();  // Eat '*'
4220                                         name += '*';
4221                                 }
4222                                 if (!parse_command(name, p, os, outer, context))
4223                                         handle_ert(os, name, context);
4224                         }
4225                 }
4226
4227                 if (flags & FLAG_LEAVE) {
4228                         flags &= ~FLAG_LEAVE;
4229                         break;
4230                 }
4231         }
4232 }
4233
4234 // }])
4235
4236
4237 } // namespace lyx