src/tex2lyx/text.cpp

   1 /**
   2  * \file tex2lyx/text.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  * \author Uwe Stöhr
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 // {[(
  14
  15 #include <config.h>
  16
  17 #include "tex2lyx.h"
  18
  19 #include "Context.h"
  20 #include "Encoding.h"
  21 #include "FloatList.h"
  22 #include "LaTeXPackages.h"
  23 #include "Layout.h"
  24 #include "Length.h"
  25 #include "Preamble.h"
  26
  27 #include "insets/ExternalTemplate.h"
  28
  29 #include "support/lassert.h"
  30 #include "support/convert.h"
  31 #include "support/FileName.h"
  32 #include "support/filetools.h"
  33 #include "support/lstrings.h"
  34 #include "support/lyxtime.h"
  35
  36 #include <algorithm>
  37 #include <iostream>
  38 #include <map>
  39 #include <sstream>
  40 #include <vector>
  41
  42 using namespace std;
  43 using namespace lyx::support;
  44
  45 namespace lyx {
  46
  47
  48 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  49                 Context const & context, InsetLayout const * layout)
  50 {
  51         bool const forcePlainLayout =
  52                 layout ? layout->forcePlainLayout() : false;
  53         Context newcontext(true, context.textclass);
  54         if (forcePlainLayout)
  55                 newcontext.layout = &context.textclass.plainLayout();
  56         else
  57                 newcontext.font = context.font;
  58         parse_text(p, os, flags, outer, newcontext);
  59         newcontext.check_end_layout(os);
  60 }
  61
  62
  63 namespace {
  64
  65 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  66                 Context const & context, string const & name)
  67 {
  68         InsetLayout const * layout = 0;
  69         DocumentClass::InsetLayouts::const_iterator it =
  70                 context.textclass.insetLayouts().find(from_ascii(name));
  71         if (it != context.textclass.insetLayouts().end())
  72                 layout = &(it->second);
  73         parse_text_in_inset(p, os, flags, outer, context, layout);
  74 }
  75
  76 /// parses a paragraph snippet, useful for example for \\emph{...}
  77 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  78                 Context & context)
  79 {
  80         Context newcontext(context);
  81         // Don't inherit the paragraph-level extra stuff
  82         newcontext.par_extra_stuff.clear();
  83         parse_text(p, os, flags, outer, newcontext);
  84         // Make sure that we don't create invalid .lyx files
  85         context.need_layout = newcontext.need_layout;
  86         context.need_end_layout = newcontext.need_end_layout;
  87 }
  88
  89
  90 /*!
  91  * Thin wrapper around parse_text_snippet() using a string.
  92  *
  93  * We completely ignore \c context.need_layout and \c context.need_end_layout,
  94  * because our return value is not used directly (otherwise the stream version
  95  * of parse_text_snippet() could be used). That means that the caller needs
  96  * to do layout management manually.
  97  * This is intended to parse text that does not create any layout changes.
  98  */
  99 string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 100                   Context & context)
 101 {
 102         Context newcontext(context);
 103         newcontext.need_layout = false;
 104         newcontext.need_end_layout = false;
 105         newcontext.new_layout_allowed = false;
 106         // Avoid warning by Context::~Context()
 107         newcontext.par_extra_stuff.clear();
 108         ostringstream os;
 109         parse_text_snippet(p, os, flags, outer, newcontext);
 110         return os.str();
 111 }
 112
 113
 114 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 115  "vpageref", "prettyref", "eqref", 0 };
 116
 117 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
 118  "vpageref", "formatted", "eqref", 0 };
 119
 120 /**
 121  * known polyglossia language names (inluding synomyms)
 122  */
 123 const char * const polyglossia_languages[] = {
 124 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 125 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 126 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 127 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 128 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 129 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 130 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 131 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 132 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 133 "marathi", "spanish", 0};
 134
 135 /**
 136  * the same as polyglossia_languages with .lyx names
 137  * please keep this in sync with polyglossia_languages line by line!
 138  */
 139 const char * const coded_polyglossia_languages[] = {
 140 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 141 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 142 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 143 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 144 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 145 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 146 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 147 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 148 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 149 "marathi", "spanish", 0};
 150
 151 string polyglossia2lyx(string const & language)
 152 {
 153         char const * const * where = is_known(language, polyglossia_languages);
 154         if (where)
 155                 return coded_polyglossia_languages[where - polyglossia_languages];
 156         return language;
 157 }
 158
 159 /*!
 160  * natbib commands.
 161  * The starred forms are also known except for "citefullauthor",
 162  * "citeyear" and "citeyearpar".
 163  */
 164 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 165 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 166 "citefullauthor", "Citet", "Citep", "Citealt", "Citealp", "Citeauthor", 0 };
 167
 168 /*!
 169  * jurabib commands.
 170  * No starred form other than "cite*" known.
 171  */
 172 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 173 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 174 // jurabib commands not (yet) supported by LyX:
 175 // "fullcite",
 176 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 177 // "footciteauthor", "footciteyear", "footciteyearpar",
 178 "citefield", "citetitle", 0 };
 179
 180 /// LaTeX names for quotes
 181 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
 182 "guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
 183 "quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
 184 "guilsinglleft", "frq", "guilsinglright", 0};
 185
 186 /// the same as known_quotes with .lyx names
 187 char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 188 "ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
 189 "gld", "grd", "gls", "ers", "fls",
 190 "fls", "frs", "frs", 0};
 191
 192 /// LaTeX names for font sizes
 193 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 194 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 195
 196 /// the same as known_sizes with .lyx names
 197 char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 198 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 199
 200 /// LaTeX 2.09 names for font families
 201 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 202
 203 /// LaTeX names for font families
 204 char const * const known_font_families[] = { "rmfamily", "sffamily",
 205 "ttfamily", 0};
 206
 207 /// LaTeX names for font family changing commands
 208 char const * const known_text_font_families[] = { "textrm", "textsf",
 209 "texttt", 0};
 210
 211 /// The same as known_old_font_families, known_font_families and
 212 /// known_text_font_families with .lyx names
 213 char const * const known_coded_font_families[] = { "roman", "sans",
 214 "typewriter", 0};
 215
 216 /// LaTeX 2.09 names for font series
 217 char const * const known_old_font_series[] = { "bf", 0};
 218
 219 /// LaTeX names for font series
 220 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 221
 222 /// LaTeX names for font series changing commands
 223 char const * const known_text_font_series[] = { "textbf", "textmd", 0};
 224
 225 /// The same as known_old_font_series, known_font_series and
 226 /// known_text_font_series with .lyx names
 227 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 228
 229 /// LaTeX 2.09 names for font shapes
 230 char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 231
 232 /// LaTeX names for font shapes
 233 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 234 "upshape", 0};
 235
 236 /// LaTeX names for font shape changing commands
 237 char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
 238 "textup", 0};
 239
 240 /// The same as known_old_font_shapes, known_font_shapes and
 241 /// known_text_font_shapes with .lyx names
 242 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 243 "smallcaps", "up", 0};
 244
 245 /// Known special characters which need skip_spaces_braces() afterwards
 246 char const * const known_special_chars[] = {"ldots", "lyxarrow",
 247 "textcompwordmark", "slash", 0};
 248
 249 /// the same as known_special_chars with .lyx names
 250 char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
 251 "textcompwordmark{}", "slash{}", 0};
 252
 253 /*!
 254  * Graphics file extensions known by the dvips driver of the graphics package.
 255  * These extensions are used to complete the filename of an included
 256  * graphics file if it does not contain an extension.
 257  * The order must be the same that latex uses to find a file, because we
 258  * will use the first extension that matches.
 259  * This is only an approximation for the common cases. If we would want to
 260  * do it right in all cases, we would need to know which graphics driver is
 261  * used and know the extensions of every driver of the graphics package.
 262  */
 263 char const * const known_dvips_graphics_formats[] = {"eps", "ps", "eps.gz",
 264 "ps.gz", "eps.Z", "ps.Z", 0};
 265
 266 /*!
 267  * Graphics file extensions known by the pdftex driver of the graphics package.
 268  * \sa known_dvips_graphics_formats
 269  */
 270 char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 271 "mps", "tif", 0};
 272
 273 /*!
 274  * Known file extensions for TeX files as used by \\include.
 275  */
 276 char const * const known_tex_extensions[] = {"tex", 0};
 277
 278 /// spaces known by InsetSpace
 279 char const * const known_spaces[] = { " ", "space", ",",
 280 "thinspace", "quad", "qquad", "enspace", "enskip",
 281 "negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
 282 "hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
 283 "upbracefill", "downbracefill", 0};
 284
 285 /// the same as known_spaces with .lyx names
 286 char const * const known_coded_spaces[] = { "space{}", "space{}",
 287 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
 288 "negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
 289 "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
 290 "upbracefill{}", "downbracefill{}", 0};
 291
 292 /// These are translated by LyX to commands like "\\LyX{}", so we have to put
 293 /// them in ERT. "LaTeXe" must come before "LaTeX"!
 294 char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 295 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 296 int const known_phrase_lengths[] = {3, 5, 7, 0};
 297
 298 // string to store the float type to be able to determine the type of subfloats
 299 string float_type = "";
 300
 301
 302 /// splits "x=z, y=b" into a map and an ordered keyword vector
 303 void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 304 {
 305         vector<string> v;
 306         split(s, v);
 307         res.clear();
 308         keys.resize(v.size());
 309         for (size_t i = 0; i < v.size(); ++i) {
 310                 size_t const pos   = v[i].find('=');
 311                 string const index = trimSpaceAndEol(v[i].substr(0, pos));
 312                 string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
 313                 res[index] = value;
 314                 keys[i] = index;
 315         }
 316 }
 317
 318
 319 /*!
 320  * Split a LaTeX length into value and unit.
 321  * The latter can be a real unit like "pt", or a latex length variable
 322  * like "\textwidth". The unit may contain additional stuff like glue
 323  * lengths, but we don't care, because such lengths are ERT anyway.
 324  * \returns true if \p value and \p unit are valid.
 325  */
 326 bool splitLatexLength(string const & len, string & value, string & unit)
 327 {
 328         if (len.empty())
 329                 return false;
 330         const string::size_type i = len.find_first_not_of(" -+0123456789.,");
 331         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 332         string const length = subst(len, ',', '.');
 333         if (i == string::npos)
 334                 return false;
 335         if (i == 0) {
 336                 if (len[0] == '\\') {
 337                         // We had something like \textwidth without a factor
 338                         value = "1.0";
 339                 } else {
 340                         return false;
 341                 }
 342         } else {
 343                 value = trimSpaceAndEol(string(length, 0, i));
 344         }
 345         if (value == "-")
 346                 value = "-1.0";
 347         // 'cM' is a valid LaTeX length unit. Change it to 'cm'
 348         if (contains(len, '\\'))
 349                 unit = trimSpaceAndEol(string(len, i));
 350         else
 351                 unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
 352         return true;
 353 }
 354
 355
 356 /// A simple function to translate a latex length to something LyX can
 357 /// understand. Not perfect, but rather best-effort.
 358 bool translate_len(string const & length, string & valstring, string & unit)
 359 {
 360         if (!splitLatexLength(length, valstring, unit))
 361                 return false;
 362         // LyX uses percent values
 363         double value;
 364         istringstream iss(valstring);
 365         iss >> value;
 366         value *= 100;
 367         ostringstream oss;
 368         oss << value;
 369         string const percentval = oss.str();
 370         // a normal length
 371         if (unit.empty() || unit[0] != '\\')
 372                 return true;
 373         string::size_type const i = unit.find(' ');
 374         string const endlen = (i == string::npos) ? string() : string(unit, i);
 375         if (unit == "\\textwidth") {
 376                 valstring = percentval;
 377                 unit = "text%" + endlen;
 378         } else if (unit == "\\columnwidth") {
 379                 valstring = percentval;
 380                 unit = "col%" + endlen;
 381         } else if (unit == "\\paperwidth") {
 382                 valstring = percentval;
 383                 unit = "page%" + endlen;
 384         } else if (unit == "\\linewidth") {
 385                 valstring = percentval;
 386                 unit = "line%" + endlen;
 387         } else if (unit == "\\paperheight") {
 388                 valstring = percentval;
 389                 unit = "pheight%" + endlen;
 390         } else if (unit == "\\textheight") {
 391                 valstring = percentval;
 392                 unit = "theight%" + endlen;
 393         }
 394         return true;
 395 }
 396
 397 }
 398
 399
 400 string translate_len(string const & length)
 401 {
 402         string unit;
 403         string value;
 404         if (translate_len(length, value, unit))
 405                 return value + unit;
 406         // If the input is invalid, return what we have.
 407         return length;
 408 }
 409
 410
 411 namespace {
 412
 413 /*!
 414  * Translates a LaTeX length into \p value, \p unit and
 415  * \p special parts suitable for a box inset.
 416  * The difference from translate_len() is that a box inset knows about
 417  * some special "units" that are stored in \p special.
 418  */
 419 void translate_box_len(string const & length, string & value, string & unit, string & special)
 420 {
 421         if (translate_len(length, value, unit)) {
 422                 if (unit == "\\height" || unit == "\\depth" ||
 423                     unit == "\\totalheight" || unit == "\\width") {
 424                         special = unit.substr(1);
 425                         // The unit is not used, but LyX requires a dummy setting
 426                         unit = "in";
 427                 } else
 428                         special = "none";
 429         } else {
 430                 value.clear();
 431                 unit = length;
 432                 special = "none";
 433         }
 434 }
 435
 436
 437 /*!
 438  * Find a file with basename \p name in path \p path and an extension
 439  * in \p extensions.
 440  */
 441 string find_file(string const & name, string const & path,
 442                  char const * const * extensions)
 443 {
 444         for (char const * const * what = extensions; *what; ++what) {
 445                 string const trial = addExtension(name, *what);
 446                 if (makeAbsPath(trial, path).exists())
 447                         return trial;
 448         }
 449         return string();
 450 }
 451
 452
 453 void begin_inset(ostream & os, string const & name)
 454 {
 455         os << "\n\\begin_inset " << name;
 456 }
 457
 458
 459 void begin_command_inset(ostream & os, string const & name,
 460                          string const & latexname)
 461 {
 462         begin_inset(os, "CommandInset ");
 463         os << name << "\nLatexCommand " << latexname << '\n';
 464 }
 465
 466
 467 void end_inset(ostream & os)
 468 {
 469         os << "\n\\end_inset\n\n";
 470 }
 471
 472
 473 bool skip_braces(Parser & p)
 474 {
 475         if (p.next_token().cat() != catBegin)
 476                 return false;
 477         p.get_token();
 478         if (p.next_token().cat() == catEnd) {
 479                 p.get_token();
 480                 return true;
 481         }
 482         p.putback();
 483         return false;
 484 }
 485
 486
 487 /// replace LaTeX commands in \p s from the unicodesymbols file with their
 488 /// unicode points
 489 docstring convert_unicodesymbols(docstring s)
 490 {
 491         odocstringstream os;
 492         for (size_t i = 0; i < s.size();) {
 493                 if (s[i] != '\\') {
 494                         os.put(s[i++]);
 495                         continue;
 496                 }
 497                 s = s.substr(i);
 498                 bool termination;
 499                 docstring rem;
 500                 set<string> req;
 501                 docstring parsed = encodings.fromLaTeXCommand(s,
 502                                 Encodings::TEXT_CMD, termination, rem, &req);
 503                 set<string>::const_iterator it = req.begin();
 504                 set<string>::const_iterator en = req.end();
 505                 for (; it != en; ++it)
 506                         preamble.registerAutomaticallyLoadedPackage(*it);
 507                 os << parsed;
 508                 s = rem;
 509                 if (s.empty() || s[0] != '\\')
 510                         i = 0;
 511                 else
 512                         i = 1;
 513         }
 514         return os.str();
 515 }
 516
 517
 518 /// try to convert \p s to a valid InsetCommand argument
 519 string convert_command_inset_arg(string s)
 520 {
 521         if (isAscii(s))
 522                 // since we don't know the input encoding we can't use from_utf8
 523                 s = to_utf8(convert_unicodesymbols(from_ascii(s)));
 524         // LyX cannot handle newlines in a latex command
 525         return subst(s, "\n", " ");
 526 }
 527
 528
 529 void handle_backslash(ostream & os, string const & s)
 530 {
 531         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 532                 if (*it == '\\')
 533                         os << "\n\\backslash\n";
 534                 else
 535                         os << *it;
 536         }
 537 }
 538
 539
 540 void handle_ert(ostream & os, string const & s, Context & context)
 541 {
 542         // We must have a valid layout before outputting the ERT inset.
 543         context.check_layout(os);
 544         Context newcontext(true, context.textclass);
 545         begin_inset(os, "ERT");
 546         os << "\nstatus collapsed\n";
 547         newcontext.check_layout(os);
 548         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 549                 if (*it == '\\')
 550                         os << "\n\\backslash\n";
 551                 else if (*it == '\n') {
 552                         newcontext.new_paragraph(os);
 553                         newcontext.check_layout(os);
 554                 } else
 555                         os << *it;
 556         }
 557         newcontext.check_end_layout(os);
 558         end_inset(os);
 559 }
 560
 561
 562 void handle_comment(ostream & os, string const & s, Context & context)
 563 {
 564         // TODO: Handle this better
 565         Context newcontext(true, context.textclass);
 566         begin_inset(os, "ERT");
 567         os << "\nstatus collapsed\n";
 568         newcontext.check_layout(os);
 569         handle_backslash(os, s);
 570         // make sure that our comment is the last thing on the line
 571         newcontext.new_paragraph(os);
 572         newcontext.check_layout(os);
 573         newcontext.check_end_layout(os);
 574         end_inset(os);
 575 }
 576
 577
 578 Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
 579 {
 580         Layout const * layout = findLayoutWithoutModule(textclass, name, command);
 581         if (layout)
 582                 return layout;
 583         if (checkModule(name, command))
 584                 return findLayoutWithoutModule(textclass, name, command);
 585         return layout;
 586 }
 587
 588
 589 InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 590 {
 591         InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
 592         if (insetlayout)
 593                 return insetlayout;
 594         if (checkModule(name, command))
 595                 return findInsetLayoutWithoutModule(textclass, name, command);
 596         return insetlayout;
 597 }
 598
 599
 600 void eat_whitespace(Parser &, ostream &, Context &, bool);
 601
 602
 603 /*!
 604  * Skips whitespace and braces.
 605  * This should be called after a command has been parsed that is not put into
 606  * ERT, and where LyX adds "{}" if needed.
 607  */
 608 void skip_spaces_braces(Parser & p, bool keepws = false)
 609 {
 610         /* The following four examples produce the same typeset output and
 611            should be handled by this function:
 612            - abc \j{} xyz
 613            - abc \j {} xyz
 614            - abc \j
 615              {} xyz
 616            - abc \j %comment
 617              {} xyz
 618          */
 619         // Unfortunately we need to skip comments, too.
 620         // We can't use eat_whitespace since writing them after the {}
 621         // results in different output in some cases.
 622         bool const skipped_spaces = p.skip_spaces(true);
 623         bool const skipped_braces = skip_braces(p);
 624         if (keepws && skipped_spaces && !skipped_braces)
 625                 // put back the space (it is better handled by check_space)
 626                 p.unskip_spaces(true);
 627 }
 628
 629
 630 void output_command_layout(ostream & os, Parser & p, bool outer,
 631                            Context & parent_context,
 632                            Layout const * newlayout)
 633 {
 634         TeXFont const oldFont = parent_context.font;
 635         // save the current font size
 636         string const size = oldFont.size;
 637         // reset the font size to default, because the font size switches
 638         // don't affect section headings and the like
 639         parent_context.font.size = Context::normalfont.size;
 640         // we only need to write the font change if we have an open layout
 641         if (!parent_context.atParagraphStart())
 642                 output_font_change(os, oldFont, parent_context.font);
 643         parent_context.check_end_layout(os);
 644         Context context(true, parent_context.textclass, newlayout,
 645                         parent_context.layout, parent_context.font);
 646         if (parent_context.deeper_paragraph) {
 647                 // We are beginning a nested environment after a
 648                 // deeper paragraph inside the outer list environment.
 649                 // Therefore we don't need to output a "begin deeper".
 650                 context.need_end_deeper = true;
 651         }
 652         context.check_deeper(os);
 653         context.check_layout(os);
 654         unsigned int optargs = 0;
 655         while (optargs < context.layout->optargs) {
 656                 eat_whitespace(p, os, context, false);
 657                 if (p.next_token().cat() == catEscape ||
 658                     p.next_token().character() != '[')
 659                         break;
 660                 p.get_token(); // eat '['
 661                 begin_inset(os, "Argument\n");
 662                 os << "status collapsed\n\n";
 663                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 664                 end_inset(os);
 665                 eat_whitespace(p, os, context, false);
 666                 ++optargs;
 667         }
 668         unsigned int reqargs = 0;
 669         while (reqargs < context.layout->reqargs) {
 670                 eat_whitespace(p, os, context, false);
 671                 if (p.next_token().cat() != catBegin)
 672                         break;
 673                 p.get_token(); // eat '{'
 674                 begin_inset(os, "Argument\n");
 675                 os << "status collapsed\n\n";
 676                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
 677                 end_inset(os);
 678                 eat_whitespace(p, os, context, false);
 679                 ++reqargs;
 680         }
 681         parse_text(p, os, FLAG_ITEM, outer, context);
 682         context.check_end_layout(os);
 683         if (parent_context.deeper_paragraph) {
 684                 // We must suppress the "end deeper" because we
 685                 // suppressed the "begin deeper" above.
 686                 context.need_end_deeper = false;
 687         }
 688         context.check_end_deeper(os);
 689         // We don't need really a new paragraph, but
 690         // we must make sure that the next item gets a \begin_layout.
 691         parent_context.new_paragraph(os);
 692         // Set the font size to the original value. No need to output it here
 693         // (Context::begin_layout() will do that if needed)
 694         parent_context.font.size = size;
 695 }
 696
 697
 698 /*!
 699  * Output a space if necessary.
 700  * This function gets called for every whitespace token.
 701  *
 702  * We have three cases here:
 703  * 1. A space must be suppressed. Example: The lyxcode case below
 704  * 2. A space may be suppressed. Example: Spaces before "\par"
 705  * 3. A space must not be suppressed. Example: A space between two words
 706  *
 707  * We currently handle only 1. and 3 and from 2. only the case of
 708  * spaces before newlines as a side effect.
 709  *
 710  * 2. could be used to suppress as many spaces as possible. This has two effects:
 711  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 712  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 713  * The drawback is that the logic inside the function becomes
 714  * complicated, and that is the reason why it is not implemented.
 715  */
 716 void check_space(Parser & p, ostream & os, Context & context)
 717 {
 718         Token const next = p.next_token();
 719         Token const curr = p.curr_token();
 720         // A space before a single newline and vice versa must be ignored
 721         // LyX emits a newline before \end{lyxcode}.
 722         // This newline must be ignored,
 723         // otherwise LyX will add an additional protected space.
 724         if (next.cat() == catSpace ||
 725             next.cat() == catNewline ||
 726             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 727                 return;
 728         }
 729         context.check_layout(os);
 730         os << ' ';
 731 }
 732
 733
 734 /*!
 735  * Parse all arguments of \p command
 736  */
 737 void parse_arguments(string const & command,
 738                      vector<ArgumentType> const & template_arguments,
 739                      Parser & p, ostream & os, bool outer, Context & context)
 740 {
 741         string ert = command;
 742         size_t no_arguments = template_arguments.size();
 743         for (size_t i = 0; i < no_arguments; ++i) {
 744                 switch (template_arguments[i]) {
 745                 case required:
 746                 case req_group:
 747                         // This argument contains regular LaTeX
 748                         handle_ert(os, ert + '{', context);
 749                         eat_whitespace(p, os, context, false);
 750                         if (template_arguments[i] == required)
 751                                 parse_text(p, os, FLAG_ITEM, outer, context);
 752                         else
 753                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 754                         ert = "}";
 755                         break;
 756                 case item:
 757                         // This argument consists only of a single item.
 758                         // The presence of '{' or not must be preserved.
 759                         p.skip_spaces();
 760                         if (p.next_token().cat() == catBegin)
 761                                 ert += '{' + p.verbatim_item() + '}';
 762                         else
 763                                 ert += p.verbatim_item();
 764                         break;
 765                 case displaymath:
 766                 case verbatim:
 767                         // This argument may contain special characters
 768                         ert += '{' + p.verbatim_item() + '}';
 769                         break;
 770                 case optional:
 771                 case opt_group:
 772                         // true because we must not eat whitespace
 773                         // if an optional arg follows we must not strip the
 774                         // brackets from this one
 775                         if (i < no_arguments - 1 &&
 776                             template_arguments[i+1] == optional)
 777                                 ert += p.getFullOpt(true);
 778                         else
 779                                 ert += p.getOpt(true);
 780                         break;
 781                 }
 782         }
 783         handle_ert(os, ert, context);
 784 }
 785
 786
 787 /*!
 788  * Check whether \p command is a known command. If yes,
 789  * handle the command with all arguments.
 790  * \return true if the command was parsed, false otherwise.
 791  */
 792 bool parse_command(string const & command, Parser & p, ostream & os,
 793                    bool outer, Context & context)
 794 {
 795         if (known_commands.find(command) != known_commands.end()) {
 796                 parse_arguments(command, known_commands[command], p, os,
 797                                 outer, context);
 798                 return true;
 799         }
 800         return false;
 801 }
 802
 803
 804 /// Parses a minipage or parbox
 805 void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 806                unsigned inner_flags, bool outer, Context & parent_context,
 807                string const & outer_type, string const & special,
 808                string const & inner_type)
 809 {
 810         string position;
 811         string inner_pos;
 812         string hor_pos = "c";
 813         // We need to set the height to the LaTeX default of 1\\totalheight
 814         // for the case when no height argument is given
 815         string height_value = "1";
 816         string height_unit = "in";
 817         string height_special = "totalheight";
 818         string latex_height;
 819         string width_value;
 820         string width_unit;
 821         string latex_width;
 822         string width_special = "none";
 823         if (!inner_type.empty() && p.hasOpt()) {
 824                 if (inner_type != "makebox")
 825                         position = p.getArg('[', ']');
 826                 else {
 827                         latex_width = p.getArg('[', ']');
 828                         translate_box_len(latex_width, width_value, width_unit, width_special);
 829                         position = "t";
 830                 }
 831                 if (position != "t" && position != "c" && position != "b") {
 832                         cerr << "invalid position " << position << " for "
 833                              << inner_type << endl;
 834                         position = "c";
 835                 }
 836                 if (p.hasOpt()) {
 837                         if (inner_type != "makebox") {
 838                                 latex_height = p.getArg('[', ']');
 839                                 translate_box_len(latex_height, height_value, height_unit, height_special);
 840                         } else
 841                                 hor_pos = p.getArg('[', ']');
 842
 843                         if (p.hasOpt()) {
 844                                 inner_pos = p.getArg('[', ']');
 845                                 if (inner_pos != "c" && inner_pos != "t" &&
 846                                     inner_pos != "b" && inner_pos != "s") {
 847                                         cerr << "invalid inner_pos "
 848                                              << inner_pos << " for "
 849                                              << inner_type << endl;
 850                                         inner_pos = position;
 851                                 }
 852                         }
 853                 }
 854         }
 855         if (inner_type.empty()) {
 856                 if (special.empty() && outer_type != "framebox")
 857                         latex_width = "1\\columnwidth";
 858                 else {
 859                         Parser p2(special);
 860                         latex_width = p2.getArg('[', ']');
 861                         string const opt = p2.getArg('[', ']');
 862                         if (!opt.empty()) {
 863                                 hor_pos = opt;
 864                                 if (hor_pos != "l" && hor_pos != "c" &&
 865                                     hor_pos != "r") {
 866                                         cerr << "invalid hor_pos " << hor_pos
 867                                              << " for " << outer_type << endl;
 868                                         hor_pos = "c";
 869                                 }
 870                         }
 871                 }
 872         } else if (inner_type != "makebox")
 873                 latex_width = p.verbatim_item();
 874         // if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 875         // as this is LyX's standard for such cases (except for makebox)
 876         // \framebox is more special and handled below
 877         if (latex_width.empty() && inner_type != "makebox"
 878                 && outer_type != "framebox")
 879                 latex_width = "1\\columnwidth";
 880
 881         translate_len(latex_width, width_value, width_unit);
 882
 883         bool shadedparbox = false;
 884         if (inner_type == "shaded") {
 885                 eat_whitespace(p, os, parent_context, false);
 886                 if (outer_type == "parbox") {
 887                         // Eat '{'
 888                         if (p.next_token().cat() == catBegin)
 889                                 p.get_token();
 890                         eat_whitespace(p, os, parent_context, false);
 891                         shadedparbox = true;
 892                 }
 893                 p.get_token();
 894                 p.getArg('{', '}');
 895         }
 896         // If we already read the inner box we have to push the inner env
 897         if (!outer_type.empty() && !inner_type.empty() &&
 898             (inner_flags & FLAG_END))
 899                 active_environments.push_back(inner_type);
 900         // LyX can't handle length variables
 901         bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
 902         if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
 903                 // Look whether there is some content after the end of the
 904                 // inner box, but before the end of the outer box.
 905                 // If yes, we need to output ERT.
 906                 p.pushPosition();
 907                 if (inner_flags & FLAG_END)
 908                         p.verbatimEnvironment(inner_type);
 909                 else
 910                         p.verbatim_item();
 911                 p.skip_spaces(true);
 912                 bool const outer_env(outer_type == "framed" || outer_type == "minipage");
 913                 if ((outer_env && p.next_token().asInput() != "\\end") ||
 914                     (!outer_env && p.next_token().cat() != catEnd)) {
 915                         // something is between the end of the inner box and
 916                         // the end of the outer box, so we need to use ERT.
 917                         use_ert = true;
 918                 }
 919                 p.popPosition();
 920         }
 921         // if only \makebox{content} was used we can set its width to 1\width
 922         // because this identic and also identic to \mbox
 923         // this doesn't work for \framebox{content}, thus we have to use ERT for this
 924         if (latex_width.empty() && inner_type == "makebox") {
 925                 width_value = "1";
 926                 width_unit = "in";
 927                 width_special = "width";
 928         } else if (latex_width.empty() && outer_type == "framebox") {
 929                 use_ert = true;
 930         }
 931         if (use_ert) {
 932                 ostringstream ss;
 933                 if (!outer_type.empty()) {
 934                         if (outer_flags & FLAG_END)
 935                                 ss << "\\begin{" << outer_type << '}';
 936                         else {
 937                                 ss << '\\' << outer_type << '{';
 938                                 if (!special.empty())
 939                                         ss << special;
 940                         }
 941                 }
 942                 if (!inner_type.empty()) {
 943                         if (inner_type != "shaded") {
 944                                 if (inner_flags & FLAG_END)
 945                                         ss << "\\begin{" << inner_type << '}';
 946                                 else
 947                                         ss << '\\' << inner_type;
 948                         }
 949                         if (!position.empty())
 950                                 ss << '[' << position << ']';
 951                         if (!latex_height.empty())
 952                                 ss << '[' << latex_height << ']';
 953                         if (!inner_pos.empty())
 954                                 ss << '[' << inner_pos << ']';
 955                         ss << '{' << latex_width << '}';
 956                         if (!(inner_flags & FLAG_END))
 957                                 ss << '{';
 958                 }
 959                 if (inner_type == "shaded")
 960                         ss << "\\begin{shaded}";
 961                 handle_ert(os, ss.str(), parent_context);
 962                 if (!inner_type.empty()) {
 963                         parse_text(p, os, inner_flags, outer, parent_context);
 964                         if (inner_flags & FLAG_END)
 965                                 handle_ert(os, "\\end{" + inner_type + '}',
 966                                            parent_context);
 967                         else
 968                                 handle_ert(os, "}", parent_context);
 969                 }
 970                 if (!outer_type.empty()) {
 971                         // If we already read the inner box we have to pop
 972                         // the inner env
 973                         if (!inner_type.empty() && (inner_flags & FLAG_END))
 974                                 active_environments.pop_back();
 975
 976                         // Ensure that the end of the outer box is parsed correctly:
 977                         // The opening brace has been eaten by parse_outer_box()
 978                         if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
 979                                 outer_flags &= ~FLAG_ITEM;
 980                                 outer_flags |= FLAG_BRACE_LAST;
 981                         }
 982                         parse_text(p, os, outer_flags, outer, parent_context);
 983                         if (outer_flags & FLAG_END)
 984                                 handle_ert(os, "\\end{" + outer_type + '}',
 985                                            parent_context);
 986                         else if (inner_type.empty() && outer_type == "framebox")
 987                                 // in this case it is already closed later
 988                                 ;
 989                         else
 990                                 handle_ert(os, "}", parent_context);
 991                 }
 992         } else {
 993                 // LyX does not like empty positions, so we have
 994                 // to set them to the LaTeX default values here.
 995                 if (position.empty())
 996                         position = "c";
 997                 if (inner_pos.empty())
 998                         inner_pos = position;
 999                 parent_context.check_layout(os);
1000                 begin_inset(os, "Box ");
1001                 if (outer_type == "framed")
1002                         os << "Framed\n";
1003                 else if (outer_type == "framebox")
1004                         os << "Boxed\n";
1005                 else if (outer_type == "shadowbox")
1006                         os << "Shadowbox\n";
1007                 else if ((outer_type == "shaded" && inner_type.empty()) ||
1008                              (outer_type == "minipage" && inner_type == "shaded") ||
1009                              (outer_type == "parbox" && inner_type == "shaded")) {
1010                         os << "Shaded\n";
1011                         preamble.registerAutomaticallyLoadedPackage("color");
1012                 } else if (outer_type == "doublebox")
1013                         os << "Doublebox\n";
1014                 else if (outer_type.empty())
1015                         os << "Frameless\n";
1016                 else
1017                         os << outer_type << '\n';
1018                 os << "position \"" << position << "\"\n";
1019                 os << "hor_pos \"" << hor_pos << "\"\n";
1020                 os << "has_inner_box " << !inner_type.empty() << "\n";
1021                 os << "inner_pos \"" << inner_pos << "\"\n";
1022                 os << "use_parbox " << (inner_type == "parbox" || shadedparbox)
1023                    << '\n';
1024                 os << "use_makebox " << (inner_type == "makebox") << '\n';
1025                 os << "width \"" << width_value << width_unit << "\"\n";
1026                 os << "special \"" << width_special << "\"\n";
1027                 os << "height \"" << height_value << height_unit << "\"\n";
1028                 os << "height_special \"" << height_special << "\"\n";
1029                 os << "status open\n\n";
1030
1031                 // Unfortunately we can't use parse_text_in_inset:
1032                 // InsetBox::forcePlainLayout() is hard coded and does not
1033                 // use the inset layout. Apart from that do we call parse_text
1034                 // up to two times, but need only one check_end_layout.
1035                 bool const forcePlainLayout =
1036                         (!inner_type.empty() || inner_type == "makebox") &&
1037                         outer_type != "shaded" && outer_type != "framed";
1038                 Context context(true, parent_context.textclass);
1039                 if (forcePlainLayout)
1040                         context.layout = &context.textclass.plainLayout();
1041                 else
1042                         context.font = parent_context.font;
1043
1044                 // If we have no inner box the contents will be read with the outer box
1045                 if (!inner_type.empty())
1046                         parse_text(p, os, inner_flags, outer, context);
1047
1048                 // Ensure that the end of the outer box is parsed correctly:
1049                 // The opening brace has been eaten by parse_outer_box()
1050                 if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
1051                         outer_flags &= ~FLAG_ITEM;
1052                         outer_flags |= FLAG_BRACE_LAST;
1053                 }
1054
1055                 // Find end of outer box, output contents if inner_type is
1056                 // empty and output possible comments
1057                 if (!outer_type.empty()) {
1058                         // If we already read the inner box we have to pop
1059                         // the inner env
1060                         if (!inner_type.empty() && (inner_flags & FLAG_END))
1061                                 active_environments.pop_back();
1062                         // This does not output anything but comments if
1063                         // inner_type is not empty (see use_ert)
1064                         parse_text(p, os, outer_flags, outer, context);
1065                 }
1066
1067                 context.check_end_layout(os);
1068                 end_inset(os);
1069 #ifdef PRESERVE_LAYOUT
1070                 // LyX puts a % after the end of the minipage
1071                 if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
1072                         // new paragraph
1073                         //handle_comment(os, "%dummy", parent_context);
1074                         p.get_token();
1075                         p.skip_spaces();
1076                         parent_context.new_paragraph(os);
1077                 }
1078                 else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
1079                         //handle_comment(os, "%dummy", parent_context);
1080                         p.get_token();
1081                         p.skip_spaces();
1082                         // We add a protected space if something real follows
1083                         if (p.good() && p.next_token().cat() != catComment) {
1084                                 begin_inset(os, "space ~\n");
1085                                 end_inset(os);
1086                         }
1087                 }
1088 #endif
1089         }
1090 }
1091
1092
1093 void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
1094                      Context & parent_context, string const & outer_type,
1095                      string const & special)
1096 {
1097         eat_whitespace(p, os, parent_context, false);
1098         if (flags & FLAG_ITEM) {
1099                 // Eat '{'
1100                 if (p.next_token().cat() == catBegin)
1101                         p.get_token();
1102                 else
1103                         cerr << "Warning: Ignoring missing '{' after \\"
1104                              << outer_type << '.' << endl;
1105                 eat_whitespace(p, os, parent_context, false);
1106         }
1107         string inner;
1108         unsigned int inner_flags = 0;
1109         p.pushPosition();
1110         if (outer_type == "minipage" || outer_type == "parbox") {
1111                 p.skip_spaces(true);
1112                 while (p.hasOpt()) {
1113                         p.getArg('[', ']');
1114                         p.skip_spaces(true);
1115                 }
1116                 p.getArg('{', '}');
1117                 p.skip_spaces(true);
1118                 if (outer_type == "parbox") {
1119                         // Eat '{'
1120                         if (p.next_token().cat() == catBegin)
1121                                 p.get_token();
1122                         p.skip_spaces(true);
1123                 }
1124         }
1125         if (outer_type == "shaded") {
1126                 // These boxes never have an inner box
1127                 ;
1128         } else if (p.next_token().asInput() == "\\parbox") {
1129                 inner = p.get_token().cs();
1130                 inner_flags = FLAG_ITEM;
1131         } else if (p.next_token().asInput() == "\\begin") {
1132                 // Is this a minipage or shaded box?
1133                 p.pushPosition();
1134                 p.get_token();
1135                 inner = p.getArg('{', '}');
1136                 p.popPosition();
1137                 if (inner == "minipage" || inner == "shaded")
1138                         inner_flags = FLAG_END;
1139                 else
1140                         inner = "";
1141         }
1142         p.popPosition();
1143         if (inner_flags == FLAG_END) {
1144                 if (inner != "shaded")
1145                 {
1146                         p.get_token();
1147                         p.getArg('{', '}');
1148                         eat_whitespace(p, os, parent_context, false);
1149                 }
1150                 parse_box(p, os, flags, FLAG_END, outer, parent_context,
1151                           outer_type, special, inner);
1152         } else {
1153                 if (inner_flags == FLAG_ITEM) {
1154                         p.get_token();
1155                         eat_whitespace(p, os, parent_context, false);
1156                 }
1157                 parse_box(p, os, flags, inner_flags, outer, parent_context,
1158                           outer_type, special, inner);
1159         }
1160 }
1161
1162
1163 void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
1164 {
1165         parent_context.check_layout(os);
1166         begin_inset(os, "listings\n");
1167         if (p.hasOpt()) {
1168                 string arg = p.verbatimOption();
1169                 os << "lstparams " << '"' << arg << '"' << '\n';
1170         }
1171         if (in_line)
1172                 os << "inline true\n";
1173         else
1174                 os << "inline false\n";
1175         os << "status collapsed\n";
1176         Context context(true, parent_context.textclass);
1177         context.layout = &parent_context.textclass.plainLayout();
1178         string s;
1179         if (in_line) {
1180                 s = p.plainCommand('!', '!', "lstinline");
1181                 context.new_paragraph(os);
1182                 context.check_layout(os);
1183         } else
1184                 s = p.plainEnvironment("lstlisting");
1185         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1186                 if (*it == '\\')
1187                         os << "\n\\backslash\n";
1188                 else if (*it == '\n') {
1189                         // avoid adding an empty paragraph at the end
1190                         if (it + 1 != et) {
1191                                 context.new_paragraph(os);
1192                                 context.check_layout(os);
1193                         }
1194                 } else
1195                         os << *it;
1196         }
1197         context.check_end_layout(os);
1198         end_inset(os);
1199 }
1200
1201
1202 /// parse an unknown environment
1203 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
1204                                unsigned flags, bool outer,
1205                                Context & parent_context)
1206 {
1207         if (name == "tabbing")
1208                 // We need to remember that we have to handle '\=' specially
1209                 flags |= FLAG_TABBING;
1210
1211         // We need to translate font changes and paragraphs inside the
1212         // environment to ERT if we have a non standard font.
1213         // Otherwise things like
1214         // \large\begin{foo}\huge bar\end{foo}
1215         // will not work.
1216         bool const specialfont =
1217                 (parent_context.font != parent_context.normalfont);
1218         bool const new_layout_allowed = parent_context.new_layout_allowed;
1219         if (specialfont)
1220                 parent_context.new_layout_allowed = false;
1221         handle_ert(os, "\\begin{" + name + "}", parent_context);
1222         parse_text_snippet(p, os, flags, outer, parent_context);
1223         handle_ert(os, "\\end{" + name + "}", parent_context);
1224         if (specialfont)
1225                 parent_context.new_layout_allowed = new_layout_allowed;
1226 }
1227
1228
1229 void parse_environment(Parser & p, ostream & os, bool outer,
1230                        string & last_env, Context & parent_context)
1231 {
1232         Layout const * newlayout;
1233         InsetLayout const * newinsetlayout = 0;
1234         string const name = p.getArg('{', '}');
1235         const bool is_starred = suffixIs(name, '*');
1236         string const unstarred_name = rtrim(name, "*");
1237         active_environments.push_back(name);
1238
1239         if (is_math_env(name)) {
1240                 parent_context.check_layout(os);
1241                 begin_inset(os, "Formula ");
1242                 os << "\\begin{" << name << "}";
1243                 parse_math(p, os, FLAG_END, MATH_MODE);
1244                 os << "\\end{" << name << "}";
1245                 end_inset(os);
1246                 if (is_display_math_env(name)) {
1247                         // Prevent the conversion of a line break to a space
1248                         // (bug 7668). This does not change the output, but
1249                         // looks ugly in LyX.
1250                         eat_whitespace(p, os, parent_context, false);
1251                 }
1252         }
1253
1254         else if (is_known(name, polyglossia_languages)) {
1255                 parent_context.check_layout(os);
1256                 parent_context.font.language = polyglossia2lyx(name);
1257                 os << "\n\\lang " << parent_context.font.language << "\n";
1258                 p.skip_spaces();
1259         }
1260
1261         else if (unstarred_name == "tabular" || name == "longtable") {
1262                 eat_whitespace(p, os, parent_context, false);
1263                 string width = "0pt";
1264                 if (name == "tabular*") {
1265                         width = lyx::translate_len(p.getArg('{', '}'));
1266                         eat_whitespace(p, os, parent_context, false);
1267                 }
1268                 parent_context.check_layout(os);
1269                 begin_inset(os, "Tabular ");
1270                 handle_tabular(p, os, name, width, parent_context);
1271                 end_inset(os);
1272                 p.skip_spaces();
1273         }
1274
1275         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
1276                 eat_whitespace(p, os, parent_context, false);
1277                 string const opt = p.hasOpt() ? p.getArg('[', ']') : string();
1278                 eat_whitespace(p, os, parent_context, false);
1279                 parent_context.check_layout(os);
1280                 begin_inset(os, "Float " + unstarred_name + "\n");
1281                 // store the float type for subfloats
1282                 // subfloats only work with figures and tables
1283                 if (unstarred_name == "figure")
1284                         float_type = unstarred_name;
1285                 else if (unstarred_name == "table")
1286                         float_type = unstarred_name;
1287                 else
1288                         float_type = "";
1289                 if (!opt.empty())
1290                         os << "placement " << opt << '\n';
1291                 if (contains(opt, "H"))
1292                         preamble.registerAutomaticallyLoadedPackage("float");
1293                 else {
1294                         Floating const & fl = parent_context.textclass.floats()
1295                                 .getType(unstarred_name);
1296                         if (!fl.floattype().empty() && fl.usesFloatPkg())
1297                                 preamble.registerAutomaticallyLoadedPackage("float");
1298                 }
1299
1300                 os << "wide " << convert<string>(is_starred)
1301                    << "\nsideways false"
1302                    << "\nstatus open\n\n";
1303                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1304                 end_inset(os);
1305                 // We don't need really a new paragraph, but
1306                 // we must make sure that the next item gets a \begin_layout.
1307                 parent_context.new_paragraph(os);
1308                 p.skip_spaces();
1309                 // the float is parsed thus delete the type
1310                 float_type = "";
1311         }
1312
1313         else if (unstarred_name == "sidewaysfigure"
1314                 || unstarred_name == "sidewaystable") {
1315                 eat_whitespace(p, os, parent_context, false);
1316                 parent_context.check_layout(os);
1317                 if (unstarred_name == "sidewaysfigure")
1318                         begin_inset(os, "Float figure\n");
1319                 else
1320                         begin_inset(os, "Float table\n");
1321                 os << "wide " << convert<string>(is_starred)
1322                    << "\nsideways true"
1323                    << "\nstatus open\n\n";
1324                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1325                 end_inset(os);
1326                 // We don't need really a new paragraph, but
1327                 // we must make sure that the next item gets a \begin_layout.
1328                 parent_context.new_paragraph(os);
1329                 p.skip_spaces();
1330                 preamble.registerAutomaticallyLoadedPackage("rotfloat");
1331         }
1332
1333         else if (name == "wrapfigure" || name == "wraptable") {
1334                 // syntax is \begin{wrapfigure}[lines]{placement}[overhang]{width}
1335                 eat_whitespace(p, os, parent_context, false);
1336                 parent_context.check_layout(os);
1337                 // default values
1338                 string lines = "0";
1339                 string overhang = "0col%";
1340                 // parse
1341                 if (p.hasOpt())
1342                         lines = p.getArg('[', ']');
1343                 string const placement = p.getArg('{', '}');
1344                 if (p.hasOpt())
1345                         overhang = p.getArg('[', ']');
1346                 string const width = p.getArg('{', '}');
1347                 // write
1348                 if (name == "wrapfigure")
1349                         begin_inset(os, "Wrap figure\n");
1350                 else
1351                         begin_inset(os, "Wrap table\n");
1352                 os << "lines " << lines
1353                    << "\nplacement " << placement
1354                    << "\noverhang " << lyx::translate_len(overhang)
1355                    << "\nwidth " << lyx::translate_len(width)
1356                    << "\nstatus open\n\n";
1357                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1358                 end_inset(os);
1359                 // We don't need really a new paragraph, but
1360                 // we must make sure that the next item gets a \begin_layout.
1361                 parent_context.new_paragraph(os);
1362                 p.skip_spaces();
1363                 preamble.registerAutomaticallyLoadedPackage("wrapfig");
1364         }
1365
1366         else if (name == "minipage") {
1367                 eat_whitespace(p, os, parent_context, false);
1368                 // Test whether this is an outer box of a shaded box
1369                 p.pushPosition();
1370                 // swallow arguments
1371                 while (p.hasOpt()) {
1372                         p.getArg('[', ']');
1373                         p.skip_spaces(true);
1374                 }
1375                 p.getArg('{', '}');
1376                 p.skip_spaces(true);
1377                 Token t = p.get_token();
1378                 bool shaded = false;
1379                 if (t.asInput() == "\\begin") {
1380                         p.skip_spaces(true);
1381                         if (p.getArg('{', '}') == "shaded")
1382                                 shaded = true;
1383                 }
1384                 p.popPosition();
1385                 if (shaded)
1386                         parse_outer_box(p, os, FLAG_END, outer,
1387                                         parent_context, name, "shaded");
1388                 else
1389                         parse_box(p, os, 0, FLAG_END, outer, parent_context,
1390                                   "", "", name);
1391                 p.skip_spaces();
1392         }
1393
1394         else if (name == "comment") {
1395                 eat_whitespace(p, os, parent_context, false);
1396                 parent_context.check_layout(os);
1397                 begin_inset(os, "Note Comment\n");
1398                 os << "status open\n";
1399                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1400                 end_inset(os);
1401                 p.skip_spaces();
1402                 skip_braces(p); // eat {} that might by set by LyX behind comments
1403                 preamble.registerAutomaticallyLoadedPackage("verbatim");
1404         }
1405
1406         else if (name == "verbatim") {
1407                 os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
1408                 string const s = p.plainEnvironment("verbatim");
1409                 string::const_iterator it2 = s.begin();
1410                 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1411                         if (*it == '\\')
1412                                 os << "\\backslash ";
1413                         else if (*it == '\n') {
1414                                 it2 = it + 1;
1415                                 // avoid adding an empty paragraph at the end
1416                                 // FIXME: if there are 2 consecutive spaces at the end ignore it
1417                                 // because LyX will re-add a \n
1418                                 // This hack must be removed once bug 8049 is fixed!
1419                                 if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
1420                                         os << "\n\\end_layout\n\\begin_layout Verbatim\n";
1421                         } else
1422                                 os << *it;
1423                 }
1424                 os << "\n\\end_layout\n\n";
1425                 p.skip_spaces();
1426                 // reset to Standard layout
1427                 os << "\n\\begin_layout Standard\n";
1428         }
1429
1430         else if (name == "lyxgreyedout") {
1431                 eat_whitespace(p, os, parent_context, false);
1432                 parent_context.check_layout(os);
1433                 begin_inset(os, "Note Greyedout\n");
1434                 os << "status open\n";
1435                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1436                 end_inset(os);
1437                 p.skip_spaces();
1438                 if (!preamble.notefontcolor().empty())
1439                         preamble.registerAutomaticallyLoadedPackage("color");
1440         }
1441
1442         else if (name == "framed" || name == "shaded") {
1443                 eat_whitespace(p, os, parent_context, false);
1444                 parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
1445                 p.skip_spaces();
1446         }
1447
1448         else if (name == "lstlisting") {
1449                 eat_whitespace(p, os, parent_context, false);
1450                 // FIXME handle the automatic color package loading
1451                 // uwestoehr asks: In what case color is loaded?
1452                 parse_listings(p, os, parent_context, false);
1453                 p.skip_spaces();
1454         }
1455
1456         else if (!parent_context.new_layout_allowed)
1457                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1458                                           parent_context);
1459
1460         // Alignment and spacing settings
1461         // FIXME (bug xxxx): These settings can span multiple paragraphs and
1462         //                                       therefore are totally broken!
1463         // Note that \centering, raggedright, and raggedleft cannot be handled, as
1464         // they are commands not environments. They are furthermore switches that
1465         // can be ended by another switches, but also by commands like \footnote or
1466         // \parbox. So the only safe way is to leave them untouched.
1467         else if (name == "center" || name == "centering" ||
1468                  name == "flushleft" || name == "flushright" ||
1469                  name == "singlespace" || name == "onehalfspace" ||
1470                  name == "doublespace" || name == "spacing") {
1471                 eat_whitespace(p, os, parent_context, false);
1472                 // We must begin a new paragraph if not already done
1473                 if (! parent_context.atParagraphStart()) {
1474                         parent_context.check_end_layout(os);
1475                         parent_context.new_paragraph(os);
1476                 }
1477                 if (name == "flushleft")
1478                         parent_context.add_extra_stuff("\\align left\n");
1479                 else if (name == "flushright")
1480                         parent_context.add_extra_stuff("\\align right\n");
1481                 else if (name == "center" || name == "centering")
1482                         parent_context.add_extra_stuff("\\align center\n");
1483                 else if (name == "singlespace")
1484                         parent_context.add_extra_stuff("\\paragraph_spacing single\n");
1485                 else if (name == "onehalfspace") {
1486                         parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
1487                         preamble.registerAutomaticallyLoadedPackage("setspace");
1488                 } else if (name == "doublespace") {
1489                         parent_context.add_extra_stuff("\\paragraph_spacing double\n");
1490                         preamble.registerAutomaticallyLoadedPackage("setspace");
1491                 } else if (name == "spacing") {
1492                         parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
1493                         preamble.registerAutomaticallyLoadedPackage("setspace");
1494                 }
1495                 parse_text(p, os, FLAG_END, outer, parent_context);
1496                 // Just in case the environment is empty
1497                 parent_context.extra_stuff.erase();
1498                 // We must begin a new paragraph to reset the alignment
1499                 parent_context.new_paragraph(os);
1500                 p.skip_spaces();
1501         }
1502
1503         // The single '=' is meant here.
1504         else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
1505                 eat_whitespace(p, os, parent_context, false);
1506                 Context context(true, parent_context.textclass, newlayout,
1507                                 parent_context.layout, parent_context.font);
1508                 if (parent_context.deeper_paragraph) {
1509                         // We are beginning a nested environment after a
1510                         // deeper paragraph inside the outer list environment.
1511                         // Therefore we don't need to output a "begin deeper".
1512                         context.need_end_deeper = true;
1513                 }
1514                 parent_context.check_end_layout(os);
1515                 if (last_env == name) {
1516                         // we need to output a separator since LyX would export
1517                         // the two environments as one otherwise (bug 5716)
1518                         docstring const sep = from_ascii("--Separator--");
1519                         TeX2LyXDocClass const & textclass(parent_context.textclass);
1520                         if (textclass.hasLayout(sep)) {
1521                                 Context newcontext(parent_context);
1522                                 newcontext.layout = &(textclass[sep]);
1523                                 newcontext.check_layout(os);
1524                                 newcontext.check_end_layout(os);
1525                         } else {
1526                                 parent_context.check_layout(os);
1527                                 begin_inset(os, "Note Note\n");
1528                                 os << "status closed\n";
1529                                 Context newcontext(true, textclass,
1530                                                 &(textclass.defaultLayout()));
1531                                 newcontext.check_layout(os);
1532                                 newcontext.check_end_layout(os);
1533                                 end_inset(os);
1534                                 parent_context.check_end_layout(os);
1535                         }
1536                 }
1537                 switch (context.layout->latextype) {
1538                 case  LATEX_LIST_ENVIRONMENT:
1539                         context.add_par_extra_stuff("\\labelwidthstring "
1540                                                     + p.verbatim_item() + '\n');
1541                         p.skip_spaces();
1542                         break;
1543                 case  LATEX_BIB_ENVIRONMENT:
1544                         p.verbatim_item(); // swallow next arg
1545                         p.skip_spaces();
1546                         break;
1547                 default:
1548                         break;
1549                 }
1550                 context.check_deeper(os);
1551                 // handle known optional and required arguments
1552                 // layouts require all optional arguments before the required ones
1553                 // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
1554                 // It is impossible to place anything after the environment name,
1555                 // but before the first \\item.
1556                 if (context.layout->latextype == LATEX_ENVIRONMENT) {
1557                         bool need_layout = true;
1558                         unsigned int optargs = 0;
1559                         while (optargs < context.layout->optargs) {
1560                                 eat_whitespace(p, os, context, false);
1561                                 if (p.next_token().cat() == catEscape ||
1562                                     p.next_token().character() != '[')
1563                                         break;
1564                                 p.get_token(); // eat '['
1565                                 if (need_layout) {
1566                                         context.check_layout(os);
1567                                         need_layout = false;
1568                                 }
1569                                 begin_inset(os, "Argument\n");
1570                                 os << "status collapsed\n\n";
1571                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
1572                                 end_inset(os);
1573                                 eat_whitespace(p, os, context, false);
1574                                 ++optargs;
1575                         }
1576                         unsigned int reqargs = 0;
1577                         while (reqargs < context.layout->reqargs) {
1578                                 eat_whitespace(p, os, context, false);
1579                                 if (p.next_token().cat() != catBegin)
1580                                         break;
1581                                 p.get_token(); // eat '{'
1582                                 if (need_layout) {
1583                                         context.check_layout(os);
1584                                         need_layout = false;
1585                                 }
1586                                 begin_inset(os, "Argument\n");
1587                                 os << "status collapsed\n\n";
1588                                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
1589                                 end_inset(os);
1590                                 eat_whitespace(p, os, context, false);
1591                                 ++reqargs;
1592                         }
1593                 }
1594                 parse_text(p, os, FLAG_END, outer, context);
1595                 context.check_end_layout(os);
1596                 if (parent_context.deeper_paragraph) {
1597                         // We must suppress the "end deeper" because we
1598                         // suppressed the "begin deeper" above.
1599                         context.need_end_deeper = false;
1600                 }
1601                 context.check_end_deeper(os);
1602                 parent_context.new_paragraph(os);
1603                 p.skip_spaces();
1604                 if (!preamble.titleLayoutFound())
1605                         preamble.titleLayoutFound(newlayout->intitle);
1606                 set<string> const & req = newlayout->requires();
1607                 set<string>::const_iterator it = req.begin();
1608                 set<string>::const_iterator en = req.end();
1609                 for (; it != en; ++it)
1610                         preamble.registerAutomaticallyLoadedPackage(*it);
1611         }
1612
1613         // The single '=' is meant here.
1614         else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
1615                 eat_whitespace(p, os, parent_context, false);
1616                 parent_context.check_layout(os);
1617                 begin_inset(os, "Flex ");
1618                 os << to_utf8(newinsetlayout->name()) << '\n'
1619                    << "status collapsed\n";
1620                 parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
1621                 end_inset(os);
1622         }
1623
1624         else if (name == "appendix") {
1625                 // This is no good latex style, but it works and is used in some documents...
1626                 eat_whitespace(p, os, parent_context, false);
1627                 parent_context.check_end_layout(os);
1628                 Context context(true, parent_context.textclass, parent_context.layout,
1629                                 parent_context.layout, parent_context.font);
1630                 context.check_layout(os);
1631                 os << "\\start_of_appendix\n";
1632                 parse_text(p, os, FLAG_END, outer, context);
1633                 context.check_end_layout(os);
1634                 p.skip_spaces();
1635         }
1636
1637         else if (known_environments.find(name) != known_environments.end()) {
1638                 vector<ArgumentType> arguments = known_environments[name];
1639                 // The last "argument" denotes wether we may translate the
1640                 // environment contents to LyX
1641                 // The default required if no argument is given makes us
1642                 // compatible with the reLyXre environment.
1643                 ArgumentType contents = arguments.empty() ?
1644                         required :
1645                         arguments.back();
1646                 if (!arguments.empty())
1647                         arguments.pop_back();
1648                 // See comment in parse_unknown_environment()
1649                 bool const specialfont =
1650                         (parent_context.font != parent_context.normalfont);
1651                 bool const new_layout_allowed =
1652                         parent_context.new_layout_allowed;
1653                 if (specialfont)
1654                         parent_context.new_layout_allowed = false;
1655                 parse_arguments("\\begin{" + name + "}", arguments, p, os,
1656                                 outer, parent_context);
1657                 if (contents == verbatim)
1658                         handle_ert(os, p.verbatimEnvironment(name),
1659                                    parent_context);
1660                 else
1661                         parse_text_snippet(p, os, FLAG_END, outer,
1662                                            parent_context);
1663                 handle_ert(os, "\\end{" + name + "}", parent_context);
1664                 if (specialfont)
1665                         parent_context.new_layout_allowed = new_layout_allowed;
1666         }
1667
1668         else
1669                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1670                                           parent_context);
1671
1672         last_env = name;
1673         active_environments.pop_back();
1674 }
1675
1676
1677 /// parses a comment and outputs it to \p os.
1678 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
1679 {
1680         LASSERT(t.cat() == catComment, return);
1681         if (!t.cs().empty()) {
1682                 context.check_layout(os);
1683                 handle_comment(os, '%' + t.cs(), context);
1684                 if (p.next_token().cat() == catNewline) {
1685                         // A newline after a comment line starts a new
1686                         // paragraph
1687                         if (context.new_layout_allowed) {
1688                                 if(!context.atParagraphStart())
1689                                         // Only start a new paragraph if not already
1690                                         // done (we might get called recursively)
1691                                         context.new_paragraph(os);
1692                         } else
1693                                 handle_ert(os, "\n", context);
1694                         eat_whitespace(p, os, context, true);
1695                 }
1696         } else {
1697                 // "%\n" combination
1698                 p.skip_spaces();
1699         }
1700 }
1701
1702
1703 /*!
1704  * Reads spaces and comments until the first non-space, non-comment token.
1705  * New paragraphs (double newlines or \\par) are handled like simple spaces
1706  * if \p eatParagraph is true.
1707  * Spaces are skipped, but comments are written to \p os.
1708  */
1709 void eat_whitespace(Parser & p, ostream & os, Context & context,
1710                     bool eatParagraph)
1711 {
1712         while (p.good()) {
1713                 Token const & t = p.get_token();
1714                 if (t.cat() == catComment)
1715                         parse_comment(p, os, t, context);
1716                 else if ((! eatParagraph && p.isParagraph()) ||
1717                          (t.cat() != catSpace && t.cat() != catNewline)) {
1718                         p.putback();
1719                         return;
1720                 }
1721         }
1722 }
1723
1724
1725 /*!
1726  * Set a font attribute, parse text and reset the font attribute.
1727  * \param attribute Attribute name (e.g. \\family, \\shape etc.)
1728  * \param currentvalue Current value of the attribute. Is set to the new
1729  * value during parsing.
1730  * \param newvalue New value of the attribute
1731  */
1732 void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
1733                            Context & context, string const & attribute,
1734                            string & currentvalue, string const & newvalue)
1735 {
1736         context.check_layout(os);
1737         string const oldvalue = currentvalue;
1738         currentvalue = newvalue;
1739         os << '\n' << attribute << ' ' << newvalue << "\n";
1740         parse_text_snippet(p, os, flags, outer, context);
1741         context.check_layout(os);
1742         os << '\n' << attribute << ' ' << oldvalue << "\n";
1743         currentvalue = oldvalue;
1744 }
1745
1746
1747 /// get the arguments of a natbib or jurabib citation command
1748 void get_cite_arguments(Parser & p, bool natbibOrder,
1749         string & before, string & after)
1750 {
1751         // We need to distinguish "" and "[]", so we can't use p.getOpt().
1752
1753         // text before the citation
1754         before.clear();
1755         // text after the citation
1756         after = p.getFullOpt();
1757
1758         if (!after.empty()) {
1759                 before = p.getFullOpt();
1760                 if (natbibOrder && !before.empty())
1761                         swap(before, after);
1762         }
1763 }
1764
1765
1766 /// Convert filenames with TeX macros and/or quotes to something LyX
1767 /// can understand
1768 string const normalize_filename(string const & name)
1769 {
1770         Parser p(trim(name, "\""));
1771         ostringstream os;
1772         while (p.good()) {
1773                 Token const & t = p.get_token();
1774                 if (t.cat() != catEscape)
1775                         os << t.asInput();
1776                 else if (t.cs() == "lyxdot") {
1777                         // This is used by LyX for simple dots in relative
1778                         // names
1779                         os << '.';
1780                         p.skip_spaces();
1781                 } else if (t.cs() == "space") {
1782                         os << ' ';
1783                         p.skip_spaces();
1784                 } else
1785                         os << t.asInput();
1786         }
1787         return os.str();
1788 }
1789
1790
1791 /// Convert \p name from TeX convention (relative to master file) to LyX
1792 /// convention (relative to .lyx file) if it is relative
1793 void fix_relative_filename(string & name)
1794 {
1795         if (FileName::isAbsolute(name))
1796                 return;
1797
1798         name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
1799                                    from_utf8(getParentFilePath())));
1800 }
1801
1802
1803 /// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
1804 void parse_noweb(Parser & p, ostream & os, Context & context)
1805 {
1806         // assemble the rest of the keyword
1807         string name("<<");
1808         bool scrap = false;
1809         while (p.good()) {
1810                 Token const & t = p.get_token();
1811                 if (t.asInput() == ">" && p.next_token().asInput() == ">") {
1812                         name += ">>";
1813                         p.get_token();
1814                         scrap = (p.good() && p.next_token().asInput() == "=");
1815                         if (scrap)
1816                                 name += p.get_token().asInput();
1817                         break;
1818                 }
1819                 name += t.asInput();
1820         }
1821
1822         if (!scrap || !context.new_layout_allowed ||
1823             !context.textclass.hasLayout(from_ascii("Scrap"))) {
1824                 cerr << "Warning: Could not interpret '" << name
1825                      << "'. Ignoring it." << endl;
1826                 return;
1827         }
1828
1829         // We use new_paragraph instead of check_end_layout because the stuff
1830         // following the noweb chunk needs to start with a \begin_layout.
1831         // This may create a new paragraph even if there was none in the
1832         // noweb file, but the alternative is an invalid LyX file. Since
1833         // noweb code chunks are implemented with a layout style in LyX they
1834         // always must be in an own paragraph.
1835         context.new_paragraph(os);
1836         Context newcontext(true, context.textclass,
1837                 &context.textclass[from_ascii("Scrap")]);
1838         newcontext.check_layout(os);
1839         os << name;
1840         while (p.good()) {
1841                 Token const & t = p.get_token();
1842                 // We abuse the parser a bit, because this is no TeX syntax
1843                 // at all.
1844                 if (t.cat() == catEscape)
1845                         os << subst(t.asInput(), "\\", "\n\\backslash\n");
1846                 else {
1847                         ostringstream oss;
1848                         Context tmp(false, context.textclass,
1849                                     &context.textclass[from_ascii("Scrap")]);
1850                         tmp.need_end_layout = true;
1851                         tmp.check_layout(oss);
1852                         os << subst(t.asInput(), "\n", oss.str());
1853                 }
1854                 // The scrap chunk is ended by an @ at the beginning of a line.
1855                 // After the @ the line may contain a comment and/or
1856                 // whitespace, but nothing else.
1857                 if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
1858                     (p.next_token().cat() == catSpace ||
1859                      p.next_token().cat() == catNewline ||
1860                      p.next_token().cat() == catComment)) {
1861                         while (p.good() && p.next_token().cat() == catSpace)
1862                                 os << p.get_token().asInput();
1863                         if (p.next_token().cat() == catComment)
1864                                 // The comment includes a final '\n'
1865                                 os << p.get_token().asInput();
1866                         else {
1867                                 if (p.next_token().cat() == catNewline)
1868                                         p.get_token();
1869                                 os << '\n';
1870                         }
1871                         break;
1872                 }
1873         }
1874         newcontext.check_end_layout(os);
1875 }
1876
1877
1878 /// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
1879 bool is_macro(Parser & p)
1880 {
1881         Token first = p.curr_token();
1882         if (first.cat() != catEscape || !p.good())
1883                 return false;
1884         if (first.cs() == "def")
1885                 return true;
1886         if (first.cs() != "global" && first.cs() != "long")
1887                 return false;
1888         Token second = p.get_token();
1889         int pos = 1;
1890         while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
1891                second.cat() == catNewline || second.cat() == catComment)) {
1892                 second = p.get_token();
1893                 pos++;
1894         }
1895         bool secondvalid = second.cat() == catEscape;
1896         Token third;
1897         bool thirdvalid = false;
1898         if (p.good() && first.cs() == "global" && secondvalid &&
1899             second.cs() == "long") {
1900                 third = p.get_token();
1901                 pos++;
1902                 while (p.good() && !p.isParagraph() &&
1903                        (third.cat() == catSpace ||
1904                         third.cat() == catNewline ||
1905                         third.cat() == catComment)) {
1906                         third = p.get_token();
1907                         pos++;
1908                 }
1909                 thirdvalid = third.cat() == catEscape;
1910         }
1911         for (int i = 0; i < pos; ++i)
1912                 p.putback();
1913         if (!secondvalid)
1914                 return false;
1915         if (!thirdvalid)
1916                 return (first.cs() == "global" || first.cs() == "long") &&
1917                        second.cs() == "def";
1918         return first.cs() == "global" && second.cs() == "long" &&
1919                third.cs() == "def";
1920 }
1921
1922
1923 /// Parse a macro definition (assumes that is_macro() returned true)
1924 void parse_macro(Parser & p, ostream & os, Context & context)
1925 {
1926         context.check_layout(os);
1927         Token first = p.curr_token();
1928         Token second;
1929         Token third;
1930         string command = first.asInput();
1931         if (first.cs() != "def") {
1932                 p.get_token();
1933                 eat_whitespace(p, os, context, false);
1934                 second = p.curr_token();
1935                 command += second.asInput();
1936                 if (second.cs() != "def") {
1937                         p.get_token();
1938                         eat_whitespace(p, os, context, false);
1939                         third = p.curr_token();
1940                         command += third.asInput();
1941                 }
1942         }
1943         eat_whitespace(p, os, context, false);
1944         string const name = p.get_token().cs();
1945         eat_whitespace(p, os, context, false);
1946
1947         // parameter text
1948         bool simple = true;
1949         string paramtext;
1950         int arity = 0;
1951         while (p.next_token().cat() != catBegin) {
1952                 if (p.next_token().cat() == catParameter) {
1953                         // # found
1954                         p.get_token();
1955                         paramtext += "#";
1956
1957                         // followed by number?
1958                         if (p.next_token().cat() == catOther) {
1959                                 char c = p.getChar();
1960                                 paramtext += c;
1961                                 // number = current arity + 1?
1962                                 if (c == arity + '0' + 1)
1963                                         ++arity;
1964                                 else
1965                                         simple = false;
1966                         } else
1967                                 paramtext += p.get_token().cs();
1968                 } else {
1969                         paramtext += p.get_token().cs();
1970                         simple = false;
1971                 }
1972         }
1973
1974         // only output simple (i.e. compatible) macro as FormulaMacros
1975         string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
1976         if (simple) {
1977                 context.check_layout(os);
1978                 begin_inset(os, "FormulaMacro");
1979                 os << "\n\\def" << ert;
1980                 end_inset(os);
1981         } else
1982                 handle_ert(os, command + ert, context);
1983 }
1984
1985
1986 void registerExternalTemplatePackages(string const & name)
1987 {
1988         external::TemplateManager const & etm = external::TemplateManager::get();
1989         external::Template const * const et = etm.getTemplateByName(name);
1990         if (!et)
1991                 return;
1992         external::Template::Formats::const_iterator cit = et->formats.end();
1993         if (pdflatex)
1994                 cit = et->formats.find("PDFLaTeX");
1995         if (cit == et->formats.end())
1996                 // If the template has not specified a PDFLaTeX output,
1997                 // we try the LaTeX format.
1998                 cit = et->formats.find("LaTeX");
1999         if (cit == et->formats.end())
2000                 return;
2001         vector<string>::const_iterator qit = cit->second.requirements.begin();
2002         vector<string>::const_iterator qend = cit->second.requirements.end();
2003         for (; qit != qend; ++qit)
2004                 preamble.registerAutomaticallyLoadedPackage(*qit);
2005 }
2006
2007 } // anonymous namespace
2008
2009
2010 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
2011                 Context & context)
2012 {
2013         Layout const * newlayout = 0;
2014         InsetLayout const * newinsetlayout = 0;
2015         char const * const * where = 0;
2016         // Store the latest bibliographystyle and nocite{*} option
2017         // (needed for bibtex inset)
2018         string btprint;
2019         string bibliographystyle = "default";
2020         bool const use_natbib = preamble.isPackageUsed("natbib");
2021         bool const use_jurabib = preamble.isPackageUsed("jurabib");
2022         string last_env;
2023         while (p.good()) {
2024                 Token const & t = p.get_token();
2025
2026 #ifdef FILEDEBUG
2027                 debugToken(cerr, t, flags);
2028 #endif
2029
2030                 if (flags & FLAG_ITEM) {
2031                         if (t.cat() == catSpace)
2032                                 continue;
2033
2034                         flags &= ~FLAG_ITEM;
2035                         if (t.cat() == catBegin) {
2036                                 // skip the brace and collect everything to the next matching
2037                                 // closing brace
2038                                 flags |= FLAG_BRACE_LAST;
2039                                 continue;
2040                         }
2041
2042                         // handle only this single token, leave the loop if done
2043                         flags |= FLAG_LEAVE;
2044                 }
2045
2046                 if (t.cat() != catEscape && t.character() == ']' &&
2047                     (flags & FLAG_BRACK_LAST))
2048                         return;
2049                 if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
2050                         return;
2051
2052                 // If there is anything between \end{env} and \begin{env} we
2053                 // don't need to output a separator.
2054                 if (t.cat() != catSpace && t.cat() != catNewline &&
2055                     t.asInput() != "\\begin")
2056                         last_env = "";
2057
2058                 //
2059                 // cat codes
2060                 //
2061                 if (t.cat() == catMath) {
2062                         // we are inside some text mode thingy, so opening new math is allowed
2063                         context.check_layout(os);
2064                         begin_inset(os, "Formula ");
2065                         Token const & n = p.get_token();
2066                         bool const display(n.cat() == catMath && outer);
2067                         if (display) {
2068                                 // TeX's $$...$$ syntax for displayed math
2069                                 os << "\\[";
2070                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2071                                 os << "\\]";
2072                                 p.get_token(); // skip the second '$' token
2073                         } else {
2074                                 // simple $...$  stuff
2075                                 p.putback();
2076                                 os << '$';
2077                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2078                                 os << '$';
2079                         }
2080                         end_inset(os);
2081                         if (display) {
2082                                 // Prevent the conversion of a line break to a
2083                                 // space (bug 7668). This does not change the
2084                                 // output, but looks ugly in LyX.
2085                                 eat_whitespace(p, os, context, false);
2086                         }
2087                 }
2088
2089                 else if (t.cat() == catSuper || t.cat() == catSub)
2090                         cerr << "catcode " << t << " illegal in text mode\n";
2091
2092                 // Basic support for english quotes. This should be
2093                 // extended to other quotes, but is not so easy (a
2094                 // left english quote is the same as a right german
2095                 // quote...)
2096                 else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
2097                         context.check_layout(os);
2098                         begin_inset(os, "Quotes ");
2099                         os << "eld";
2100                         end_inset(os);
2101                         p.get_token();
2102                         skip_braces(p);
2103                 }
2104                 else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
2105                         context.check_layout(os);
2106                         begin_inset(os, "Quotes ");
2107                         os << "erd";
2108                         end_inset(os);
2109                         p.get_token();
2110                         skip_braces(p);
2111                 }
2112
2113                 else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
2114                         context.check_layout(os);
2115                         begin_inset(os, "Quotes ");
2116                         os << "ald";
2117                         end_inset(os);
2118                         p.get_token();
2119                         skip_braces(p);
2120                 }
2121
2122                 else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
2123                         context.check_layout(os);
2124                         begin_inset(os, "Quotes ");
2125                         os << "ard";
2126                         end_inset(os);
2127                         p.get_token();
2128                         skip_braces(p);
2129                 }
2130
2131                 else if (t.asInput() == "<"
2132                          && p.next_token().asInput() == "<" && noweb_mode) {
2133                         p.get_token();
2134                         parse_noweb(p, os, context);
2135                 }
2136
2137                 else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
2138                         check_space(p, os, context);
2139
2140                 else if (t.character() == '[' && noweb_mode &&
2141                          p.next_token().character() == '[') {
2142                         // These can contain underscores
2143                         p.putback();
2144                         string const s = p.getFullOpt() + ']';
2145                         if (p.next_token().character() == ']')
2146                                 p.get_token();
2147                         else
2148                                 cerr << "Warning: Inserting missing ']' in '"
2149                                      << s << "'." << endl;
2150                         handle_ert(os, s, context);
2151                 }
2152
2153                 else if (t.cat() == catLetter) {
2154                         context.check_layout(os);
2155                         // Workaround for bug 4752.
2156                         // FIXME: This whole code block needs to be removed
2157                         //        when the bug is fixed and tex2lyx produces
2158                         //        the updated file format.
2159                         // The replacement algorithm in LyX is so stupid that
2160                         // it even translates a phrase if it is part of a word.
2161                         bool handled = false;
2162                         for (int const * l = known_phrase_lengths; *l; ++l) {
2163                                 string phrase = t.cs();
2164                                 for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
2165                                         phrase += p.get_token().cs();
2166                                 if (is_known(phrase, known_coded_phrases)) {
2167                                         handle_ert(os, phrase, context);
2168                                         handled = true;
2169                                         break;
2170                                 } else {
2171                                         for (size_t i = 1; i < phrase.length(); ++i)
2172                                                 p.putback();
2173                                 }
2174                         }
2175                         if (!handled)
2176                                 os << t.cs();
2177                 }
2178
2179                 else if (t.cat() == catOther ||
2180                                t.cat() == catAlign ||
2181                                t.cat() == catParameter) {
2182                         // This translates "&" to "\\&" which may be wrong...
2183                         context.check_layout(os);
2184                         os << t.cs();
2185                 }
2186
2187                 else if (p.isParagraph()) {
2188                         if (context.new_layout_allowed)
2189                                 context.new_paragraph(os);
2190                         else
2191                                 handle_ert(os, "\\par ", context);
2192                         eat_whitespace(p, os, context, true);
2193                 }
2194
2195                 else if (t.cat() == catActive) {
2196                         context.check_layout(os);
2197                         if (t.character() == '~') {
2198                                 if (context.layout->free_spacing)
2199                                         os << ' ';
2200                                 else {
2201                                         begin_inset(os, "space ~\n");
2202                                         end_inset(os);
2203                                 }
2204                         } else
2205                                 os << t.cs();
2206                 }
2207
2208                 else if (t.cat() == catBegin) {
2209                         Token const next = p.next_token();
2210                         Token const end = p.next_next_token();
2211                         if (next.cat() == catEnd) {
2212                         // {}
2213                         Token const prev = p.prev_token();
2214                         p.get_token();
2215                         if (p.next_token().character() == '`' ||
2216                             (prev.character() == '-' &&
2217                              p.next_token().character() == '-'))
2218                                 ; // ignore it in {}`` or -{}-
2219                         else
2220                                 handle_ert(os, "{}", context);
2221                         } else if (next.cat() == catEscape &&
2222                                    is_known(next.cs(), known_quotes) &&
2223                                    end.cat() == catEnd) {
2224                                 // Something like {\textquoteright} (e.g.
2225                                 // from writer2latex). LyX writes
2226                                 // \textquoteright{}, so we may skip the
2227                                 // braces here for better readability.
2228                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2229                                                    outer, context);
2230                         } else {
2231                         context.check_layout(os);
2232                         // special handling of font attribute changes
2233                         Token const prev = p.prev_token();
2234                         TeXFont const oldFont = context.font;
2235                         if (next.character() == '[' ||
2236                             next.character() == ']' ||
2237                             next.character() == '*') {
2238                                 p.get_token();
2239                                 if (p.next_token().cat() == catEnd) {
2240                                         os << next.cs();
2241                                         p.get_token();
2242                                 } else {
2243                                         p.putback();
2244                                         handle_ert(os, "{", context);
2245                                         parse_text_snippet(p, os,
2246                                                         FLAG_BRACE_LAST,
2247                                                         outer, context);
2248                                         handle_ert(os, "}", context);
2249                                 }
2250                         } else if (! context.new_layout_allowed) {
2251                                 handle_ert(os, "{", context);
2252                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2253                                                    outer, context);
2254                                 handle_ert(os, "}", context);
2255                         } else if (is_known(next.cs(), known_sizes)) {
2256                                 // next will change the size, so we must
2257                                 // reset it here
2258                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2259                                                    outer, context);
2260                                 if (!context.atParagraphStart())
2261                                         os << "\n\\size "
2262                                            << context.font.size << "\n";
2263                         } else if (is_known(next.cs(), known_font_families)) {
2264                                 // next will change the font family, so we
2265                                 // must reset it here
2266                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2267                                                    outer, context);
2268                                 if (!context.atParagraphStart())
2269                                         os << "\n\\family "
2270                                            << context.font.family << "\n";
2271                         } else if (is_known(next.cs(), known_font_series)) {
2272                                 // next will change the font series, so we
2273                                 // must reset it here
2274                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2275                                                    outer, context);
2276                                 if (!context.atParagraphStart())
2277                                         os << "\n\\series "
2278                                            << context.font.series << "\n";
2279                         } else if (is_known(next.cs(), known_font_shapes)) {
2280                                 // next will change the font shape, so we
2281                                 // must reset it here
2282                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2283                                                    outer, context);
2284                                 if (!context.atParagraphStart())
2285                                         os << "\n\\shape "
2286                                            << context.font.shape << "\n";
2287                         } else if (is_known(next.cs(), known_old_font_families) ||
2288                                    is_known(next.cs(), known_old_font_series) ||
2289                                    is_known(next.cs(), known_old_font_shapes)) {
2290                                 // next will change the font family, series
2291                                 // and shape, so we must reset it here
2292                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2293                                                    outer, context);
2294                                 if (!context.atParagraphStart())
2295                                         os <<  "\n\\family "
2296                                            << context.font.family
2297                                            << "\n\\series "
2298                                            << context.font.series
2299                                            << "\n\\shape "
2300                                            << context.font.shape << "\n";
2301                         } else {
2302                                 handle_ert(os, "{", context);
2303                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2304                                                    outer, context);
2305                                 handle_ert(os, "}", context);
2306                                 }
2307                         }
2308                 }
2309
2310                 else if (t.cat() == catEnd) {
2311                         if (flags & FLAG_BRACE_LAST) {
2312                                 return;
2313                         }
2314                         cerr << "stray '}' in text\n";
2315                         handle_ert(os, "}", context);
2316                 }
2317
2318                 else if (t.cat() == catComment)
2319                         parse_comment(p, os, t, context);
2320
2321                 //
2322                 // control sequences
2323                 //
2324
2325                 else if (t.cs() == "(") {
2326                         context.check_layout(os);
2327                         begin_inset(os, "Formula");
2328                         os << " \\(";
2329                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
2330                         os << "\\)";
2331                         end_inset(os);
2332                 }
2333
2334                 else if (t.cs() == "[") {
2335                         context.check_layout(os);
2336                         begin_inset(os, "Formula");
2337                         os << " \\[";
2338                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
2339                         os << "\\]";
2340                         end_inset(os);
2341                         // Prevent the conversion of a line break to a space
2342                         // (bug 7668). This does not change the output, but
2343                         // looks ugly in LyX.
2344                         eat_whitespace(p, os, context, false);
2345                 }
2346
2347                 else if (t.cs() == "begin")
2348                         parse_environment(p, os, outer, last_env,
2349                                           context);
2350
2351                 else if (t.cs() == "end") {
2352                         if (flags & FLAG_END) {
2353                                 // eat environment name
2354                                 string const name = p.getArg('{', '}');
2355                                 if (name != active_environment())
2356                                         cerr << "\\end{" + name + "} does not match \\begin{"
2357                                                 + active_environment() + "}\n";
2358                                 return;
2359                         }
2360                         p.error("found 'end' unexpectedly");
2361                 }
2362
2363                 else if (t.cs() == "item") {
2364                         string s;
2365                         bool const optarg = p.hasOpt();
2366                         if (optarg) {
2367                                 // FIXME: This swallows comments, but we cannot use
2368                                 //        eat_whitespace() since we must not output
2369                                 //        anything before the item.
2370                                 p.skip_spaces(true);
2371                                 s = p.verbatimOption();
2372                         } else
2373                                 p.skip_spaces(false);
2374                         context.set_item();
2375                         context.check_layout(os);
2376                         if (context.has_item) {
2377                                 // An item in an unknown list-like environment
2378                                 // FIXME: Do this in check_layout()!
2379                                 context.has_item = false;
2380                                 if (optarg)
2381                                         handle_ert(os, "\\item", context);
2382                                 else
2383                                         handle_ert(os, "\\item ", context);
2384                         }
2385                         if (optarg) {
2386                                 if (context.layout->labeltype != LABEL_MANUAL) {
2387                                         // LyX does not support \item[\mybullet]
2388                                         // in itemize environments
2389                                         Parser p2(s + ']');
2390                                         os << parse_text_snippet(p2,
2391                                                 FLAG_BRACK_LAST, outer, context);
2392                                 } else if (!s.empty()) {
2393                                         // LyX adds braces around the argument,
2394                                         // so we need to remove them here.
2395                                         if (s.size() > 2 && s[0] == '{' &&
2396                                             s[s.size()-1] == '}')
2397                                                 s = s.substr(1, s.size()-2);
2398                                         // If the argument contains a space we
2399                                         // must put it into ERT: Otherwise LyX
2400                                         // would misinterpret the space as
2401                                         // item delimiter (bug 7663)
2402                                         if (contains(s, ' ')) {
2403                                                 handle_ert(os, s, context);
2404                                         } else {
2405                                                 Parser p2(s + ']');
2406                                                 os << parse_text_snippet(p2,
2407                                                         FLAG_BRACK_LAST,
2408                                                         outer, context);
2409                                         }
2410                                         // The space is needed to separate the
2411                                         // item from the rest of the sentence.
2412                                         os << ' ';
2413                                         eat_whitespace(p, os, context, false);
2414                                 }
2415                         }
2416                 }
2417
2418                 else if (t.cs() == "bibitem") {
2419                         context.set_item();
2420                         context.check_layout(os);
2421                         eat_whitespace(p, os, context, false);
2422                         string label = convert_command_inset_arg(p.verbatimOption());
2423                         string key = convert_command_inset_arg(p.verbatim_item());
2424                         if (contains(label, '\\') || contains(key, '\\')) {
2425                                 // LyX can't handle LaTeX commands in labels or keys
2426                                 handle_ert(os, t.asInput() + '[' + label +
2427                                                "]{" + p.verbatim_item() + '}',
2428                                            context);
2429                         } else {
2430                                 begin_command_inset(os, "bibitem", "bibitem");
2431                                 os << "label \"" << label << "\"\n"
2432                                       "key \"" << key << "\"\n";
2433                                 end_inset(os);
2434                         }
2435                 }
2436
2437                 else if (is_macro(p)) {
2438                         // catch the case of \def\inputGnumericTable
2439                         bool macro = true;
2440                         if (t.cs() == "def") {
2441                                 Token second = p.next_token();
2442                                 if (second.cs() == "inputGnumericTable") {
2443                                         p.pushPosition();
2444                                         p.get_token();
2445                                         skip_braces(p);
2446                                         Token third = p.get_token();
2447                                         p.popPosition();
2448                                         if (third.cs() == "input") {
2449                                                 p.get_token();
2450                                                 skip_braces(p);
2451                                                 p.get_token();
2452                                                 string name = normalize_filename(p.verbatim_item());
2453                                                 string const path = getMasterFilePath();
2454                                                 // We want to preserve relative / absolute filenames,
2455                                                 // therefore path is only used for testing
2456                                                 // The file extension is in every case ".tex".
2457                                                 // So we need to remove this extension and check for
2458                                                 // the original one.
2459                                                 name = removeExtension(name);
2460                                                 if (!makeAbsPath(name, path).exists()) {
2461                                                         char const * const Gnumeric_formats[] = {"gnumeric",
2462                                                                 "ods", "xls", 0};
2463                                                         string const Gnumeric_name =
2464                                                                 find_file(name, path, Gnumeric_formats);
2465                                                         if (!Gnumeric_name.empty())
2466                                                                 name = Gnumeric_name;
2467                                                 }
2468                                                 if (makeAbsPath(name, path).exists())
2469                                                         fix_relative_filename(name);
2470                                                 else
2471                                                         cerr << "Warning: Could not find file '"
2472                                                              << name << "'." << endl;
2473                                                 context.check_layout(os);
2474                                                 begin_inset(os, "External\n\ttemplate ");
2475                                                 os << "GnumericSpreadsheet\n\tfilename "
2476                                                    << name << "\n";
2477                                                 end_inset(os);
2478                                                 context.check_layout(os);
2479                                                 macro = false;
2480                                                 // register the packages that are automatically reloaded
2481                                                 // by the Gnumeric template
2482                                                 registerExternalTemplatePackages("GnumericSpreadsheet");
2483                                         }
2484                                 }
2485                         }
2486                         if (macro)
2487                                 parse_macro(p, os, context);
2488                 }
2489
2490                 else if (t.cs() == "noindent") {
2491                         p.skip_spaces();
2492                         context.add_par_extra_stuff("\\noindent\n");
2493                 }
2494
2495                 else if (t.cs() == "appendix") {
2496                         context.add_par_extra_stuff("\\start_of_appendix\n");
2497                         // We need to start a new paragraph. Otherwise the
2498                         // appendix in 'bla\appendix\chapter{' would start
2499                         // too late.
2500                         context.new_paragraph(os);
2501                         // We need to make sure that the paragraph is
2502                         // generated even if it is empty. Otherwise the
2503                         // appendix in '\par\appendix\par\chapter{' would
2504                         // start too late.
2505                         context.check_layout(os);
2506                         // FIXME: This is a hack to prevent paragraph
2507                         // deletion if it is empty. Handle this better!
2508                         handle_comment(os,
2509                                 "%dummy comment inserted by tex2lyx to "
2510                                 "ensure that this paragraph is not empty",
2511                                 context);
2512                         // Both measures above may generate an additional
2513                         // empty paragraph, but that does not hurt, because
2514                         // whitespace does not matter here.
2515                         eat_whitespace(p, os, context, true);
2516                 }
2517
2518                 // Must catch empty dates before findLayout is called below
2519                 else if (t.cs() == "date") {
2520                         eat_whitespace(p, os, context, false);
2521                         p.pushPosition();
2522                         string const date = p.verbatim_item();
2523                         p.popPosition();
2524                         if (date.empty()) {
2525                                 preamble.suppressDate(true);
2526                                 p.verbatim_item();
2527                         } else {
2528                                 preamble.suppressDate(false);
2529                                 if (context.new_layout_allowed &&
2530                                     (newlayout = findLayout(context.textclass,
2531                                                             t.cs(), true))) {
2532                                         // write the layout
2533                                         output_command_layout(os, p, outer,
2534                                                         context, newlayout);
2535                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2536                                         if (!preamble.titleLayoutFound())
2537                                                 preamble.titleLayoutFound(newlayout->intitle);
2538                                         set<string> const & req = newlayout->requires();
2539                                         set<string>::const_iterator it = req.begin();
2540                                         set<string>::const_iterator en = req.end();
2541                                         for (; it != en; ++it)
2542                                                 preamble.registerAutomaticallyLoadedPackage(*it);
2543                                 } else
2544                                         handle_ert(os,
2545                                                 "\\date{" + p.verbatim_item() + '}',
2546                                                 context);
2547                         }
2548                 }
2549
2550                 // Starred section headings
2551                 // Must attempt to parse "Section*" before "Section".
2552                 else if ((p.next_token().asInput() == "*") &&
2553                          context.new_layout_allowed &&
2554                          (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
2555                         // write the layout
2556                         p.get_token();
2557                         output_command_layout(os, p, outer, context, newlayout);
2558                         p.skip_spaces();
2559                         if (!preamble.titleLayoutFound())
2560                                 preamble.titleLayoutFound(newlayout->intitle);
2561                         set<string> const & req = newlayout->requires();
2562                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2563                                 preamble.registerAutomaticallyLoadedPackage(*it);
2564                 }
2565
2566                 // Section headings and the like
2567                 else if (context.new_layout_allowed &&
2568                          (newlayout = findLayout(context.textclass, t.cs(), true))) {
2569                         // write the layout
2570                         output_command_layout(os, p, outer, context, newlayout);
2571                         p.skip_spaces();
2572                         if (!preamble.titleLayoutFound())
2573                                 preamble.titleLayoutFound(newlayout->intitle);
2574                         set<string> const & req = newlayout->requires();
2575                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2576                                 preamble.registerAutomaticallyLoadedPackage(*it);
2577                 }
2578
2579                 else if (t.cs() == "caption") {
2580                         p.skip_spaces();
2581                         context.check_layout(os);
2582                         p.skip_spaces();
2583                         begin_inset(os, "Caption\n");
2584                         Context newcontext(true, context.textclass);
2585                         newcontext.font = context.font;
2586                         newcontext.check_layout(os);
2587                         if (p.next_token().cat() != catEscape &&
2588                             p.next_token().character() == '[') {
2589                                 p.get_token(); // eat '['
2590                                 begin_inset(os, "Argument\n");
2591                                 os << "status collapsed\n";
2592                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
2593                                 end_inset(os);
2594                                 eat_whitespace(p, os, context, false);
2595                         }
2596                         parse_text(p, os, FLAG_ITEM, outer, context);
2597                         context.check_end_layout(os);
2598                         // We don't need really a new paragraph, but
2599                         // we must make sure that the next item gets a \begin_layout.
2600                         context.new_paragraph(os);
2601                         end_inset(os);
2602                         p.skip_spaces();
2603                         newcontext.check_end_layout(os);
2604                 }
2605
2606                 else if (t.cs() == "subfloat") {
2607                         // the syntax is \subfloat[caption]{content}
2608                         // if it is a table of figure depends on the surrounding float
2609                         bool has_caption = false;
2610                         p.skip_spaces();
2611                         // do nothing if there is no outer float
2612                         if (!float_type.empty()) {
2613                                 context.check_layout(os);
2614                                 p.skip_spaces();
2615                                 begin_inset(os, "Float " + float_type + "\n");
2616                                 os << "wide false"
2617                                    << "\nsideways false"
2618                                    << "\nstatus collapsed\n\n";
2619                                 // test for caption
2620                                 string caption;
2621                                 if (p.next_token().cat() != catEscape &&
2622                                                 p.next_token().character() == '[') {
2623                                                         p.get_token(); // eat '['
2624                                                         caption = parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
2625                                                         has_caption = true;
2626                                 }
2627                                 // the content
2628                                 parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
2629                                 // the caption comes always as the last
2630                                 if (has_caption) {
2631                                         // we must make sure that the caption gets a \begin_layout
2632                                         os << "\n\\begin_layout Plain Layout";
2633                                         p.skip_spaces();
2634                                         begin_inset(os, "Caption\n");
2635                                         Context newcontext(true, context.textclass);
2636                                         newcontext.font = context.font;
2637                                         newcontext.check_layout(os);
2638                                         os << caption << "\n";
2639                                         newcontext.check_end_layout(os);
2640                                         // We don't need really a new paragraph, but
2641                                         // we must make sure that the next item gets a \begin_layout.
2642                                         //newcontext.new_paragraph(os);
2643                                         end_inset(os);
2644                                         p.skip_spaces();
2645                                 }
2646                                 // We don't need really a new paragraph, but
2647                                 // we must make sure that the next item gets a \begin_layout.
2648                                 if (has_caption)
2649                                         context.new_paragraph(os);
2650                                 end_inset(os);
2651                                 p.skip_spaces();
2652                                 context.check_end_layout(os);
2653                                 // close the layout we opened
2654                                 if (has_caption)
2655                                         os << "\n\\end_layout\n";
2656                         } else {
2657                                 // if the float type is not supported or there is no surrounding float
2658                                 // output it as ERT
2659                                 if (p.hasOpt()) {
2660                                         string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
2661                                         handle_ert(os, t.asInput() + '[' + opt_arg +
2662                                                "]{" + p.verbatim_item() + '}', context);
2663                                 } else
2664                                         handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
2665                         }
2666                 }
2667
2668                 else if (t.cs() == "includegraphics") {
2669                         bool const clip = p.next_token().asInput() == "*";
2670                         if (clip)
2671                                 p.get_token();
2672                         string const arg = p.getArg('[', ']');
2673                         map<string, string> opts;
2674                         vector<string> keys;
2675                         split_map(arg, opts, keys);
2676                         if (clip)
2677                                 opts["clip"] = string();
2678                         string name = normalize_filename(p.verbatim_item());
2679
2680                         string const path = getMasterFilePath();
2681                         // We want to preserve relative / absolute filenames,
2682                         // therefore path is only used for testing
2683                         if (!makeAbsPath(name, path).exists()) {
2684                                 // The file extension is probably missing.
2685                                 // Now try to find it out.
2686                                 string const dvips_name =
2687                                         find_file(name, path,
2688                                                   known_dvips_graphics_formats);
2689                                 string const pdftex_name =
2690                                         find_file(name, path,
2691                                                   known_pdftex_graphics_formats);
2692                                 if (!dvips_name.empty()) {
2693                                         if (!pdftex_name.empty()) {
2694                                                 cerr << "This file contains the "
2695                                                         "latex snippet\n"
2696                                                         "\"\\includegraphics{"
2697                                                      << name << "}\".\n"
2698                                                         "However, files\n\""
2699                                                      << dvips_name << "\" and\n\""
2700                                                      << pdftex_name << "\"\n"
2701                                                         "both exist, so I had to make a "
2702                                                         "choice and took the first one.\n"
2703                                                         "Please move the unwanted one "
2704                                                         "someplace else and try again\n"
2705                                                         "if my choice was wrong."
2706                                                      << endl;
2707                                         }
2708                                         name = dvips_name;
2709                                 } else if (!pdftex_name.empty()) {
2710                                         name = pdftex_name;
2711                                         pdflatex = true;
2712                                 }
2713                         }
2714
2715                         if (makeAbsPath(name, path).exists())
2716                                 fix_relative_filename(name);
2717                         else
2718                                 cerr << "Warning: Could not find graphics file '"
2719                                      << name << "'." << endl;
2720
2721                         context.check_layout(os);
2722                         begin_inset(os, "Graphics ");
2723                         os << "\n\tfilename " << name << '\n';
2724                         if (opts.find("width") != opts.end())
2725                                 os << "\twidth "
2726                                    << translate_len(opts["width"]) << '\n';
2727                         if (opts.find("height") != opts.end())
2728                                 os << "\theight "
2729                                    << translate_len(opts["height"]) << '\n';
2730                         if (opts.find("scale") != opts.end()) {
2731                                 istringstream iss(opts["scale"]);
2732                                 double val;
2733                                 iss >> val;
2734                                 val = val*100;
2735                                 os << "\tscale " << val << '\n';
2736                         }
2737                         if (opts.find("angle") != opts.end()) {
2738                                 os << "\trotateAngle "
2739                                    << opts["angle"] << '\n';
2740                                 vector<string>::const_iterator a =
2741                                         find(keys.begin(), keys.end(), "angle");
2742                                 vector<string>::const_iterator s =
2743                                         find(keys.begin(), keys.end(), "width");
2744                                 if (s == keys.end())
2745                                         s = find(keys.begin(), keys.end(), "height");
2746                                 if (s == keys.end())
2747                                         s = find(keys.begin(), keys.end(), "scale");
2748                                 if (s != keys.end() && distance(s, a) > 0)
2749                                         os << "\tscaleBeforeRotation\n";
2750                         }
2751                         if (opts.find("origin") != opts.end()) {
2752                                 ostringstream ss;
2753                                 string const opt = opts["origin"];
2754                                 if (opt.find('l') != string::npos) ss << "left";
2755                                 if (opt.find('r') != string::npos) ss << "right";
2756                                 if (opt.find('c') != string::npos) ss << "center";
2757                                 if (opt.find('t') != string::npos) ss << "Top";
2758                                 if (opt.find('b') != string::npos) ss << "Bottom";
2759                                 if (opt.find('B') != string::npos) ss << "Baseline";
2760                                 if (!ss.str().empty())
2761                                         os << "\trotateOrigin " << ss.str() << '\n';
2762                                 else
2763                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
2764                         }
2765                         if (opts.find("keepaspectratio") != opts.end())
2766                                 os << "\tkeepAspectRatio\n";
2767                         if (opts.find("clip") != opts.end())
2768                                 os << "\tclip\n";
2769                         if (opts.find("draft") != opts.end())
2770                                 os << "\tdraft\n";
2771                         if (opts.find("bb") != opts.end())
2772                                 os << "\tBoundingBox "
2773                                    << opts["bb"] << '\n';
2774                         int numberOfbbOptions = 0;
2775                         if (opts.find("bbllx") != opts.end())
2776                                 numberOfbbOptions++;
2777                         if (opts.find("bblly") != opts.end())
2778                                 numberOfbbOptions++;
2779                         if (opts.find("bburx") != opts.end())
2780                                 numberOfbbOptions++;
2781                         if (opts.find("bbury") != opts.end())
2782                                 numberOfbbOptions++;
2783                         if (numberOfbbOptions == 4)
2784                                 os << "\tBoundingBox "
2785                                    << opts["bbllx"] << " " << opts["bblly"] << " "
2786                                    << opts["bburx"] << " " << opts["bbury"] << '\n';
2787                         else if (numberOfbbOptions > 0)
2788                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2789                         numberOfbbOptions = 0;
2790                         if (opts.find("natwidth") != opts.end())
2791                                 numberOfbbOptions++;
2792                         if (opts.find("natheight") != opts.end())
2793                                 numberOfbbOptions++;
2794                         if (numberOfbbOptions == 2)
2795                                 os << "\tBoundingBox 0bp 0bp "
2796                                    << opts["natwidth"] << " " << opts["natheight"] << '\n';
2797                         else if (numberOfbbOptions > 0)
2798                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2799                         ostringstream special;
2800                         if (opts.find("hiresbb") != opts.end())
2801                                 special << "hiresbb,";
2802                         if (opts.find("trim") != opts.end())
2803                                 special << "trim,";
2804                         if (opts.find("viewport") != opts.end())
2805                                 special << "viewport=" << opts["viewport"] << ',';
2806                         if (opts.find("totalheight") != opts.end())
2807                                 special << "totalheight=" << opts["totalheight"] << ',';
2808                         if (opts.find("type") != opts.end())
2809                                 special << "type=" << opts["type"] << ',';
2810                         if (opts.find("ext") != opts.end())
2811                                 special << "ext=" << opts["ext"] << ',';
2812                         if (opts.find("read") != opts.end())
2813                                 special << "read=" << opts["read"] << ',';
2814                         if (opts.find("command") != opts.end())
2815                                 special << "command=" << opts["command"] << ',';
2816                         string s_special = special.str();
2817                         if (!s_special.empty()) {
2818                                 // We had special arguments. Remove the trailing ','.
2819                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
2820                         }
2821                         // TODO: Handle the unknown settings better.
2822                         // Warn about invalid options.
2823                         // Check whether some option was given twice.
2824                         end_inset(os);
2825                         preamble.registerAutomaticallyLoadedPackage("graphicx");
2826                 }
2827
2828                 else if (t.cs() == "footnote" ||
2829                          (t.cs() == "thanks" && context.layout->intitle)) {
2830                         p.skip_spaces();
2831                         context.check_layout(os);
2832                         begin_inset(os, "Foot\n");
2833                         os << "status collapsed\n\n";
2834                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2835                         end_inset(os);
2836                 }
2837
2838                 else if (t.cs() == "marginpar") {
2839                         p.skip_spaces();
2840                         context.check_layout(os);
2841                         begin_inset(os, "Marginal\n");
2842                         os << "status collapsed\n\n";
2843                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2844                         end_inset(os);
2845                 }
2846
2847                 else if (t.cs() == "lstinline") {
2848                         p.skip_spaces();
2849                         parse_listings(p, os, context, true);
2850                 }
2851
2852                 else if (t.cs() == "ensuremath") {
2853                         p.skip_spaces();
2854                         context.check_layout(os);
2855                         string const s = p.verbatim_item();
2856                         //FIXME: this never triggers in UTF8
2857                         if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
2858                                 os << s;
2859                         else
2860                                 handle_ert(os, "\\ensuremath{" + s + "}",
2861                                            context);
2862                 }
2863
2864                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
2865                         if (preamble.titleLayoutFound()) {
2866                                 // swallow this
2867                                 skip_spaces_braces(p);
2868                         } else
2869                                 handle_ert(os, t.asInput(), context);
2870                 }
2871
2872                 else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
2873                         context.check_layout(os);
2874                         begin_command_inset(os, "toc", t.cs());
2875                         end_inset(os);
2876                         skip_spaces_braces(p);
2877                         if (t.cs() == "lstlistoflistings")
2878                                 preamble.registerAutomaticallyLoadedPackage("listings");
2879                 }
2880
2881                 else if (t.cs() == "listoffigures") {
2882                         context.check_layout(os);
2883                         begin_inset(os, "FloatList figure\n");
2884                         end_inset(os);
2885                         skip_spaces_braces(p);
2886                 }
2887
2888                 else if (t.cs() == "listoftables") {
2889                         context.check_layout(os);
2890                         begin_inset(os, "FloatList table\n");
2891                         end_inset(os);
2892                         skip_spaces_braces(p);
2893                 }
2894
2895                 else if (t.cs() == "listof") {
2896                         p.skip_spaces(true);
2897                         string const name = p.get_token().cs();
2898                         if (context.textclass.floats().typeExist(name)) {
2899                                 context.check_layout(os);
2900                                 begin_inset(os, "FloatList ");
2901                                 os << name << "\n";
2902                                 end_inset(os);
2903                                 p.get_token(); // swallow second arg
2904                         } else
2905                                 handle_ert(os, "\\listof{" + name + "}", context);
2906                 }
2907
2908                 else if ((where = is_known(t.cs(), known_text_font_families)))
2909                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2910                                 context, "\\family", context.font.family,
2911                                 known_coded_font_families[where - known_text_font_families]);
2912
2913                 else if ((where = is_known(t.cs(), known_text_font_series)))
2914                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2915                                 context, "\\series", context.font.series,
2916                                 known_coded_font_series[where - known_text_font_series]);
2917
2918                 else if ((where = is_known(t.cs(), known_text_font_shapes)))
2919                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2920                                 context, "\\shape", context.font.shape,
2921                                 known_coded_font_shapes[where - known_text_font_shapes]);
2922
2923                 else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
2924                         context.check_layout(os);
2925                         TeXFont oldFont = context.font;
2926                         context.font.init();
2927                         context.font.size = oldFont.size;
2928                         os << "\n\\family " << context.font.family << "\n";
2929                         os << "\n\\series " << context.font.series << "\n";
2930                         os << "\n\\shape " << context.font.shape << "\n";
2931                         if (t.cs() == "textnormal") {
2932                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2933                                 output_font_change(os, context.font, oldFont);
2934                                 context.font = oldFont;
2935                         } else
2936                                 eat_whitespace(p, os, context, false);
2937                 }
2938
2939                 else if (t.cs() == "textcolor") {
2940                         // scheme is \textcolor{color name}{text}
2941                         string const color = p.verbatim_item();
2942                         // we only support the predefined colors of the color package
2943                         if (color == "black" || color == "blue" || color == "cyan"
2944                                 || color == "green" || color == "magenta" || color == "red"
2945                                 || color == "white" || color == "yellow") {
2946                                         context.check_layout(os);
2947                                         os << "\n\\color " << color << "\n";
2948                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2949                                         context.check_layout(os);
2950                                         os << "\n\\color inherit\n";
2951                                         preamble.registerAutomaticallyLoadedPackage("color");
2952                         } else
2953                                 // for custom defined colors
2954                                 handle_ert(os, t.asInput() + "{" + color + "}", context);
2955                 }
2956
2957                 else if (t.cs() == "underbar" || t.cs() == "uline") {
2958                         // \underbar is not 100% correct (LyX outputs \uline
2959                         // of ulem.sty). The difference is that \ulem allows
2960                         // line breaks, and \underbar does not.
2961                         // Do NOT handle \underline.
2962                         // \underbar cuts through y, g, q, p etc.,
2963                         // \underline does not.
2964                         context.check_layout(os);
2965                         os << "\n\\bar under\n";
2966                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2967                         context.check_layout(os);
2968                         os << "\n\\bar default\n";
2969                         preamble.registerAutomaticallyLoadedPackage("ulem");
2970                 }
2971
2972                 else if (t.cs() == "sout") {
2973                         context.check_layout(os);
2974                         os << "\n\\strikeout on\n";
2975                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2976                         context.check_layout(os);
2977                         os << "\n\\strikeout default\n";
2978                         preamble.registerAutomaticallyLoadedPackage("ulem");
2979                 }
2980
2981                 else if (t.cs() == "uuline" || t.cs() == "uwave" ||
2982                          t.cs() == "emph" || t.cs() == "noun") {
2983                         context.check_layout(os);
2984                         os << "\n\\" << t.cs() << " on\n";
2985                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2986                         context.check_layout(os);
2987                         os << "\n\\" << t.cs() << " default\n";
2988                         if (t.cs() == "uuline" || t.cs() == "uwave")
2989                                 preamble.registerAutomaticallyLoadedPackage("ulem");
2990                 }
2991
2992                 else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
2993                         context.check_layout(os);
2994                         string name = p.getArg('{', '}');
2995                         string localtime = p.getArg('{', '}');
2996                         preamble.registerAuthor(name);
2997                         Author const & author = preamble.getAuthor(name);
2998                         // from_ctime() will fail if LyX decides to output the
2999                         // time in the text language. It might also use a wrong
3000                         // time zone (if the original LyX document was exported
3001                         // with a different time zone).
3002                         time_t ptime = from_ctime(localtime);
3003                         if (ptime == static_cast<time_t>(-1)) {
3004                                 cerr << "Warning: Could not parse time `" << localtime
3005                                      << "´ for change tracking, using current time instead.\n";
3006                                 ptime = current_time();
3007                         }
3008                         if (t.cs() == "lyxadded")
3009                                 os << "\n\\change_inserted ";
3010                         else
3011                                 os << "\n\\change_deleted ";
3012                         os << author.bufferId() << ' ' << ptime << '\n';
3013                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3014                         bool dvipost    = LaTeXPackages::isAvailable("dvipost");
3015                         bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
3016                                           LaTeXPackages::isAvailable("xcolor");
3017                         // No need to test for luatex, since luatex comes in
3018                         // two flavours (dvi and pdf), like latex, and those
3019                         // are detected by pdflatex.
3020                         if (pdflatex || xetex) {
3021                                 if (xcolorulem) {
3022                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3023                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3024                                         preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
3025                                 }
3026                         } else {
3027                                 if (dvipost) {
3028                                         preamble.registerAutomaticallyLoadedPackage("dvipost");
3029                                 } else if (xcolorulem) {
3030                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3031                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3032                                 }
3033                         }
3034                 }
3035
3036                 else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
3037                              t.cs() == "vphantom") {
3038                         context.check_layout(os);
3039                         if (t.cs() == "phantom")
3040                                 begin_inset(os, "Phantom Phantom\n");
3041                         if (t.cs() == "hphantom")
3042                                 begin_inset(os, "Phantom HPhantom\n");
3043                         if (t.cs() == "vphantom")
3044                                 begin_inset(os, "Phantom VPhantom\n");
3045                         os << "status open\n";
3046                         parse_text_in_inset(p, os, FLAG_ITEM, outer, context,
3047                                             "Phantom");
3048                         end_inset(os);
3049                 }
3050
3051                 else if (t.cs() == "href") {
3052                         context.check_layout(os);
3053                         string target = p.getArg('{', '}');
3054                         string name = p.getArg('{', '}');
3055                         string type;
3056                         size_t i = target.find(':');
3057                         if (i != string::npos) {
3058                                 type = target.substr(0, i + 1);
3059                                 if (type == "mailto:" || type == "file:")
3060                                         target = target.substr(i + 1);
3061                                 // handle the case that name is equal to target, except of "http://"
3062                                 else if (target.substr(i + 3) == name && type == "http:")
3063                                         target = name;
3064                         }
3065                         begin_command_inset(os, "href", "href");
3066                         if (name != target)
3067                                 os << "name \"" << name << "\"\n";
3068                         os << "target \"" << target << "\"\n";
3069                         if (type == "mailto:" || type == "file:")
3070                                 os << "type \"" << type << "\"\n";
3071                         end_inset(os);
3072                         skip_spaces_braces(p);
3073                 }
3074
3075                 else if (t.cs() == "lyxline") {
3076                         // swallow size argument (it is not used anyway)
3077                         p.getArg('{', '}');
3078                         if (!context.atParagraphStart()) {
3079                                 // so our line is in the middle of a paragraph
3080                                 // we need to add a new line, lest this line
3081                                 // follow the other content on that line and
3082                                 // run off the side of the page
3083                                 // FIXME: This may create an empty paragraph,
3084                                 //        but without that it would not be
3085                                 //        possible to set noindent below.
3086                                 //        Fortunately LaTeX does not care
3087                                 //        about the empty paragraph.
3088                                 context.new_paragraph(os);
3089                         }
3090                         if (preamble.indentParagraphs()) {
3091                                 // we need to unindent, lest the line be too long
3092                                 context.add_par_extra_stuff("\\noindent\n");
3093                         }
3094                         context.check_layout(os);
3095                         begin_command_inset(os, "line", "rule");
3096                         os << "offset \"0.5ex\"\n"
3097                               "width \"100line%\"\n"
3098                               "height \"1pt\"\n";
3099                         end_inset(os);
3100                 }
3101
3102                 else if (t.cs() == "rule") {
3103                         string const offset = (p.hasOpt() ? p.getArg('[', ']') : string());
3104                         string const width = p.getArg('{', '}');
3105                         string const thickness = p.getArg('{', '}');
3106                         context.check_layout(os);
3107                         begin_command_inset(os, "line", "rule");
3108                         if (!offset.empty())
3109                                 os << "offset \"" << translate_len(offset) << "\"\n";
3110                         os << "width \"" << translate_len(width) << "\"\n"
3111                                   "height \"" << translate_len(thickness) << "\"\n";
3112                         end_inset(os);
3113                 }
3114
3115                 else if (is_known(t.cs(), known_phrases) ||
3116                          (t.cs() == "protect" &&
3117                           p.next_token().cat() == catEscape &&
3118                           is_known(p.next_token().cs(), known_phrases))) {
3119                         // LyX sometimes puts a \protect in front, so we have to ignore it
3120                         // FIXME: This needs to be changed when bug 4752 is fixed.
3121                         where = is_known(
3122                                 t.cs() == "protect" ? p.get_token().cs() : t.cs(),
3123                                 known_phrases);
3124                         context.check_layout(os);
3125                         os << known_coded_phrases[where - known_phrases];
3126                         skip_spaces_braces(p);
3127                 }
3128
3129                 else if ((where = is_known(t.cs(), known_ref_commands))) {
3130                         string const opt = p.getOpt();
3131                         if (opt.empty()) {
3132                                 context.check_layout(os);
3133                                 begin_command_inset(os, "ref",
3134                                         known_coded_ref_commands[where - known_ref_commands]);
3135                                 os << "reference \""
3136                                    << convert_command_inset_arg(p.verbatim_item())
3137                                    << "\"\n";
3138                                 end_inset(os);
3139                                 if (t.cs() == "vref" || t.cs() == "vpageref")
3140                                         preamble.registerAutomaticallyLoadedPackage("varioref");
3141
3142                         } else {
3143                                 // LyX does not support optional arguments of ref commands
3144                                 handle_ert(os, t.asInput() + '[' + opt + "]{" +
3145                                                p.verbatim_item() + "}", context);
3146                         }
3147                 }
3148
3149                 else if (use_natbib &&
3150                          is_known(t.cs(), known_natbib_commands) &&
3151                          ((t.cs() != "citefullauthor" &&
3152                            t.cs() != "citeyear" &&
3153                            t.cs() != "citeyearpar") ||
3154                           p.next_token().asInput() != "*")) {
3155                         context.check_layout(os);
3156                         string command = t.cs();
3157                         if (p.next_token().asInput() == "*") {
3158                                 command += '*';
3159                                 p.get_token();
3160                         }
3161                         if (command == "citefullauthor")
3162                                 // alternative name for "\\citeauthor*"
3163                                 command = "citeauthor*";
3164
3165                         // text before the citation
3166                         string before;
3167                         // text after the citation
3168                         string after;
3169                         get_cite_arguments(p, true, before, after);
3170
3171                         if (command == "cite") {
3172                                 // \cite without optional argument means
3173                                 // \citet, \cite with at least one optional
3174                                 // argument means \citep.
3175                                 if (before.empty() && after.empty())
3176                                         command = "citet";
3177                                 else
3178                                         command = "citep";
3179                         }
3180                         if (before.empty() && after == "[]")
3181                                 // avoid \citet[]{a}
3182                                 after.erase();
3183                         else if (before == "[]" && after == "[]") {
3184                                 // avoid \citet[][]{a}
3185                                 before.erase();
3186                                 after.erase();
3187                         }
3188                         // remove the brackets around after and before
3189                         if (!after.empty()) {
3190                                 after.erase(0, 1);
3191                                 after.erase(after.length() - 1, 1);
3192                                 after = convert_command_inset_arg(after);
3193                         }
3194                         if (!before.empty()) {
3195                                 before.erase(0, 1);
3196                                 before.erase(before.length() - 1, 1);
3197                                 before = convert_command_inset_arg(before);
3198                         }
3199                         begin_command_inset(os, "citation", command);
3200                         os << "after " << '"' << after << '"' << "\n";
3201                         os << "before " << '"' << before << '"' << "\n";
3202                         os << "key \""
3203                            << convert_command_inset_arg(p.verbatim_item())
3204                            << "\"\n";
3205                         end_inset(os);
3206                 }
3207
3208                 else if (use_jurabib &&
3209                          is_known(t.cs(), known_jurabib_commands) &&
3210                          (t.cs() == "cite" || p.next_token().asInput() != "*")) {
3211                         context.check_layout(os);
3212                         string command = t.cs();
3213                         if (p.next_token().asInput() == "*") {
3214                                 command += '*';
3215                                 p.get_token();
3216                         }
3217                         char argumentOrder = '\0';
3218                         vector<string> const options =
3219                                 preamble.getPackageOptions("jurabib");
3220                         if (find(options.begin(), options.end(),
3221                                       "natbiborder") != options.end())
3222                                 argumentOrder = 'n';
3223                         else if (find(options.begin(), options.end(),
3224                                            "jurabiborder") != options.end())
3225                                 argumentOrder = 'j';
3226
3227                         // text before the citation
3228                         string before;
3229                         // text after the citation
3230                         string after;
3231                         get_cite_arguments(p, argumentOrder != 'j', before, after);
3232
3233                         string const citation = p.verbatim_item();
3234                         if (!before.empty() && argumentOrder == '\0') {
3235                                 cerr << "Warning: Assuming argument order "
3236                                         "of jurabib version 0.6 for\n'"
3237                                      << command << before << after << '{'
3238                                      << citation << "}'.\n"
3239                                         "Add 'jurabiborder' to the jurabib "
3240                                         "package options if you used an\n"
3241                                         "earlier jurabib version." << endl;
3242                         }
3243                         if (!after.empty()) {
3244                                 after.erase(0, 1);
3245                                 after.erase(after.length() - 1, 1);
3246                         }
3247                         if (!before.empty()) {
3248                                 before.erase(0, 1);
3249                                 before.erase(before.length() - 1, 1);
3250                         }
3251                         begin_command_inset(os, "citation", command);
3252                         os << "after " << '"' << after << '"' << "\n";
3253                         os << "before " << '"' << before << '"' << "\n";
3254                         os << "key " << '"' << citation << '"' << "\n";
3255                         end_inset(os);
3256                 }
3257
3258                 else if (t.cs() == "cite"
3259                         || t.cs() == "nocite") {
3260                         context.check_layout(os);
3261                         string after = convert_command_inset_arg(p.getArg('[', ']'));
3262                         string key = convert_command_inset_arg(p.verbatim_item());
3263                         // store the case that it is "\nocite{*}" to use it later for
3264                         // the BibTeX inset
3265                         if (key != "*") {
3266                                 begin_command_inset(os, "citation", t.cs());
3267                                 os << "after " << '"' << after << '"' << "\n";
3268                                 os << "key " << '"' << key << '"' << "\n";
3269                                 end_inset(os);
3270                         } else if (t.cs() == "nocite")
3271                                 btprint = key;
3272                 }
3273
3274                 else if (t.cs() == "index" ||
3275                          (t.cs() == "sindex" && preamble.use_indices() == "true")) {
3276                         context.check_layout(os);
3277                         string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
3278                                 p.getArg('[', ']') : "";
3279                         string const kind = arg.empty() ? "idx" : arg;
3280                         begin_inset(os, "Index ");
3281                         os << kind << "\nstatus collapsed\n";
3282                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
3283                         end_inset(os);
3284                         if (kind != "idx")
3285                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3286                 }
3287
3288                 else if (t.cs() == "nomenclature") {
3289                         context.check_layout(os);
3290                         begin_command_inset(os, "nomenclature", "nomenclature");
3291                         string prefix = convert_command_inset_arg(p.getArg('[', ']'));
3292                         if (!prefix.empty())
3293                                 os << "prefix " << '"' << prefix << '"' << "\n";
3294                         os << "symbol " << '"'
3295                            << convert_command_inset_arg(p.verbatim_item());
3296                         os << "\"\ndescription \""
3297                            << convert_command_inset_arg(p.verbatim_item())
3298                            << "\"\n";
3299                         end_inset(os);
3300                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3301                 }
3302
3303                 else if (t.cs() == "label") {
3304                         context.check_layout(os);
3305                         begin_command_inset(os, "label", "label");
3306                         os << "name \""
3307                            << convert_command_inset_arg(p.verbatim_item())
3308                            << "\"\n";
3309                         end_inset(os);
3310                 }
3311
3312                 else if (t.cs() == "printindex") {
3313                         context.check_layout(os);
3314                         begin_command_inset(os, "index_print", "printindex");
3315                         os << "type \"idx\"\n";
3316                         end_inset(os);
3317                         skip_spaces_braces(p);
3318                         preamble.registerAutomaticallyLoadedPackage("makeidx");
3319                         if (preamble.use_indices() == "true")
3320                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3321                 }
3322
3323                 else if (t.cs() == "printnomenclature") {
3324                         string width = "";
3325                         string width_type = "";
3326                         context.check_layout(os);
3327                         begin_command_inset(os, "nomencl_print", "printnomenclature");
3328                         // case of a custom width
3329                         if (p.hasOpt()) {
3330                                 width = p.getArg('[', ']');
3331                                 width = translate_len(width);
3332                                 width_type = "custom";
3333                         }
3334                         // case of no custom width
3335                         // the case of no custom width but the width set
3336                         // via \settowidth{\nomlabelwidth}{***} cannot be supported
3337                         // because the user could have set anything, not only the width
3338                         // of the longest label (which would be width_type = "auto")
3339                         string label = convert_command_inset_arg(p.getArg('{', '}'));
3340                         if (label.empty() && width_type.empty())
3341                                 width_type = "none";
3342                         os << "set_width \"" << width_type << "\"\n";
3343                         if (width_type == "custom")
3344                                 os << "width \"" << width << '\"';
3345                         end_inset(os);
3346                         skip_spaces_braces(p);
3347                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3348                 }
3349
3350                 else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
3351                         context.check_layout(os);
3352                         begin_inset(os, "script ");
3353                         os << t.cs().substr(4) << '\n';
3354                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
3355                         end_inset(os);
3356                         if (t.cs() == "textsubscript")
3357                                 preamble.registerAutomaticallyLoadedPackage("subscript");
3358                 }
3359
3360                 else if ((where = is_known(t.cs(), known_quotes))) {
3361                         context.check_layout(os);
3362                         begin_inset(os, "Quotes ");
3363                         os << known_coded_quotes[where - known_quotes];
3364                         end_inset(os);
3365                         // LyX adds {} after the quote, so we have to eat
3366                         // spaces here if there are any before a possible
3367                         // {} pair.
3368                         eat_whitespace(p, os, context, false);
3369                         skip_braces(p);
3370                 }
3371
3372                 else if ((where = is_known(t.cs(), known_sizes)) &&
3373                          context.new_layout_allowed) {
3374                         context.check_layout(os);
3375                         TeXFont const oldFont = context.font;
3376                         context.font.size = known_coded_sizes[where - known_sizes];
3377                         output_font_change(os, oldFont, context.font);
3378                         eat_whitespace(p, os, context, false);
3379                 }
3380
3381                 else if ((where = is_known(t.cs(), known_font_families)) &&
3382                          context.new_layout_allowed) {
3383                         context.check_layout(os);
3384                         TeXFont const oldFont = context.font;
3385                         context.font.family =
3386                                 known_coded_font_families[where - known_font_families];
3387                         output_font_change(os, oldFont, context.font);
3388                         eat_whitespace(p, os, context, false);
3389                 }
3390
3391                 else if ((where = is_known(t.cs(), known_font_series)) &&
3392                          context.new_layout_allowed) {
3393                         context.check_layout(os);
3394                         TeXFont const oldFont = context.font;
3395                         context.font.series =
3396                                 known_coded_font_series[where - known_font_series];
3397                         output_font_change(os, oldFont, context.font);
3398                         eat_whitespace(p, os, context, false);
3399                 }
3400
3401                 else if ((where = is_known(t.cs(), known_font_shapes)) &&
3402                          context.new_layout_allowed) {
3403                         context.check_layout(os);
3404                         TeXFont const oldFont = context.font;
3405                         context.font.shape =
3406                                 known_coded_font_shapes[where - known_font_shapes];
3407                         output_font_change(os, oldFont, context.font);
3408                         eat_whitespace(p, os, context, false);
3409                 }
3410                 else if ((where = is_known(t.cs(), known_old_font_families)) &&
3411                          context.new_layout_allowed) {
3412                         context.check_layout(os);
3413                         TeXFont const oldFont = context.font;
3414                         context.font.init();
3415                         context.font.size = oldFont.size;
3416                         context.font.family =
3417                                 known_coded_font_families[where - known_old_font_families];
3418                         output_font_change(os, oldFont, context.font);
3419                         eat_whitespace(p, os, context, false);
3420                 }
3421
3422                 else if ((where = is_known(t.cs(), known_old_font_series)) &&
3423                          context.new_layout_allowed) {
3424                         context.check_layout(os);
3425                         TeXFont const oldFont = context.font;
3426                         context.font.init();
3427                         context.font.size = oldFont.size;
3428                         context.font.series =
3429                                 known_coded_font_series[where - known_old_font_series];
3430                         output_font_change(os, oldFont, context.font);
3431                         eat_whitespace(p, os, context, false);
3432                 }
3433
3434                 else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
3435                          context.new_layout_allowed) {
3436                         context.check_layout(os);
3437                         TeXFont const oldFont = context.font;
3438                         context.font.init();
3439                         context.font.size = oldFont.size;
3440                         context.font.shape =
3441                                 known_coded_font_shapes[where - known_old_font_shapes];
3442                         output_font_change(os, oldFont, context.font);
3443                         eat_whitespace(p, os, context, false);
3444                 }
3445
3446                 else if (t.cs() == "selectlanguage") {
3447                         context.check_layout(os);
3448                         // save the language for the case that a
3449                         // \foreignlanguage is used
3450                         context.font.language = babel2lyx(p.verbatim_item());
3451                         os << "\n\\lang " << context.font.language << "\n";
3452                 }
3453
3454                 else if (t.cs() == "foreignlanguage") {
3455                         string const lang = babel2lyx(p.verbatim_item());
3456                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3457                                               context, "\\lang",
3458                                               context.font.language, lang);
3459                 }
3460
3461                 else if (t.cs() == "inputencoding") {
3462                         // nothing to write here
3463                         string const enc = subst(p.verbatim_item(), "\n", " ");
3464                         p.setEncoding(enc);
3465                 }
3466
3467                 else if ((where = is_known(t.cs(), known_special_chars))) {
3468                         context.check_layout(os);
3469                         os << "\\SpecialChar \\"
3470                            << known_coded_special_chars[where - known_special_chars]
3471                            << '\n';
3472                         skip_spaces_braces(p);
3473                 }
3474
3475                 else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
3476                         context.check_layout(os);
3477                         os << "\\SpecialChar \\nobreakdash-\n";
3478                         p.get_token();
3479                 }
3480
3481                 else if (t.cs() == "textquotedbl") {
3482                         context.check_layout(os);
3483                         os << "\"";
3484                         skip_braces(p);
3485                 }
3486
3487                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
3488                         context.check_layout(os);
3489                         os << "\\SpecialChar \\@.\n";
3490                         p.get_token();
3491                 }
3492
3493                 else if (t.cs() == "-") {
3494                         context.check_layout(os);
3495                         os << "\\SpecialChar \\-\n";
3496                 }
3497
3498                 else if (t.cs() == "textasciitilde") {
3499                         context.check_layout(os);
3500                         os << '~';
3501                         skip_spaces_braces(p);
3502                 }
3503
3504                 else if (t.cs() == "textasciicircum") {
3505                         context.check_layout(os);
3506                         os << '^';
3507                         skip_spaces_braces(p);
3508                 }
3509
3510                 else if (t.cs() == "textbackslash") {
3511                         context.check_layout(os);
3512                         os << "\n\\backslash\n";
3513                         skip_spaces_braces(p);
3514                 }
3515
3516                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
3517                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
3518                             || t.cs() == "%") {
3519                         context.check_layout(os);
3520                         os << t.cs();
3521                 }
3522
3523                 else if (t.cs() == "char") {
3524                         context.check_layout(os);
3525                         if (p.next_token().character() == '`') {
3526                                 p.get_token();
3527                                 if (p.next_token().cs() == "\"") {
3528                                         p.get_token();
3529                                         os << '"';
3530                                         skip_braces(p);
3531                                 } else {
3532                                         handle_ert(os, "\\char`", context);
3533                                 }
3534                         } else {
3535                                 handle_ert(os, "\\char", context);
3536                         }
3537                 }
3538
3539                 else if (t.cs() == "verb") {
3540                         context.check_layout(os);
3541                         char const delimiter = p.next_token().character();
3542                         string const arg = p.getArg(delimiter, delimiter);
3543                         ostringstream oss;
3544                         oss << "\\verb" << delimiter << arg << delimiter;
3545                         handle_ert(os, oss.str(), context);
3546                 }
3547
3548                 // Problem: \= creates a tabstop inside the tabbing environment
3549                 // and else an accent. In the latter case we really would want
3550                 // \={o} instead of \= o.
3551                 else if (t.cs() == "=" && (flags & FLAG_TABBING))
3552                         handle_ert(os, t.asInput(), context);
3553
3554                 // accents (see Table 6 in Comprehensive LaTeX Symbol List)
3555                 else if (t.cs().size() == 1
3556                          && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
3557                         context.check_layout(os);
3558                         // try to see whether the string is in unicodesymbols
3559                         bool termination;
3560                         docstring rem;
3561                         string command = t.asInput() + "{"
3562                                 + trimSpaceAndEol(p.verbatim_item())
3563                                 + "}";
3564                         set<string> req;
3565                         docstring s = encodings.fromLaTeXCommand(from_utf8(command),
3566                                 Encodings::TEXT_CMD | Encodings::MATH_CMD,
3567                                 termination, rem, &req);
3568                         if (!s.empty()) {
3569                                 if (!rem.empty())
3570                                         cerr << "When parsing " << command
3571                                              << ", result is " << to_utf8(s)
3572                                              << "+" << to_utf8(rem) << endl;
3573                                 os << to_utf8(s);
3574                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
3575                                         preamble.registerAutomaticallyLoadedPackage(*it);
3576                         } else
3577                                 // we did not find a non-ert version
3578                                 handle_ert(os, command, context);
3579                 }
3580
3581                 else if (t.cs() == "\\") {
3582                         context.check_layout(os);
3583                         if (p.hasOpt())
3584                                 handle_ert(os, "\\\\" + p.getOpt(), context);
3585                         else if (p.next_token().asInput() == "*") {
3586                                 p.get_token();
3587                                 // getOpt() eats the following space if there
3588                                 // is no optional argument, but that is OK
3589                                 // here since it has no effect in the output.
3590                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
3591                         }
3592                         else {
3593                                 begin_inset(os, "Newline newline");
3594                                 end_inset(os);
3595                         }
3596                 }
3597
3598                 else if (t.cs() == "newline" ||
3599                          (t.cs() == "linebreak" && !p.hasOpt())) {
3600                         context.check_layout(os);
3601                         begin_inset(os, "Newline ");
3602                         os << t.cs();
3603                         end_inset(os);
3604                         skip_spaces_braces(p);
3605                 }
3606
3607                 else if (t.cs() == "input" || t.cs() == "include"
3608                          || t.cs() == "verbatiminput") {
3609                         string name = t.cs();
3610                         if (t.cs() == "verbatiminput"
3611                             && p.next_token().asInput() == "*")
3612                                 name += p.get_token().asInput();
3613                         context.check_layout(os);
3614                         string filename(normalize_filename(p.getArg('{', '}')));
3615                         string const path = getMasterFilePath();
3616                         // We want to preserve relative / absolute filenames,
3617                         // therefore path is only used for testing
3618                         if ((t.cs() == "include" || t.cs() == "input") &&
3619                             !makeAbsPath(filename, path).exists()) {
3620                                 // The file extension is probably missing.
3621                                 // Now try to find it out.
3622                                 string const tex_name =
3623                                         find_file(filename, path,
3624                                                   known_tex_extensions);
3625                                 if (!tex_name.empty())
3626                                         filename = tex_name;
3627                         }
3628                         bool external = false;
3629                         string outname;
3630                         if (makeAbsPath(filename, path).exists()) {
3631                                 string const abstexname =
3632                                         makeAbsPath(filename, path).absFileName();
3633                                 string const abslyxname =
3634                                         changeExtension(abstexname, ".lyx");
3635                                 string const absfigname =
3636                                         changeExtension(abstexname, ".fig");
3637                                 fix_relative_filename(filename);
3638                                 string const lyxname =
3639                                         changeExtension(filename, ".lyx");
3640                                 bool xfig = false;
3641                                 external = FileName(absfigname).exists();
3642                                 if (t.cs() == "input") {
3643                                         string const ext = getExtension(abstexname);
3644
3645                                         // Combined PS/LaTeX:
3646                                         // x.eps, x.pstex_t (old xfig)
3647                                         // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
3648                                         FileName const absepsname(
3649                                                 changeExtension(abstexname, ".eps"));
3650                                         FileName const abspstexname(
3651                                                 changeExtension(abstexname, ".pstex"));
3652                                         bool const xfigeps =
3653                                                 (absepsname.exists() ||
3654                                                  abspstexname.exists()) &&
3655                                                 ext == "pstex_t";
3656
3657                                         // Combined PDF/LaTeX:
3658                                         // x.pdf, x.pdftex_t (old xfig)
3659                                         // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
3660                                         FileName const abspdfname(
3661                                                 changeExtension(abstexname, ".pdf"));
3662                                         bool const xfigpdf =
3663                                                 abspdfname.exists() &&
3664                                                 (ext == "pdftex_t" || ext == "pdf_t");
3665                                         if (xfigpdf)
3666                                                 pdflatex = true;
3667
3668                                         // Combined PS/PDF/LaTeX:
3669                                         // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
3670                                         string const absbase2(
3671                                                 removeExtension(abstexname) + "_pspdftex");
3672                                         FileName const abseps2name(
3673                                                 addExtension(absbase2, ".eps"));
3674                                         FileName const abspdf2name(
3675                                                 addExtension(absbase2, ".pdf"));
3676                                         bool const xfigboth =
3677                                                 abspdf2name.exists() &&
3678                                                 abseps2name.exists() && ext == "pspdftex";
3679
3680                                         xfig = xfigpdf || xfigeps || xfigboth;
3681                                         external = external && xfig;
3682                                 }
3683                                 if (external) {
3684                                         outname = changeExtension(filename, ".fig");
3685                                 } else if (xfig) {
3686                                         // Don't try to convert, the result
3687                                         // would be full of ERT.
3688                                         outname = filename;
3689                                 } else if (t.cs() != "verbatiminput" &&
3690                                     tex2lyx(abstexname, FileName(abslyxname),
3691                                             p.getEncoding())) {
3692                                         outname = lyxname;
3693                                 } else {
3694                                         outname = filename;
3695                                 }
3696                         } else {
3697                                 cerr << "Warning: Could not find included file '"
3698                                      << filename << "'." << endl;
3699                                 outname = filename;
3700                         }
3701                         if (external) {
3702                                 begin_inset(os, "External\n");
3703                                 os << "\ttemplate XFig\n"
3704                                    << "\tfilename " << outname << '\n';
3705                                 registerExternalTemplatePackages("XFig");
3706                         } else {
3707                                 begin_command_inset(os, "include", name);
3708                                 os << "preview false\n"
3709                                       "filename \"" << outname << "\"\n";
3710                                 if (t.cs() == "verbatiminput")
3711                                         preamble.registerAutomaticallyLoadedPackage("verbatim");
3712                         }
3713                         end_inset(os);
3714                 }
3715
3716                 else if (t.cs() == "bibliographystyle") {
3717                         // store new bibliographystyle
3718                         bibliographystyle = p.verbatim_item();
3719                         // If any other command than \bibliography and
3720                         // \nocite{*} follows, we need to output the style
3721                         // (because it might be used by that command).
3722                         // Otherwise, it will automatically be output by LyX.
3723                         p.pushPosition();
3724                         bool output = true;
3725                         for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
3726                                 if (t2.cat() == catBegin)
3727                                         break;
3728                                 if (t2.cat() != catEscape)
3729                                         continue;
3730                                 if (t2.cs() == "nocite") {
3731                                         if (p.getArg('{', '}') == "*")
3732                                                 continue;
3733                                 } else if (t2.cs() == "bibliography")
3734                                         output = false;
3735                                 break;
3736                         }
3737                         p.popPosition();
3738                         if (output) {
3739                                 handle_ert(os,
3740                                         "\\bibliographystyle{" + bibliographystyle + '}',
3741                                         context);
3742                         }
3743                 }
3744
3745                 else if (t.cs() == "bibliography") {
3746                         context.check_layout(os);
3747                         begin_command_inset(os, "bibtex", "bibtex");
3748                         if (!btprint.empty()) {
3749                                 os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
3750                                 // clear the string because the next BibTeX inset can be without the
3751                                 // \nocite{*} option
3752                                 btprint.clear();
3753                         }
3754                         os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
3755                         // Do we have a bibliographystyle set?
3756                         if (!bibliographystyle.empty())
3757                                 os << "options " << '"' << bibliographystyle << '"' << "\n";
3758                         end_inset(os);
3759                 }
3760
3761                 else if (t.cs() == "parbox") {
3762                         // Test whether this is an outer box of a shaded box
3763                         p.pushPosition();
3764                         // swallow arguments
3765                         while (p.hasOpt()) {
3766                                 p.getArg('[', ']');
3767                                 p.skip_spaces(true);
3768                         }
3769                         p.getArg('{', '}');
3770                         p.skip_spaces(true);
3771                         // eat the '{'
3772                         if (p.next_token().cat() == catBegin)
3773                                 p.get_token();
3774                         p.skip_spaces(true);
3775                         Token to = p.get_token();
3776                         bool shaded = false;
3777                         if (to.asInput() == "\\begin") {
3778                                 p.skip_spaces(true);
3779                                 if (p.getArg('{', '}') == "shaded")
3780                                         shaded = true;
3781                         }
3782                         p.popPosition();
3783                         if (shaded) {
3784                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3785                                                 context, "parbox", "shaded");
3786                         } else
3787                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3788                                           "", "", t.cs());
3789                 }
3790
3791                 else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
3792                          t.cs() == "shadowbox" || t.cs() == "doublebox")
3793                         parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
3794
3795                 else if (t.cs() == "framebox") {
3796                         if (p.next_token().character() == '(') {
3797                                 //the syntax is: \framebox(x,y)[position]{content}
3798                                 string arg = t.asInput();
3799                                 arg += p.getFullParentheseArg();
3800                                 arg += p.getFullOpt();
3801                                 eat_whitespace(p, os, context, false);
3802                                 handle_ert(os, arg + '{', context);
3803                                 eat_whitespace(p, os, context, false);
3804                                 parse_text(p, os, FLAG_ITEM, outer, context);
3805                                 handle_ert(os, "}", context);
3806                         } else {
3807                                 string special = p.getFullOpt();
3808                                 special += p.getOpt();
3809                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3810                                                 context, t.cs(), special);
3811                         }
3812                 }
3813
3814                 //\makebox() is part of the picture environment and different from \makebox{}
3815                 //\makebox{} will be parsed by parse_box
3816                 else if (t.cs() == "makebox") {
3817                         if (p.next_token().character() == '(') {
3818                                 //the syntax is: \makebox(x,y)[position]{content}
3819                                 string arg = t.asInput();
3820                                 arg += p.getFullParentheseArg();
3821                                 arg += p.getFullOpt();
3822                                 eat_whitespace(p, os, context, false);
3823                                 handle_ert(os, arg + '{', context);
3824                                 eat_whitespace(p, os, context, false);
3825                                 parse_text(p, os, FLAG_ITEM, outer, context);
3826                                 handle_ert(os, "}", context);
3827                         } else
3828                                 //the syntax is: \makebox[width][position]{content}
3829                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3830                                           "", "", t.cs());
3831                 }
3832
3833                 else if (t.cs() == "smallskip" ||
3834                          t.cs() == "medskip" ||
3835                          t.cs() == "bigskip" ||
3836                          t.cs() == "vfill") {
3837                         context.check_layout(os);
3838                         begin_inset(os, "VSpace ");
3839                         os << t.cs();
3840                         end_inset(os);
3841                         skip_spaces_braces(p);
3842                 }
3843
3844                 else if ((where = is_known(t.cs(), known_spaces))) {
3845                         context.check_layout(os);
3846                         begin_inset(os, "space ");
3847                         os << '\\' << known_coded_spaces[where - known_spaces]
3848                            << '\n';
3849                         end_inset(os);
3850                         // LaTeX swallows whitespace after all spaces except
3851                         // "\\,". We have to do that here, too, because LyX
3852                         // adds "{}" which would make the spaces significant.
3853                         if (t.cs() !=  ",")
3854                                 eat_whitespace(p, os, context, false);
3855                         // LyX adds "{}" after all spaces except "\\ " and
3856                         // "\\,", so we have to remove "{}".
3857                         // "\\,{}" is equivalent to "\\," in LaTeX, so we
3858                         // remove the braces after "\\,", too.
3859                         if (t.cs() != " ")
3860                                 skip_braces(p);
3861                 }
3862
3863                 else if (t.cs() == "newpage" ||
3864                          (t.cs() == "pagebreak" && !p.hasOpt()) ||
3865                          t.cs() == "clearpage" ||
3866                          t.cs() == "cleardoublepage") {
3867                         context.check_layout(os);
3868                         begin_inset(os, "Newpage ");
3869                         os << t.cs();
3870                         end_inset(os);
3871                         skip_spaces_braces(p);
3872                 }
3873
3874                 else if (t.cs() == "DeclareRobustCommand" ||
3875                          t.cs() == "DeclareRobustCommandx" ||
3876                          t.cs() == "newcommand" ||
3877                          t.cs() == "newcommandx" ||
3878                          t.cs() == "providecommand" ||
3879                          t.cs() == "providecommandx" ||
3880                          t.cs() == "renewcommand" ||
3881                          t.cs() == "renewcommandx") {
3882                         // DeclareRobustCommand, DeclareRobustCommandx,
3883                         // providecommand and providecommandx could be handled
3884                         // by parse_command(), but we need to call
3885                         // add_known_command() here.
3886                         string name = t.asInput();
3887                         if (p.next_token().asInput() == "*") {
3888                                 // Starred form. Eat '*'
3889                                 p.get_token();
3890                                 name += '*';
3891                         }
3892                         string const command = p.verbatim_item();
3893                         string const opt1 = p.getFullOpt();
3894                         string const opt2 = p.getFullOpt();
3895                         add_known_command(command, opt1, !opt2.empty());
3896                         string const ert = name + '{' + command + '}' +
3897                                            opt1 + opt2 +
3898                                            '{' + p.verbatim_item() + '}';
3899
3900                         if (t.cs() == "DeclareRobustCommand" ||
3901                             t.cs() == "DeclareRobustCommandx" ||
3902                             t.cs() == "providecommand" ||
3903                             t.cs() == "providecommandx" ||
3904                             name[name.length()-1] == '*')
3905                                 handle_ert(os, ert, context);
3906                         else {
3907                                 context.check_layout(os);
3908                                 begin_inset(os, "FormulaMacro");
3909                                 os << "\n" << ert;
3910                                 end_inset(os);
3911                         }
3912                 }
3913
3914                 else if (t.cs() == "let" && p.next_token().asInput() != "*") {
3915                         // let could be handled by parse_command(),
3916                         // but we need to call add_known_command() here.
3917                         string ert = t.asInput();
3918                         string name;
3919                         p.skip_spaces();
3920                         if (p.next_token().cat() == catBegin) {
3921                                 name = p.verbatim_item();
3922                                 ert += '{' + name + '}';
3923                         } else {
3924                                 name = p.verbatim_item();
3925                                 ert += name;
3926                         }
3927                         string command;
3928                         p.skip_spaces();
3929                         if (p.next_token().cat() == catBegin) {
3930                                 command = p.verbatim_item();
3931                                 ert += '{' + command + '}';
3932                         } else {
3933                                 command = p.verbatim_item();
3934                                 ert += command;
3935                         }
3936                         // If command is known, make name known too, to parse
3937                         // its arguments correctly. For this reason we also
3938                         // have commands in syntax.default that are hardcoded.
3939                         CommandMap::iterator it = known_commands.find(command);
3940                         if (it != known_commands.end())
3941                                 known_commands[t.asInput()] = it->second;
3942                         handle_ert(os, ert, context);
3943                 }
3944
3945                 else if (t.cs() == "hspace" || t.cs() == "vspace") {
3946                         bool starred = false;
3947                         if (p.next_token().asInput() == "*") {
3948                                 p.get_token();
3949                                 starred = true;
3950                         }
3951                         string name = t.asInput();
3952                         string const length = p.verbatim_item();
3953                         string unit;
3954                         string valstring;
3955                         bool valid = splitLatexLength(length, valstring, unit);
3956                         bool known_hspace = false;
3957                         bool known_vspace = false;
3958                         bool known_unit = false;
3959                         double value;
3960                         if (valid) {
3961                                 istringstream iss(valstring);
3962                                 iss >> value;
3963                                 if (value == 1.0) {
3964                                         if (t.cs()[0] == 'h') {
3965                                                 if (unit == "\\fill") {
3966                                                         if (!starred) {
3967                                                                 unit = "";
3968                                                                 name = "\\hfill";
3969                                                         }
3970                                                         known_hspace = true;
3971                                                 }
3972                                         } else {
3973                                                 if (unit == "\\smallskipamount") {
3974                                                         unit = "smallskip";
3975                                                         known_vspace = true;
3976                                                 } else if (unit == "\\medskipamount") {
3977                                                         unit = "medskip";
3978                                                         known_vspace = true;
3979                                                 } else if (unit == "\\bigskipamount") {
3980                                                         unit = "bigskip";
3981                                                         known_vspace = true;
3982                                                 } else if (unit == "\\fill") {
3983                                                         unit = "vfill";
3984                                                         known_vspace = true;
3985                                                 }
3986                                         }
3987                                 }
3988                                 if (!known_hspace && !known_vspace) {
3989                                         switch (unitFromString(unit)) {
3990                                         case Length::SP:
3991                                         case Length::PT:
3992                                         case Length::BP:
3993                                         case Length::DD:
3994                                         case Length::MM:
3995                                         case Length::PC:
3996                                         case Length::CC:
3997                                         case Length::CM:
3998                                         case Length::IN:
3999                                         case Length::EX:
4000                                         case Length::EM:
4001                                         case Length::MU:
4002                                                 known_unit = true;
4003                                                 break;
4004                                         default:
4005                                                 break;
4006                                         }
4007                                 }
4008                         }
4009
4010                         if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
4011                                 // Literal horizontal length or known variable
4012                                 context.check_layout(os);
4013                                 begin_inset(os, "space ");
4014                                 os << name;
4015                                 if (starred)
4016                                         os << '*';
4017                                 os << '{';
4018                                 if (known_hspace)
4019                                         os << unit;
4020                                 os << "}";
4021                                 if (known_unit && !known_hspace)
4022                                         os << "\n\\length "
4023                                            << translate_len(length);
4024                                 end_inset(os);
4025                         } else if (known_unit || known_vspace) {
4026                                 // Literal vertical length or known variable
4027                                 context.check_layout(os);
4028                                 begin_inset(os, "VSpace ");
4029                                 if (known_unit)
4030                                         os << value;
4031                                 os << unit;
4032                                 if (starred)
4033                                         os << '*';
4034                                 end_inset(os);
4035                         } else {
4036                                 // LyX can't handle other length variables in Inset VSpace/space
4037                                 if (starred)
4038                                         name += '*';
4039                                 if (valid) {
4040                                         if (value == 1.0)
4041                                                 handle_ert(os, name + '{' + unit + '}', context);
4042                                         else if (value == -1.0)
4043                                                 handle_ert(os, name + "{-" + unit + '}', context);
4044                                         else
4045                                                 handle_ert(os, name + '{' + valstring + unit + '}', context);
4046                                 } else
4047                                         handle_ert(os, name + '{' + length + '}', context);
4048                         }
4049                 }
4050
4051                 // The single '=' is meant here.
4052                 else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
4053                         p.skip_spaces();
4054                         context.check_layout(os);
4055                         begin_inset(os, "Flex ");
4056                         os << to_utf8(newinsetlayout->name()) << '\n'
4057                            << "status collapsed\n";
4058                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
4059                         end_inset(os);
4060                 }
4061
4062                 else if (t.cs() == "includepdf") {
4063                         p.skip_spaces();
4064                         string const arg = p.getArg('[', ']');
4065                         map<string, string> opts;
4066                         vector<string> keys;
4067                         split_map(arg, opts, keys);
4068                         string name = normalize_filename(p.verbatim_item());
4069                         string const path = getMasterFilePath();
4070                         // We want to preserve relative / absolute filenames,
4071                         // therefore path is only used for testing
4072                         if (!makeAbsPath(name, path).exists()) {
4073                                 // The file extension is probably missing.
4074                                 // Now try to find it out.
4075                                 char const * const pdfpages_format[] = {"pdf", 0};
4076                                 string const pdftex_name =
4077                                         find_file(name, path, pdfpages_format);
4078                                 if (!pdftex_name.empty()) {
4079                                         name = pdftex_name;
4080                                         pdflatex = true;
4081                                 }
4082                         }
4083                         if (makeAbsPath(name, path).exists())
4084                                 fix_relative_filename(name);
4085                         else
4086                                 cerr << "Warning: Could not find file '"
4087                                      << name << "'." << endl;
4088                         // write output
4089                         context.check_layout(os);
4090                         begin_inset(os, "External\n\ttemplate ");
4091                         os << "PDFPages\n\tfilename "
4092                            << name << "\n";
4093                         // parse the options
4094                         if (opts.find("pages") != opts.end())
4095                                 os << "\textra LaTeX \"pages="
4096                                    << opts["pages"] << "\"\n";
4097                         if (opts.find("angle") != opts.end())
4098                                 os << "\trotateAngle "
4099                                    << opts["angle"] << '\n';
4100                         if (opts.find("origin") != opts.end()) {
4101                                 ostringstream ss;
4102                                 string const opt = opts["origin"];
4103                                 if (opt == "tl") ss << "topleft";
4104                                 if (opt == "bl") ss << "bottomleft";
4105                                 if (opt == "Bl") ss << "baselineleft";
4106                                 if (opt == "c") ss << "center";
4107                                 if (opt == "tc") ss << "topcenter";
4108                                 if (opt == "bc") ss << "bottomcenter";
4109                                 if (opt == "Bc") ss << "baselinecenter";
4110                                 if (opt == "tr") ss << "topright";
4111                                 if (opt == "br") ss << "bottomright";
4112                                 if (opt == "Br") ss << "baselineright";
4113                                 if (!ss.str().empty())
4114                                         os << "\trotateOrigin " << ss.str() << '\n';
4115                                 else
4116                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
4117                         }
4118                         if (opts.find("width") != opts.end())
4119                                 os << "\twidth "
4120                                    << translate_len(opts["width"]) << '\n';
4121                         if (opts.find("height") != opts.end())
4122                                 os << "\theight "
4123                                    << translate_len(opts["height"]) << '\n';
4124                         if (opts.find("keepaspectratio") != opts.end())
4125                                 os << "\tkeepAspectRatio\n";
4126                         end_inset(os);
4127                         context.check_layout(os);
4128                         registerExternalTemplatePackages("PDFPages");
4129                 }
4130
4131                 else if (t.cs() == "loadgame") {
4132                         p.skip_spaces();
4133                         string name = normalize_filename(p.verbatim_item());
4134                         string const path = getMasterFilePath();
4135                         // We want to preserve relative / absolute filenames,
4136                         // therefore path is only used for testing
4137                         if (!makeAbsPath(name, path).exists()) {
4138                                 // The file extension is probably missing.
4139                                 // Now try to find it out.
4140                                 char const * const lyxskak_format[] = {"fen", 0};
4141                                 string const lyxskak_name =
4142                                         find_file(name, path, lyxskak_format);
4143                                 if (!lyxskak_name.empty())
4144                                         name = lyxskak_name;
4145                         }
4146                         if (makeAbsPath(name, path).exists())
4147                                 fix_relative_filename(name);
4148                         else
4149                                 cerr << "Warning: Could not find file '"
4150                                      << name << "'." << endl;
4151                         context.check_layout(os);
4152                         begin_inset(os, "External\n\ttemplate ");
4153                         os << "ChessDiagram\n\tfilename "
4154                            << name << "\n";
4155                         end_inset(os);
4156                         context.check_layout(os);
4157                         // after a \loadgame follows a \showboard
4158                         if (p.get_token().asInput() == "showboard")
4159                                 p.get_token();
4160                         registerExternalTemplatePackages("ChessDiagram");
4161                 }
4162
4163                 else {
4164                         // try to see whether the string is in unicodesymbols
4165                         // Only use text mode commands, since we are in text mode here,
4166                         // and math commands may be invalid (bug 6797)
4167                         bool termination;
4168                         docstring rem;
4169                         set<string> req;
4170                         docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
4171                                         Encodings::TEXT_CMD, termination, rem, &req);
4172                         if (!s.empty()) {
4173                                 if (!rem.empty())
4174                                         cerr << "When parsing " << t.cs()
4175                                              << ", result is " << to_utf8(s)
4176                                              << "+" << to_utf8(rem) << endl;
4177                                 context.check_layout(os);
4178                                 os << to_utf8(s);
4179                                 if (termination)
4180                                         skip_spaces_braces(p);
4181                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
4182                                         preamble.registerAutomaticallyLoadedPackage(*it);
4183                         }
4184                         //cerr << "#: " << t << " mode: " << mode << endl;
4185                         // heuristic: read up to next non-nested space
4186                         /*
4187                         string s = t.asInput();
4188                         string z = p.verbatim_item();
4189                         while (p.good() && z != " " && z.size()) {
4190                                 //cerr << "read: " << z << endl;
4191                                 s += z;
4192                                 z = p.verbatim_item();
4193                         }
4194                         cerr << "found ERT: " << s << endl;
4195                         handle_ert(os, s + ' ', context);
4196                         */
4197                         else {
4198                                 string name = t.asInput();
4199                                 if (p.next_token().asInput() == "*") {
4200                                         // Starred commands like \vspace*{}
4201                                         p.get_token();  // Eat '*'
4202                                         name += '*';
4203                                 }
4204                                 if (!parse_command(name, p, os, outer, context))
4205                                         handle_ert(os, name, context);
4206                         }
4207                 }
4208
4209                 if (flags & FLAG_LEAVE) {
4210                         flags &= ~FLAG_LEAVE;
4211                         break;
4212                 }
4213         }
4214 }
4215
4216 // }])
4217
4218
4219 } // namespace lyx