src/tex2lyx/text.C

   1 /**
   2  * \file tex2lyx/text.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  *
   9  * Full author contact details are available in file CREDITS.
  10  */
  11
  12 // {[(
  13
  14 #include <config.h>
  15
  16 #include "tex2lyx.h"
  17 #include "context.h"
  18 #include "FloatList.h"
  19 #include "support/lstrings.h"
  20 #include "support/tostr.h"
  21 #include "support/filetools.h"
  22
  23 #include <iostream>
  24 #include <map>
  25 #include <sstream>
  26 #include <vector>
  27
  28 using std::cerr;
  29 using std::endl;
  30
  31 using std::map;
  32 using std::ostream;
  33 using std::ostringstream;
  34 using std::istringstream;
  35 using std::string;
  36 using std::vector;
  37
  38 using lyx::support::rtrim;
  39 using lyx::support::suffixIs;
  40
  41
  42 // thin wrapper around parse_text using a string
  43 string parse_text(Parser & p, unsigned flags, const bool outer,
  44                   Context & context)
  45 {
  46         ostringstream os;
  47         parse_text(p, os, flags, outer, context);
  48         return os.str();
  49 }
  50
  51 // parses a subdocument, usually useful in insets (whence the name)
  52 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  53                 Context & context)
  54 {
  55         Context newcontext(true, context.textclass);
  56         parse_text(p, os, flags, outer, newcontext);
  57         newcontext.check_end_layout(os);
  58 }
  59
  60
  61 // parses a paragraph snippet, useful for example for \emph{...}
  62 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  63                 Context & context)
  64 {
  65         Context newcontext(false, context.textclass);
  66         parse_text(p, os, flags, outer, newcontext);
  67         // should not be needed
  68         newcontext.check_end_layout(os);
  69 }
  70
  71
  72 namespace {
  73
  74 char const * known_latex_commands[] = { "ref", "cite", "label", "index",
  75 "printindex", "pageref", "url", "vref", "vpageref", "prettyref", "eqref", 0 };
  76
  77 // LaTeX names for quotes
  78 char const * known_quotes[] = { "glqq", "grqq", "quotedblbase",
  79 "textquotedblleft", "quotesinglbase", "guilsinglleft", "guilsinglright", 0};
  80
  81 // the same as known_quotes with .lyx names
  82 char const * known_coded_quotes[] = { "gld", "grd", "gld",
  83 "grd", "gls", "fls", "frd", 0};
  84
  85 char const * known_sizes[] = { "tiny", "scriptsize", "footnotesize",
  86 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
  87
  88 char const * known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
  89 "small", "normal", "large", "larger", "largest",  "huge", "giant", 0};
  90
  91 // splits "x=z, y=b" into a map
  92 map<string, string> split_map(string const & s)
  93 {
  94         map<string, string> res;
  95         vector<string> v;
  96         split(s, v);
  97         for (size_t i = 0; i < v.size(); ++i) {
  98                 size_t const pos   = v[i].find('=');
  99                 string const index = v[i].substr(0, pos);
 100                 string const value = v[i].substr(pos + 1, string::npos);
 101                 res[trim(index)] = trim(value);
 102         }
 103         return res;
 104 }
 105
 106 // A simple function to translate a latex length to something lyx can
 107 // understand. Not perfect, but rather best-effort.
 108 string translate_len(string const & len)
 109 {
 110         const string::size_type i = len.find_first_not_of(" -0123456789.,");
 111         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 112         string const length = lyx::support::subst(len, ',', '.');
 113         // a normal length
 114         if (i == string::npos || len[i]  != '\\')
 115                 return length;
 116         double val;
 117         if (i == 0) {
 118                 // We had something like \textwidth without a factor
 119                 val = 100;
 120         } else {
 121                 istringstream iss(string(length, 0, i));
 122                 iss >> val;
 123                 val = val * 100;
 124         }
 125         ostringstream oss;
 126         oss << val;
 127         string const valstring = oss.str();
 128         const string::size_type i2 = length.find(" ", i);
 129         string const unit = string(len, i, i2 - i);
 130         string const endlen = (i2 == string::npos) ? string() : string(len, i2);
 131         if (unit == "\\textwidth")
 132                 return valstring + "text%" + endlen;
 133         else if (unit == "\\columnwidth")
 134                 return valstring + "col%" + endlen;
 135         else if (unit == "\\paperwidth")
 136                 return valstring + "page%" + endlen;
 137         else if (unit == "\\linewidth")
 138                 return valstring + "line%" + endlen;
 139         else if (unit == "\\paperheight")
 140                 return valstring + "pheight%" + endlen;
 141         else if (unit == "\\textheight")
 142                 return valstring + "theight%" + endlen;
 143         else
 144                 return length;
 145 }
 146
 147
 148 void begin_inset(ostream & os, string const & name)
 149 {
 150         os << "\n\\begin_inset " << name;
 151 }
 152
 153
 154 void end_inset(ostream & os)
 155 {
 156         os << "\n\\end_inset \n\n";
 157 }
 158
 159
 160 void skip_braces(Parser & p)
 161 {
 162         if (p.next_token().cat() != catBegin)
 163                 return;
 164         p.get_token();
 165         if (p.next_token().cat() == catEnd) {
 166                 p.get_token();
 167                 return;
 168         }
 169         p.putback();
 170 }
 171
 172
 173 void handle_ert(ostream & os, string const & s, Context & context, bool check_layout = true)
 174 {
 175         if (check_layout) {
 176                 // We must have a valid layout before outputting the ERT inset.
 177                 context.check_layout(os);
 178         }
 179         Context newcontext(true, context.textclass);
 180         begin_inset(os, "ERT");
 181         os << "\nstatus Collapsed\n";
 182         newcontext.check_layout(os);
 183         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 184                 if (*it == '\\')
 185                         os << "\n\\backslash \n";
 186                 else
 187                         os << *it;
 188         }
 189         newcontext.check_end_layout(os);
 190         end_inset(os);
 191 }
 192
 193
 194 void handle_comment(ostream & os, string const & s, Context & context)
 195 {
 196         // TODO: Handle this better
 197         Context newcontext(true, context.textclass);
 198         begin_inset(os, "ERT");
 199         os << "\nstatus Collapsed\n";
 200         newcontext.check_layout(os);
 201         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 202                 if (*it == '\\')
 203                         os << "\n\\backslash \n";
 204                 else
 205                         os << *it;
 206         }
 207         // make sure that our comment is the last thing on the line
 208         os << "\n\\newline";
 209         newcontext.check_end_layout(os);
 210         end_inset(os);
 211 }
 212
 213
 214 struct isLayout {
 215         isLayout(string const name) : name_(name) {}
 216         bool operator()(LyXLayout_ptr const & ptr) {
 217                 return ptr.get() && ptr->latexname() == name_;
 218         }
 219 private:
 220         string const name_;
 221 };
 222
 223
 224 LyXLayout_ptr findLayout(LyXTextClass const & textclass,
 225                          string const & name)
 226 {
 227         LyXTextClass::const_iterator it  = textclass.begin();
 228         LyXTextClass::const_iterator end = textclass.end();
 229         it = std::find_if(it, end, isLayout(name));
 230         return (it == end) ? LyXLayout_ptr() : *it;
 231 }
 232
 233
 234 void output_command_layout(ostream & os, Parser & p, bool outer,
 235                            Context & parent_context,
 236                            LyXLayout_ptr newlayout)
 237 {
 238         parent_context.check_end_layout(os);
 239         Context context(true, parent_context.textclass, newlayout,
 240                         parent_context.layout);
 241         context.check_deeper(os);
 242         context.check_layout(os);
 243         if (context.layout->optionalargs > 0) {
 244                 p.skip_spaces();
 245                 if (p.next_token().character() == '[') {
 246                         p.get_token(); // eat '['
 247                         begin_inset(os, "OptArg\n");
 248                         os << "collapsed true\n\n";
 249                         parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 250                         end_inset(os);
 251                 }
 252         }
 253         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 254         context.check_end_layout(os);
 255         context.check_end_deeper(os);
 256         // We don't need really a new paragraph, but
 257         // we must make sure that the next item gets a \begin_layout.
 258         parent_context.new_paragraph(os);
 259 }
 260
 261
 262 /*!
 263  * Output a space if necessary.
 264  * This function gets called for every whitespace token.
 265  *
 266  * We have three cases here:
 267  * 1. A space must be suppressed. Example: The lyxcode case below
 268  * 2. A space may be suppressed. Example: Spaces before "\par"
 269  * 3. A space must not be suppressed. Example: A space between two words
 270  *
 271  * We currently handle only 1. and 3 and from 2. only the case of
 272  * spaces before newlines as a side effect.
 273  *
 274  * 2. could be used to suppress as many spaces as possible. This has two effects:
 275  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 276  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 277  * The drawback is that the logic inside the function becomes
 278  * complicated, and that is the reason why it is not implemented.
 279  */
 280 void check_space(Parser const & p, ostream & os, Context & context)
 281 {
 282         Token const next = p.next_token();
 283         Token const curr = p.curr_token();
 284         // A space before a single newline and vice versa must be ignored
 285         // LyX emits a newline before \end{lyxcode}.
 286         // This newline must be ignored,
 287         // otherwise LyX will add an additional protected space.
 288         if (next.cat() == catSpace ||
 289             next.cat() == catNewline ||
 290             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 291                 return;
 292         }
 293         context.check_layout(os);
 294         os << ' ';
 295 }
 296
 297 void parse_environment(Parser & p, ostream & os, bool outer,
 298                        Context & parent_context)
 299 {
 300         LyXLayout_ptr newlayout;
 301         string const name = p.getArg('{', '}');
 302         const bool is_starred = suffixIs(name, '*');
 303         string const unstarred_name = rtrim(name, "*");
 304         active_environments.push_back(name);
 305         p.skip_spaces();
 306
 307         if (is_math_env(name)) {
 308                 parent_context.check_layout(os);
 309                 begin_inset(os, "Formula ");
 310                 os << "\\begin{" << name << "}";
 311                 parse_math(p, os, FLAG_END, MATH_MODE);
 312                 os << "\\end{" << name << "}";
 313                 end_inset(os);
 314         }
 315
 316         else if (name == "tabular") {
 317                 parent_context.check_layout(os);
 318                 begin_inset(os, "Tabular ");
 319                 handle_tabular(p, os, parent_context);
 320                 end_inset(os);
 321         }
 322
 323         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
 324                 parent_context.check_layout(os);
 325                 begin_inset(os, "Float " + unstarred_name + "\n");
 326                 if (p.next_token().asInput() == "[") {
 327                         os << "placement " << p.getArg('[', ']') << '\n';
 328                 }
 329                 os << "wide " << tostr(is_starred)
 330                    << "\ncollapsed false\n\n";
 331                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
 332                 end_inset(os);
 333                 // We don't need really a new paragraph, but
 334                 // we must make sure that the next item gets a \begin_layout.
 335                 parent_context.new_paragraph(os);
 336         }
 337
 338         else if (name == "minipage") {
 339                 string position = "1";
 340                 string inner_pos = "0";
 341                 string height = "0pt";
 342                 string latex_position;
 343                 string latex_inner_pos;
 344                 string latex_height;
 345                 if (p.next_token().asInput() == "[") {
 346                         latex_position = p.getArg('[', ']');
 347                         switch(latex_position[0]) {
 348                         case 't': position = "0"; break;
 349                         case 'c': position = "1"; break;
 350                         case 'b': position = "2"; break;
 351                         default:
 352                                 cerr << "invalid position for minipage"
 353                                      << endl;
 354                                 break;
 355                         }
 356                         if (p.next_token().asInput() == "[") {
 357                                 latex_height = p.getArg('[', ']');
 358                                 height = translate_len(latex_height);
 359
 360                                 if (p.next_token().asInput() == "[") {
 361                                         latex_inner_pos = p.getArg('[', ']');
 362                                         switch(latex_inner_pos[0]) {
 363                                         case 'c': inner_pos = "0"; break;
 364                                         case 't': inner_pos = "1"; break;
 365                                         case 'b': inner_pos = "2"; break;
 366                                         case 's': inner_pos = "3"; break;
 367                                         default:
 368                                                 cerr << "invalid inner_pos for minipage"
 369                                                      << endl;
 370                                                 break;
 371                                         }
 372                                 }
 373                         }
 374                 }
 375                 string width = translate_len(p.verbatim_item());
 376                 if (width[0] == '\\') {
 377                         // lyx can't handle length variables
 378                         ostringstream ss;
 379                         ss << "\\begin{minipage}";
 380                         if (latex_position.size())
 381                                 ss << '[' << latex_position << ']';
 382                         if (latex_height.size())
 383                                 ss << '[' << latex_height << ']';
 384                         if (latex_inner_pos.size())
 385                                 ss << '[' << latex_inner_pos << ']';
 386                         ss << "{" << width << "}";
 387                         handle_ert(os, ss.str(), parent_context);
 388                         parent_context.new_paragraph(os);
 389                         parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
 390                         handle_ert(os, "\\end{minipage}", parent_context);
 391                 } else {
 392                         parent_context.check_layout(os);
 393                         begin_inset(os, "Minipage\n");
 394                         os << "position " << position << '\n';
 395                         os << "inner_position " << inner_pos << '\n';
 396                         os << "height \"" << height << "\"\n";
 397                         os << "width \"" << width << "\"\n";
 398                         os << "collapsed false\n\n";
 399                         parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
 400                         end_inset(os);
 401                 }
 402         }
 403
 404         // Alignment settings
 405         else if (name == "center" || name == "flushleft" || name == "flushright" ||
 406                  name == "centering" || name == "raggedright" || name == "raggedleft") {
 407                 // We must begin a new paragraph if not already done
 408                 if (! parent_context.atParagraphStart()) {
 409                         parent_context.check_end_layout(os);
 410                         parent_context.new_paragraph(os);
 411                 }
 412                 if (name == "flushleft" || name == "raggedright")
 413                         parent_context.extra_stuff += "\\align left ";
 414                 else if (name == "flushright" || name == "raggedleft")
 415                         parent_context.extra_stuff += "\\align right ";
 416                 else
 417                         parent_context.extra_stuff += "\\align center ";
 418                 parse_text(p, os, FLAG_END, outer, parent_context);
 419                 // Just in case the environment is empty ..
 420                 parent_context.extra_stuff.erase();
 421                 // We must begin a new paragraph to reset the alignment
 422                 parent_context.new_paragraph(os);
 423         }
 424
 425         // The single '=' is meant here.
 426         else if ((newlayout = findLayout(parent_context.textclass, name)).get() &&
 427                    newlayout->isEnvironment()) {
 428                 Context context(true, parent_context.textclass, newlayout,
 429                                 parent_context.layout);
 430                 parent_context.check_end_layout(os);
 431                 switch (context.layout->latextype) {
 432                 case  LATEX_LIST_ENVIRONMENT:
 433                         context.extra_stuff = "\\labelwidthstring "
 434                                 + p.verbatim_item() + '\n';
 435                         p.skip_spaces();
 436                         break;
 437                 case  LATEX_BIB_ENVIRONMENT:
 438                         p.verbatim_item(); // swallow next arg
 439                         p.skip_spaces();
 440                         break;
 441                 default:
 442                         break;
 443                 }
 444                 context.check_deeper(os);
 445                 parse_text(p, os, FLAG_END, outer, context);
 446                 context.check_end_layout(os);
 447                 context.check_end_deeper(os);
 448                 parent_context.new_paragraph(os);
 449         }
 450
 451         else if (name == "appendix") {
 452                 // This is no good latex style, but it works and is used in some documents...
 453                 parent_context.check_end_layout(os);
 454                 Context context(true, parent_context.textclass, parent_context.layout,
 455                                 parent_context.layout);
 456                 context.check_layout(os);
 457                 os << "\\start_of_appendix\n";
 458                 parse_text(p, os, FLAG_END, outer, context);
 459                 context.check_end_layout(os);
 460         }
 461
 462         else if (name == "comment") {
 463                 parent_context.check_layout(os);
 464                 begin_inset(os, "Comment\n");
 465                 os << "collapsed false\n";
 466                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
 467                 end_inset(os);
 468         }
 469
 470         else if (name == "tabbing") {
 471                 // We need to remember that we have to handle '\=' specially
 472                 handle_ert(os, "\\begin{" + name + "}", parent_context);
 473                 parse_text_snippet(p, os, FLAG_END | FLAG_TABBING, outer, parent_context);
 474                 handle_ert(os, "\\end{" + name + "}", parent_context);
 475         }
 476
 477         else {
 478                 handle_ert(os, "\\begin{" + name + "}", parent_context);
 479                 parse_text_snippet(p, os, FLAG_END, outer, parent_context);
 480                 handle_ert(os, "\\end{" + name + "}", parent_context);
 481         }
 482
 483         active_environments.pop_back();
 484         if (name != "math")
 485                 p.skip_spaces();
 486 }
 487
 488 } // anonymous namespace
 489
 490
 491
 492
 493 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 494                 Context & context)
 495 {
 496         LyXLayout_ptr newlayout;
 497         // Store the latest bibliographystyle (needed for bibtex inset)
 498         string bibliographystyle;
 499         while (p.good()) {
 500                 Token const & t = p.get_token();
 501
 502 #ifdef FILEDEBUG
 503                 cerr << "t: " << t << " flags: " << flags << "\n";
 504 #endif
 505
 506                 if (flags & FLAG_ITEM) {
 507                         if (t.cat() == catSpace)
 508                                 continue;
 509
 510                         flags &= ~FLAG_ITEM;
 511                         if (t.cat() == catBegin) {
 512                                 // skip the brace and collect everything to the next matching
 513                                 // closing brace
 514                                 flags |= FLAG_BRACE_LAST;
 515                                 continue;
 516                         }
 517
 518                         // handle only this single token, leave the loop if done
 519                         flags |= FLAG_LEAVE;
 520                 }
 521
 522                 if (t.character() == ']' && (flags & FLAG_BRACK_LAST))
 523                         return;
 524
 525                 //
 526                 // cat codes
 527                 //
 528                 if (t.cat() == catMath) {
 529                         // we are inside some text mode thingy, so opening new math is allowed
 530                         context.check_layout(os);
 531                         begin_inset(os, "Formula ");
 532                         Token const & n = p.get_token();
 533                         if (n.cat() == catMath && outer) {
 534                                 // TeX's $$...$$ syntax for displayed math
 535                                 os << "\\[";
 536                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
 537                                 os << "\\]";
 538                                 p.get_token(); // skip the second '$' token
 539                         } else {
 540                                 // simple $...$  stuff
 541                                 p.putback();
 542                                 os << '$';
 543                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
 544                                 os << '$';
 545                         }
 546                         end_inset(os);
 547                 }
 548
 549                 else if (t.cat() == catSuper || t.cat() == catSub)
 550                         cerr << "catcode " << t << " illegal in text mode\n";
 551
 552                 // Basic support for english quotes. This should be
 553                 // extended to other quotes, but is not so easy (a
 554                 // left english quote is the same as a right german
 555                 // quote...)
 556                 else if (t.asInput() == "`"
 557                          && p.next_token().asInput() == "`") {
 558                         context.check_layout(os);
 559                         begin_inset(os, "Quotes ");
 560                         os << "eld";
 561                         end_inset(os);
 562                         p.get_token();
 563                         skip_braces(p);
 564                 }
 565                 else if (t.asInput() == "'"
 566                          && p.next_token().asInput() == "'") {
 567                         context.check_layout(os);
 568                         begin_inset(os, "Quotes ");
 569                         os << "erd";
 570                         end_inset(os);
 571                         p.get_token();
 572                         skip_braces(p);
 573                 }
 574
 575                 else if (t.cat() == catSpace || (t.cat() == catNewline && t.cs().size() == 1))
 576                         check_space(p, os, context);
 577
 578                 else if (t.cat() == catLetter ||
 579                                t.cat() == catOther ||
 580                                t.cat() == catAlign ||
 581                                t.cat() == catParameter) {
 582                         context.check_layout(os);
 583                         os << t.character();
 584                 }
 585
 586                 else if (t.cat() == catNewline || (t.cat() == catEscape && t.cs() == "par")) {
 587                         p.skip_spaces();
 588                         context.new_paragraph(os);
 589                 }
 590
 591                 else if (t.cat() == catActive) {
 592                         context.check_layout(os);
 593                         if (t.character() == '~') {
 594                                 if (context.layout->free_spacing)
 595                                         os << ' ';
 596                                 else
 597                                         os << "\\InsetSpace ~\n";
 598                         } else
 599                                 os << t.character();
 600                 }
 601
 602                 else if (t.cat() == catBegin) {
 603                         // special handling of size changes
 604                         context.check_layout(os);
 605                         bool const is_size = is_known(p.next_token().cs(), known_sizes);
 606                         Token const prev = p.prev_token();
 607                         string const s = parse_text(p, FLAG_BRACE_LAST, outer, context);
 608                         if (s.empty() && (p.next_token().character() == '`' ||
 609                                           (prev.character() == '-' && p.next_token().character())))
 610                                 ; // ignore it in {}`` or -{}-
 611                         else if (is_size || s == "[" || s == "]" || s == "*")
 612                                 os << s;
 613                         else {
 614                                 handle_ert(os, "{", context, false);
 615                                 // s will end the current layout and begin a new one if necessary
 616                                 os << s;
 617                                 handle_ert(os, "}", context);
 618                         }
 619                 }
 620
 621                 else if (t.cat() == catEnd) {
 622                         if (flags & FLAG_BRACE_LAST) {
 623                                 return;
 624                         }
 625                         cerr << "stray '}' in text\n";
 626                         handle_ert(os, "}", context);
 627                 }
 628
 629                 else if (t.cat() == catComment) {
 630                         context.check_layout(os);
 631                         if (t.cs().size()) {
 632                                 handle_comment(os, '%' + t.cs(), context);
 633                                 if (p.next_token().cat() == catNewline) {
 634                                         // A newline after a comment line starts a new paragraph
 635                                         context.new_paragraph(os);
 636                                         p.skip_spaces();
 637                                 }
 638                         } else {
 639                                 // "%\n" combination
 640                                 p.skip_spaces();
 641                         }
 642                 }
 643
 644                 //
 645                 // control sequences
 646                 //
 647
 648                 else if (t.cs() == "(") {
 649                         context.check_layout(os);
 650                         begin_inset(os, "Formula");
 651                         os << " \\(";
 652                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
 653                         os << "\\)";
 654                         end_inset(os);
 655                 }
 656
 657                 else if (t.cs() == "[") {
 658                         context.check_layout(os);
 659                         begin_inset(os, "Formula");
 660                         os << " \\[";
 661                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
 662                         os << "\\]";
 663                         end_inset(os);
 664                 }
 665
 666                 else if (t.cs() == "begin")
 667                         parse_environment(p, os, outer, context);
 668
 669                 else if (t.cs() == "end") {
 670                         if (flags & FLAG_END) {
 671                                 // eat environment name
 672                                 string const name = p.getArg('{', '}');
 673                                 if (name != active_environment())
 674                                         cerr << "\\end{" + name + "} does not match \\begin{"
 675                                                 + active_environment() + "}\n";
 676                                 return;
 677                         }
 678                         p.error("found 'end' unexpectedly");
 679                 }
 680
 681                 else if (t.cs() == "item") {
 682                         p.skip_spaces();
 683                         string s;
 684                         bool optarg = false;
 685                         if (p.next_token().character() == '[') {
 686                                 p.get_token(); // eat '['
 687                                 Context newcontext(false, context.textclass);
 688                                 s = parse_text(p, FLAG_BRACK_LAST, outer, newcontext);
 689                                 optarg = true;
 690                         }
 691                         context.set_item();
 692                         context.check_layout(os);
 693                         if (optarg) {
 694                                 if (context.layout->labeltype != LABEL_MANUAL) {
 695                                         // lyx does not support \item[\mybullet] in itemize environments
 696                                         handle_ert(os, "[", context);
 697                                         os << s;
 698                                         handle_ert(os, "]", context);
 699                                 } else if (s.size()) {
 700                                         // The space is needed to separate the item from the rest of the sentence.
 701                                         os << s << ' ';
 702                                         p.skip_spaces();
 703                                 }
 704                         }
 705                 }
 706
 707                 else if (t.cs() == "bibitem") {
 708                         context.set_item();
 709                         context.check_layout(os);
 710                         os << "\\bibitem ";
 711                         os << p.getOpt();
 712                         os << '{' << p.verbatim_item() << '}' << "\n";
 713                 }
 714
 715                 else if (t.cs() == "def") {
 716                         p.skip_spaces();
 717                         context.check_layout(os);
 718                         string name = p.get_token().cs();
 719                         while (p.next_token().cat() != catBegin)
 720                                 name += p.get_token().asString();
 721                         handle_ert(os, "\\def\\" + name + '{' + p.verbatim_item() + '}', context);
 722                 }
 723
 724                 else if (t.cs() == "noindent") {
 725                         p.skip_spaces();
 726                         context.extra_stuff += "\\noindent ";
 727                 }
 728
 729                 else if (t.cs() == "appendix") {
 730                         p.skip_spaces();
 731                         context.extra_stuff += "\\start_of_appendix ";
 732                 }
 733
 734                 // Must attempt to parse "Section*" before "Section".
 735                 else if ((p.next_token().asInput() == "*") &&
 736                          // The single '=' is meant here.
 737                          (newlayout = findLayout(context.textclass,
 738                                                  t.cs() + '*')).get() &&
 739                          newlayout->isCommand()) {
 740                         p.get_token();
 741                         output_command_layout(os, p, outer, context, newlayout);
 742                         p.skip_spaces();
 743                 }
 744
 745                 // The single '=' is meant here.
 746                 else if ((newlayout = findLayout(context.textclass, t.cs())).get() &&
 747                          newlayout->isCommand()) {
 748                         output_command_layout(os, p, outer, context, newlayout);
 749                         p.skip_spaces();
 750                 }
 751
 752                 else if (t.cs() == "includegraphics") {
 753                         map<string, string> opts = split_map(p.getArg('[', ']'));
 754                         string name = p.verbatim_item();
 755
 756                         context.check_layout(os);
 757                         begin_inset(os, "Graphics ");
 758                         os << "\n\tfilename " << name << '\n';
 759                         if (opts.find("width") != opts.end())
 760                                 os << "\twidth "
 761                                    << translate_len(opts["width"]) << '\n';
 762                         if (opts.find("height") != opts.end())
 763                                 os << "\theight "
 764                                    << translate_len(opts["height"]) << '\n';
 765                         if (opts.find("scale") != opts.end()) {
 766                                 istringstream iss(opts["scale"]);
 767                                 double val;
 768                                 iss >> val;
 769                                 val = val*100;
 770                                 os << "\tscale " << val << '\n';
 771                         }
 772                         if (opts.find("angle") != opts.end())
 773                                 os << "\trotateAngle "
 774                                    << opts["angle"] << '\n';
 775                         if (opts.find("origin") != opts.end()) {
 776                                 ostringstream ss;
 777                                 string const opt = opts["origin"];
 778                                 if (opt.find('l') != string::npos) ss << "left";
 779                                 if (opt.find('r') != string::npos) ss << "right";
 780                                 if (opt.find('c') != string::npos) ss << "center";
 781                                 if (opt.find('t') != string::npos) ss << "Top";
 782                                 if (opt.find('b') != string::npos) ss << "Bottom";
 783                                 if (opt.find('B') != string::npos) ss << "Baseline";
 784                                 if (ss.str().size())
 785                                         os << "\trotateOrigin " << ss.str() << '\n';
 786                                 else
 787                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
 788                         }
 789                         if (opts.find("keepaspectratio") != opts.end())
 790                                 os << "\tkeepAspectRatio\n";
 791                         if (opts.find("clip") != opts.end())
 792                                 os << "\tclip\n";
 793                         if (opts.find("draft") != opts.end())
 794                                 os << "\tdraft\n";
 795                         if (opts.find("bb") != opts.end())
 796                                 os << "\tBoundingBox "
 797                                    << opts["bb"] << '\n';
 798                         int numberOfbbOptions = 0;
 799                         if (opts.find("bbllx") != opts.end())
 800                                 numberOfbbOptions++;
 801                         if (opts.find("bblly") != opts.end())
 802                                 numberOfbbOptions++;
 803                         if (opts.find("bburx") != opts.end())
 804                                 numberOfbbOptions++;
 805                         if (opts.find("bbury") != opts.end())
 806                                 numberOfbbOptions++;
 807                         if (numberOfbbOptions == 4)
 808                                 os << "\tBoundingBox "
 809                                    << opts["bbllx"] << opts["bblly"]
 810                                    << opts["bburx"] << opts["bbury"] << '\n';
 811                         else if (numberOfbbOptions > 0)
 812                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
 813                         numberOfbbOptions = 0;
 814                         if (opts.find("natwidth") != opts.end())
 815                                 numberOfbbOptions++;
 816                         if (opts.find("natheight") != opts.end())
 817                                 numberOfbbOptions++;
 818                         if (numberOfbbOptions == 2)
 819                                 os << "\tBoundingBox 0bp 0bp "
 820                                    << opts["natwidth"] << opts["natheight"] << '\n';
 821                         else if (numberOfbbOptions > 0)
 822                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
 823                         ostringstream special;
 824                         if (opts.find("hiresbb") != opts.end())
 825                                 special << "hiresbb,";
 826                         if (opts.find("trim") != opts.end())
 827                                 special << "trim,";
 828                         if (opts.find("viewport") != opts.end())
 829                                 special << "viewport=" << opts["viewport"] << ',';
 830                         if (opts.find("totalheight") != opts.end())
 831                                 special << "totalheight=" << opts["totalheight"] << ',';
 832                         if (opts.find("type") != opts.end())
 833                                 special << "type=" << opts["type"] << ',';
 834                         if (opts.find("ext") != opts.end())
 835                                 special << "ext=" << opts["ext"] << ',';
 836                         if (opts.find("read") != opts.end())
 837                                 special << "read=" << opts["read"] << ',';
 838                         if (opts.find("command") != opts.end())
 839                                 special << "command=" << opts["command"] << ',';
 840                         string s_special = special.str();
 841                         if (s_special.size()) {
 842                                 // We had special arguments. Remove the trailing ','.
 843                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
 844                         }
 845                         // TODO: Handle the unknown settings better.
 846                         // Warn about invalid options.
 847                         // Check wether some option was given twice.
 848                         end_inset(os);
 849                 }
 850
 851                 else if (t.cs() == "footnote") {
 852                         p.skip_spaces();
 853                         context.check_layout(os);
 854                         begin_inset(os, "Foot\n");
 855                         os << "collapsed true\n\n";
 856                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
 857                         end_inset(os);
 858                 }
 859
 860                 else if (t.cs() == "marginpar") {
 861                         p.skip_spaces();
 862                         context.check_layout(os);
 863                         begin_inset(os, "Marginal\n");
 864                         os << "collapsed true\n\n";
 865                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
 866                         end_inset(os);
 867                 }
 868
 869                 else if (t.cs() == "ensuremath") {
 870                         p.skip_spaces();
 871                         context.check_layout(os);
 872                         Context newcontext(false, context.textclass);
 873                         string s = parse_text(p, FLAG_ITEM, false, newcontext);
 874                         if (s == "±" || s == "³" || s == "²" || s == "µ")
 875                                 os << s;
 876                         else
 877                                 handle_ert(os, "\\ensuremath{" + s + "}",
 878                                            context);
 879                 }
 880
 881                 else if (t.cs() == "hfill") {
 882                         context.check_layout(os);
 883                         os << "\n\\hfill\n";
 884                         skip_braces(p);
 885                         p.skip_spaces();
 886                 }
 887
 888                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
 889                         p.skip_spaces();
 890                         skip_braces(p); // swallow this
 891                 }
 892
 893                 else if (t.cs() == "tableofcontents") {
 894                         p.skip_spaces();
 895                         context.check_layout(os);
 896                         begin_inset(os, "LatexCommand \\tableofcontents\n");
 897                         end_inset(os);
 898                         skip_braces(p); // swallow this
 899                 }
 900
 901                 else if (t.cs() == "listoffigures") {
 902                         p.skip_spaces();
 903                         context.check_layout(os);
 904                         begin_inset(os, "FloatList figure\n");
 905                         end_inset(os);
 906                         skip_braces(p); // swallow this
 907                 }
 908
 909                 else if (t.cs() == "listoftables") {
 910                         p.skip_spaces();
 911                         context.check_layout(os);
 912                         begin_inset(os, "FloatList table\n");
 913                         end_inset(os);
 914                         skip_braces(p); // swallow this
 915                 }
 916
 917                 else if (t.cs() == "listof") {
 918                         p.skip_spaces(true);
 919                         string const name = p.get_token().asString();
 920                         if (context.textclass.floats().typeExist(name)) {
 921                                 context.check_layout(os);
 922                                 begin_inset(os, "FloatList ");
 923                                 os << name << "\n";
 924                                 end_inset(os);
 925                                 p.get_token(); // swallow second arg
 926                         } else
 927                                 handle_ert(os, "\\listof{" + name + "}", context);
 928                 }
 929
 930                 else if (t.cs() == "textrm") {
 931                         context.check_layout(os);
 932                         os << "\n\\family roman \n";
 933                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 934                         os << "\n\\family default \n";
 935                 }
 936
 937                 else if (t.cs() == "textsf") {
 938                         context.check_layout(os);
 939                         os << "\n\\family sans \n";
 940                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 941                         os << "\n\\family default \n";
 942                 }
 943
 944                 else if (t.cs() == "textsl") {
 945                         context.check_layout(os);
 946                         os << "\n\\shape slanted \n";
 947                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 948                         os << "\n\\shape default \n";
 949                 }
 950
 951                 else if (t.cs() == "texttt") {
 952                         context.check_layout(os);
 953                         os << "\n\\family typewriter \n";
 954                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 955                         os << "\n\\family default \n";
 956                 }
 957
 958                 else if (t.cs() == "textit") {
 959                         context.check_layout(os);
 960                         os << "\n\\shape italic \n";
 961                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 962                         os << "\n\\shape default \n";
 963                 }
 964
 965                 else if (t.cs() == "textsc") {
 966                         context.check_layout(os);
 967                         os << "\n\\noun on \n";
 968                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 969                         os << "\n\\noun default \n";
 970                 }
 971
 972                 else if (t.cs() == "textbf") {
 973                         context.check_layout(os);
 974                         os << "\n\\series bold \n";
 975                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 976                         os << "\n\\series default \n";
 977                 }
 978
 979                 else if (t.cs() == "underbar") {
 980                         context.check_layout(os);
 981                         os << "\n\\bar under \n";
 982                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 983                         os << "\n\\bar default \n";
 984                 }
 985
 986                 else if (t.cs() == "emph" || t.cs() == "noun") {
 987                         context.check_layout(os);
 988                         os << "\n\\" << t.cs() << " on \n";
 989                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 990                         os << "\n\\" << t.cs() << " default \n";
 991                 }
 992
 993                 else if (is_known(t.cs(), known_latex_commands)) {
 994                         context.check_layout(os);
 995                         begin_inset(os, "LatexCommand ");
 996                         os << '\\' << t.cs();
 997                         os << p.getOpt();
 998                         os << p.getOpt();
 999                         os << '{' << p.verbatim_item() << "}\n";
1000                         end_inset(os);
1001                 }
1002
1003                 else if (is_known(t.cs(), known_quotes)) {
1004                         char const ** where = is_known(t.cs(), known_quotes);
1005                         context.check_layout(os);
1006                         begin_inset(os, "Quotes ");
1007                         os << known_coded_quotes[where - known_quotes];
1008                         end_inset(os);
1009                         skip_braces(p);
1010                 }
1011
1012                 else if (is_known(t.cs(), known_sizes)) {
1013                         char const ** where = is_known(t.cs(), known_sizes);
1014                         context.check_layout(os);
1015                         os << "\n\\size " << known_coded_sizes[where - known_sizes] << "\n";
1016                         p.skip_spaces();
1017                 }
1018
1019                 else if (t.cs() == "LyX" || t.cs() == "TeX"
1020                          || t.cs() == "LaTeX") {
1021                         context.check_layout(os);
1022                         os << t.cs();
1023                         skip_braces(p); // eat {}
1024                 }
1025
1026                 else if (t.cs() == "LaTeXe") {
1027                         context.check_layout(os);
1028                         os << "LaTeX2e";
1029                         skip_braces(p); // eat {}
1030                 }
1031
1032                 else if (t.cs() == "ldots") {
1033                         context.check_layout(os);
1034                         skip_braces(p);
1035                         os << "\\SpecialChar \\ldots{}\n";
1036                 }
1037
1038                 else if (t.cs() == "lyxarrow") {
1039                         context.check_layout(os);
1040                         os << "\\SpecialChar \\menuseparator\n";
1041                         skip_braces(p);
1042                 }
1043
1044                 else if (t.cs() == "textcompwordmark") {
1045                         context.check_layout(os);
1046                         os << "\\SpecialChar \\textcompwordmark{}\n";
1047                         skip_braces(p);
1048                 }
1049
1050                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
1051                         context.check_layout(os);
1052                         os << "\\SpecialChar \\@.\n";
1053                         p.get_token();
1054                 }
1055
1056                 else if (t.cs() == "-") {
1057                         context.check_layout(os);
1058                         os << "\\SpecialChar \\-\n";
1059                 }
1060
1061                 else if (t.cs() == "textasciitilde") {
1062                         context.check_layout(os);
1063                         os << '~';
1064                         skip_braces(p);
1065                 }
1066
1067                 else if (t.cs() == "textasciicircum") {
1068                         context.check_layout(os);
1069                         os << '^';
1070                         skip_braces(p);
1071                 }
1072
1073                 else if (t.cs() == "textbackslash") {
1074                         context.check_layout(os);
1075                         os << "\n\\backslash \n";
1076                         skip_braces(p);
1077                 }
1078
1079                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
1080                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
1081                             || t.cs() == "%") {
1082                         context.check_layout(os);
1083                         os << t.cs();
1084                 }
1085
1086                 else if (t.cs() == "char") {
1087                         context.check_layout(os);
1088                         if (p.next_token().character() == '`') {
1089                                 p.get_token();
1090                                 if (p.next_token().cs() == "\"") {
1091                                         p.get_token();
1092                                         os << '"';
1093                                         skip_braces(p);
1094                                 } else {
1095                                         handle_ert(os, "\\char`", context);
1096                                 }
1097                         } else {
1098                                 handle_ert(os, "\\char", context);
1099                         }
1100                 }
1101
1102                 else if (t.cs() == "\"") {
1103                         context.check_layout(os);
1104                         string const name = p.verbatim_item();
1105                              if (name == "a") os << 'ä';
1106                         else if (name == "o") os << 'ö';
1107                         else if (name == "u") os << 'ü';
1108                         else if (name == "A") os << 'Ä';
1109                         else if (name == "O") os << 'Ö';
1110                         else if (name == "U") os << 'Ü';
1111                         else handle_ert(os, "\"{" + name + "}", context);
1112                 }
1113
1114                 // Problem: \= creates a tabstop inside the tabbing environment
1115                 // and else an accent. In the latter case we really would want
1116                 // \={o} instead of \= o.
1117                 else if (t.cs() == "H" || t.cs() == "c" || t.cs() == "^" || t.cs() == "'"
1118                       || t.cs() == "~" || t.cs() == "." || (t.cs() == "=" && ! (flags & FLAG_TABBING))) {
1119                         // we need the trim as the LyX parser chokes on such spaces
1120                         context.check_layout(os);
1121                         os << "\n\\i \\" << t.cs() << "{"
1122                            << trim(parse_text(p, FLAG_ITEM, outer, context), " ") << "}\n";
1123                 }
1124
1125                 else if (t.cs() == "ss") {
1126                         context.check_layout(os);
1127                         os << "ß";
1128                 }
1129
1130                 else if (t.cs() == "i" || t.cs() == "j") {
1131                         context.check_layout(os);
1132                         os << "\\" << t.cs() << ' ';
1133                 }
1134
1135                 else if (t.cs() == "\\") {
1136                         context.check_layout(os);
1137                         string const next = p.next_token().asInput();
1138                         if (next == "[")
1139                                 handle_ert(os, "\\\\" + p.getOpt(), context);
1140                         else if (next == "*") {
1141                                 p.get_token();
1142                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
1143                         }
1144                         else {
1145                                 os << "\n\\newline \n";
1146                         }
1147                 }
1148
1149                 else if (t.cs() == "input" || t.cs() == "include"
1150                          || t.cs() == "verbatiminput") {
1151                         string name = '\\' + t.cs();
1152                         if (t.cs() == "verbatiminput"
1153                             && p.next_token().asInput() == "*")
1154                                 name += p.get_token().asInput();
1155                         context.check_layout(os);
1156                         begin_inset(os, "Include ");
1157                         string filename(p.getArg('{', '}'));
1158                         string lyxname(lyx::support::ChangeExtension(filename, ".lyx"));
1159                         if (tex2lyx(filename, lyxname)) {
1160                                 os << name << '{' << lyxname << "}\n";
1161                         } else {
1162                                 os << name << '{' << filename << "}\n";
1163                         }
1164                         os << "preview false\n";
1165                         end_inset(os);
1166                 }
1167
1168                 else if (t.cs() == "fancyhead") {
1169                         context.check_layout(os);
1170                         ostringstream ss;
1171                         ss << "\\fancyhead";
1172                         ss << p.getOpt();
1173                         ss << '{' << p.verbatim_item() << "}\n";
1174                         handle_ert(os, ss.str(), context);
1175                 }
1176
1177                 else if (t.cs() == "bibliographystyle") {
1178                         // store new bibliographystyle
1179                         bibliographystyle = p.verbatim_item();
1180                         // output new bibliographystyle.
1181                         // This is only necessary if used in some other macro than \bibliography.
1182                         handle_ert(os, "\\bibliographystyle{" + bibliographystyle + "}", context);
1183                 }
1184
1185                 else if (t.cs() == "bibliography") {
1186                         context.check_layout(os);
1187                         begin_inset(os, "LatexCommand ");
1188                         os << "\\bibtex";
1189                         // Do we have a bibliographystyle set?
1190                         if (bibliographystyle.size()) {
1191                                 os << '[' << bibliographystyle << ']';
1192                         }
1193                         os << '{' << p.verbatim_item() << "}\n";
1194                         end_inset(os);
1195                 }
1196
1197                 else if ( t.cs() == "smallskip" ||
1198                           t.cs() == "medskip" ||
1199                           t.cs() == "bigskip" ||
1200                           t.cs() == "vfill" ||
1201                          (t.cs() == "vspace" && p.next_token().asInput() != "*")) {
1202                         string arg;
1203                         if (t.cs() == "vspace")
1204                                 arg = p.getArg('{', '}');
1205                         else
1206                                 arg = t.cs();
1207                         // We may only add the vspace to the current context if the
1208                         // current paragraph is not empty.
1209                         if (context.atParagraphStart()
1210                             && (p.next_token().cat() != catNewline || p.next_token().cs().size() == 1)
1211                             && (! (p.next_token().cat() == catEscape && p.next_token().cs() == "end"))
1212                             && (! (p.next_token().cat() == catEscape && p.next_token().cs() == "par"))) {
1213                                 context.extra_stuff += "\\added_space_top " + arg + " ";
1214                                 p.skip_spaces();
1215                         } else {
1216                                 if (t.cs() == "vspace")
1217                                         handle_ert(os, t.asInput() + '{' + arg + '}', context);
1218                                 else
1219                                         handle_ert(os, t.asInput(), context);
1220                         }
1221                         // Would be nice to recognize added_space_bottom too...
1222                         // At the moment this is parsed as added_space_top of the
1223                         // next paragraph.
1224                 }
1225
1226                 else if (t.cs() == "psfrag") {
1227                         // psfrag{ps-text}[ps-pos][tex-pos]{tex-text}
1228                         // TODO: Generalize this!
1229                         string arguments = p.getArg('{', '}');
1230                         arguments += '}';
1231                         arguments += p.getOpt();
1232                         arguments += p.getOpt();
1233                         handle_ert(os, "\\psfrag{" + arguments, context);
1234                 }
1235
1236                 else {
1237                         //cerr << "#: " << t << " mode: " << mode << endl;
1238                         // heuristic: read up to next non-nested space
1239                         /*
1240                         string s = t.asInput();
1241                         string z = p.verbatim_item();
1242                         while (p.good() && z != " " && z.size()) {
1243                                 //cerr << "read: " << z << endl;
1244                                 s += z;
1245                                 z = p.verbatim_item();
1246                         }
1247                         cerr << "found ERT: " << s << endl;
1248                         handle_ert(os, s + ' ', context);
1249                         */
1250                         context.check_layout(os);
1251                         string name = t.asInput();
1252                         if (p.next_token().asInput() == "*") {
1253                                 // Starred commands like \vspace*{}
1254                                 p.get_token();                          // Eat '*'
1255                                 name += '*';
1256                         }
1257                         handle_ert(os, name, context);
1258                 }
1259
1260                 if (flags & FLAG_LEAVE) {
1261                         flags &= ~FLAG_LEAVE;
1262                         break;
1263                 }
1264         }
1265 }
1266
1267
1268 // }])