src/tex2lyx/tex2lyx.C

   1 /** The .tex to .lyx converter
   2     \author André Pönitz (2003)
   3  */
   4
   5 // {[(
   6
   7 #include <config.h>
   8
   9 #include <algorithm>
  10 #include <cctype>
  11 #include <fstream>
  12 #include <iostream>
  13 #include <map>
  14 #include <stack>
  15 #include <string>
  16 #include <vector>
  17
  18 #include "Lsstream.h"
  19
  20 #include "texparser.h"
  21
  22 using std::count_if;
  23 using std::cout;
  24 using std::cerr;
  25 using std::endl;
  26 using std::fill;
  27 using std::getline;
  28 using std::ios;
  29 using std::ifstream;
  30 using std::istream;
  31 using std::istringstream;
  32 using std::map;
  33 using std::ostream;
  34 using std::ostringstream;
  35 using std::stack;
  36 using std::string;
  37 using std::vector;
  38
  39
  40 namespace {
  41
  42 void parse_preamble(Parser & p, ostream & os);
  43
  44 void parse(Parser & p, ostream & os, unsigned flags, const mode_type mode);
  45
  46 char const OPEN = '<';
  47 char const CLOSE = '>';
  48
  49 // rather brutish way to code table structure in a string:
  50 //
  51 //  \begin{tabular}{ccc}
  52 //    1 & 2 & 3\\ \hline
  53 //    \multicolumn{2}{c}{4} & 5\\
  54 //    6 & 7 \\
  55 //  \end{tabular}
  56 //
  57 // gets "translated" to:
  58 //
  59 //  1 TAB 2 TAB 3 LINE
  60 //  HLINE 2 MULT c MULT 4 TAB 5 LINE
  61 //  5 TAB 7 LINE
  62
  63 char const TAB   = '\001';
  64 char const LINE  = '\002';
  65 char const MULT  = '\003';
  66 char const HLINE = '\004';
  67
  68 const char * known_languages[] = { "austrian", "babel", "bahasa", "basque",
  69 "breton", "british", "bulgarian", "catalan", "croatian", "czech", "danish",
  70 "dutch", "english", "esperanto", "estonian", "finnish", "francais",
  71 "frenchb", "galician", "german", "germanb", "greek", "hebcal", "hebfont",
  72 "hebrew", "hebrew_newcode", "hebrew_oldcode", "hebrew_p", "hyphen",
  73 "icelandic", "irish", "italian", "latin", "lgrcmr", "lgrcmro", "lgrcmss",
  74 "lgrcmtt", "lgrenc", "lgrlcmss", "lgrlcmtt", "lheclas", "lhecmr",
  75 "lhecmss", "lhecmtt", "lhecrml", "lheenc", "lhefr", "lheredis", "lheshold",
  76 "lheshscr", "lheshstk", "lsorbian", "magyar", "naustrian", "ngermanb",
  77 "ngerman", "norsk", "polish", "portuges", "rlbabel", "romanian",
  78 "russianb", "samin", "scottish", "serbian", "slovak", "slovene", "spanish",
  79 "swedish", "turkish", "ukraineb", "usorbian", "welsh", 0};
  80
  81 char const * known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
  82
  83 char const * known_headings[] = { "caption", "title", "author", "date",
  84 "paragraph", "chapter", "section", "subsection", "subsubsection", 0 };
  85
  86 char const * known_math_envs[] = { "equation", "equation*",
  87 "eqnarray", "eqnarray*", "align", "align*", 0};
  88
  89 char const * known_latex_commands[] = { "ref", "cite", "label", "index",
  90 "printindex", 0 };
  91
  92 // LaTeX names for quotes
  93 char const * known_quotes[] = { "glqq", "grqq", "quotedblbase",
  94 "textquotedblleft", 0};
  95
  96 // the same as known_quotes with .lyx names
  97 char const * known_coded_quotes[] = { "gld", "grd", "gld", "grd", 0};
  98
  99
 100
 101 // some ugly stuff
 102 ostringstream h_preamble;
 103 string h_textclass               = "article";
 104 string h_options                 = "";
 105 string h_language                = "english";
 106 string h_inputencoding           = "latin1";
 107 string h_fontscheme              = "default";
 108 string h_graphics                = "default";
 109 string h_paperfontsize           = "default";
 110 string h_spacing                 = "single";
 111 string h_papersize               = "default";
 112 string h_paperpackage            = "default";
 113 string h_use_geometry            = "0";
 114 string h_use_amsmath             = "0";
 115 string h_use_natbib              = "0";
 116 string h_use_numerical_citations = "0";
 117 string h_paperorientation        = "portrait";
 118 string h_secnumdepth             = "3";
 119 string h_tocdepth                = "3";
 120 string h_paragraph_separation    = "indent";
 121 string h_defskip                 = "medskip";
 122 string h_quotes_language         = "english";
 123 string h_quotes_times            = "2";
 124 string h_papercolumns            = "1";
 125 string h_papersides              = "1";
 126 string h_paperpagestyle          = "default";
 127 string h_tracking_changes        = "0";
 128
 129 // current stack of nested environments
 130 stack<string> active_environments;
 131
 132
 133 string cap(string s)
 134 {
 135         if (s.size())
 136                 s[0] = toupper(s[0]);
 137         return s;
 138 }
 139
 140
 141 string const trim(string const & a, char const * p = " \t\n\r")
 142 {
 143         // lyx::Assert(p);
 144
 145         if (a.empty() || !*p)
 146                 return a;
 147
 148         string::size_type r = a.find_last_not_of(p);
 149         string::size_type l = a.find_first_not_of(p);
 150
 151         // Is this the minimal test? (lgb)
 152         if (r == string::npos && l == string::npos)
 153                 return string();
 154
 155         return a.substr(l, r - l + 1);
 156 }
 157
 158
 159 void split(string const & s, vector<string> & result, char delim = ',')
 160 {
 161         istringstream is(s);
 162         string t;
 163         while (getline(is, t, delim))
 164                 result.push_back(t);
 165 }
 166
 167
 168 // splits "x=z, y=b" into a map
 169 map<string, string> split_map(string const & s)
 170 {
 171         map<string, string> res;
 172         vector<string> v;
 173         split(s, v);
 174         for (size_t i = 0; i < v.size(); ++i) {
 175                 size_t const pos   = v[i].find('=');
 176                 string const index = v[i].substr(0, pos);
 177                 string const value = v[i].substr(pos + 1, string::npos);
 178                 res[trim(index)] = trim(value);
 179         }
 180         return res;
 181 }
 182
 183
 184 string join(vector<string> const & input, char const * delim)
 185 {
 186         ostringstream os;
 187         for (size_t i = 0; i != input.size(); ++i) {
 188                 if (i)
 189                         os << delim;
 190                 os << input[i];
 191         }
 192         return os.str();
 193 }
 194
 195
 196 char const ** is_known(string const & str, char const ** what)
 197 {
 198         for ( ; *what; ++what)
 199                 if (str == *what)
 200                         return what;
 201         return 0;
 202 }
 203
 204
 205 void handle_opt(vector<string> & opts, char const ** what, string & target)
 206 {
 207         if (opts.empty())
 208                 return;
 209
 210         for ( ; *what; ++what) {
 211                 vector<string>::iterator it = find(opts.begin(), opts.end(), *what);
 212                 if (it != opts.end()) {
 213                         //cerr << "### found option '" << *what << "'\n";
 214                         target = *what;
 215                         opts.erase(it);
 216                         return;
 217                 }
 218         }
 219 }
 220
 221
 222 bool is_math_env(string const & name)
 223 {
 224         for (char const ** what = known_math_envs; *what; ++what)
 225                 if (*what == name)
 226                         return true;
 227         return false;
 228 }
 229
 230
 231 void begin_inset(ostream & os, string const & name)
 232 {
 233         os << "\n\\begin_inset " << name;
 234 }
 235
 236
 237 void end_inset(ostream & os)
 238 {
 239         os << "\n\\end_inset\n\n";
 240 }
 241
 242
 243 string curr_env()
 244 {
 245         return active_environments.empty() ? string() : active_environments.top();
 246 }
 247
 248
 249 void handle_ert(ostream & os, string const & s)
 250 {
 251         begin_inset(os, "ERT");
 252         os << "\nstatus Collapsed\n\n\\layout Standard\n\n";
 253         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 254                 if (*it == '\\')
 255                         os << "\n\\backslash\n";
 256                 else
 257                         os << *it;
 258         }
 259         end_inset(os);
 260 }
 261
 262
 263 void handle_par(ostream & os)
 264 {
 265         if (active_environments.empty())
 266                 return;
 267         os << "\n\\layout ";
 268         string s = curr_env();
 269         if (s == "document" || s == "table")
 270                 os << "Standard\n\n";
 271         else if (s == "lyxcode")
 272                 os << "LyX-Code\n\n";
 273         else if (s == "lyxlist")
 274                 os << "List\n\n";
 275         else if (s == "thebibliography")
 276                 os << "Bibliography\n\n";
 277         else
 278                 os << cap(s) << "\n\n";
 279 }
 280
 281
 282 void handle_package(string const & name, string const & options)
 283 {
 284         //cerr << "handle_package: '" << name << "'\n";
 285         if (name == "a4wide") {
 286                 h_papersize = "a4paper";
 287                 h_paperpackage = "widemarginsa4";
 288         } else if (name == "ae")
 289                 h_fontscheme = "ae";
 290         else if (name == "aecompl")
 291                 h_fontscheme = "ae";
 292         else if (name == "amsmath")
 293                 h_use_amsmath = "1";
 294         else if (name == "amssymb")
 295                 h_use_amsmath = "1";
 296         else if (name == "babel")
 297                 ; // ignore this
 298         else if (name == "fontenc")
 299                 ; // ignore this
 300         else if (name == "inputenc")
 301                 h_inputencoding = options;
 302         else if (name == "makeidx")
 303                 ; // ignore this
 304         else if (name == "verbatim")
 305                 ; // ignore this
 306         else if (is_known(name, known_languages)) {
 307                 h_language = name;
 308                 h_quotes_language = name;
 309         } else {
 310                 if (options.size())
 311                         h_preamble << "\\usepackage[" << options << "]{" << name << "}\n";
 312                 else
 313                         h_preamble << "\\usepackage{" << name << "}\n";
 314         }
 315 }
 316
 317
 318 vector<string> extract_col_align(string const & s)
 319 {
 320         vector<string> res;
 321         string t;
 322         for (size_t i = 0; i < s.size(); ++i) {
 323                 switch (s[i]) {
 324                         case 'c': res.push_back("center"); break;
 325                         case 'l': res.push_back("left");   break;
 326                         case 'r': res.push_back("right");  break;
 327                         //case '|': cerr << "ignoring vertical separator\n";  break;
 328                         //default : cerr << "ignoring special separator '" << s[i] << "'\n"; break;
 329                         default: break;
 330                 }
 331         }
 332         return res;
 333 }
 334
 335
 336 void handle_tabular(Parser & p, ostream & os, mode_type mode)
 337 {
 338         begin_inset(os, "Tabular \n");
 339         string posopts = p.getOpt();
 340         if (posopts.size())
 341                 cerr << "vertical tabular positioning '" << posopts << "' ignored\n";
 342         string colopts = p.verbatimItem();
 343         vector<string> colalign = extract_col_align(colopts);
 344         ostringstream ss;
 345         parse(p, ss, FLAG_END, mode);
 346         vector<string> lines;
 347         split(ss.str(), lines, LINE);
 348         const size_t cols = colalign.size();
 349         const size_t rows = lines.size();
 350         os << "<lyxtabular version=\"3\" rows=\"" << rows
 351                  << "\" columns=\"" << cols << "\">\n"
 352                  << "<features>\n";
 353         for (size_t c = 0; c < cols; ++c)
 354                 os << "<column alignment=\"" << colalign[c] << "\""
 355                          << " valignment=\"top\""
 356                          << " width=\"0pt\""
 357                          << ">\n";
 358         for (size_t r = 0; r < rows; ++r) {
 359                 vector<string> dummy;
 360                 // handle hlines
 361                 split(lines[r], dummy, HLINE);
 362                 int hlines = dummy.size() - 1;
 363                 lines[r] = join(dummy, "");
 364                 // handle almost empty line
 365                 if (lines[r].empty() && hlines && r > 0) {
 366                         cerr << "last hline lost\n";
 367                         continue;
 368                 }
 369                 // handle cells
 370                 vector<string> cells;
 371                 split(lines[r], cells, TAB);
 372                 while (cells.size() < cols)
 373                         cells.push_back(string());
 374                 os << "<row";
 375                 if (hlines)
 376                         os << " topline=\"true\"";
 377                 os << ">\n";
 378                 for (size_t c = 0; c < cols; ++c) {
 379                         os << "<cell";
 380                         string alignment = "center";
 381                         vector<string> parts;
 382                         split(cells[c], parts, MULT);
 383                         if (parts.size() > 2) {
 384                                 os << " multicolumn=\"" << parts[0] << "\"";
 385                                 alignment = parts[1];
 386                         }
 387                         os << " alignment=\"" << alignment << "\""
 388                                  << " valignment=\"top\""
 389                                  << " topline=\"true\""
 390                                  << " leftline=\"true\""
 391                                  << " usebox=\"none\""
 392                                  << ">";
 393                         begin_inset(os, "Text");
 394                         os << "\n\n\\layout Standard\n\n";
 395                         if (parts.size())
 396                                 os << parts.back();
 397                         end_inset(os);
 398                         os << "</cell>\n";
 399                 }
 400                 os << "</row>\n";
 401         }
 402         os << "</lyxtabular>\n";
 403         end_inset(os);
 404 }
 405
 406
 407 string wrap(string const & cmd, string const & str)
 408 {
 409         return OPEN + cmd + ' ' + str + CLOSE;
 410 }
 411
 412
 413 void end_preamble(ostream & os)
 414 {
 415         os << "# tex2lyx 0.0.2 created this file\n"
 416            << "\\lyxformat 222\n"
 417            << "\\textclass " << h_textclass << "\n"
 418            << "\\begin_preamble\n" << h_preamble.str() << "\n\\end_preamble\n"
 419            << "\\options " << h_options << "\n"
 420            << "\\language " << h_language << "\n"
 421            << "\\inputencoding " << h_inputencoding << "\n"
 422            << "\\fontscheme " << h_fontscheme << "\n"
 423            << "\\graphics " << h_graphics << "\n"
 424            << "\\paperfontsize " << h_paperfontsize << "\n"
 425            << "\\spacing " << h_spacing << "\n"
 426            << "\\papersize " << h_papersize << "\n"
 427            << "\\paperpackage " << h_paperpackage << "\n"
 428            << "\\use_geometry " << h_use_geometry << "\n"
 429            << "\\use_amsmath " << h_use_amsmath << "\n"
 430            << "\\use_natbib " << h_use_natbib << "\n"
 431            << "\\use_numerical_citations " << h_use_numerical_citations << "\n"
 432            << "\\paperorientation " << h_paperorientation << "\n"
 433            << "\\secnumdepth " << h_secnumdepth << "\n"
 434            << "\\tocdepth " << h_tocdepth << "\n"
 435            << "\\paragraph_separation " << h_paragraph_separation << "\n"
 436            << "\\defskip " << h_defskip << "\n"
 437            << "\\quotes_language " << h_quotes_language << "\n"
 438            << "\\quotes_times " << h_quotes_times << "\n"
 439            << "\\papercolumns " << h_papercolumns << "\n"
 440            << "\\papersides " << h_papersides << "\n"
 441            << "\\paperpagestyle " << h_paperpagestyle << "\n"
 442            << "\\tracking_changes " << h_tracking_changes << "\n";
 443 }
 444
 445
 446 void parse_preamble(Parser & p, ostream & os)
 447 {
 448         while (p.good()) {
 449                 Token const & t = p.getToken();
 450
 451 #ifdef FILEDEBUG
 452                 cerr << "t: " << t << " flags: " << flags << "\n";
 453                 //cell->dump();
 454 #endif
 455
 456                 //
 457                 // cat codes
 458                 //
 459                 if (t.cat() == catLetter ||
 460                           t.cat() == catSpace ||
 461                           t.cat() == catSuper ||
 462                           t.cat() == catSub ||
 463                           t.cat() == catOther ||
 464                           t.cat() == catMath ||
 465                           t.cat() == catActive ||
 466                           t.cat() == catBegin ||
 467                           t.cat() == catEnd ||
 468                           t.cat() == catAlign ||
 469                           t.cat() == catNewline ||
 470                           t.cat() == catParameter)
 471                 h_preamble << t.character();
 472
 473                 else if (t.cat() == catComment) {
 474                         string s;
 475                         while (p.good()) {
 476                                 Token const & t = p.getToken();
 477                                 if (t.cat() == catNewline)
 478                                         break;
 479                                 s += t.asString();
 480                         }
 481                         //os << wrap("comment", s);
 482                         p.skipSpaces();
 483                 }
 484
 485                 else if (t.cs() == "pagestyle")
 486                         h_paperpagestyle == p.verbatimItem();
 487
 488                 else if (t.cs() == "makeatletter") {
 489                         p.setCatCode('@', catLetter);
 490                         h_preamble << "\\makeatletter\n";
 491                 }
 492
 493                 else if (t.cs() == "makeatother") {
 494                         p.setCatCode('@', catOther);
 495                         h_preamble << "\\makeatother\n";
 496                 }
 497
 498                 else if (t.cs() == "newcommand" || t.cs() == "renewcommand"
 499                             || t.cs() == "providecommand") {
 500                         string const name = p.verbatimItem();
 501                         string const opts = p.getOpt();
 502                         string const body = p.verbatimItem();
 503                         // only non-lyxspecific stuff
 504                         if (name != "\\noun "
 505                                   && name != "\\tabularnewline "
 506                             && name != "\\LyX "
 507                                   && name != "\\lyxline "
 508                                   && name != "\\lyxaddress "
 509                                   && name != "\\lyxrightaddress "
 510                                   && name != "\\boldsymbol "
 511                                   && name != "\\lyxarrow ") {
 512                                 ostringstream ss;
 513                                 ss << '\\' << t.cs() << '{' << name << '}'
 514                                         << opts << '{' << body << "}\n";
 515                                 h_preamble << ss.str();
 516 /*
 517                                 ostream & out = in_preamble ? h_preamble : os;
 518                                 out << "\\" << t.cs() << "{" << name << "}"
 519                                     << opts << "{" << body << "}\n";
 520                                 if (!in_preamble)
 521                                         end_inset(os);
 522 */
 523                         }
 524                 }
 525
 526                 else if (t.cs() == "documentclass") {
 527                         vector<string> opts;
 528                         split(p.getArg('[', ']'), opts, ',');
 529                         handle_opt(opts, known_languages, h_language);
 530                         handle_opt(opts, known_fontsizes, h_paperfontsize);
 531                         h_quotes_language = h_language;
 532                         h_options = join(opts, ",");
 533                         h_textclass = p.getArg('{', '}');
 534                 }
 535
 536                 else if (t.cs() == "usepackage") {
 537                         string const options = p.getArg('[', ']');
 538                         string const name = p.getArg('{', '}');
 539                         if (options.empty() && name.find(',')) {
 540                                 vector<string> vecnames;
 541                                 split(name, vecnames, ',');
 542                                 vector<string>::const_iterator it  = vecnames.begin();
 543                                 vector<string>::const_iterator end = vecnames.end();
 544                                 for (; it != end; ++it)
 545                                         handle_package(trim(*it), string());
 546                         } else {
 547                                 handle_package(name, options);
 548                         }
 549                 }
 550
 551                 else if (t.cs() == "newenvironment") {
 552                         string const name = p.getArg('{', '}');
 553                         ostringstream ss;
 554                         ss << "\\newenvironment{" << name << "}";
 555                         ss << p.getOpt();
 556                         ss << p.getOpt();
 557                         ss << '{' << p.verbatimItem() << '}';
 558                         ss << '{' << p.verbatimItem() << '}';
 559                         ss << '\n';
 560                         if (name != "lyxcode" && name != "lyxlist"
 561                                         && name != "lyxrightadress" && name != "lyxaddress")
 562                                 h_preamble << ss.str();
 563                 }
 564
 565                 else if (t.cs() == "def") {
 566                         string name = p.getToken().cs();
 567                         while (p.nextToken().cat() != catBegin)
 568                                 name += p.getToken().asString();
 569                         h_preamble << "\\def\\" << name << '{' << p.verbatimItem() << "}\n";
 570                 }
 571
 572                 else if (t.cs() == "setcounter") {
 573                         string const name = p.getArg('{', '}');
 574                         string const content = p.getArg('{', '}');
 575                         if (name == "secnumdepth")
 576                                 h_secnumdepth = content;
 577                         else if (name == "tocdepth")
 578                                 h_tocdepth = content;
 579                         else
 580                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}\n";
 581                 }
 582
 583                 else if (t.cs() == "setlength") {
 584                         string const name = p.verbatimItem();
 585                         string const content = p.verbatimItem();
 586                         if (name == "parskip")
 587                                 h_paragraph_separation = "skip";
 588                         else if (name == "parindent")
 589                                 h_paragraph_separation = "skip";
 590                         else
 591                                 h_preamble << "\\setlength{" + name + "}{" + content + "}\n";
 592                 }
 593
 594                 else if (t.cs() == "par")
 595                         h_preamble << '\n';
 596
 597                 else if (t.cs() == "begin") {
 598                         string const name = p.getArg('{', '}');
 599                         if (name == "document") {
 600                                 end_preamble(os);
 601                                 os << "\n\n\\layout Standard\n\n";
 602                                 return;
 603                         }
 604                         h_preamble << "\\begin{" << name << "}";
 605                 }
 606
 607                 else if (t.cs().size())
 608                         h_preamble << '\\' << t.cs() << ' ';
 609         }
 610 }
 611
 612
 613 void parse(Parser & p, ostream & os, unsigned flags, const mode_type mode)
 614 {
 615         while (p.good()) {
 616                 Token const & t = p.getToken();
 617
 618 #ifdef FILEDEBUG
 619                 cerr << "t: " << t << " flags: " << flags << "\n";
 620                 //cell->dump();
 621 #endif
 622
 623                 if (flags & FLAG_ITEM) {
 624                         if (t.cat() == catSpace)
 625                                 continue;
 626
 627                         flags &= ~FLAG_ITEM;
 628                         if (t.cat() == catBegin) {
 629                                 // skip the brace and collect everything to the next matching
 630                                 // closing brace
 631                                 flags |= FLAG_BRACE_LAST;
 632                                 continue;
 633                         }
 634
 635                         // handle only this single token, leave the loop if done
 636                         flags |= FLAG_LEAVE;
 637                 }
 638
 639
 640                 if (flags & FLAG_BRACED) {
 641                         if (t.cat() == catSpace)
 642                                 continue;
 643
 644                         if (t.cat() != catBegin) {
 645                                 p.error("opening brace expected");
 646                                 return;
 647                         }
 648
 649                         // skip the brace and collect everything to the next matching
 650                         // closing brace
 651                         flags = FLAG_BRACE_LAST;
 652                 }
 653
 654
 655                 if (flags & FLAG_OPTION) {
 656                         if (t.cat() == catOther && t.character() == '[') {
 657                                 parse(p, os, FLAG_BRACK_LAST, mode);
 658                         } else {
 659                                 // no option found, put back token and we are done
 660                                 p.putback();
 661                         }
 662                         return;
 663                 }
 664
 665                 //
 666                 // cat codes
 667                 //
 668                 if (t.cat() == catMath) {
 669                         if (mode == TEXT_MODE || mode == MATHTEXT_MODE) {
 670                                 // we are inside some text mode thingy, so opening new math is allowed
 671                                 if (mode == TEXT_MODE)
 672                                         begin_inset(os, "Formula ");
 673                                 Token const & n = p.getToken();
 674                                 if (n.cat() == catMath) {
 675                                         // TeX's $$...$$ syntax for displayed math
 676                                         os << "\\[";
 677                                         parse(p, os, FLAG_SIMPLE, MATH_MODE);
 678                                         os << "\\]";
 679                                         p.getToken(); // skip the second '$' token
 680                                 } else {
 681                                         // simple $...$  stuff
 682                                         p.putback();
 683                                         os << '$';
 684                                         parse(p, os, FLAG_SIMPLE, MATH_MODE);
 685                                         os << '$';
 686                                 }
 687                                 if (mode == TEXT_MODE)
 688                                         end_inset(os);
 689                         }
 690
 691                         else if (flags & FLAG_SIMPLE) {
 692                                 // this is the end of the formula
 693                                 return;
 694                         }
 695
 696                         else {
 697                                 cerr << "mode: " << mode << endl;
 698                                 p.error("something strange in the parser\n");
 699                                 break;
 700                         }
 701                 }
 702
 703                 else if (t.cat() == catLetter ||
 704                                t.cat() == catSpace ||
 705                          t.cat() == catSuper ||
 706                                t.cat() == catSub ||
 707                                t.cat() == catOther ||
 708                                t.cat() == catParameter)
 709                         os << t.character();
 710
 711                 else if (t.cat() == catNewline)
 712                         os << ' ';
 713
 714                 else if (t.cat() == catActive) {
 715                         if (t.character() == '~') {
 716                                 if (curr_env() == "lyxcode")
 717                                         os << ' ';
 718                                 else if (mode == TEXT_MODE)
 719                                         os << "\\SpecialChar ~\n";
 720                                 else
 721                                         os << '~';
 722                         } else
 723                                 os << t.character();
 724                 }
 725
 726                 else if (t.cat() == catBegin) {
 727                         if (mode == TEXT_MODE)
 728                                 handle_ert(os, "{");
 729                         else
 730                                 os << '{';
 731                 }
 732
 733                 else if (t.cat() == catEnd) {
 734                         if (flags & FLAG_BRACE_LAST)
 735                                 return;
 736                         if (mode == TEXT_MODE)
 737                                 handle_ert(os, "}");
 738                         else
 739                                 os << '}';
 740                 }
 741
 742                 else if (t.cat() == catAlign) {
 743                         if (mode == TEXT_MODE)
 744                                 os << TAB;
 745                         else
 746                                 os << t.character();
 747                 }
 748
 749                 else if (t.cs() == "tabularnewline") {
 750                         if (mode == TEXT_MODE)
 751                                 os << LINE;
 752                         else
 753                                 os << t.asInput();
 754                 }
 755
 756                 else if (t.cs() == "\\" && mode == MATH_MODE)
 757                         os << t.asInput();
 758
 759                 else if (t.cs() == "\\" && mode == TEXT_MODE && curr_env() == "tabular")
 760                         os << LINE;
 761
 762                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 763                         //cerr << "finished reading option\n";
 764                         return;
 765                 }
 766
 767                 else if (t.cat() == catOther)
 768                         os << string(1, t.character());
 769
 770                 else if (t.cat() == catComment) {
 771                         string s;
 772                         while (p.good()) {
 773                                 Token const & t = p.getToken();
 774                                 if (t.cat() == catNewline)
 775                                         break;
 776                                 s += t.asString();
 777                         }
 778                         //os << wrap("comment", s);
 779                         p.skipSpaces();
 780                 }
 781
 782                 //
 783                 // control sequences
 784                 //
 785
 786                 else if (t.cs() == "ldots" && mode == MATH_MODE)
 787                         os << "\n\\SpecialChar \\ldots{}\n";
 788
 789                 else if (t.cs() == "lyxlock")
 790                         ; // ignored
 791
 792                 else if (t.cs() == "makeatletter") {
 793                         p.setCatCode('@', catLetter);
 794                         handle_ert(os, "\\makeatletter\n");
 795                 }
 796
 797                 else if (t.cs() == "makeatother") {
 798                         p.setCatCode('@', catOther);
 799                         handle_ert(os, "\\makeatother\n");
 800                 }
 801
 802                 else if (t.cs() == "newcommand" || t.cs() == "renewcommand"
 803                             || t.cs() == "providecommand") {
 804                         string const name = p.verbatimItem();
 805                         string const opts = p.getOpt();
 806                         string const body = p.verbatimItem();
 807                         // only non-lyxspecific stuff
 808                         if (name != "\\noun " && name != "\\tabularnewline ") {
 809                                 ostringstream ss;
 810                                 ss << '\\' << t.cs() << '{' << name << '}'
 811                                         << opts << '{' << body << "}\n";
 812                                 handle_ert(os, ss.str());
 813 /*
 814                                 ostream & out = in_preamble ? h_preamble : os;
 815                                 if (!in_preamble)
 816                                         begin_inset(os, "FormulaMacro\n");
 817                                 out << "\\" << t.cs() << "{" << name << "}"
 818                                     << opts << "{" << body << "}\n";
 819                                 if (!in_preamble)
 820                                         end_inset(os);
 821 */
 822                         }
 823                 }
 824
 825                 else if (t.cs() == "newtheorem") {
 826                         ostringstream ss;
 827                         ss << "\\newtheorem";
 828                         ss << '{' << p.verbatimItem() << '}';
 829                         ss << p.getOpt();
 830                         ss << '{' << p.verbatimItem() << '}';
 831                         ss << p.getOpt();
 832                         ss << '\n';
 833                         handle_ert(os, ss.str());
 834                 }
 835
 836                 else if (t.cs() == "(") {
 837                         begin_inset(os, "Formula");
 838                         os << " \\(";
 839                         parse(p, os, FLAG_SIMPLE2, MATH_MODE);
 840                         os << "\\)";
 841                         end_inset(os);
 842                 }
 843
 844                 else if (t.cs() == "[") {
 845                         begin_inset(os, "Formula");
 846                         os << " \\[";
 847                         parse(p, os, FLAG_EQUATION, MATH_MODE);
 848                         os << "\\]";
 849                         end_inset(os);
 850                 }
 851
 852                 else if (t.cs() == "protect")
 853                         // ignore \\protect, will hopefully be re-added during output
 854                         ;
 855
 856                 else if (t.cs() == "begin") {
 857                         string const name = p.getArg('{', '}');
 858                         active_environments.push(name);
 859                         if (name == "abstract") {
 860                                 handle_par(os);
 861                                 parse(p, os, FLAG_END, mode);
 862                         } else if (is_math_env(name)) {
 863                                 begin_inset(os, "Formula ");
 864                                 os << "\\begin{" << name << "}";
 865                                 parse(p, os, FLAG_END, MATH_MODE);
 866                                 os << "\\end{" << name << "}";
 867                                 end_inset(os);
 868                         } else if (name == "tabular") {
 869                                 if (mode == TEXT_MODE)
 870                                         handle_tabular(p, os, mode);
 871                                 else {
 872                                         os << "\\begin{" << name << "}";
 873                                         parse(p, os, FLAG_END, MATHTEXT_MODE);
 874                                         os << "\\end{" << name << "}";
 875                                 }
 876                         } else if (name == "table" || name == "figure") {
 877                                 string opts = p.getOpt();
 878                                 begin_inset(os, "Float " + name + "\n");
 879                                 if (opts.size())
 880                                         os << "placement " << opts << '\n';
 881                                 os << "wide false\n"
 882                                          << "collapsed false\n"
 883                                          << "\n"
 884                                          << "\\layout Standard\n";
 885                                 parse(p, os, FLAG_END, mode);
 886                                 end_inset(os);
 887                         } else if (name == "thebibliography") {
 888                                 p.verbatimItem(); // swallow next arg
 889                                 parse(p, os, FLAG_END, mode);
 890                                 os << "\n\\layout Bibliography\n\n";
 891                         } else if (mode == MATH_MODE || mode == MATHTEXT_MODE) {
 892                                 os << "\\begin{" << name << "}";
 893                                 parse(p, os, FLAG_END, mode);
 894                                 os << "\\end{" << name << "}";
 895                         } else {
 896                                 parse(p, os, FLAG_END, mode);
 897                         }
 898                 }
 899
 900                 else if (t.cs() == "end") {
 901                         if (flags & FLAG_END) {
 902                                 // eat environment name
 903                                 string const name = p.getArg('{', '}');
 904                                 if (name != curr_env())
 905                                         p.error("\\end{" + name + "} does not match \\begin{"
 906                                                 + curr_env() + "}");
 907                                 active_environments.pop();
 908                                 return;
 909                         }
 910                         p.error("found 'end' unexpectedly");
 911                 }
 912
 913                 else if (t.cs() == "item")
 914                         handle_par(os);
 915
 916                 else if (t.cs() == ")") {
 917                         if (flags & FLAG_SIMPLE2)
 918                                 return;
 919                         p.error("found '\\)' unexpectedly");
 920                 }
 921
 922                 else if (t.cs() == "]") {
 923                         if (flags & FLAG_EQUATION)
 924                                 return;
 925                         p.error("found '\\]' unexpectedly");
 926                 }
 927
 928                 else if (t.cs() == "documentclass") {
 929                         vector<string> opts;
 930                         split(p.getArg('[', ']'), opts, ',');
 931                         handle_opt(opts, known_languages, h_language);
 932                         handle_opt(opts, known_fontsizes, h_paperfontsize);
 933                         h_quotes_language = h_language;
 934                         h_options = join(opts, ",");
 935                         h_textclass = p.getArg('{', '}');
 936                 }
 937
 938                 else if (t.cs() == "usepackage") {
 939                         string const options = p.getArg('[', ']');
 940                         string const name = p.getArg('{', '}');
 941                         if (options.empty() && name.find(',')) {
 942                                 vector<string> vecnames;
 943                                 split(name, vecnames, ',');
 944                                 vector<string>::const_iterator it  = vecnames.begin();
 945                                 vector<string>::const_iterator end = vecnames.end();
 946                                 for (; it != end; ++it)
 947                                         handle_package(trim(*it), string());
 948                         } else {
 949                                 handle_package(name, options);
 950                         }
 951                 }
 952
 953                 else if (t.cs() == "def") {
 954                         string name = p.getToken().cs();
 955                         while (p.nextToken().cat() != catBegin)
 956                                 name += p.getToken().asString();
 957                         handle_ert(os, "\\def\\" + name + '{' + p.verbatimItem() + '}');
 958                 }
 959
 960                 else if (t.cs() == "par")
 961                         handle_par(os);
 962
 963                 else if (is_known(t.cs(), known_headings)) {
 964                         string name = t.cs();
 965                         if (p.nextToken().asInput() == "*") {
 966                                 p.getToken();
 967                                 name += "*";
 968                         }
 969                         os << "\n\n\\layout " << cap(name) << "\n\n";
 970                         string opt = p.getOpt();
 971                         if (opt.size()) {
 972                                 begin_inset(os, "OptArg\n");
 973                                 os << "collapsed true\n\n\\layout Standard\n\n" << opt;
 974                                 end_inset(os);
 975                         }
 976                         parse(p, os, FLAG_ITEM, mode);
 977                         os << "\n\n\\layout Standard\n\n";
 978                 }
 979
 980                 else if (t.cs() == "includegraphics") {
 981                         if (mode == TEXT_MODE) {
 982                                 map<string, string> opts = split_map(p.getArg('[', ']'));
 983                                 string name = p.verbatimItem();
 984                                 begin_inset(os, "Graphics ");
 985                                 os << "\n\tfilename " << name << '\n';
 986                                 if (opts.find("width") != opts.end())
 987                                         os << "\twidth " << opts["width"] << '\n';
 988                                 if (opts.find("height") != opts.end())
 989                                         os << "\theight " << opts["height"] << '\n';
 990                                 end_inset(os);
 991                         } else {
 992                                 os << "\\includegraphics ";
 993                         }
 994                 }
 995
 996                 else if (t.cs() == "makeindex" || t.cs() == "maketitle")
 997                         ; // swallow this
 998
 999                 else if (t.cs() == "tableofcontents")
1000                         p.verbatimItem(); // swallow this
1001
1002                 else if (t.cs() == "multicolumn" && mode == TEXT_MODE) {
1003                         // brutish...
1004                         parse(p, os, FLAG_ITEM, mode);
1005                         os << MULT;
1006                         parse(p, os, FLAG_ITEM, mode);
1007                         os << MULT;
1008                         parse(p, os, FLAG_ITEM, mode);
1009                 }
1010
1011                 else if (t.cs() == "hline" && mode == TEXT_MODE)
1012                         os << HLINE;
1013
1014                 else if (t.cs() == "textrm") {
1015                         if (mode == TEXT_MODE) {
1016                                 os << "\n\\family roman\n";
1017                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1018                                 os << "\n\\family default\n";
1019                         } else {
1020                                 os << '\\' << t.cs() << '{';
1021                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1022                                 os << '}';
1023                         }
1024                 }
1025
1026                 else if (t.cs() == "textsf") {
1027                         if (mode == TEXT_MODE) {
1028                                 os << "\n\\family sans\n";
1029                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1030                                 os << "\n\\family default\n";
1031                         } else {
1032                                 os << '\\' << t.cs() << '{';
1033                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1034                                 os << '}';
1035                         }
1036                 }
1037
1038                 else if (t.cs() == "texttt") {
1039                         if (mode == TEXT_MODE) {
1040                                 os << "\n\\family typewriter\n";
1041                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1042                                 os << "\n\\family default\n";
1043                         } else {
1044                                 os << '\\' << t.cs() << '{';
1045                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1046                                 os << '}';
1047                         }
1048                 }
1049
1050                 else if (t.cs() == "textsc") {
1051                         if (mode == TEXT_MODE) {
1052                                 os << "\n\\noun on\n";
1053                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1054                                 os << "\n\\noun default\n";
1055                         } else {
1056                                 os << '\\' << t.cs() << '{';
1057                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1058                                 os << '}';
1059                         }
1060                 }
1061
1062                 else if (t.cs() == "textbf") {
1063                         if (mode == TEXT_MODE) {
1064                                 os << "\n\\series bold\n";
1065                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1066                                 os << "\n\\series default\n";
1067                         } else {
1068                                 os << '\\' << t.cs() << '{';
1069                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1070                                 os << '}';
1071                         }
1072                 }
1073
1074                 else if (t.cs() == "underbar") {
1075                         if (mode == TEXT_MODE) {
1076                                 os << "\n\\bar under\n";
1077                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1078                                 os << "\n\\bar default\n";
1079                         } else {
1080                                 os << '\\' << t.cs() << '{';
1081                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1082                                 os << '}';
1083                         }
1084                 }
1085
1086                 else if ((t.cs() == "emph" || t.cs() == "noun") && mode == TEXT_MODE) {
1087                         os << "\n\\" << t.cs() << " on\n";
1088                         parse(p, os, FLAG_ITEM, mode);
1089                         os << "\n\\" << t.cs() << " default\n";
1090                 }
1091
1092                 else if (is_known(t.cs(), known_latex_commands) && mode == TEXT_MODE) {
1093                         begin_inset(os, "LatexCommand ");
1094                         os << '\\' << t.cs();
1095                         os << p.getOpt();
1096                         os << p.getOpt();
1097                         os << '{' << p.verbatimItem() << '}';
1098                         end_inset(os);
1099                 }
1100
1101                 else if (t.cs() == "bibitem") {
1102                         os << "\n\\layout Bibliography\n\\bibitem ";
1103                         os << p.getOpt();
1104                         os << '{' << p.verbatimItem() << '}' << "\n\n";
1105                 }
1106
1107                 else if (mode == TEXT_MODE && is_known(t.cs(), known_quotes)) {
1108                   char const ** where = is_known(t.cs(), known_quotes);
1109                         begin_inset(os, "Quotes ");
1110                         os << known_coded_quotes[where - known_quotes];
1111                         end_inset(os);
1112                 }
1113
1114                 else if (t.cs() == "LyX") {
1115                         p.verbatimItem(); // eat {}
1116                         os << "LyX";
1117                 }
1118
1119                 else if (t.cs() == "TeX") {
1120                         p.verbatimItem(); // eat {}
1121                         os << "TeX";
1122                 }
1123
1124                 else if (t.cs() == "LaTeX") {
1125                         p.verbatimItem(); // eat {}
1126                         os << "LaTeX";
1127                 }
1128
1129                 else if (t.cs() == "LaTeXe") {
1130                         p.verbatimItem(); // eat {}
1131                         os << "LaTeXe";
1132                 }
1133
1134                 else if (t.cs() == "textasciitilde")
1135                         os << '~';
1136
1137                 else if (t.cs() == "_" && mode == TEXT_MODE)
1138                         os << '_';
1139
1140                 else if (t.cs() == "&" && mode == TEXT_MODE)
1141                         os << '&';
1142
1143                 else if (t.cs() == "\"") {
1144                         string const name = p.verbatimItem();
1145                              if (name == "a") os << 'ä';
1146                         else if (name == "o") os << 'ö';
1147                         else if (name == "u") os << 'ü';
1148                         else if (name == "A") os << 'Ä';
1149                         else if (name == "O") os << 'Ö';
1150                         else if (name == "U") os << 'Ü';
1151                         else handle_ert(os, "\"{" + name + "}");
1152                 }
1153
1154                 else if (t.cs() == "ss")
1155                         os << "ß";
1156
1157                 else if (t.cs() == "input")
1158                         handle_ert(os, "\\input{" + p.verbatimItem() + "}\n");
1159
1160                 else if (t.cs() == "fancyhead") {
1161                         ostringstream ss;
1162                         ss << "\\fancyhead";
1163                         ss << p.getOpt();
1164                         ss << '{' << p.verbatimItem() << "}\n";
1165                         handle_ert(os, ss.str());
1166                 }
1167
1168                 else {
1169                         //cerr << "#: " << t << " mode: " << mode << endl;
1170                         if (mode == MATH_MODE || mode == MATHTEXT_MODE) {
1171                                 os << t.asInput();
1172                                 //cerr << "#: writing: '" << t.asInput() << "'\n";
1173                         } else {
1174                                 // heuristic: read up to next non-nested space
1175                                 /*
1176                                 string s = t.asInput();
1177                                 string z = p.verbatimItem();
1178                                 while (p.good() && z != " " && z.size()) {
1179                                         //cerr << "read: " << z << endl;
1180                                         s += z;
1181                                         z = p.verbatimItem();
1182                                 }
1183                                 cerr << "found ERT: " << s << endl;
1184                                 handle_ert(os, s + ' ');
1185                                 */
1186                                 handle_ert(os, t.asInput() + ' ');
1187                         }
1188                 }
1189
1190                 if (flags & FLAG_LEAVE) {
1191                         flags &= ~FLAG_LEAVE;
1192                         break;
1193                 }
1194         }
1195 }
1196
1197
1198
1199 } // anonymous namespace
1200
1201
1202 int main(int argc, char * argv[])
1203 {
1204         if (argc <= 1) {
1205                 cerr << "Usage: " << argv[0] << " <infile.tex>" << endl;
1206                 return 2;
1207         }
1208
1209         ifstream is(argv[1]);
1210         Parser p(is);
1211         parse_preamble(p, cout);
1212         active_environments.push("document");
1213         parse(p, cout, FLAG_END, TEXT_MODE);
1214         cout << "\n\\the_end";
1215
1216         return 0;
1217 }
1218
1219 // }])