src/tex2lyx/tex2lyx.C

   1 /** The .tex to .lyx converter
   2     \author André Pönitz (2003)
   3  */
   4
   5 // {[(
   6
   7 #include <config.h>
   8
   9 #include <algorithm>
  10 #include <cctype>
  11 #include <fstream>
  12 #include <iostream>
  13 #include <map>
  14 #include <stack>
  15 #include <string>
  16 #include <vector>
  17
  18 #include "Lsstream.h"
  19
  20 #include "texparser.h"
  21
  22 using std::count_if;
  23 using std::cout;
  24 using std::cerr;
  25 using std::endl;
  26 using std::fill;
  27 using std::getline;
  28 using std::ios;
  29 using std::ifstream;
  30 using std::istream;
  31 using std::istringstream;
  32 using std::map;
  33 using std::ostream;
  34 using std::ostringstream;
  35 using std::stack;
  36 using std::string;
  37 using std::vector;
  38
  39
  40 namespace {
  41
  42 void parse_preamble(Parser & p, ostream & os);
  43
  44 void parse(Parser & p, ostream & os, unsigned flags, const mode_type mode);
  45
  46 char const OPEN = '<';
  47 char const CLOSE = '>';
  48
  49 /* rather brutish way to code table structure in a string:
  50
  51   \begin{tabular}{ccc}
  52     1 & 2 & 3\\ \hline
  53     \multicolumn{2}{c}{4} & 5 //
  54     6 & 7 \\
  55   \end{tabular}
  56
  57  gets "translated" to:
  58
  59   1 TAB 2 TAB 3 LINE
  60   HLINE 2 MULT c MULT 4 TAB 5 LINE
  61   5 TAB 7 LINE
  62 */
  63
  64 char const TAB   = '\001';
  65 char const LINE  = '\002';
  66 char const MULT  = '\003';
  67 char const HLINE = '\004';
  68
  69 const char * known_languages[] = { "austrian", "babel", "bahasa", "basque",
  70 "breton", "british", "bulgarian", "catalan", "croatian", "czech", "danish",
  71 "dutch", "english", "esperanto", "estonian", "finnish", "francais",
  72 "frenchb", "galician", "german", "germanb", "greek", "hebcal", "hebfont",
  73 "hebrew", "hebrew_newcode", "hebrew_oldcode", "hebrew_p", "hyphen",
  74 "icelandic", "irish", "italian", "latin", "lgrcmr", "lgrcmro", "lgrcmss",
  75 "lgrcmtt", "lgrenc", "lgrlcmss", "lgrlcmtt", "lheclas", "lhecmr",
  76 "lhecmss", "lhecmtt", "lhecrml", "lheenc", "lhefr", "lheredis", "lheshold",
  77 "lheshscr", "lheshstk", "lsorbian", "magyar", "naustrian", "ngermanb",
  78 "ngerman", "norsk", "polish", "portuges", "rlbabel", "romanian",
  79 "russianb", "samin", "scottish", "serbian", "slovak", "slovene", "spanish",
  80 "swedish", "turkish", "ukraineb", "usorbian", "welsh", 0};
  81
  82 char const * known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
  83
  84 char const * known_headings[] = { "caption", "title", "author", "date",
  85 "paragraph", "chapter", "section", "subsection", "subsubsection", 0 };
  86
  87 char const * known_math_envs[] = { "equation", "equation*",
  88 "eqnarray", "eqnarray*", "align", "align*", 0};
  89
  90 char const * known_latex_commands[] = { "ref", "cite", "label", "index",
  91 "printindex", 0 };
  92
  93 // LaTeX names for quotes
  94 char const * known_quotes[] = { "glqq", "grqq", "quotedblbase",
  95 "textquotedblleft", 0};
  96
  97 // the same as known_quotes with .lyx names
  98 char const * known_coded_quotes[] = { "gld", "grd", "gld", "grd", 0};
  99
 100
 101
 102 // some ugly stuff
 103 ostringstream h_preamble;
 104 string h_textclass               = "article";
 105 string h_options                 = "";
 106 string h_language                = "english";
 107 string h_inputencoding           = "latin1";
 108 string h_fontscheme              = "default";
 109 string h_graphics                = "default";
 110 string h_paperfontsize           = "default";
 111 string h_spacing                 = "single";
 112 string h_papersize               = "default";
 113 string h_paperpackage            = "default";
 114 string h_use_geometry            = "0";
 115 string h_use_amsmath             = "0";
 116 string h_use_natbib              = "0";
 117 string h_use_numerical_citations = "0";
 118 string h_paperorientation        = "portrait";
 119 string h_secnumdepth             = "3";
 120 string h_tocdepth                = "3";
 121 string h_paragraph_separation    = "indent";
 122 string h_defskip                 = "medskip";
 123 string h_quotes_language         = "english";
 124 string h_quotes_times            = "2";
 125 string h_papercolumns            = "1";
 126 string h_papersides              = "1";
 127 string h_paperpagestyle          = "default";
 128 string h_tracking_changes        = "0";
 129
 130 // current stack of nested environments
 131 stack<string> active_environments;
 132
 133
 134 string cap(string s)
 135 {
 136         if (s.size())
 137                 s[0] = toupper(s[0]);
 138         return s;
 139 }
 140
 141
 142 string const trim(string const & a, char const * p = " \t\n\r")
 143 {
 144         // lyx::Assert(p);
 145
 146         if (a.empty() || !*p)
 147                 return a;
 148
 149         string::size_type r = a.find_last_not_of(p);
 150         string::size_type l = a.find_first_not_of(p);
 151
 152         // Is this the minimal test? (lgb)
 153         if (r == string::npos && l == string::npos)
 154                 return string();
 155
 156         return a.substr(l, r - l + 1);
 157 }
 158
 159
 160 void split(string const & s, vector<string> & result, char delim = ',')
 161 {
 162         istringstream is(s);
 163         string t;
 164         while (getline(is, t, delim))
 165                 result.push_back(t);
 166 }
 167
 168
 169 // splits "x=z, y=b" into a map
 170 map<string, string> split_map(string const & s)
 171 {
 172         map<string, string> res;
 173         vector<string> v;
 174         split(s, v);
 175         for (size_t i = 0; i < v.size(); ++i) {
 176                 size_t const pos   = v[i].find('=');
 177                 string const index = v[i].substr(0, pos);
 178                 string const value = v[i].substr(pos + 1, string::npos);
 179                 res[trim(index)] = trim(value);
 180         }
 181         return res;
 182 }
 183
 184
 185 string join(vector<string> const & input, char const * delim)
 186 {
 187         ostringstream os;
 188         for (size_t i = 0; i != input.size(); ++i) {
 189                 if (i)
 190                         os << delim;
 191                 os << input[i];
 192         }
 193         return os.str();
 194 }
 195
 196
 197 char const ** is_known(string const & str, char const ** what)
 198 {
 199         for ( ; *what; ++what)
 200                 if (str == *what)
 201                         return what;
 202         return 0;
 203 }
 204
 205
 206 void handle_opt(vector<string> & opts, char const ** what, string & target)
 207 {
 208         if (opts.empty())
 209                 return;
 210
 211         for ( ; *what; ++what) {
 212                 vector<string>::iterator it = find(opts.begin(), opts.end(), *what);
 213                 if (it != opts.end()) {
 214                         //cerr << "### found option '" << *what << "'\n";
 215                         target = *what;
 216                         opts.erase(it);
 217                         return;
 218                 }
 219         }
 220 }
 221
 222
 223 bool is_math_env(string const & name)
 224 {
 225         for (char const ** what = known_math_envs; *what; ++what)
 226                 if (*what == name)
 227                         return true;
 228         return false;
 229 }
 230
 231
 232 void begin_inset(ostream & os, string const & name)
 233 {
 234         os << "\n\\begin_inset " << name;
 235 }
 236
 237
 238 void end_inset(ostream & os)
 239 {
 240         os << "\n\\end_inset\n\n";
 241 }
 242
 243
 244 string curr_env()
 245 {
 246         return active_environments.empty() ? string() : active_environments.top();
 247 }
 248
 249
 250 void handle_ert(ostream & os, string const & s)
 251 {
 252         begin_inset(os, "ERT");
 253         os << "\nstatus Collapsed\n\n\\layout Standard\n\n";
 254         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 255                 if (*it == '\\')
 256                         os << "\n\\backslash\n";
 257                 else
 258                         os << *it;
 259         }
 260         end_inset(os);
 261 }
 262
 263
 264 void handle_par(ostream & os)
 265 {
 266         if (active_environments.empty())
 267                 return;
 268         os << "\n\\layout ";
 269         string s = curr_env();
 270         if (s == "document" || s == "table")
 271                 os << "Standard\n\n";
 272         else if (s == "lyxcode")
 273                 os << "LyX-Code\n\n";
 274         else if (s == "lyxlist")
 275                 os << "List\n\n";
 276         else if (s == "thebibliography")
 277                 os << "Bibliography\n\n";
 278         else
 279                 os << cap(s) << "\n\n";
 280 }
 281
 282
 283 void handle_package(string const & name, string const & options)
 284 {
 285         //cerr << "handle_package: '" << name << "'\n";
 286         if (name == "a4wide") {
 287                 h_papersize = "a4paper";
 288                 h_paperpackage = "widemarginsa4";
 289         } else if (name == "ae")
 290                 h_fontscheme = "ae";
 291         else if (name == "aecompl")
 292                 h_fontscheme = "ae";
 293         else if (name == "amsmath")
 294                 h_use_amsmath = "1";
 295         else if (name == "amssymb")
 296                 h_use_amsmath = "1";
 297         else if (name == "babel")
 298                 ; // ignore this
 299         else if (name == "fontenc")
 300                 ; // ignore this
 301         else if (name == "inputenc")
 302                 h_inputencoding = options;
 303         else if (name == "makeidx")
 304                 ; // ignore this
 305         else if (name == "verbatim")
 306                 ; // ignore this
 307         else if (is_known(name, known_languages)) {
 308                 h_language = name;
 309                 h_quotes_language = name;
 310         } else {
 311                 if (options.size())
 312                         h_preamble << "\\usepackage[" << options << "]{" << name << "}\n";
 313                 else
 314                         h_preamble << "\\usepackage{" << name << "}\n";
 315         }
 316 }
 317
 318
 319 vector<string> extract_col_align(string const & s)
 320 {
 321         vector<string> res;
 322         string t;
 323         for (size_t i = 0; i < s.size(); ++i) {
 324                 switch (s[i]) {
 325                         case 'c': res.push_back("center"); break;
 326                         case 'l': res.push_back("left");   break;
 327                         case 'r': res.push_back("right");  break;
 328                         //case '|': cerr << "ignoring vertical separator\n";  break;
 329                         //default : cerr << "ignoring special separator '" << s[i] << "'\n"; break;
 330                         default: break;
 331                 }
 332         }
 333         return res;
 334 }
 335
 336
 337 void handle_tabular(Parser & p, ostream & os, mode_type mode)
 338 {
 339         begin_inset(os, "Tabular \n");
 340         string posopts = p.getOpt();
 341         if (posopts.size())
 342                 cerr << "vertical tabular positioning '" << posopts << "' ignored\n";
 343         string colopts = p.verbatimItem();
 344         vector<string> colalign = extract_col_align(colopts);
 345         ostringstream ss;
 346         parse(p, ss, FLAG_END, mode);
 347         vector<string> lines;
 348         split(ss.str(), lines, LINE);
 349         const size_t cols = colalign.size();
 350         const size_t rows = lines.size();
 351         os << "<lyxtabular version=\"3\" rows=\"" << rows
 352                  << "\" columns=\"" << cols << "\">\n"
 353                  << "<features>\n";
 354         for (size_t c = 0; c < cols; ++c)
 355                 os << "<column alignment=\"" << colalign[c] << "\""
 356                          << " valignment=\"top\""
 357                          << " width=\"0pt\""
 358                          << ">\n";
 359         for (size_t r = 0; r < rows; ++r) {
 360                 vector<string> dummy;
 361                 // handle hlines
 362                 split(lines[r], dummy, HLINE);
 363                 int hlines = dummy.size() - 1;
 364                 lines[r] = join(dummy, "");
 365                 // handle almost empty line
 366                 if (lines[r].empty() && hlines && r > 0) {
 367                         cerr << "last hline lost\n";
 368                         continue;
 369                 }
 370                 // handle cells
 371                 vector<string> cells;
 372                 split(lines[r], cells, TAB);
 373                 while (cells.size() < cols)
 374                         cells.push_back(string());
 375                 os << "<row";
 376                 if (hlines)
 377                         os << " topline=\"true\"";
 378                 os << ">\n";
 379                 for (size_t c = 0; c < cols; ++c) {
 380                         os << "<cell";
 381                         string alignment = "center";
 382                         vector<string> parts;
 383                         split(cells[c], parts, MULT);
 384                         if (parts.size() > 2) {
 385                                 os << " multicolumn=\"" << parts[0] << "\"";
 386                                 alignment = parts[1];
 387                         }
 388                         os << " alignment=\"" << alignment << "\""
 389                                  << " valignment=\"top\""
 390                                  << " topline=\"true\""
 391                                  << " leftline=\"true\""
 392                                  << " usebox=\"none\""
 393                                  << ">";
 394                         begin_inset(os, "Text");
 395                         os << "\n\n\\layout Standard\n\n";
 396                         if (parts.size())
 397                                 os << parts.back();
 398                         end_inset(os);
 399                         os << "</cell>\n";
 400                 }
 401                 os << "</row>\n";
 402         }
 403         os << "</lyxtabular>\n";
 404         end_inset(os);
 405 }
 406
 407
 408 string wrap(string const & cmd, string const & str)
 409 {
 410         return OPEN + cmd + ' ' + str + CLOSE;
 411 }
 412
 413
 414 void end_preamble(ostream & os)
 415 {
 416         os << "# tex2lyx 0.0.2 created this file\n"
 417            << "\\lyxformat 222\n"
 418            << "\\textclass " << h_textclass << "\n"
 419            << "\\begin_preamble\n" << h_preamble.str() << "\n\\end_preamble\n"
 420            << "\\options " << h_options << "\n"
 421            << "\\language " << h_language << "\n"
 422            << "\\inputencoding " << h_inputencoding << "\n"
 423            << "\\fontscheme " << h_fontscheme << "\n"
 424            << "\\graphics " << h_graphics << "\n"
 425            << "\\paperfontsize " << h_paperfontsize << "\n"
 426            << "\\spacing " << h_spacing << "\n"
 427            << "\\papersize " << h_papersize << "\n"
 428            << "\\paperpackage " << h_paperpackage << "\n"
 429            << "\\use_geometry " << h_use_geometry << "\n"
 430            << "\\use_amsmath " << h_use_amsmath << "\n"
 431            << "\\use_natbib " << h_use_natbib << "\n"
 432            << "\\use_numerical_citations " << h_use_numerical_citations << "\n"
 433            << "\\paperorientation " << h_paperorientation << "\n"
 434            << "\\secnumdepth " << h_secnumdepth << "\n"
 435            << "\\tocdepth " << h_tocdepth << "\n"
 436            << "\\paragraph_separation " << h_paragraph_separation << "\n"
 437            << "\\defskip " << h_defskip << "\n"
 438            << "\\quotes_language " << h_quotes_language << "\n"
 439            << "\\quotes_times " << h_quotes_times << "\n"
 440            << "\\papercolumns " << h_papercolumns << "\n"
 441            << "\\papersides " << h_papersides << "\n"
 442            << "\\paperpagestyle " << h_paperpagestyle << "\n"
 443            << "\\tracking_changes " << h_tracking_changes << "\n";
 444 }
 445
 446
 447 void parse_preamble(Parser & p, ostream & os)
 448 {
 449         while (p.good()) {
 450                 Token const & t = p.getToken();
 451
 452 #ifdef FILEDEBUG
 453                 cerr << "t: " << t << " flags: " << flags << "\n";
 454                 //cell->dump();
 455 #endif
 456
 457                 //
 458                 // cat codes
 459                 //
 460                 if (t.cat() == catLetter ||
 461                           t.cat() == catSpace ||
 462                           t.cat() == catSuper ||
 463                           t.cat() == catSub ||
 464                           t.cat() == catOther ||
 465                           t.cat() == catMath ||
 466                           t.cat() == catActive ||
 467                           t.cat() == catBegin ||
 468                           t.cat() == catEnd ||
 469                           t.cat() == catAlign ||
 470                           t.cat() == catNewline ||
 471                           t.cat() == catParameter)
 472                 h_preamble << t.character();
 473
 474                 else if (t.cat() == catComment) {
 475                         string s;
 476                         while (p.good()) {
 477                                 Token const & t = p.getToken();
 478                                 if (t.cat() == catNewline)
 479                                         break;
 480                                 s += t.asString();
 481                         }
 482                         //os << wrap("comment", s);
 483                         p.skipSpaces();
 484                 }
 485
 486                 else if (t.cs() == "pagestyle")
 487                         h_paperpagestyle == p.verbatimItem();
 488
 489                 else if (t.cs() == "makeatletter") {
 490                         p.setCatCode('@', catLetter);
 491                         h_preamble << "\\makeatletter\n";
 492                 }
 493
 494                 else if (t.cs() == "makeatother") {
 495                         p.setCatCode('@', catOther);
 496                         h_preamble << "\\makeatother\n";
 497                 }
 498
 499                 else if (t.cs() == "newcommand" || t.cs() == "renewcommand"
 500                             || t.cs() == "providecommand") {
 501                         string const name = p.verbatimItem();
 502                         string const opts = p.getOpt();
 503                         string const body = p.verbatimItem();
 504                         // only non-lyxspecific stuff
 505                         if (name != "\\noun "
 506                                   && name != "\\tabularnewline "
 507                             && name != "\\LyX "
 508                                   && name != "\\lyxline "
 509                                   && name != "\\lyxaddress "
 510                                   && name != "\\lyxrightaddress "
 511                                   && name != "\\boldsymbol "
 512                                   && name != "\\lyxarrow ") {
 513                                 ostringstream ss;
 514                                 ss << '\\' << t.cs() << '{' << name << '}'
 515                                         << opts << '{' << body << "}\n";
 516                                 h_preamble << ss.str();
 517 /*
 518                                 ostream & out = in_preamble ? h_preamble : os;
 519                                 out << "\\" << t.cs() << "{" << name << "}"
 520                                     << opts << "{" << body << "}\n";
 521                                 if (!in_preamble)
 522                                         end_inset(os);
 523 */
 524                         }
 525                 }
 526
 527                 else if (t.cs() == "documentclass") {
 528                         vector<string> opts;
 529                         split(p.getArg('[', ']'), opts, ',');
 530                         handle_opt(opts, known_languages, h_language);
 531                         handle_opt(opts, known_fontsizes, h_paperfontsize);
 532                         h_quotes_language = h_language;
 533                         h_options = join(opts, ",");
 534                         h_textclass = p.getArg('{', '}');
 535                 }
 536
 537                 else if (t.cs() == "usepackage") {
 538                         string const options = p.getArg('[', ']');
 539                         string const name = p.getArg('{', '}');
 540                         if (options.empty() && name.find(',')) {
 541                                 vector<string> vecnames;
 542                                 split(name, vecnames, ',');
 543                                 vector<string>::const_iterator it  = vecnames.begin();
 544                                 vector<string>::const_iterator end = vecnames.end();
 545                                 for (; it != end; ++it)
 546                                         handle_package(trim(*it), string());
 547                         } else {
 548                                 handle_package(name, options);
 549                         }
 550                 }
 551
 552                 else if (t.cs() == "newenvironment") {
 553                         string const name = p.getArg('{', '}');
 554                         ostringstream ss;
 555                         ss << "\\newenvironment{" << name << "}";
 556                         ss << p.getOpt();
 557                         ss << p.getOpt();
 558                         ss << '{' << p.verbatimItem() << '}';
 559                         ss << '{' << p.verbatimItem() << '}';
 560                         ss << '\n';
 561                         if (name != "lyxcode" && name != "lyxlist"
 562                                         && name != "lyxrightadress" && name != "lyxaddress")
 563                                 h_preamble << ss.str();
 564                 }
 565
 566                 else if (t.cs() == "def") {
 567                         string name = p.getToken().cs();
 568                         while (p.nextToken().cat() != catBegin)
 569                                 name += p.getToken().asString();
 570                         h_preamble << "\\def\\" << name << '{' << p.verbatimItem() << "}\n";
 571                 }
 572
 573                 else if (t.cs() == "setcounter") {
 574                         string const name = p.getArg('{', '}');
 575                         string const content = p.getArg('{', '}');
 576                         if (name == "secnumdepth")
 577                                 h_secnumdepth = content;
 578                         else if (name == "tocdepth")
 579                                 h_tocdepth = content;
 580                         else
 581                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}\n";
 582                 }
 583
 584                 else if (t.cs() == "setlength") {
 585                         string const name = p.verbatimItem();
 586                         string const content = p.verbatimItem();
 587                         if (name == "parskip")
 588                                 h_paragraph_separation = "skip";
 589                         else if (name == "parindent")
 590                                 h_paragraph_separation = "skip";
 591                         else
 592                                 h_preamble << "\\setlength{" + name + "}{" + content + "}\n";
 593                 }
 594
 595                 else if (t.cs() == "par")
 596                         h_preamble << '\n';
 597
 598                 else if (t.cs() == "begin") {
 599                         string const name = p.getArg('{', '}');
 600                         if (name == "document") {
 601                                 end_preamble(os);
 602                                 os << "\n\n\\layout Standard\n\n";
 603                                 return;
 604                         }
 605                         h_preamble << "\\begin{" << name << "}";
 606                 }
 607
 608                 else if (t.cs().size())
 609                         h_preamble << '\\' << t.cs() << ' ';
 610         }
 611 }
 612
 613
 614 void parse(Parser & p, ostream & os, unsigned flags, const mode_type mode)
 615 {
 616         while (p.good()) {
 617                 Token const & t = p.getToken();
 618
 619 #ifdef FILEDEBUG
 620                 cerr << "t: " << t << " flags: " << flags << "\n";
 621                 //cell->dump();
 622 #endif
 623
 624                 if (flags & FLAG_ITEM) {
 625                         if (t.cat() == catSpace)
 626                                 continue;
 627
 628                         flags &= ~FLAG_ITEM;
 629                         if (t.cat() == catBegin) {
 630                                 // skip the brace and collect everything to the next matching
 631                                 // closing brace
 632                                 flags |= FLAG_BRACE_LAST;
 633                                 continue;
 634                         }
 635
 636                         // handle only this single token, leave the loop if done
 637                         flags |= FLAG_LEAVE;
 638                 }
 639
 640
 641                 if (flags & FLAG_BRACED) {
 642                         if (t.cat() == catSpace)
 643                                 continue;
 644
 645                         if (t.cat() != catBegin) {
 646                                 p.error("opening brace expected");
 647                                 return;
 648                         }
 649
 650                         // skip the brace and collect everything to the next matching
 651                         // closing brace
 652                         flags = FLAG_BRACE_LAST;
 653                 }
 654
 655
 656                 if (flags & FLAG_OPTION) {
 657                         if (t.cat() == catOther && t.character() == '[') {
 658                                 parse(p, os, FLAG_BRACK_LAST, mode);
 659                         } else {
 660                                 // no option found, put back token and we are done
 661                                 p.putback();
 662                         }
 663                         return;
 664                 }
 665
 666                 //
 667                 // cat codes
 668                 //
 669                 if (t.cat() == catMath) {
 670                         if (mode == TEXT_MODE || mode == MATHTEXT_MODE) {
 671                                 // we are inside some text mode thingy, so opening new math is allowed
 672                                 if (mode == TEXT_MODE)
 673                                         begin_inset(os, "Formula ");
 674                                 Token const & n = p.getToken();
 675                                 if (n.cat() == catMath) {
 676                                         // TeX's $$...$$ syntax for displayed math
 677                                         os << "\\[";
 678                                         parse(p, os, FLAG_SIMPLE, MATH_MODE);
 679                                         os << "\\]";
 680                                         p.getToken(); // skip the second '$' token
 681                                 } else {
 682                                         // simple $...$  stuff
 683                                         p.putback();
 684                                         os << '$';
 685                                         parse(p, os, FLAG_SIMPLE, MATH_MODE);
 686                                         os << '$';
 687                                 }
 688                                 if (mode == TEXT_MODE)
 689                                         end_inset(os);
 690                         }
 691
 692                         else if (flags & FLAG_SIMPLE) {
 693                                 // this is the end of the formula
 694                                 return;
 695                         }
 696
 697                         else {
 698                                 cerr << "mode: " << mode << endl;
 699                                 p.error("something strange in the parser\n");
 700                                 break;
 701                         }
 702                 }
 703
 704                 else if (t.cat() == catLetter ||
 705                                t.cat() == catSpace ||
 706                          t.cat() == catSuper ||
 707                                t.cat() == catSub ||
 708                                t.cat() == catOther ||
 709                                t.cat() == catParameter)
 710                         os << t.character();
 711
 712                 else if (t.cat() == catNewline)
 713                         os << ' ';
 714
 715                 else if (t.cat() == catActive) {
 716                         if (t.character() == '~') {
 717                                 if (curr_env() == "lyxcode")
 718                                         os << ' ';
 719                                 else if (mode == TEXT_MODE)
 720                                         os << "\\SpecialChar ~\n";
 721                                 else
 722                                         os << '~';
 723                         } else
 724                                 os << t.character();
 725                 }
 726
 727                 else if (t.cat() == catBegin) {
 728                         if (mode == TEXT_MODE)
 729                                 handle_ert(os, "{");
 730                         else
 731                                 os << '{';
 732                 }
 733
 734                 else if (t.cat() == catEnd) {
 735                         if (flags & FLAG_BRACE_LAST)
 736                                 return;
 737                         if (mode == TEXT_MODE)
 738                                 handle_ert(os, "}");
 739                         else
 740                                 os << '}';
 741                 }
 742
 743                 else if (t.cat() == catAlign) {
 744                         if (mode == TEXT_MODE)
 745                                 os << TAB;
 746                         else
 747                                 os << t.character();
 748                 }
 749
 750                 else if (t.cs() == "tabularnewline") {
 751                         if (mode == TEXT_MODE)
 752                                 os << LINE;
 753                         else
 754                                 os << t.asInput();
 755                 }
 756
 757                 else if (t.cs() == "\\" && mode == MATH_MODE)
 758                         os << t.asInput();
 759
 760                 else if (t.cs() == "\\" && mode == TEXT_MODE && curr_env() == "tabular")
 761                         os << LINE;
 762
 763                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 764                         //cerr << "finished reading option\n";
 765                         return;
 766                 }
 767
 768                 else if (t.cat() == catOther)
 769                         os << string(1, t.character());
 770
 771                 else if (t.cat() == catComment) {
 772                         string s;
 773                         while (p.good()) {
 774                                 Token const & t = p.getToken();
 775                                 if (t.cat() == catNewline)
 776                                         break;
 777                                 s += t.asString();
 778                         }
 779                         //os << wrap("comment", s);
 780                         p.skipSpaces();
 781                 }
 782
 783                 //
 784                 // control sequences
 785                 //
 786
 787                 else if (t.cs() == "ldots" && mode == MATH_MODE)
 788                         os << "\n\\SpecialChar \\ldots{}\n";
 789
 790                 else if (t.cs() == "lyxlock")
 791                         ; // ignored
 792
 793                 else if (t.cs() == "makeatletter") {
 794                         p.setCatCode('@', catLetter);
 795                         handle_ert(os, "\\makeatletter\n");
 796                 }
 797
 798                 else if (t.cs() == "makeatother") {
 799                         p.setCatCode('@', catOther);
 800                         handle_ert(os, "\\makeatother\n");
 801                 }
 802
 803                 else if (t.cs() == "newcommand" || t.cs() == "renewcommand"
 804                             || t.cs() == "providecommand") {
 805                         string const name = p.verbatimItem();
 806                         string const opts = p.getOpt();
 807                         string const body = p.verbatimItem();
 808                         // only non-lyxspecific stuff
 809                         if (name != "\\noun " && name != "\\tabularnewline ") {
 810                                 ostringstream ss;
 811                                 ss << '\\' << t.cs() << '{' << name << '}'
 812                                         << opts << '{' << body << "}\n";
 813                                 handle_ert(os, ss.str());
 814 /*
 815                                 ostream & out = in_preamble ? h_preamble : os;
 816                                 if (!in_preamble)
 817                                         begin_inset(os, "FormulaMacro\n");
 818                                 out << "\\" << t.cs() << "{" << name << "}"
 819                                     << opts << "{" << body << "}\n";
 820                                 if (!in_preamble)
 821                                         end_inset(os);
 822 */
 823                         }
 824                 }
 825
 826                 else if (t.cs() == "newtheorem") {
 827                         ostringstream ss;
 828                         ss << "\\newtheorem";
 829                         ss << '{' << p.verbatimItem() << '}';
 830                         ss << p.getOpt();
 831                         ss << '{' << p.verbatimItem() << '}';
 832                         ss << p.getOpt();
 833                         ss << '\n';
 834                         handle_ert(os, ss.str());
 835                 }
 836
 837                 else if (t.cs() == "(") {
 838                         begin_inset(os, "Formula");
 839                         os << " \\(";
 840                         parse(p, os, FLAG_SIMPLE2, MATH_MODE);
 841                         os << "\\)";
 842                         end_inset(os);
 843                 }
 844
 845                 else if (t.cs() == "[") {
 846                         begin_inset(os, "Formula");
 847                         os << " \\[";
 848                         parse(p, os, FLAG_EQUATION, MATH_MODE);
 849                         os << "\\]";
 850                         end_inset(os);
 851                 }
 852
 853                 else if (t.cs() == "protect")
 854                         // ignore \\protect, will hopefully be re-added during output
 855                         ;
 856
 857                 else if (t.cs() == "begin") {
 858                         string const name = p.getArg('{', '}');
 859                         active_environments.push(name);
 860                         if (name == "abstract") {
 861                                 handle_par(os);
 862                                 parse(p, os, FLAG_END, mode);
 863                         } else if (is_math_env(name)) {
 864                                 begin_inset(os, "Formula ");
 865                                 os << "\\begin{" << name << "}";
 866                                 parse(p, os, FLAG_END, MATH_MODE);
 867                                 os << "\\end{" << name << "}";
 868                                 end_inset(os);
 869                         } else if (name == "tabular") {
 870                                 if (mode == TEXT_MODE)
 871                                         handle_tabular(p, os, mode);
 872                                 else {
 873                                         os << "\\begin{" << name << "}";
 874                                         parse(p, os, FLAG_END, MATHTEXT_MODE);
 875                                         os << "\\end{" << name << "}";
 876                                 }
 877                         } else if (name == "table" || name == "figure") {
 878                                 string opts = p.getOpt();
 879                                 begin_inset(os, "Float " + name + "\n");
 880                                 if (opts.size())
 881                                         os << "placement " << opts << '\n';
 882                                 os << "wide false\n"
 883                                          << "collapsed false\n"
 884                                          << "\n"
 885                                          << "\\layout Standard\n";
 886                                 parse(p, os, FLAG_END, mode);
 887                                 end_inset(os);
 888                         } else if (name == "thebibliography") {
 889                                 p.verbatimItem(); // swallow next arg
 890                                 parse(p, os, FLAG_END, mode);
 891                                 os << "\n\\layout Bibliography\n\n";
 892                         } else if (mode == MATH_MODE || mode == MATHTEXT_MODE) {
 893                                 os << "\\begin{" << name << "}";
 894                                 parse(p, os, FLAG_END, mode);
 895                                 os << "\\end{" << name << "}";
 896                         } else {
 897                                 parse(p, os, FLAG_END, mode);
 898                         }
 899                 }
 900
 901                 else if (t.cs() == "end") {
 902                         if (flags & FLAG_END) {
 903                                 // eat environment name
 904                                 string const name = p.getArg('{', '}');
 905                                 if (name != curr_env())
 906                                         p.error("\\end{" + name + "} does not match \\begin{"
 907                                                 + curr_env() + "}");
 908                                 active_environments.pop();
 909                                 return;
 910                         }
 911                         p.error("found 'end' unexpectedly");
 912                 }
 913
 914                 else if (t.cs() == "item")
 915                         handle_par(os);
 916
 917                 else if (t.cs() == ")") {
 918                         if (flags & FLAG_SIMPLE2)
 919                                 return;
 920                         p.error("found '\\)' unexpectedly");
 921                 }
 922
 923                 else if (t.cs() == "]") {
 924                         if (flags & FLAG_EQUATION)
 925                                 return;
 926                         p.error("found '\\]' unexpectedly");
 927                 }
 928
 929                 else if (t.cs() == "documentclass") {
 930                         vector<string> opts;
 931                         split(p.getArg('[', ']'), opts, ',');
 932                         handle_opt(opts, known_languages, h_language);
 933                         handle_opt(opts, known_fontsizes, h_paperfontsize);
 934                         h_quotes_language = h_language;
 935                         h_options = join(opts, ",");
 936                         h_textclass = p.getArg('{', '}');
 937                 }
 938
 939                 else if (t.cs() == "usepackage") {
 940                         string const options = p.getArg('[', ']');
 941                         string const name = p.getArg('{', '}');
 942                         if (options.empty() && name.find(',')) {
 943                                 vector<string> vecnames;
 944                                 split(name, vecnames, ',');
 945                                 vector<string>::const_iterator it  = vecnames.begin();
 946                                 vector<string>::const_iterator end = vecnames.end();
 947                                 for (; it != end; ++it)
 948                                         handle_package(trim(*it), string());
 949                         } else {
 950                                 handle_package(name, options);
 951                         }
 952                 }
 953
 954                 else if (t.cs() == "def") {
 955                         string name = p.getToken().cs();
 956                         while (p.nextToken().cat() != catBegin)
 957                                 name += p.getToken().asString();
 958                         handle_ert(os, "\\def\\" + name + '{' + p.verbatimItem() + '}');
 959                 }
 960
 961                 else if (t.cs() == "par")
 962                         handle_par(os);
 963
 964                 else if (is_known(t.cs(), known_headings)) {
 965                         string name = t.cs();
 966                         if (p.nextToken().asInput() == "*") {
 967                                 p.getToken();
 968                                 name += "*";
 969                         }
 970                         os << "\n\n\\layout " << cap(name) << "\n\n";
 971                         string opt = p.getOpt();
 972                         if (opt.size()) {
 973                                 begin_inset(os, "OptArg\n");
 974                                 os << "collapsed true\n\n\\layout Standard\n\n" << opt;
 975                                 end_inset(os);
 976                         }
 977                         parse(p, os, FLAG_ITEM, mode);
 978                         os << "\n\n\\layout Standard\n\n";
 979                 }
 980
 981                 else if (t.cs() == "includegraphics") {
 982                         if (mode == TEXT_MODE) {
 983                                 map<string, string> opts = split_map(p.getArg('[', ']'));
 984                                 string name = p.verbatimItem();
 985                                 begin_inset(os, "Graphics ");
 986                                 os << "\n\tfilename " << name << '\n';
 987                                 if (opts.find("width") != opts.end())
 988                                         os << "\twidth " << opts["width"] << '\n';
 989                                 if (opts.find("height") != opts.end())
 990                                         os << "\theight " << opts["height"] << '\n';
 991                                 end_inset(os);
 992                         } else {
 993                                 os << "\\includegraphics ";
 994                         }
 995                 }
 996
 997                 else if (t.cs() == "makeindex" || t.cs() == "maketitle")
 998                         ; // swallow this
 999
1000                 else if (t.cs() == "tableofcontents")
1001                         p.verbatimItem(); // swallow this
1002
1003                 else if (t.cs() == "multicolumn" && mode == TEXT_MODE) {
1004                         // brutish...
1005                         parse(p, os, FLAG_ITEM, mode);
1006                         os << MULT;
1007                         parse(p, os, FLAG_ITEM, mode);
1008                         os << MULT;
1009                         parse(p, os, FLAG_ITEM, mode);
1010                 }
1011
1012                 else if (t.cs() == "hline" && mode == TEXT_MODE)
1013                         os << HLINE;
1014
1015                 else if (t.cs() == "textrm") {
1016                         if (mode == TEXT_MODE) {
1017                                 os << "\n\\family roman\n";
1018                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1019                                 os << "\n\\family default\n";
1020                         } else {
1021                                 os << '\\' << t.cs() << '{';
1022                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1023                                 os << '}';
1024                         }
1025                 }
1026
1027                 else if (t.cs() == "textsf") {
1028                         if (mode == TEXT_MODE) {
1029                                 os << "\n\\family sans\n";
1030                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1031                                 os << "\n\\family default\n";
1032                         } else {
1033                                 os << '\\' << t.cs() << '{';
1034                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1035                                 os << '}';
1036                         }
1037                 }
1038
1039                 else if (t.cs() == "texttt") {
1040                         if (mode == TEXT_MODE) {
1041                                 os << "\n\\family typewriter\n";
1042                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1043                                 os << "\n\\family default\n";
1044                         } else {
1045                                 os << '\\' << t.cs() << '{';
1046                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1047                                 os << '}';
1048                         }
1049                 }
1050
1051                 else if (t.cs() == "textsc") {
1052                         if (mode == TEXT_MODE) {
1053                                 os << "\n\\noun on\n";
1054                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1055                                 os << "\n\\noun default\n";
1056                         } else {
1057                                 os << '\\' << t.cs() << '{';
1058                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1059                                 os << '}';
1060                         }
1061                 }
1062
1063                 else if (t.cs() == "textbf") {
1064                         if (mode == TEXT_MODE) {
1065                                 os << "\n\\series bold\n";
1066                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1067                                 os << "\n\\series default\n";
1068                         } else {
1069                                 os << '\\' << t.cs() << '{';
1070                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1071                                 os << '}';
1072                         }
1073                 }
1074
1075                 else if (t.cs() == "underbar") {
1076                         if (mode == TEXT_MODE) {
1077                                 os << "\n\\bar under\n";
1078                                 parse(p, os, FLAG_ITEM, TEXT_MODE);
1079                                 os << "\n\\bar default\n";
1080                         } else {
1081                                 os << '\\' << t.cs() << '{';
1082                                 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
1083                                 os << '}';
1084                         }
1085                 }
1086
1087                 else if ((t.cs() == "emph" || t.cs() == "noun") && mode == TEXT_MODE) {
1088                         os << "\n\\" << t.cs() << " on\n";
1089                         parse(p, os, FLAG_ITEM, mode);
1090                         os << "\n\\" << t.cs() << " default\n";
1091                 }
1092
1093                 else if (is_known(t.cs(), known_latex_commands) && mode == TEXT_MODE) {
1094                         begin_inset(os, "LatexCommand ");
1095                         os << '\\' << t.cs();
1096                         os << p.getOpt();
1097                         os << p.getOpt();
1098                         os << '{' << p.verbatimItem() << '}';
1099                         end_inset(os);
1100                 }
1101
1102                 else if (t.cs() == "bibitem") {
1103                         os << "\n\\layout Bibliography\n\\bibitem ";
1104                         os << p.getOpt();
1105                         os << '{' << p.verbatimItem() << '}' << "\n\n";
1106                 }
1107
1108                 else if (mode == TEXT_MODE && is_known(t.cs(), known_quotes)) {
1109                   char const ** where = is_known(t.cs(), known_quotes);
1110                         begin_inset(os, "Quotes ");
1111                         os << known_coded_quotes[where - known_quotes];
1112                         end_inset(os);
1113                 }
1114
1115                 else if (t.cs() == "LyX") {
1116                         p.verbatimItem(); // eat {}
1117                         os << "LyX";
1118                 }
1119
1120                 else if (t.cs() == "TeX") {
1121                         p.verbatimItem(); // eat {}
1122                         os << "TeX";
1123                 }
1124
1125                 else if (t.cs() == "LaTeX") {
1126                         p.verbatimItem(); // eat {}
1127                         os << "LaTeX";
1128                 }
1129
1130                 else if (t.cs() == "LaTeXe") {
1131                         p.verbatimItem(); // eat {}
1132                         os << "LaTeXe";
1133                 }
1134
1135                 else if (t.cs() == "textasciitilde")
1136                         os << '~';
1137
1138                 else if (t.cs() == "_" && mode == TEXT_MODE)
1139                         os << '_';
1140
1141                 else if (t.cs() == "&" && mode == TEXT_MODE)
1142                         os << '&';
1143
1144                 else if (t.cs() == "\"") {
1145                         string const name = p.verbatimItem();
1146                              if (name == "a") os << 'ä';
1147                         else if (name == "o") os << 'ö';
1148                         else if (name == "u") os << 'ü';
1149                         else if (name == "A") os << 'Ä';
1150                         else if (name == "O") os << 'Ö';
1151                         else if (name == "U") os << 'Ü';
1152                         else handle_ert(os, "\"{" + name + "}");
1153                 }
1154
1155                 else if (t.cs() == "ss")
1156                         os << "ß";
1157
1158                 else if (t.cs() == "input")
1159                         handle_ert(os, "\\input{" + p.verbatimItem() + "}\n");
1160
1161                 else if (t.cs() == "fancyhead") {
1162                         ostringstream ss;
1163                         ss << "\\fancyhead";
1164                         ss << p.getOpt();
1165                         ss << '{' << p.verbatimItem() << "}\n";
1166                         handle_ert(os, ss.str());
1167                 }
1168
1169                 else {
1170                         //cerr << "#: " << t << " mode: " << mode << endl;
1171                         if (mode == MATH_MODE || mode == MATHTEXT_MODE) {
1172                                 os << t.asInput();
1173                                 //cerr << "#: writing: '" << t.asInput() << "'\n";
1174                         } else {
1175                                 // heuristic: read up to next non-nested space
1176                                 /*
1177                                 string s = t.asInput();
1178                                 string z = p.verbatimItem();
1179                                 while (p.good() && z != " " && z.size()) {
1180                                         //cerr << "read: " << z << endl;
1181                                         s += z;
1182                                         z = p.verbatimItem();
1183                                 }
1184                                 cerr << "found ERT: " << s << endl;
1185                                 handle_ert(os, s + ' ');
1186                                 */
1187                                 handle_ert(os, t.asInput() + ' ');
1188                         }
1189                 }
1190
1191                 if (flags & FLAG_LEAVE) {
1192                         flags &= ~FLAG_LEAVE;
1193                         break;
1194                 }
1195         }
1196 }
1197
1198
1199
1200 } // anonymous namespace
1201
1202
1203 int main(int argc, char * argv[])
1204 {
1205         if (argc <= 1) {
1206                 cerr << "Usage: " << argv[0] << " <infile.tex>" << endl;
1207                 return 2;
1208         }
1209
1210         ifstream is(argv[1]);
1211         Parser p(is);
1212         parse_preamble(p, cout);
1213         active_environments.push("document");
1214         parse(p, cout, FLAG_END, TEXT_MODE);
1215         cout << "\n\\the_end";
1216
1217         return 0;
1218 }
1219
1220 // }])