src/tex2lyx/tex2lyx.C

   1 /** The .tex to .lyx converter
   2     \author André Pönitz (2003)
   3  */
   4
   5 // {[(
   6
   7 #include <config.h>
   8
   9 #include <algorithm>
  10 #include <cctype>
  11 #include <fstream>
  12 #include <iostream>
  13 #include <sstream>
  14 #include <stack>
  15 #include <string>
  16 #include <vector>
  17
  18 #include "texparser.h"
  19
  20 using std::cout;
  21 using std::cerr;
  22 using std::endl;
  23 using std::fill;
  24 using std::getline;
  25 using std::ios;
  26 using std::ifstream;
  27 using std::istream;
  28 using std::istringstream;
  29 using std::ostream;
  30 using std::ostringstream;
  31 using std::stack;
  32 using std::string;
  33 using std::vector;
  34
  35
  36 namespace {
  37
  38 char const OPEN = '<';
  39 char const CLOSE = '>';
  40
  41 const char * known_languages[] = { "austrian", "babel", "bahasa",
  42 "basque", "breton", "bulgarian", "catalan", "croatian", "czech", "danish",
  43 "dutch", "english", "esperanto", "estonian", "finnish", "francais",
  44 "frenchb", "galician", "germanb", "greek", "hebcal", "hebfont", "hebrew",
  45 "hebrew_newcode", "hebrew_oldcode", "hebrew_p", "hyphen", "icelandic",
  46 "irish", "italian", "latin", "lgrcmr", "lgrcmro", "lgrcmss", "lgrcmtt",
  47 "lgrenc", "lgrlcmss", "lgrlcmtt", "lheclas", "lhecmr", "lhecmss",
  48 "lhecmtt", "lhecrml", "lheenc", "lhefr", "lheredis", "lheshold",
  49 "lheshscr", "lheshstk", "lsorbian", "magyar", "naustrian", "ngermanb",
  50 "ngerman", "norsk", "polish", "portuges", "rlbabel", "romanian",
  51 "russianb", "samin", "scottish", "serbian", "slovak", "slovene", "spanish",
  52 "swedish", "turkish", "ukraineb", "usorbian", "welsh", 0};
  53
  54 const char * known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
  55
  56
  57 const char * known_math_envs[] = {"equation", "eqnarray", "eqnarray*",
  58 "align", 0};
  59
  60
  61 // some ugly stuff
  62 ostringstream h_preamble;
  63 string h_textclass               = "FIXME";
  64 string h_options                 = "FIXME";
  65 string h_language                = "FIXME";
  66 string h_inputencoding           = "FIXME";
  67 string h_fontscheme              = "FIXME";
  68 string h_graphics                = "default";
  69 string h_paperfontsize           = "FIXME";
  70 string h_spacing                 = "single";
  71 string h_papersize               = "FIXME";
  72 string h_paperpackage            = "FIXME";
  73 string h_use_geometry            = "0";
  74 string h_use_amsmath             = "0";
  75 string h_use_natbib              = "0";
  76 string h_use_numerical_citations = "0";
  77 string h_paperorientation        = "portrait";
  78 string h_secnumdepth             = "3";
  79 string h_tocdepth                = "3";
  80 string h_paragraph_separation    = "indent";
  81 string h_defskip                 = "medskip";
  82 string h_quotes_language         = "FIXME";
  83 string h_quotes_times            = "1";
  84 string h_papercolumns            = "1";
  85 string h_papersides              = "1";
  86 string h_paperpagestyle          = "default";
  87 string h_tracking_changes        = "0";
  88
  89 // indicates whether we are in the preamble
  90 bool in_preamble = true;
  91
  92 // current stack of nested environments
  93 stack<string> active_environments;
  94
  95
  96 string cap(string s)
  97 {
  98         if (s.size())
  99                 s[0] = toupper(s[0]);
 100         return s;
 101 }
 102
 103
 104 string const trim(string const & a, char const * p = " ")
 105 {
 106         // lyx::Assert(p);
 107
 108         if (a.empty() || !*p)
 109                 return a;
 110
 111         string::size_type r = a.find_last_not_of(p);
 112         string::size_type l = a.find_first_not_of(p);
 113
 114         // Is this the minimal test? (lgb)
 115         if (r == string::npos && l == string::npos)
 116                 return string();
 117
 118         return a.substr(l, r - l + 1);
 119 }
 120
 121
 122 void split(string const & s, vector<string> & result, char delim)
 123 {
 124         istringstream is(s);
 125         string t;
 126         while (getline(is, t, delim))
 127                 result.push_back(t);
 128 }
 129
 130
 131 string join(vector<string> const & input, char delim)
 132 {
 133         ostringstream os;
 134         for (size_t i = 0; i != input.size(); ++i) {
 135                 if (i)
 136                         os << delim;
 137                 os << input[i];
 138         }
 139         return os.str();
 140 }
 141
 142
 143 void handle_opt(vector<string> & opts, char const ** what, string & target)
 144 {
 145         if (opts.empty())
 146                 return;
 147
 148         for ( ; what; ++what) {
 149                 vector<string>::iterator it = find(opts.begin(), opts.end(), *what);
 150                 if (it != opts.end()) {
 151                         //cerr << "### found option '" << *what << "'\n";
 152                         target = *what;
 153                         opts.erase(it);
 154                         return;
 155                 }
 156         }
 157 }
 158
 159
 160 bool is_math_env(string const & name)
 161 {
 162         for (char const ** what = known_math_envs; *what; ++what)
 163                 if (*what == name)
 164                         return true;
 165         return false;
 166 }
 167
 168
 169 bool is_heading(string const & name)
 170 {
 171         return
 172                 name == "caption" ||
 173                 name == "title" ||
 174                 name == "author" ||
 175                 name == "paragraph" ||
 176                 name == "chapter" ||
 177                 name == "chapter*" ||
 178                 name == "section" ||
 179                 name == "section*" ||
 180                 name == "subsection" ||
 181                 name == "subsection*" ||
 182                 name == "subsubsection" ||
 183                 name == "subsubsection*";
 184 }
 185
 186
 187 bool is_latex_command(string const & name)
 188 {
 189         return
 190                 name == "label" ||
 191                 name == "index" ||
 192                 name == "printindex";
 193 }
 194
 195
 196 void begin_inset(ostream & os, string const & name)
 197 {
 198         os << "\n\\begin_inset " << name;
 199 }
 200
 201
 202 void end_inset(ostream & os)
 203 {
 204         os << "\n\\end_inset\n";
 205 }
 206
 207
 208 void handle_ert(ostream & os, string const & s)
 209 {
 210         begin_inset(os, "ERT");
 211         os << "\nstatus Collapsed\n\n\\layout Standard\n\n";
 212         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 213                 if (*it == '\\')
 214                         os << "\n\\backslash\n";
 215                 else
 216                         os << *it;
 217         }
 218         end_inset(os);
 219 }
 220
 221
 222 void handle_par(ostream & os)
 223 {
 224         if (active_environments.empty())
 225                 return;
 226         os << "\n\\layout ";
 227         string s = active_environments.top();
 228         if (s == "document") {
 229                 os << "Standard\n\n";
 230                 return;
 231         }
 232         if (s == "lyxcode") {
 233                 os << "LyX-Code\n\n";
 234                 return;
 235         }
 236         os << cap(s) << "\n\n";
 237 }
 238
 239
 240 void handle_package(string const & name, string const & options)
 241 {
 242         if (name == "a4wide") {
 243                 h_papersize = "a4paper";
 244                 h_paperpackage = "widemarginsa4";
 245         } else if (name == "ae")
 246                 h_fontscheme = "ae";
 247         else if (name == "aecompl")
 248                 h_fontscheme = "ae";
 249         else if (name == "amsmath")
 250                 h_use_amsmath = "1";
 251         else if (name == "amssymb")
 252                 h_use_amsmath = "1";
 253         else if (name == "babel")
 254                 ; // ignore this
 255         else if (name == "fontenc")
 256                 ; // ignore this
 257         else if (name == "inputenc")
 258                 h_inputencoding = options;
 259         else if (name == "makeidx")
 260                 ; // ignore this
 261         else if (name == "verbatim")
 262                 ; // ignore this
 263         else {
 264                 if (options.size())
 265                         h_preamble << "\\usepackage[" << options << "]{" << name << "}\n";
 266                 else
 267                         h_preamble << "\\usepackage{" << name << "}\n";
 268         }
 269 }
 270
 271
 272 void handle_table(Parser & p, ostream & os)
 273 {
 274         // \begin{table} has been read
 275         //parse(end
 276 }
 277
 278
 279 string wrap(string const & cmd, string const & str)
 280 {
 281         return OPEN + cmd + ' ' + str + CLOSE;
 282 }
 283
 284
 285 void end_preamble(ostream & os)
 286 {
 287         in_preamble = false;
 288         os << "# tex2lyx 0.0.2 created this file\n"
 289            << "\\lyxformat 222\n"
 290            << "\\textclass " << h_textclass << "\n"
 291            << "\\begin_preamble\n" << h_preamble.str() << "\n\\end_preamble\n"
 292            << "\\options " << h_options << "\n"
 293            << "\\language " << h_language << "\n"
 294            << "\\inputencoding " << h_inputencoding << "\n"
 295            << "\\fontscheme " << h_fontscheme << "\n"
 296            << "\\graphics " << h_graphics << "\n"
 297            << "\\paperfontsize " << h_paperfontsize << "\n"
 298            << "\\spacing " << h_spacing << "\n"
 299            << "\\papersize " << h_papersize << "\n"
 300            << "\\paperpackage " << h_paperpackage << "\n"
 301            << "\\use_geometry " << h_use_geometry << "\n"
 302            << "\\use_amsmath " << h_use_amsmath << "\n"
 303            << "\\use_natbib " << h_use_natbib << "\n"
 304            << "\\use_numerical_citations " << h_use_numerical_citations << "\n"
 305            << "\\paperorientation " << h_paperorientation << "\n"
 306            << "\\secnumdepth " << h_secnumdepth << "\n"
 307            << "\\tocdepth " << h_tocdepth << "\n"
 308            << "\\paragraph_separation " << h_paragraph_separation << "\n"
 309            << "\\defskip " << h_defskip << "\n"
 310            << "\\quotes_language " << h_quotes_language << "\n"
 311            << "\\quotes_times " << h_quotes_times << "\n"
 312            << "\\papercolumns " << h_papercolumns << "\n"
 313            << "\\papersides " << h_papersides << "\n"
 314            << "\\paperpagestyle " << h_paperpagestyle << "\n"
 315            << "\\tracking_changes " << h_tracking_changes << "\n";
 316 }
 317
 318
 319 void parse(Parser & p, ostream & os, unsigned flags, mode_type mode)
 320 {
 321         while (p.good()) {
 322                 Token const & t = p.getToken();
 323
 324 #ifdef FILEDEBUG
 325                 cerr << "t: " << t << " flags: " << flags << "\n";
 326                 //cell->dump();
 327 #endif
 328
 329                 if (flags & FLAG_ITEM) {
 330                         if (t.cat() == catSpace)
 331                                 continue;
 332
 333                         flags &= ~FLAG_ITEM;
 334                         if (t.cat() == catBegin) {
 335                                 // skip the brace and collect everything to the next matching
 336                                 // closing brace
 337                                 flags |= FLAG_BRACE_LAST;
 338                                 continue;
 339                         }
 340
 341                         // handle only this single token, leave the loop if done
 342                         flags |= FLAG_LEAVE;
 343                 }
 344
 345
 346                 if (flags & FLAG_BRACED) {
 347                         if (t.cat() == catSpace)
 348                                 continue;
 349
 350                         if (t.cat() != catBegin) {
 351                                 p.error("opening brace expected");
 352                                 return;
 353                         }
 354
 355                         // skip the brace and collect everything to the next matching
 356                         // closing brace
 357                         flags = FLAG_BRACE_LAST;
 358                 }
 359
 360
 361                 if (flags & FLAG_OPTION) {
 362                         if (t.cat() == catOther && t.character() == '[') {
 363                                 parse(p, os, FLAG_BRACK_LAST, mode);
 364                         } else {
 365                                 // no option found, put back token and we are done
 366                                 p.putback();
 367                         }
 368                         return;
 369                 }
 370
 371                 //
 372                 // cat codes
 373                 //
 374                 if (t.cat() == catMath) {
 375                         if (mode == TEXT_MODE || mode == MATHTEXT_MODE) {
 376                                 // we are inside some text mode thingy, so opening new math is allowed
 377                                 if (mode == TEXT_MODE)
 378                                         begin_inset(os, "Formula ");
 379                                 Token const & n = p.getToken();
 380                                 if (n.cat() == catMath) {
 381                                         // TeX's $$...$$ syntax for displayed math
 382                                         os << "\\[";
 383                                         parse(p, os, FLAG_SIMPLE, MATH_MODE);
 384                                         os << "\\]";
 385                                         p.getToken(); // skip the second '$' token
 386                                 } else {
 387                                         // simple $...$  stuff
 388                                         p.putback();
 389                                         os << '$';
 390                                         parse(p, os, FLAG_SIMPLE, MATH_MODE);
 391                                         os << '$';
 392                                 }
 393                                 if (mode == TEXT_MODE)
 394                                         end_inset(os);
 395                         }
 396
 397                         else if (flags & FLAG_SIMPLE) {
 398                                 // this is the end of the formula
 399                                 return;
 400                         }
 401
 402                         else {
 403                                 cerr << "mode: " << mode << endl;
 404                                 p.error("something strange in the parser\n");
 405                                 break;
 406                         }
 407                 }
 408
 409                 else if (t.cat() == catLetter)
 410                         os << t.character();
 411
 412                 else if (t.cat() == catSpace)
 413                         os << t.character();
 414
 415                 else if (t.cat() == catNewline)
 416                         os << ' ';
 417
 418                 else if (t.cat() == catSuper)
 419                         os << t.character();
 420
 421                 else if (t.cat() == catSub)
 422                         os << t.character();
 423
 424                 else if (t.cat() == catParameter) {
 425                         Token const & n = p.getToken();
 426                         os << wrap("macroarg", string(1, n.character()));
 427                 }
 428
 429                 else if (t.cat() == catActive)
 430                         os << wrap("active", string(1, t.character()));
 431
 432                 else if (t.cat() == catBegin) {
 433                         os << '{';
 434                         parse(p, os, FLAG_BRACE_LAST, mode);
 435                         os << '}';
 436                 }
 437
 438                 else if (t.cat() == catEnd) {
 439                         if (flags & FLAG_BRACE_LAST)
 440                                 return;
 441                         p.error("found '}' unexpectedly");
 442                         //lyx::Assert(0);
 443                         //add(cell, '}', LM_TC_TEX);
 444                 }
 445
 446                 else if (t.cat() == catAlign)
 447                         os << t.character();
 448
 449 /*
 450                         ++cellcol;
 451                         //cerr << " column now " << cellcol << " max: " << grid.ncols() << "\n";
 452                         if (cellcol == grid.ncols()) {
 453                                 //cerr << "adding column " << cellcol << "\n";
 454                                 grid.addCol(cellcol - 1);
 455                         }
 456                         cell = &grid.cell(grid.index(cellrow, cellcol));
 457                 }
 458 */
 459
 460                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 461                         //cerr << "finished reading option\n";
 462                         return;
 463                 }
 464
 465                 else if (t.cat() == catOther)
 466                         os << string(1, t.character());
 467
 468                 else if (t.cat() == catComment) {
 469                         string s;
 470                         while (p.good()) {
 471                                 Token const & t = p.getToken();
 472                                 if (t.cat() == catNewline)
 473                                         break;
 474                                 s += t.asString();
 475                         }
 476                         //os << wrap("comment", s);
 477                         p.skipSpaces();
 478                 }
 479
 480                 //
 481                 // control sequences
 482                 //
 483
 484                 else if (t.cs() == "lyxlock") {
 485                         // ignored;
 486                 }
 487
 488                 else if (t.cs() == "newcommand" || t.cs() == "providecommand") {
 489                         string const name = p.verbatimItem();
 490                         string const opts = p.getArg('[', ']');
 491                         string const body = p.verbatimItem();
 492                         // only non-lyxspecific stuff
 493                         if (name != "noun" && name != "tabularnewline") {
 494                                 ostream & out = in_preamble ? h_preamble : os;
 495                                 if (!in_preamble)
 496                                         begin_inset(os, "FormulaMacro\n");
 497                                 out << "\\" << t.cs() << "{" << name << "}";
 498                                 if (opts.size())
 499                                         out << "[" << opts << "]";
 500                                 out << "{" << body << "}";
 501                                 if (!in_preamble)
 502                                         end_inset(os);
 503                         }
 504                 }
 505
 506                 else if (t.cs() == "(") {
 507                         begin_inset(os, "Formula");
 508                         os << " \\(";
 509                         parse(p, os, FLAG_SIMPLE2, MATH_MODE);
 510                         os << "\\)";
 511                         end_inset(os);
 512                 }
 513
 514                 else if (t.cs() == "[") {
 515                         begin_inset(os, "Formula");
 516                         os << " \\[";
 517                         parse(p, os, FLAG_EQUATION, MATH_MODE);
 518                         os << "\\]";
 519                         end_inset(os);
 520                 }
 521
 522                 else if (t.cs() == "protect")
 523                         // ignore \\protect, will hopefully be re-added during output
 524                         ;
 525
 526                 else if (t.cs() == "begin") {
 527                         string const name = p.getArg('{', '}');
 528                         active_environments.push(name);
 529                         if (name == "document") {
 530                                 end_preamble(os);
 531                                 parse(p, os, FLAG_END, mode);
 532                         } else if (name == "abstract") {
 533                                 handle_par(os);
 534                                 parse(p, os, FLAG_END, mode);
 535                         } else if (is_math_env(name)) {
 536                                 begin_inset(os, "Formula ");
 537                                 os << "\\begin{" << name << "}";
 538                                 parse(p, os, FLAG_END, MATH_MODE);
 539                         } else if (name == "table") {
 540                                 handle_table(p, os);
 541                                 parse(p, os, FLAG_END, mode);
 542                         } else if (name == "thebibliography") {
 543                                 p.verbatimItem(); // swallow next arg
 544                                 handle_table(p, os);
 545                                 parse(p, os, FLAG_END, mode);
 546                         } else {
 547                                 os << "\\begin{" << name << "}";
 548                                 parse(p, os, FLAG_END, mode);
 549                         }
 550                 }
 551
 552                 else if (t.cs() == "end") {
 553                         if (flags & FLAG_END) {
 554                                 // eat environment name
 555                                 string const name = p.getArg('{', '}');
 556                                 if (name != active_environments.top())
 557                                         p.error("\\end{" + name + "} does not match \\begin{"
 558                                                 + active_environments.top() + "}");
 559                                 active_environments.pop();
 560                                 if (name == "document" || name == "abstract")
 561                                         ;
 562                                 else if (name == "table")
 563                                         ;
 564                                 else if (name == "thebibliography")
 565                                         ;
 566                                 else if (is_math_env(name)) {
 567                                         end_inset(os);
 568                                         os << "\\end{" << name << "}";
 569                                 } else
 570                                         os << "\\end{" << name << "}";
 571                                 return;
 572                         }
 573                         p.error("found 'end' unexpectedly");
 574                 }
 575
 576                 else if (t.cs() == ")") {
 577                         if (flags & FLAG_SIMPLE2)
 578                                 return;
 579                         p.error("found '\\)' unexpectedly");
 580                 }
 581
 582                 else if (t.cs() == "]") {
 583                         if (flags & FLAG_EQUATION)
 584                                 return;
 585                         p.error("found '\\]' unexpectedly");
 586                 }
 587
 588                 else if (t.cs() == "documentclass") {
 589                         vector<string> opts;
 590                         split(p.getArg('[', ']'), opts, ',');
 591                         handle_opt(opts, known_languages, h_language);
 592                         handle_opt(opts, known_fontsizes, h_paperfontsize);
 593                         h_quotes_language = h_language;
 594                         h_options = join(opts, ',');
 595                         h_textclass = p.getArg('{', '}');
 596                 }
 597
 598                 else if (t.cs() == "usepackage") {
 599                         string const options = p.getArg('[', ']');
 600                         string const name = p.getArg('{', '}');
 601                         if (options.empty() && name.find(',')) {
 602                                 vector<string> vecnames;
 603                                 split(name, vecnames, ',');
 604                                 vector<string>::const_iterator it  = vecnames.begin();
 605                                 vector<string>::const_iterator end = vecnames.end();
 606                                 for (; it != end; ++it)
 607                                         handle_package(trim(*it), string());
 608                         } else {
 609                                 handle_package(name, options);
 610                         }
 611                 }
 612
 613                 else if (t.cs() == "newenvironment") {
 614                         string const name = p.getArg('{', '}');
 615                         p.skipSpaces();
 616                         string const begin = p.verbatimItem();
 617                         p.skipSpaces();
 618                         string const end = p.verbatimItem();
 619                         // ignore out mess
 620                         if (name != "lyxcode")
 621                                 os << wrap("newenvironment", begin + end);
 622                 }
 623
 624                 else if (t.cs() == "def") {
 625                         string name = p.getToken().cs();
 626                         while (p.nextToken().cat() != catBegin)
 627                                 name += p.getToken().asString();
 628                         handle_ert(os, "\\def\\" + name + '{' + p.verbatimItem() + '}');
 629                 }
 630
 631                 else if (t.cs() == "setcounter") {
 632                         string const name = p.getArg('{', '}');
 633                         string const content = p.getArg('{', '}');
 634                         if (name == "secnumdepth")
 635                                 h_secnumdepth = content;
 636                         else if (name == "tocdepth")
 637                                 h_tocdepth = content;
 638                         else
 639                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}\n";
 640                 }
 641
 642                 else if (t.cs() == "setlength") {
 643                         string const name = p.getToken().cs();
 644                         string const content = p.getArg('{', '}');
 645                         if (name == "parskip")
 646                                 h_paragraph_separation = "skip";
 647                         else if (name == "parindent")
 648                                 h_paragraph_separation = "skip";
 649                         else
 650                                 h_preamble << "\\setlength{" << name << "}{" << content << "}\n";
 651                 }
 652
 653                 else if (t.cs() == "par")
 654                         handle_par(os);
 655
 656                 else if (is_heading(t.cs())) {
 657                         os << "\\layout " << cap(t.cs()) << "\n\n";
 658                         parse(p, os, FLAG_ITEM, mode);
 659                 }
 660
 661                 else if (t.cs() == "makeindex" || t.cs() == "maketitle")
 662                         ; // swallow this
 663
 664                 else if (t.cs() == "tableofcontents")
 665                         p.verbatimItem(); // swallow this
 666
 667                 else if (t.cs() == "textrm") {
 668                         os << '\\' << t.cs() << '{';
 669                         parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
 670                         os << '}';
 671                 }
 672
 673                 else if (t.cs() == "emph" && mode == TEXT_MODE) {
 674                         os << "\n\\emph on\n";
 675                         parse(p, os, FLAG_ITEM, mode);
 676                         os << "\n\\emph default\n";
 677                 }
 678
 679                 else if (is_latex_command(t.cs()) && mode == TEXT_MODE) {
 680                         begin_inset(os, "LatexCommand ");
 681                         os << '\\' << t.cs() << '{';
 682                         parse(p, os, FLAG_ITEM, TEXT_MODE);
 683                         os << '}';
 684                         end_inset(os);
 685                 }
 686
 687                 else if (t.cs() == "bibitem") {
 688                         os << "\n\\layout Bibliography\n\\bibitem ";
 689                         string opt = p.getArg('[',']');
 690                         if (opt.size())
 691                                 os << '[' << opt << ']';
 692                         os << '{' << p.getArg('{','}') << '}' << "\n\n";
 693                 }
 694
 695                 else
 696                         (in_preamble ? h_preamble : os) << t.asInput();
 697
 698                 if (flags & FLAG_LEAVE) {
 699                         flags &= ~FLAG_LEAVE;
 700                         break;
 701                 }
 702         }
 703 }
 704
 705
 706
 707 } // anonymous namespace
 708
 709
 710 int main(int argc, char * argv[])
 711 {
 712         if (argc <= 1) {
 713                 cerr << "Usage: " << argv[0] << " <infile.tex>" << endl;
 714                 return 2;
 715         }
 716
 717         ifstream is(argv[1]);
 718         Parser p(is);
 719         parse(p, cout, 0, TEXT_MODE);
 720         cout << "\n\\the_end";
 721
 722         return 0;
 723 }
 724
 725 // }])