1 /** The .tex to .lyx converter
2 \author André Pönitz (2003)
20 #include "texparser.h"
31 using std::istringstream;
34 using std::ostringstream;
42 void parse(Parser & p, ostream & os, unsigned flags, const mode_type mode);
44 char const OPEN = '<';
45 char const CLOSE = '>';
47 // rather brutish way to code table structure in a string:
49 // \begin{tabular}{ccc}
51 // \multicolumn{2}{c}{4} & 5\\
55 // gets "translated" to:
58 // 2 MULT c MULT 4 TAB 5 LINE
61 char const TAB = '\001';
62 char const LINE = '\002';
63 char const MULT = '\003';
65 const char * known_languages[] = { "austrian", "babel", "bahasa", "basque",
66 "breton", "british", "bulgarian", "catalan", "croatian", "czech", "danish",
67 "dutch", "english", "esperanto", "estonian", "finnish", "francais",
68 "frenchb", "galician", "german", "germanb", "greek", "hebcal", "hebfont",
69 "hebrew", "hebrew_newcode", "hebrew_oldcode", "hebrew_p", "hyphen",
70 "icelandic", "irish", "italian", "latin", "lgrcmr", "lgrcmro", "lgrcmss",
71 "lgrcmtt", "lgrenc", "lgrlcmss", "lgrlcmtt", "lheclas", "lhecmr",
72 "lhecmss", "lhecmtt", "lhecrml", "lheenc", "lhefr", "lheredis", "lheshold",
73 "lheshscr", "lheshstk", "lsorbian", "magyar", "naustrian", "ngermanb",
74 "ngerman", "norsk", "polish", "portuges", "rlbabel", "romanian",
75 "russianb", "samin", "scottish", "serbian", "slovak", "slovene", "spanish",
76 "swedish", "turkish", "ukraineb", "usorbian", "welsh", 0};
78 char const * known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
80 char const * known_headings[] = { "caption", "title", "author", "date",
81 "paragraph", "chapter", "section", "subsection", "subsubsection", 0 };
83 char const * known_math_envs[] = { "equation", "equation*",
84 "eqnarray", "eqnarray*", "align", "align*", 0};
86 char const * known_latex_commands[] = { "ref", "cite", "label", "index",
89 // LaTeX names for quotes
90 char const * known_quotes[] = { "glqq", "grqq", 0};
92 // the same as known_quotes with .lyx names
93 char const * known_coded_quotes[] = { "gld", "grd", 0};
98 ostringstream h_preamble;
99 string h_textclass = "article";
100 string h_options = "";
101 string h_language = "english";
102 string h_inputencoding = "latin1";
103 string h_fontscheme = "default";
104 string h_graphics = "default";
105 string h_paperfontsize = "default";
106 string h_spacing = "single";
107 string h_papersize = "default";
108 string h_paperpackage = "default";
109 string h_use_geometry = "0";
110 string h_use_amsmath = "0";
111 string h_use_natbib = "0";
112 string h_use_numerical_citations = "0";
113 string h_paperorientation = "portrait";
114 string h_secnumdepth = "3";
115 string h_tocdepth = "3";
116 string h_paragraph_separation = "indent";
117 string h_defskip = "medskip";
118 string h_quotes_language = "english";
119 string h_quotes_times = "1";
120 string h_papercolumns = "1";
121 string h_papersides = "1";
122 string h_paperpagestyle = "default";
123 string h_tracking_changes = "0";
125 // indicates whether we are in the preamble
126 bool in_preamble = true;
128 // current stack of nested environments
129 stack<string> active_environments;
135 s[0] = toupper(s[0]);
140 string const trim(string const & a, char const * p = " \t\n\r")
144 if (a.empty() || !*p)
147 string::size_type r = a.find_last_not_of(p);
148 string::size_type l = a.find_first_not_of(p);
150 // Is this the minimal test? (lgb)
151 if (r == string::npos && l == string::npos)
154 return a.substr(l, r - l + 1);
158 void split(string const & s, vector<string> & result, char delim = ',')
162 while (getline(is, t, delim))
167 // splits "x=z, y=b" into a map
168 map<string, string> split_map(string const & s)
170 map<string, string> res;
173 for (size_t i = 0; i < v.size(); ++i) {
174 size_t const pos = v[i].find('=');
175 string const index = v[i].substr(0, pos);
176 string const value = v[i].substr(pos + 1, string::npos);
177 res[trim(index)] = trim(value);
183 string join(vector<string> const & input, char delim)
186 for (size_t i = 0; i != input.size(); ++i) {
195 char const ** is_known(string const & str, char const ** what)
197 for ( ; *what; ++what)
204 void handle_opt(vector<string> & opts, char const ** what, string & target)
209 for ( ; *what; ++what) {
210 vector<string>::iterator it = find(opts.begin(), opts.end(), *what);
211 if (it != opts.end()) {
212 //cerr << "### found option '" << *what << "'\n";
221 bool is_math_env(string const & name)
223 for (char const ** what = known_math_envs; *what; ++what)
230 void begin_inset(ostream & os, string const & name)
232 os << "\n\\begin_inset " << name;
236 void end_inset(ostream & os)
238 os << "\n\\end_inset\n\n";
244 return active_environments.empty() ? string() : active_environments.top();
248 void handle_ert(ostream & os, string const & s)
250 begin_inset(os, "ERT");
251 os << "\nstatus Collapsed\n\n\\layout Standard\n\n";
252 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
254 os << "\n\\backslash\n";
262 void handle_tex(ostream & os, string const & s)
264 //os << "handle_tex(" << s << ")\n";
272 void handle_par(ostream & os)
274 if (active_environments.empty())
277 string s = curr_env();
278 if (s == "document" || s == "table") {
279 os << "Standard\n\n";
282 if (s == "lyxcode") {
283 os << "LyX-Code\n\n";
286 os << cap(s) << "\n\n";
290 void handle_package(string const & name, string const & options)
292 //cerr << "handle_package: '" << name << "'\n";
293 if (name == "a4wide") {
294 h_papersize = "a4paper";
295 h_paperpackage = "widemarginsa4";
296 } else if (name == "ae")
298 else if (name == "aecompl")
300 else if (name == "amsmath")
302 else if (name == "amssymb")
304 else if (name == "babel")
306 else if (name == "fontenc")
308 else if (name == "inputenc")
309 h_inputencoding = options;
310 else if (name == "makeidx")
312 else if (name == "verbatim")
314 else if (is_known(name, known_languages)) {
316 h_quotes_language = name;
319 h_preamble << "\\usepackage[" << options << "]{" << name << "}\n";
321 h_preamble << "\\usepackage{" << name << "}\n";
326 vector<string> extract_col_align(string const & s)
329 for (size_t i = 0; i < s.size(); ++i) {
331 case 'c': res.push_back("center"); break;
332 case 'l': res.push_back("left"); break;
333 case 'r': res.push_back("right"); break;
334 default : res.push_back("right"); break;
341 void handle_tabular(Parser & p, ostream & os, mode_type mode)
343 begin_inset(os, "Tabular \n");
344 string colopts = p.verbatimItem();
345 vector<string> colalign = extract_col_align(colopts);
347 parse(p, ss, FLAG_END, mode);
348 vector<string> lines;
349 split(ss.str(), lines, LINE);
350 const size_t cols = colalign.size();
351 const size_t rows = lines.size();
352 os << "<lyxtabular version=\"3\" rows=\"" << rows
353 << "\" columns=\"" << cols << "\">\n"
355 for (size_t c = 0; c < cols; ++c)
356 os << "<column alignment=\"" << colalign[c] << "\""
357 << " valignment=\"top\""
360 for (size_t r = 0; r < rows; ++r) {
361 vector<string> cells;
362 split(lines[r], cells, TAB);
363 while (cells.size() < cols)
364 cells.push_back(string());
365 //os << "<row bottomline=\"true\">\n";
367 for (size_t c = 0; c < cols; ++c) {
369 string alignment = "center";
370 vector<string> parts;
371 split(cells[c], parts, MULT);
372 if (parts.size() > 2) {
373 os << " multicolumn=\"" << parts[0] << "\"";
374 alignment = parts[1];
376 os << " alignment=\"" << alignment << "\""
377 << " valignment=\"top\""
378 << " topline=\"true\""
379 << " leftline=\"true\""
380 << " usebox=\"none\""
382 begin_inset(os, "Text");
383 os << "\n\n\\layout Standard\n\n";
391 os << "</lyxtabular>\n";
396 string wrap(string const & cmd, string const & str)
398 return OPEN + cmd + ' ' + str + CLOSE;
402 void end_preamble(ostream & os)
405 os << "# tex2lyx 0.0.2 created this file\n"
406 << "\\lyxformat 222\n"
407 << "\\textclass " << h_textclass << "\n"
408 << "\\begin_preamble\n" << h_preamble.str() << "\n\\end_preamble\n"
409 << "\\options " << h_options << "\n"
410 << "\\language " << h_language << "\n"
411 << "\\inputencoding " << h_inputencoding << "\n"
412 << "\\fontscheme " << h_fontscheme << "\n"
413 << "\\graphics " << h_graphics << "\n"
414 << "\\paperfontsize " << h_paperfontsize << "\n"
415 << "\\spacing " << h_spacing << "\n"
416 << "\\papersize " << h_papersize << "\n"
417 << "\\paperpackage " << h_paperpackage << "\n"
418 << "\\use_geometry " << h_use_geometry << "\n"
419 << "\\use_amsmath " << h_use_amsmath << "\n"
420 << "\\use_natbib " << h_use_natbib << "\n"
421 << "\\use_numerical_citations " << h_use_numerical_citations << "\n"
422 << "\\paperorientation " << h_paperorientation << "\n"
423 << "\\secnumdepth " << h_secnumdepth << "\n"
424 << "\\tocdepth " << h_tocdepth << "\n"
425 << "\\paragraph_separation " << h_paragraph_separation << "\n"
426 << "\\defskip " << h_defskip << "\n"
427 << "\\quotes_language " << h_quotes_language << "\n"
428 << "\\quotes_times " << h_quotes_times << "\n"
429 << "\\papercolumns " << h_papercolumns << "\n"
430 << "\\papersides " << h_papersides << "\n"
431 << "\\paperpagestyle " << h_paperpagestyle << "\n"
432 << "\\tracking_changes " << h_tracking_changes << "\n";
436 void parse(Parser & p, ostream & os, unsigned flags, const mode_type mode)
439 Token const & t = p.getToken();
442 cerr << "t: " << t << " flags: " << flags << "\n";
446 if (flags & FLAG_ITEM) {
447 if (t.cat() == catSpace)
451 if (t.cat() == catBegin) {
452 // skip the brace and collect everything to the next matching
454 flags |= FLAG_BRACE_LAST;
458 // handle only this single token, leave the loop if done
463 if (flags & FLAG_BRACED) {
464 if (t.cat() == catSpace)
467 if (t.cat() != catBegin) {
468 p.error("opening brace expected");
472 // skip the brace and collect everything to the next matching
474 flags = FLAG_BRACE_LAST;
478 if (flags & FLAG_OPTION) {
479 if (t.cat() == catOther && t.character() == '[') {
480 parse(p, os, FLAG_BRACK_LAST, mode);
482 // no option found, put back token and we are done
491 if (t.cat() == catMath) {
492 if (mode == TEXT_MODE || mode == MATHTEXT_MODE) {
493 // we are inside some text mode thingy, so opening new math is allowed
494 if (mode == TEXT_MODE)
495 begin_inset(os, "Formula ");
496 Token const & n = p.getToken();
497 if (n.cat() == catMath) {
498 // TeX's $$...$$ syntax for displayed math
500 parse(p, os, FLAG_SIMPLE, MATH_MODE);
502 p.getToken(); // skip the second '$' token
504 // simple $...$ stuff
507 parse(p, os, FLAG_SIMPLE, MATH_MODE);
510 if (mode == TEXT_MODE)
514 else if (flags & FLAG_SIMPLE) {
515 // this is the end of the formula
520 cerr << "mode: " << mode << endl;
521 p.error("something strange in the parser\n");
526 else if (t.cat() == catLetter)
529 else if (t.cat() == catSpace)
532 else if (t.cat() == catNewline)
535 else if (t.cat() == catSuper)
538 else if (t.cat() == catSub)
541 else if (t.cat() == catParameter) {
542 Token const & n = p.getToken();
543 os << wrap("macroarg", string(1, n.character()));
546 else if (t.cat() == catActive) {
547 if (t.character() == '~')
548 os << (curr_env() == "lyxcode" ? ' ' : '~');
553 else if (t.cat() == catBegin) {
554 if (mode == TEXT_MODE)
560 else if (t.cat() == catEnd) {
561 if (flags & FLAG_BRACE_LAST)
563 if (mode == TEXT_MODE)
569 else if (t.cat() == catAlign) {
570 if (mode == TEXT_MODE)
576 else if (t.cs() == "tabularnewline") {
577 if (mode == TEXT_MODE)
583 else if (t.cs() == "\\" && mode == MATH_MODE)
586 else if (t.cs() == "\\" && mode == TEXT_MODE && curr_env() == "tabular")
589 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
590 //cerr << "finished reading option\n";
594 else if (t.cat() == catOther)
595 os << string(1, t.character());
597 else if (t.cat() == catComment) {
600 Token const & t = p.getToken();
601 if (t.cat() == catNewline)
605 //os << wrap("comment", s);
613 else if (t.cs() == "lyxlock")
616 else if (t.cs() == "makeatletter") {
617 p.setCatCode('@', catLetter);
618 handle_tex(os, "\\makeatletter\n");
621 else if (t.cs() == "makeatother") {
622 p.setCatCode('@', catOther);
623 handle_tex(os, "\\makeatother\n");
626 else if (t.cs() == "newcommand" || t.cs() == "renewcommand"
627 || t.cs() == "providecommand") {
628 string const name = p.verbatimItem();
629 string const opts = p.getOpt();
630 string const body = p.verbatimItem();
631 // only non-lyxspecific stuff
632 if (name != "\\noun " && name != "\\tabularnewline ") {
634 ss << '\\' << t.cs() << '{' << name << '}'
635 << opts << '{' << body << "}\n";
636 handle_tex(os, ss.str());
638 ostream & out = in_preamble ? h_preamble : os;
640 begin_inset(os, "FormulaMacro\n");
641 out << "\\" << t.cs() << "{" << name << "}"
642 << opts << "{" << body << "}\n";
649 else if (t.cs() == "newtheorem") {
651 ss << "\\newtheorem";
652 ss << '{' << p.verbatimItem() << '}';
654 ss << '{' << p.verbatimItem() << '}';
657 handle_tex(os, ss.str());
660 else if (t.cs() == "(") {
661 begin_inset(os, "Formula");
663 parse(p, os, FLAG_SIMPLE2, MATH_MODE);
668 else if (t.cs() == "[") {
669 begin_inset(os, "Formula");
671 parse(p, os, FLAG_EQUATION, MATH_MODE);
676 else if (t.cs() == "protect")
677 // ignore \\protect, will hopefully be re-added during output
680 else if (t.cs() == "begin") {
681 string const name = p.getArg('{', '}');
682 active_environments.push(name);
683 if (name == "document") {
685 os << "\n\n\\layout Standard\n\n";
686 parse(p, os, FLAG_END, mode);
687 } else if (name == "abstract") {
689 parse(p, os, FLAG_END, mode);
690 } else if (is_math_env(name)) {
691 begin_inset(os, "Formula ");
692 os << "\\begin{" << name << "}";
693 parse(p, os, FLAG_END, MATH_MODE);
694 os << "\\end{" << name << "}";
696 } else if (name == "tabular") {
697 if (mode == TEXT_MODE)
698 handle_tabular(p, os, mode);
700 os << "\\begin{" << name << "}";
701 parse(p, os, FLAG_END, MATHTEXT_MODE);
702 os << "\\end{" << name << "}";
704 } else if (name == "table" || name == "figure") {
705 string opts = p.getOpt();
706 begin_inset(os, "Float " + name + "\n");
708 os << "placement " << opts << '\n';
710 << "collapsed false\n"
712 << "\\layout Standard\n";
713 parse(p, os, FLAG_END, mode);
715 } else if (name == "thebibliography") {
716 p.verbatimItem(); // swallow next arg
717 parse(p, os, FLAG_END, mode);
718 os << "\n\\layout Standard\n\n";
719 } else if (mode == MATH_MODE || mode == MATHTEXT_MODE) {
720 os << "\\begin{" << name << "}";
721 parse(p, os, FLAG_END, mode);
722 os << "\\end{" << name << "}";
724 parse(p, os, FLAG_END, mode);
728 else if (t.cs() == "end") {
729 if (flags & FLAG_END) {
730 // eat environment name
731 string const name = p.getArg('{', '}');
732 if (name != curr_env())
733 p.error("\\end{" + name + "} does not match \\begin{"
735 active_environments.pop();
738 p.error("found 'end' unexpectedly");
742 else if (t.cs() == "item")
745 else if (t.cs() == ")") {
746 if (flags & FLAG_SIMPLE2)
748 p.error("found '\\)' unexpectedly");
751 else if (t.cs() == "]") {
752 if (flags & FLAG_EQUATION)
754 p.error("found '\\]' unexpectedly");
757 else if (t.cs() == "documentclass") {
759 split(p.getArg('[', ']'), opts, ',');
760 handle_opt(opts, known_languages, h_language);
761 handle_opt(opts, known_fontsizes, h_paperfontsize);
762 h_quotes_language = h_language;
763 h_options = join(opts, ',');
764 h_textclass = p.getArg('{', '}');
767 else if (t.cs() == "usepackage") {
768 string const options = p.getArg('[', ']');
769 string const name = p.getArg('{', '}');
770 if (options.empty() && name.find(',')) {
771 vector<string> vecnames;
772 split(name, vecnames, ',');
773 vector<string>::const_iterator it = vecnames.begin();
774 vector<string>::const_iterator end = vecnames.end();
775 for (; it != end; ++it)
776 handle_package(trim(*it), string());
778 handle_package(name, options);
782 else if (t.cs() == "newenvironment") {
783 string const name = p.getArg('{', '}');
784 if (name != "lyxcode") {
786 ss << "\\newenvironment{" << name << "}";
789 ss << '{' << p.verbatimItem() << '}';
790 ss << '{' << p.verbatimItem() << '}';
792 handle_tex(os, ss.str());
796 else if (t.cs() == "newfont") {
799 ss << '{' << p.verbatimItem() << '}';
800 ss << '{' << p.verbatimItem() << '}';
802 handle_tex(os, ss.str());
805 else if (t.cs() == "newcounter") {
807 ss << "\\newcounter";
808 ss << '{' << p.verbatimItem() << '}';
811 handle_tex(os, ss.str());
814 else if (t.cs() == "def") {
815 string name = p.getToken().cs();
816 while (p.nextToken().cat() != catBegin)
817 name += p.getToken().asString();
818 handle_ert(os, "\\def\\" + name + '{' + p.verbatimItem() + '}');
821 else if (t.cs() == "setcounter") {
822 string const name = p.getArg('{', '}');
823 string const content = p.getArg('{', '}');
824 if (name == "secnumdepth")
825 h_secnumdepth = content;
826 else if (name == "tocdepth")
827 h_tocdepth = content;
829 h_preamble << "\\setcounter{" << name << "}{" << content << "}\n";
832 else if (t.cs() == "setlength") {
833 string const name = p.verbatimItem();
834 string const content = p.verbatimItem();
835 if (in_preamble && name == "parskip")
836 h_paragraph_separation = "skip";
837 else if (in_preamble && name == "parindent")
838 h_paragraph_separation = "skip";
840 handle_tex(os, "\\setlength{" + name + "}{" + content + "}\n");
843 else if (t.cs() == "par")
846 else if (is_known(t.cs(), known_headings)) {
847 string name = t.cs();
848 if (p.nextToken().asInput() == "*") {
852 os << "\n\n\\layout " << cap(name) << "\n\n";
853 parse(p, os, FLAG_ITEM, mode);
854 os << "\n\n\\layout Standard\n\n";
857 else if (t.cs() == "includegraphics") {
858 if (mode == TEXT_MODE) {
859 map<string, string> opts = split_map(p.getArg('[', ']'));
860 string name = p.verbatimItem();
861 begin_inset(os, "Graphics ");
862 os << "\n\tfilename " << name << '\n';
863 if (opts.find("width") != opts.end())
864 os << "\twidth " << opts["width"] << '\n';
865 if (opts.find("height") != opts.end())
866 os << "\theight " << opts["height"] << '\n';
869 os << "\\includegraphics ";
873 else if (t.cs() == "makeindex" || t.cs() == "maketitle")
876 else if (t.cs() == "tableofcontents")
877 p.verbatimItem(); // swallow this
879 else if (t.cs() == "multicolumn" && mode == TEXT_MODE) {
881 parse(p, os, FLAG_ITEM, mode);
883 parse(p, os, FLAG_ITEM, mode);
885 parse(p, os, FLAG_ITEM, mode);
888 else if (t.cs() == "textrm") {
889 os << '\\' << t.cs() << '{';
890 parse(p, os, FLAG_ITEM, MATHTEXT_MODE);
894 else if ((t.cs() == "emph" || t.cs() == "noun") && mode == TEXT_MODE) {
895 os << "\n\\" << t.cs() << " on\n";
896 parse(p, os, FLAG_ITEM, mode);
897 os << "\n\\" << t.cs() << " default\n";
900 else if (is_known(t.cs(), known_latex_commands) && mode == TEXT_MODE) {
901 begin_inset(os, "LatexCommand ");
902 os << '\\' << t.cs();
905 os << '{' << p.verbatimItem() << '}';
909 else if (t.cs() == "bibitem") {
910 os << "\n\\layout Bibliography\n\\bibitem ";
912 os << '{' << p.verbatimItem() << '}' << "\n\n";
915 else if (char const ** where = is_known(t.cs(), known_quotes)) {
916 begin_inset(os, "Quotes ");
917 os << known_coded_quotes[where - known_quotes];
921 else if (t.cs() == "textasciitilde")
924 else if (t.cs() == "_" && mode == TEXT_MODE)
927 else if (t.cs() == "&" && mode == TEXT_MODE)
930 else if (t.cs() == "\"") {
931 string const name = p.verbatimItem();
932 if (name == "a") os << 'ä';
933 else if (name == "o") os << 'ö';
934 else if (name == "u") os << 'ü';
935 else if (name == "A") os << 'Ä';
936 else if (name == "O") os << 'Ö';
937 else if (name == "U") os << 'Ü';
938 else handle_ert(os, "\"{" + name + "}");
941 else if (t.cs() == "ss")
944 else if (t.cs() == "input")
945 handle_tex(os, "\\input{" + p.verbatimItem() + "}\n");
947 else if (t.cs() == "pagestyle" && in_preamble)
948 h_paperpagestyle == p.verbatimItem();
950 else if (t.cs() == "fancyhead") {
954 ss << '{' << p.verbatimItem() << "}\n";
955 handle_tex(os, ss.str());
959 //cerr << "#: " << t << " mode: " << mode << endl;
960 if (mode == MATH_MODE || mode == MATHTEXT_MODE) {
962 //cerr << "#: writing: '" << t.asInput() << "'\n";
964 else if (in_preamble)
965 h_preamble << t.asInput();
967 // heuristic: read up to next non-nested space
969 string s = t.asInput();
970 string z = p.verbatimItem();
971 while (p.good() && z != " " && z.size()) {
972 //cerr << "read: " << z << endl;
974 z = p.verbatimItem();
976 cerr << "found ERT: " << s << endl;
977 handle_ert(os, s + ' ');
979 handle_ert(os, t.asInput() + ' ');
983 if (flags & FLAG_LEAVE) {
984 flags &= ~FLAG_LEAVE;
992 } // anonymous namespace
995 int main(int argc, char * argv[])
998 cerr << "Usage: " << argv[0] << " <infile.tex>" << endl;
1002 ifstream is(argv[1]);
1004 parse(p, cout, 0, TEXT_MODE);
1005 cout << "\n\\the_end";