2 /** The .tex to .lyx converter
3 \author André Pönitz (2003)
31 char const OPEN = '<';
32 char const CLOSE = '>';
34 const char * known_languages[] = { "austrian", "babel", "bahasa",
35 "basque", "breton", "bulgarian", "catalan", "croatian", "czech", "danish",
36 "dutch", "english", "esperanto", "estonian", "finnish", "francais",
37 "frenchb", "galician", "germanb", "greek", "hebcal", "hebfont", "hebrew",
38 "hebrew_newcode", "hebrew_oldcode", "hebrew_p", "hyphen", "icelandic",
39 "irish", "italian", "latin", "lgrcmr", "lgrcmro", "lgrcmss", "lgrcmtt",
40 "lgrenc", "lgrlcmss", "lgrlcmtt", "lheclas", "lhecmr", "lhecmss",
41 "lhecmtt", "lhecrml", "lheenc", "lhefr", "lheredis", "lheshold",
42 "lheshscr", "lheshstk", "lsorbian", "magyar", "naustrian", "ngermanb",
43 "ngerman", "norsk", "polish", "portuges", "rlbabel", "romanian",
44 "russianb", "samin", "scottish", "serbian", "slovak", "slovene", "spanish",
45 "swedish", "turkish", "ukraineb", "usorbian", "welsh", 0};
47 const char * known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
52 string h_textclass = "FIXME";
53 string h_options = "FIXME";
54 string h_language = "FIXME";
55 string h_inputencoding = "FIXME";
56 string h_fontscheme = "FIXME";
57 string h_graphics = "default";
58 string h_paperfontsize = "FIXME";
59 string h_spacing = "single";
60 string h_papersize = "FIXME";
61 string h_paperpackage = "FIXME";
62 string h_use_geometry = "0";
63 string h_use_amsmath = "0";
64 string h_use_natbib = "0";
65 string h_use_numerical_citations = "0";
66 string h_paperorientation = "portrait";
67 string h_secnumdepth = "3";
68 string h_tocdepth = "3";
69 string h_paragraph_separation = "indent";
70 string h_defskip = "medskip";
71 string h_quotes_language = "2";
72 string h_quotes_times = "1";
73 string h_papercolumns = "1";
74 string h_papersides = "1";
75 string h_paperpagestyle = "default";
76 string h_tracking_changes = "0";
78 // indicates whether we are in the preamble
79 bool in_preamble = true;
81 // current stack of nested environments
82 stack<string> active_environments;
86 void split(string const & s, vector<string> & result, char delim)
90 while (getline(is, t, delim))
95 string join(vector<string> const & input, char delim)
98 for (size_t i = 0; i != input.size(); ++i) {
107 void handle_opt(vector<string> & opts, char const ** what, string & target)
109 for ( ; what; ++what) {
110 vector<string>::iterator it = find(opts.begin(), opts.end(), *what);
111 if (it != opts.end()) {
112 //cerr << "### found option '" << *what << "'\n";
121 void handle_ert(ostream & os, string const & s)
123 os << "\n\\begin_inset ERT\nstatus Collapsed\n\n\\layout Standard\n\n";
125 os << "\n\\end_inset\n";
129 string wrap(string const & cmd, string const & str)
131 return OPEN + cmd + ' ' + str + CLOSE;
135 string wrap(string const & cmd, string const & str, string const & str2)
137 return OPEN + cmd + ' ' + str + ' ' + str2 + CLOSE;
141 enum mode_type {UNDECIDED_MODE, TEXT_MODE, MATH_MODE};
143 mode_type asMode(mode_type oldmode, string const & str)
145 if (str == "mathmode")
147 if (str == "textmode" || str == "forcetext")
153 bool stared(string const & s)
155 string::size_type const n = s.size();
156 return n && s[n - 1] == '*';
160 // These are TeX's catcodes
162 catEscape, // 0 backslash
172 catSpace, // 10 space
173 catLetter, // 11 a-zA-Z
174 catOther, // 12 none of the above
177 catInvalid // 15 <delete>
180 CatCode theCatcode[256];
183 inline CatCode catcode(unsigned char c)
185 return theCatcode[c];
190 FLAG_BRACE_LAST = 1 << 1, // last closing brace ends the parsing
191 FLAG_RIGHT = 1 << 2, // next \\right ends the parsing process
192 FLAG_END = 1 << 3, // next \\end ends the parsing process
193 FLAG_BRACK_LAST = 1 << 4, // next closing bracket ends the parsing
194 FLAG_TEXTMODE = 1 << 5, // we are in a box
195 FLAG_ITEM = 1 << 6, // read a (possibly braced token)
196 FLAG_LEAVE = 1 << 7, // leave the loop at the end
197 FLAG_SIMPLE = 1 << 8, // next $ leaves the loop
198 FLAG_EQUATION = 1 << 9, // next \] leaves the loop
199 FLAG_SIMPLE2 = 1 << 10, // next \) leaves the loop
200 FLAG_OPTION = 1 << 11, // read [...] style option
201 FLAG_BRACED = 1 << 12 // read {...} style argument
207 fill(theCatcode, theCatcode + 256, catOther);
208 fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
209 fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
211 theCatcode['\\'] = catEscape;
212 theCatcode['{'] = catBegin;
213 theCatcode['}'] = catEnd;
214 theCatcode['$'] = cat;
215 theCatcode['&'] = catAlign;
216 theCatcode['\n'] = catNewline;
217 theCatcode['#'] = catParameter;
218 theCatcode['^'] = catSuper;
219 theCatcode['_'] = catSub;
220 theCatcode['
\7f'] = catIgnore;
221 theCatcode[' '] = catSpace;
222 theCatcode['\t'] = catSpace;
223 theCatcode['\r'] = catNewline;
224 theCatcode['~'] = catActive;
225 theCatcode['%'] = catComment;
231 // Helper class for parsing
237 Token() : cs_(), char_(0), cat_(catIgnore) {}
239 Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
241 Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
244 string const & cs() const { return cs_; }
246 CatCode cat() const { return cat_; }
248 char character() const { return char_; }
250 string asString() const { return cs_.size() ? cs_ : string(1, char_); }
261 ostream & operator<<(ostream & os, Token const & t)
264 os << '\\' << t.cs();
266 os << '[' << t.character() << ',' << t.cat() << ']';
275 Parser(istream & is);
280 string parse(unsigned flags, mode_type mode);
282 int lineno() const { return lineno_; }
285 /// dump contents to screen
290 string getArg(char left, char right);
294 void error(string const & msg);
296 void tokenize(istream & is);
298 void tokenize(string const & s);
300 void skipSpaceTokens(istream & is, char c);
302 void push_back(Token const & t);
306 Token const & prevToken() const;
308 Token const & nextToken() const;
310 Token const & getToken();
311 /// skips spaces if any
314 void lex(string const & s);
318 string parse_verbatim_item();
320 string parse_verbatim_option();
325 vector<Token> tokens_;
331 Parser::Parser(istream & is)
332 : lineno_(0), pos_(0)
338 void Parser::push_back(Token const & t)
340 tokens_.push_back(t);
344 void Parser::pop_back()
350 Token const & Parser::prevToken() const
352 static const Token dummy;
353 return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
357 Token const & Parser::nextToken() const
359 static const Token dummy;
360 return good() ? tokens_[pos_] : dummy;
364 Token const & Parser::getToken()
366 static const Token dummy;
367 //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
368 return good() ? tokens_[pos_++] : dummy;
372 void Parser::skipSpaces()
374 while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
379 void Parser::putback()
385 bool Parser::good() const
387 return pos_ < tokens_.size();
391 char Parser::getChar()
394 error("The input stream is not well...");
395 return tokens_[pos_++].character();
399 string Parser::getArg(char left, char right)
409 while ((c = getChar()) != right && good())
416 void Parser::skipSpaceTokens(istream & is, char c)
418 // skip trailing spaces
419 while (catcode(c) == catSpace || catcode(c) == catNewline)
422 //cerr << "putting back: " << c << "\n";
427 void Parser::tokenize(istream & is)
429 // eat everything up to the next \end_inset or end of stream
430 // and store it in s for further tokenization
435 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
436 s = s.substr(0, s.size() - 10);
440 // Remove the space after \end_inset
441 if (is.get(c) && c != ' ')
449 void Parser::tokenize(string const & buffer)
451 static bool init_done = false;
458 istringstream is(buffer.c_str(), ios::in | ios::binary);
462 //cerr << "reading c: " << c << "\n";
464 switch (catcode(c)) {
468 if (catcode(c) == catNewline)
469 push_back(Token("par"));
471 push_back(Token('\n', catNewline));
479 while (is.get(c) && catcode(c) != catNewline)
489 error("unexpected end of input");
492 if (catcode(c) == catLetter) {
494 while (is.get(c) && catcode(c) == catLetter)
496 skipSpaceTokens(is, c);
505 push_back(Token(c, catcode(c)));
507 skipSpaceTokens(is, c);
512 cerr << "ignoring a char: " << int(c) << "\n";
517 push_back(Token(c, catcode(c)));
527 void Parser::dump() const
529 cerr << "\nTokens: ";
530 for (unsigned i = 0; i < tokens_.size(); ++i) {
535 cerr << " pos: " << pos_ << "\n";
539 void Parser::error(string const & msg)
541 cerr << "Line ~" << lineno_ << ": parse error: " << msg << endl;
547 string Parser::parse()
550 return parse(0, UNDECIDED_MODE);
554 string Parser::parse_verbatim_option()
557 if (nextToken().character() == '[') {
558 Token t = getToken();
559 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
560 if (t.cat() == catBegin) {
562 res += '{' + parse_verbatim_item() + '}';
571 string Parser::parse_verbatim_item()
574 if (nextToken().cat() == catBegin) {
575 Token t = getToken();
576 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
577 if (t.cat() == catBegin) {
579 res += '{' + parse_verbatim_item() + '}';
589 string Parser::parse(unsigned flags, mode_type mode)
593 ostringstream result;
595 Token const & t = getToken();
598 cerr << "t: " << t << " flags: " << flags << "\n";
603 if (flags & FLAG_ITEM) {
604 if (t.cat() == catSpace)
608 if (t.cat() == catBegin) {
609 // skip the brace and collect everything to the next matching
611 flags |= FLAG_BRACE_LAST;
615 // handle only this single token, leave the loop if done
620 if (flags & FLAG_BRACED) {
621 if (t.cat() == catSpace)
624 if (t.cat() != catBegin) {
625 error("opening brace expected");
629 // skip the brace and collect everything to the next matching
631 flags = FLAG_BRACE_LAST;
635 if (flags & FLAG_OPTION) {
636 if (t.cat() == catOther && t.character() == '[') {
637 result << parse(FLAG_BRACK_LAST, mode);
639 // no option found, put back token and we are done
648 if (t.cat() == cat) {
649 if (mode != MATH_MODE) {
650 // we are inside some text mode thingy, so opening new math is allowed
651 Token const & n = getToken();
652 if (n.cat() == cat) {
653 // TeX's $$...$$ syntax for displayed math
654 result << wrap("equation", parse(FLAG_SIMPLE, MATH_MODE));
655 getToken(); // skip the second '$' token
657 // simple $...$ stuff
659 result << wrap("simple", parse(FLAG_SIMPLE, MATH_MODE));
663 else if (flags & FLAG_SIMPLE) {
664 // this is the end of the formula
669 error("something strange in the parser\n");
674 else if (t.cat() == catLetter)
675 result << t.character();
677 else if (t.cat() == catSpace && mode != MATH_MODE) {
678 //if (result.empty() || result[result.size() - 1] != ' ')
679 result << t.character();
682 else if (t.cat() == catNewline && mode != MATH_MODE)
683 result << t.character();
685 else if (t.cat() == catParameter) {
686 Token const & n = getToken();
687 result << wrap("macroarg", string(1, n.character()));
690 else if (t.cat() == catActive)
691 result << wrap("active", string(1, t.character()));
693 else if (t.cat() == catBegin)
694 result << wrap("braced", parse(FLAG_BRACE_LAST, mode));
696 else if (t.cat() == catEnd) {
697 if (flags & FLAG_BRACE_LAST)
699 error("found '}' unexpectedly");
701 //add(cell, '}', LM_TC_TEX);
705 else if (t.cat() == catAlign) {
707 //cerr << " column now " << cellcol << " max: " << grid.ncols() << "\n";
708 if (cellcol == grid.ncols()) {
709 //cerr << "adding column " << cellcol << "\n";
710 grid.addCol(cellcol - 1);
712 cell = &grid.cell(grid.index(cellrow, cellcol));
716 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
717 //cerr << "finished reading option\n";
721 else if (t.cat() == catOther)
722 result << string(1, t.character());
724 else if (t.cat() == catComment) {
727 Token const & t = getToken();
728 if (t.cat() == catNewline)
732 //result << wrap("comment", s);
740 else if (t.cs() == "lyxlock") {
744 else if (t.cs() == "newcommand" || t.cs() == "providecommand") {
745 string const name = parse_verbatim_item();
746 string const opts = getArg('[', ']');
747 string const body = parse_verbatim_item();
748 // only non-lyxspecific stuff
749 if (name != "noun" && name != "tabularnewline") {
750 h_preamble += "\\" + t.cs() + "{" + name + "}";
752 h_preamble += "[" + opts + "]";
753 h_preamble += "{" + body + "}\n";
757 else if (t.cs() == "(")
758 result << wrap("simple", parse(FLAG_SIMPLE2, MATH_MODE));
760 else if (t.cs() == "[")
761 result << wrap("equation", parse(FLAG_EQUATION, MATH_MODE));
763 else if (t.cs() == "protect")
764 // ignore \\protect, will hopefully be re-added during output
767 else if (t.cs() == "end") {
768 if (flags & FLAG_END) {
769 // eat environment name
770 string const name = getArg('{', '}');
771 if (name != active_environments.top())
772 error("\\end{" + name + "} does not match \\begin{"
773 + active_environments.top() + "}");
774 active_environments.pop();
777 error("found 'end' unexpectedly");
780 else if (t.cs() == ")") {
781 if (flags & FLAG_SIMPLE2)
783 error("found '\\)' unexpectedly");
786 else if (t.cs() == "]") {
787 if (flags & FLAG_EQUATION)
789 error("found '\\]' unexpectedly");
793 else if (t.cs() == "\\") {
794 grid.vcrskip(LyXLength(getArg('[', ']')), cellrow);
797 if (cellrow == grid.nrows())
798 grid.addRow(cellrow - 1);
799 if (grid.asHullstring())
800 grid.asHullstring()->numbered(cellrow, numbered);
801 cell = &grid.cell(grid.index(cellrow, cellcol));
804 else if (t.cs() == "documentclass") {
806 split(getArg('[', ']'), opts, ',');
807 handle_opt(opts, known_languages, h_language);
808 handle_opt(opts, known_fontsizes, h_paperfontsize);
809 h_options = join(opts, ',');
810 h_textclass = getArg('{', '}');
813 else if (t.cs() == "usepackage") {
814 string const options = getArg('[', ']');
815 string const name = getArg('{', '}');
816 if (name == "a4wide") {
818 h_paperpackage = "widemarginsa4";
819 } else if (name == "ae")
821 else if (name == "aecompl")
823 else if (name == "amsmath")
825 else if (name == "amssymb")
827 else if (name == "babel")
829 else if (name == "fontenc")
831 else if (name == "inputenc")
832 h_inputencoding = options;
833 else if (name == "makeidx")
835 else if (name == "verbatim")
839 h_preamble += "\\usepackage[" + options + "]{" + name + "}\n";
841 h_preamble += "\\usepackage{" + name + "}\n";
845 else if (t.cs() == "newenvironment") {
846 string const name = getArg('{', '}');
848 string const begin = parse_verbatim_item();
850 string const end = parse_verbatim_item();
852 if (name != "lyxcode")
853 result << wrap("newenvironment", begin + end);
856 else if (t.cs() == "def") {
857 string const name = getToken().cs();
859 while (nextToken().cat() != catBegin)
860 res += getToken().asString();
861 handle_ert(result, "\\def" + res + '{' + parse_verbatim_item() + '}');
864 else if (t.cs() == "setcounter") {
865 string const name = getArg('{', '}');
866 string const content = getArg('{', '}');
867 if (name == "secnumdepth")
868 h_secnumdepth = content;
869 else if (name == "tocdepth")
870 h_tocdepth = content;
872 h_preamble += "\\setcounter{" + name + "}{" + content + "}\n";
875 else if (t.cs() == "setlength") {
876 string const name = getToken().cs();
877 string const content = getArg('{', '}');
878 if (name == "parskip")
879 h_paragraph_separation = "skip";
880 else if (name == "parindent")
881 h_paragraph_separation = "skip";
883 h_preamble += "\\setcounter{" + name + "}{" + content + "}\n";
886 else if (t.cs() == "par") {
887 if (!active_environments.empty())
888 result << "\n\\layout " << active_environments.top() << "\n\n";
891 else if (t.cs() == "title")
892 result << "\\layout Title\n\n" + parse_verbatim_item();
894 else if (t.cs() == "author")
895 result << "\\layout Author\n\n" + parse_verbatim_item();
897 else if (t.cs() == "abstract")
898 result << "\\layout Abstract\n\n" + parse_verbatim_item();
900 else if (t.cs() == "begin") {
901 string const name = getArg('{', '}');
902 active_environments.push(name);
903 result << parse(FLAG_END, mode);
906 if (flags & FLAG_LEAVE) {
907 flags &= ~FLAG_LEAVE;
916 } // anonymous namespace
919 int main(int argc, char * argv[])
922 cerr << "Usage: " << argv[0] << " <infile.tex>" << endl;
927 ifstream is(argv[1]);
930 string s = p.parse();
931 cout << "# tex2lyx 0.0.2 created this file\n"
932 << "\\lyxformat 222\n"
933 << "\\textclass " << h_textclass << "\n"
934 << "\\begin_preamble\n" << h_preamble << "\\end_preamble\n"
935 << "\\options " << h_options << "\n"
936 << "\\language " << h_language << "\n"
937 << "\\inputencoding " << h_inputencoding << "\n"
938 << "\\fontscheme " << h_fontscheme << "\n"
939 << "\\graphics " << h_graphics << "\n"
940 << "\\paperfontsize " << h_paperfontsize << "\n"
941 << "\\spacing " << h_spacing << "\n"
942 << "\\papersize " << h_papersize << "\n"
943 << "\\paperpackage " << h_paperpackage << "\n"
944 << "\\use_geometry " << h_use_geometry << "\n"
945 << "\\use_amsmath " << h_use_amsmath << "\n"
946 << "\\use_natbib " << h_use_natbib << "\n"
947 << "\\use_numerical_citations " << h_use_numerical_citations << "\n"
948 << "\\paperorientation " << h_paperorientation << "\n"
949 << "\\secnumdepth " << h_secnumdepth << "\n"
950 << "\\tocdepth " << h_tocdepth << "\n"
951 << "\\paragraph_separation " << h_paragraph_separation << "\n"
952 << "\\defskip " << h_defskip << "\n"
953 << "\\quotes_language " << h_quotes_language << "\n"
954 << "\\quotes_times " << h_quotes_times << "\n"
955 << "\\papercolumns " << h_papercolumns << "\n"
956 << "\\papersides " << h_papersides << "\n"
957 << "\\paperpagestyle " << h_paperpagestyle << "\n"
958 << "\\tracking_changes " << h_tracking_changes << "\n"