]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/text.cpp
tex2lyx/text.cpp: cosmetic
[lyx.git] / src / tex2lyx / text.cpp
index d26b972c981d2074ac370dbee27df399d6ce9241..88a08d7e043e0bf536cff627028d3c3ae29d7294 100644 (file)
@@ -5,6 +5,7 @@
  *
  * \author André Pönitz
  * \author Jean-Marc Lasgouttes
+ * \author Uwe Stöhr
  *
  * Full author contact details are available in file CREDITS.
  */
 #include <config.h>
 
 #include "tex2lyx.h"
+
 #include "Context.h"
 #include "FloatList.h"
-#include "lengthcommon.h"
-#include "support/lstrings.h"
+#include "Layout.h"
+#include "Length.h"
+
+#include "support/assert.h"
 #include "support/convert.h"
+#include "support/FileName.h"
 #include "support/filetools.h"
+#include "support/lstrings.h"
 
-#include <boost/filesystem/operations.hpp>
-#include <boost/tuple/tuple.hpp>
-
+#include <algorithm>
 #include <iostream>
 #include <map>
 #include <sstream>
 #include <vector>
 
+using namespace std;
+using namespace lyx::support;
 
 namespace lyx {
 
-using support::addExtension;
-using support::changeExtension;
-using support::FileName;
-using support::makeAbsPath;
-using support::makeRelPath;
-using support::rtrim;
-using support::suffixIs;
-using support::contains;
-using support::subst;
-
-using std::cerr;
-using std::endl;
-
-using std::map;
-using std::ostream;
-using std::ostringstream;
-using std::istringstream;
-using std::string;
-using std::vector;
-
-namespace fs = boost::filesystem;
-
 
 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
                Context const & context)
@@ -105,8 +89,9 @@ string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 }
 
 
-char const * const known_latex_commands[] = { "ref", "cite", "label", "index",
-"printindex", "pageref", "url", "vref", "vpageref", "prettyref", "eqref", 0 };
+char const * const known_latex_commands[] = { "ref", "cite", "nocite", "label",
+ "index", "printindex", "pageref", "url", "vref", "vpageref", "prettyref",
+ "eqref", 0 };
 
 /*!
  * natbib commands.
@@ -133,20 +118,24 @@ char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 "citefield", "citetitle", "cite*", 0 };
 
 /// LaTeX names for quotes
-char const * const known_quotes[] = { "glqq", "grqq", "quotedblbase",
-"textquotedblleft", "quotesinglbase", "guilsinglleft", "guilsinglright", 0};
+char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
+"guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
+"quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
+"guilsinglleft", "frq", "guilsinglright", 0};
 
 /// the same as known_quotes with .lyx names
-char const * const known_coded_quotes[] = { "gld", "grd", "gld",
-"grd", "gls", "fls", "frs", 0};
+char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
+"ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
+"gld", "grd", "gls", "ers", "fls",
+"fls", "frs", "frs", 0};
 
 /// LaTeX names for font sizes
 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 
-/// the same as known_sizes with .lyx names
-char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
-"small", "normal", "large", "larger", "largest",  "huge", "giant", 0};
+/// the same as known_sizes with .lyx names plus a default entry
+char const * const known_coded_sizes[] = { "default", "tiny", "scriptsize", "footnotesize",
+"small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 
 /// LaTeX 2.09 names for font families
 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
@@ -262,7 +251,7 @@ bool splitLatexLength(string const & len, string & value, string & unit)
        if (contains(len, '\\'))
                unit = trim(string(len, i));
        else
-               unit = support::ascii_lowercase(trim(string(len, i)));
+               unit = ascii_lowercase(trim(string(len, i)));
        return true;
 }
 
@@ -359,7 +348,7 @@ string find_file(string const & name, string const & path,
        // expects utf8)
        for (char const * const * what = extensions; *what; ++what) {
                string const trial = addExtension(name, *what);
-               if (fs::exists(makeAbsPath(trial, path).toFilesystemEncoding()))
+               if (makeAbsPath(trial, path).exists())
                        return trial;
        }
        return string();
@@ -434,36 +423,37 @@ void handle_comment(ostream & os, string const & s, Context & context)
 }
 
 
-class isLayout : public std::unary_function<LyXLayout_ptr, bool> {
-public:
-       isLayout(string const name) : name_(name) {}
-       bool operator()(LyXLayout_ptr const & ptr) const {
-               return ptr->latexname() == name_;
-       }
-private:
-       string const name_;
-};
-
-
-LyXLayout_ptr findLayout(LyXTextClass const & textclass,
-                        string const & name)
+Layout const * findLayout(TextClass const & textclass, string const & name)
 {
-       LyXTextClass::const_iterator beg = textclass.begin();
-       LyXTextClass::const_iterator end = textclass.end();
-
-       LyXTextClass::const_iterator
-               it = std::find_if(beg, end, isLayout(name));
-
-       return (it == end) ? LyXLayout_ptr() : *it;
+       DocumentClass::const_iterator lit = textclass.begin();
+       DocumentClass::const_iterator len = textclass.end();
+       for (; lit != len; ++lit)
+               if (lit->latexname() == name)
+                       return &*lit;
+       return 0;
 }
 
 
 void eat_whitespace(Parser &, ostream &, Context &, bool);
 
 
+Layout * captionlayout()
+{
+       static Layout * lay = 0;
+       if (!lay) {
+               lay = new Layout;
+               lay->name_ = from_ascii("Caption");
+               lay->latexname_ = "caption";
+               lay->latextype = LATEX_COMMAND;
+               lay->optionalargs = 1;
+       }
+       return lay;
+}
+
+
 void output_command_layout(ostream & os, Parser & p, bool outer,
                           Context & parent_context,
-                          LyXLayout_ptr newlayout)
+                          Layout const * newlayout)
 {
        parent_context.check_end_layout(os);
        Context context(true, parent_context.textclass, newlayout,
@@ -591,9 +581,11 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
 {
        string position;
        string inner_pos;
-       string height_value = "0";
-       string height_unit = "pt";
-       string height_special = "none";
+       // We need to set the height to the LaTeX default of 1\\totalheight
+       // for the case when no height argument is given
+       string height_value = "1";
+       string height_unit = "in";
+       string height_special = "totalheight";
        string latex_height;
        if (p.next_token().asInput() == "[") {
                position = p.getArg('[', ']');
@@ -717,7 +709,7 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
 void parse_environment(Parser & p, ostream & os, bool outer,
                       Context & parent_context)
 {
-       LyXLayout_ptr newlayout;
+       Layout const * newlayout;
        string const name = p.getArg('{', '}');
        const bool is_starred = suffixIs(name, '*');
        string const unstarred_name = rtrim(name, "*");
@@ -773,6 +765,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
                end_inset(os);
                p.skip_spaces();
+               skip_braces(p); // eat {} that might by set by LyX behind comments
        }
 
        else if (name == "lyxgreyedout") {
@@ -785,27 +778,52 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                p.skip_spaces();
        }
 
+       else if (name == "framed") {
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_layout(os);
+               begin_inset(os, "Note Framed\n");
+               os << "status open\n";
+               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+               end_inset(os);
+               p.skip_spaces();
+       }
+
+       else if (name == "shaded") {
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_layout(os);
+               begin_inset(os, "Note Shaded\n");
+               os << "status open\n";
+               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+               end_inset(os);
+               p.skip_spaces();
+       }
+
        else if (!parent_context.new_layout_allowed)
                parse_unknown_environment(p, name, os, FLAG_END, outer,
                                          parent_context);
 
-       // Alignment settings
-       else if (name == "center" || name == "flushleft" || name == "flushright" ||
-                name == "centering" || name == "raggedright" || name == "raggedleft") {
+       // Alignment and spacing settings
+       // FIXME (bug xxxx): These settings can span multiple paragraphs and
+       //                                       therefore are totally broken!
+       // Note that \centering, raggedright, and raggedleft cannot be handled, as
+       // they are commands not environments. They are furthermore switches that
+       // can be ended by another switches, but also by commands like \footnote or
+       // \parbox. So the only safe way is to leave them untouched.
+       else if (name == "center" || name == "flushleft" || name == "flushright") {
                eat_whitespace(p, os, parent_context, false);
                // We must begin a new paragraph if not already done
                if (! parent_context.atParagraphStart()) {
                        parent_context.check_end_layout(os);
                        parent_context.new_paragraph(os);
                }
-               if (name == "flushleft" || name == "raggedright")
+               if (name == "flushleft")
                        parent_context.add_extra_stuff("\\align left\n");
-               else if (name == "flushright" || name == "raggedleft")
+               else if (name == "flushright")
                        parent_context.add_extra_stuff("\\align right\n");
-               else
+               else if (name == "center")
                        parent_context.add_extra_stuff("\\align center\n");
                parse_text(p, os, FLAG_END, outer, parent_context);
-               // Just in case the environment is empty ..
+               // Just in case the environment is empty
                parent_context.extra_stuff.erase();
                // We must begin a new paragraph to reset the alignment
                parent_context.new_paragraph(os);
@@ -813,7 +831,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
        }
 
        // The single '=' is meant here.
-       else if ((newlayout = findLayout(parent_context.textclass, name)).get() &&
+       else if ((newlayout = findLayout(parent_context.textclass, name)) &&
                  newlayout->isEnvironment()) {
                eat_whitespace(p, os, parent_context, false);
                Context context(true, parent_context.textclass, newlayout,
@@ -906,7 +924,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 /// parses a comment and outputs it to \p os.
 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
 {
-       BOOST_ASSERT(t.cat() == catComment);
+       LASSERT(t.cat() == catComment, return);
        if (!t.cs().empty()) {
                context.check_layout(os);
                handle_comment(os, '%' + t.cs(), context);
@@ -974,25 +992,26 @@ void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
 
 
 /// get the arguments of a natbib or jurabib citation command
-std::pair<string, string> getCiteArguments(Parser & p, bool natbibOrder)
+void get_cite_arguments(Parser & p, bool natbibOrder,
+       string & before, string & after)
 {
        // We need to distinguish "" and "[]", so we can't use p.getOpt().
 
        // text before the citation
-       string before;
+       before.clear();
        // text after the citation
-       string after = p.getFullOpt();
+       after = p.getFullOpt();
 
        if (!after.empty()) {
                before = p.getFullOpt();
                if (natbibOrder && !before.empty())
-                       std::swap(before, after);
+                       swap(before, after);
        }
-       return std::make_pair(before, after);
 }
 
 
-/// Convert filenames with TeX macros and/or quotes to something LyX can understand
+/// Convert filenames with TeX macros and/or quotes to something LyX
+/// can understand
 string const normalize_filename(string const & name)
 {
        Parser p(trim(name, "\""));
@@ -1020,12 +1039,14 @@ string const normalize_filename(string const & name)
 /// convention (relative to .lyx file) if it is relative
 void fix_relative_filename(string & name)
 {
-       if (lyx::support::absolutePath(name))
+       FileName fname(name);
+       if (fname.isAbsolute())
                return;
+
        // FIXME UNICODE encoding of name may be wrong (makeAbsPath expects
        // utf8)
        name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFilename()),
-                                  from_utf8(getParentFilePath())));
+                                  from_utf8(getParentFilePath())));
 }
 
 
@@ -1049,7 +1070,7 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
        }
 
        if (!scrap || !context.new_layout_allowed ||
-           !context.textclass.hasLayout("Scrap")) {
+           !context.textclass.hasLayout(from_ascii("Scrap"))) {
                cerr << "Warning: Could not interpret '" << name
                     << "'. Ignoring it." << endl;
                return;
@@ -1062,7 +1083,8 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
        // noweb code chunks are implemented with a layout style in LyX they
        // always must be in an own paragraph.
        context.new_paragraph(os);
-       Context newcontext(true, context.textclass, context.textclass["Scrap"]);
+       Context newcontext(true, context.textclass,
+               &context.textclass[from_ascii("Scrap")]);
        newcontext.check_layout(os);
        os << name;
        while (p.good()) {
@@ -1102,7 +1124,9 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
 {
-       LyXLayout_ptr newlayout;
+       Layout const * newlayout = 0;
+       // store the current selectlanguage to be used after \foreignlanguage
+       string selectlang;
        // Store the latest bibliographystyle (needed for bibtex inset)
        string bibliographystyle;
        bool const use_natbib = used_packages.find("natbib") != used_packages.end();
@@ -1164,8 +1188,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                // extended to other quotes, but is not so easy (a
                // left english quote is the same as a right german
                // quote...)
-               else if (t.asInput() == "`"
-                        && p.next_token().asInput() == "`") {
+               else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
                        context.check_layout(os);
                        begin_inset(os, "Quotes ");
                        os << "eld";
@@ -1173,8 +1196,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.get_token();
                        skip_braces(p);
                }
-               else if (t.asInput() == "'"
-                        && p.next_token().asInput() == "'") {
+               else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
                        context.check_layout(os);
                        begin_inset(os, "Quotes ");
                        os << "erd";
@@ -1183,6 +1205,24 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        skip_braces(p);
                }
 
+               else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
+                       context.check_layout(os);
+                       begin_inset(os, "Quotes ");
+                       os << "ald";
+                       end_inset(os);
+                       p.get_token();
+                       skip_braces(p);
+               }
+
+               else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
+                       context.check_layout(os);
+                       begin_inset(os, "Quotes ");
+                       os << "ard";
+                       end_inset(os);
+                       p.get_token();
+                       skip_braces(p);
+               }
+
                else if (t.asInput() == "<"
                         && p.next_token().asInput() == "<" && noweb_mode) {
                        p.get_token();
@@ -1193,7 +1233,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        check_space(p, os, context);
 
                else if (t.character() == '[' && noweb_mode &&
-                        p.next_token().character() == '[') {
+                        p.next_token().character() == '[') {
                        // These can contain underscores
                        p.putback();
                        string const s = p.getFullOpt() + ']';
@@ -1252,7 +1292,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // special handling of font attribute changes
                        Token const prev = p.prev_token();
                        Token const next = p.next_token();
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        if (next.character() == '[' ||
                            next.character() == ']' ||
                            next.character() == '*') {
@@ -1418,15 +1458,54 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        os << "\\bibitem ";
                        os << p.getOpt();
                        os << '{' << p.verbatim_item() << '}' << "\n";
+               } 
+               
+               else if(t.cs() == "global") {
+                       // skip global which can appear in front of e.g. "def"
                }
-
+               
                else if (t.cs() == "def") {
                        context.check_layout(os);
                        eat_whitespace(p, os, context, false);
                        string name = p.get_token().cs();
-                       while (p.next_token().cat() != catBegin)
-                               name += p.get_token().asString();
-                       handle_ert(os, "\\def\\" + name + '{' + p.verbatim_item() + '}', context);
+                       eat_whitespace(p, os, context, false);
+
+                       // parameter text
+                       bool simple = true;
+                       string paramtext;
+                       int arity = 0;
+                       while (p.next_token().cat() != catBegin) {
+                               if (p.next_token().cat() == catParameter) {
+                                       // # found
+                                       p.get_token();
+                                       paramtext += "#";
+
+                                       // followed by number?
+                                       if (p.next_token().cat() == catOther) {
+                                               char c = p.getChar();
+                                               paramtext += c;
+                                               // number = current arity + 1?
+                                               if (c == arity + '0' + 1)
+                                                       ++arity;
+                                               else
+                                                       simple = false;
+                                       } else
+                                               paramtext += p.get_token().asString();
+                               } else {
+                                       paramtext += p.get_token().asString();
+                                       simple = false;
+                               }
+                       }
+
+                       // only output simple (i.e. compatible) macro as FormulaMacros
+                       string ert = "\\def\\" + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
+                       if (simple) {
+                               context.check_layout(os);
+                               begin_inset(os, "FormulaMacro");
+                               os << "\n" << ert;
+                               end_inset(os);
+                       } else
+                               handle_ert(os, ert, context);
                }
 
                else if (t.cs() == "noindent") {
@@ -1457,23 +1536,53 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        eat_whitespace(p, os, context, true);
                }
 
+               // Starred section headings
                // Must attempt to parse "Section*" before "Section".
                else if ((p.next_token().asInput() == "*") &&
                         context.new_layout_allowed &&
-                        // The single '=' is meant here.
-                        (newlayout = findLayout(context.textclass,
-                                                t.cs() + '*')).get() &&
+                        (newlayout = findLayout(context.textclass, t.cs() + '*')) &&
                         newlayout->isCommand()) {
+                       TeXFont const oldFont = context.font;
+                       // save the current font size
+                       string const size = oldFont.size;
+                       // reset the font size to default, because the font size switches don't
+                       // affect section headings and the like
+                       context.font.size = known_coded_sizes[0];
+                       output_font_change(os, oldFont, context.font);
+                       // write the layout
                        p.get_token();
                        output_command_layout(os, p, outer, context, newlayout);
+                       // set the font size to the original value
+                       context.font.size = size;
+                       output_font_change(os, oldFont, context.font);
                        p.skip_spaces();
                }
 
-               // The single '=' is meant here.
+               // Section headings and the like
                else if (context.new_layout_allowed &&
-                        (newlayout = findLayout(context.textclass, t.cs())).get() &&
+                        (newlayout = findLayout(context.textclass, t.cs())) &&
                         newlayout->isCommand()) {
+                       TeXFont const oldFont = context.font;
+                       // save the current font size
+                       string const size = oldFont.size;
+                       // reset the font size to default, because the font size switches don't
+                       // affect section headings and the like
+                       context.font.size = known_coded_sizes[0];
+                       output_font_change(os, oldFont, context.font);
+                       // write the layout
                        output_command_layout(os, p, outer, context, newlayout);
+                       // set the font size to the original value
+                       context.font.size = size;
+                       output_font_change(os, oldFont, context.font);
+                       p.skip_spaces();
+               }
+
+               // Special handling for \caption
+               // FIXME: remove this when InsetCaption is supported.
+               else if (context.new_layout_allowed &&
+                        t.cs() == captionlayout()->latexname()) {
+                       output_command_layout(os, p, outer, context, 
+                                             captionlayout());
                        p.skip_spaces();
                }
 
@@ -1491,7 +1600,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // therefore path is only used for testing
                        // FIXME UNICODE encoding of name and path may be
                        // wrong (makeAbsPath expects utf8)
-                       if (!fs::exists(makeAbsPath(name, path).toFilesystemEncoding())) {
+                       if (!makeAbsPath(name, path).exists()) {
                                // The file extension is probably missing.
                                // Now try to find it out.
                                string const dvips_name =
@@ -1523,7 +1632,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                        // FIXME UNICODE encoding of name and path may be
                        // wrong (makeAbsPath expects utf8)
-                       if (fs::exists(makeAbsPath(name, path).toFilesystemEncoding()))
+                       if (makeAbsPath(name, path).exists())
                                fix_relative_filename(name);
                        else
                                cerr << "Warning: Could not find graphics file '"
@@ -1582,8 +1691,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                numberOfbbOptions++;
                        if (numberOfbbOptions == 4)
                                os << "\tBoundingBox "
-                                  << opts["bbllx"] << opts["bblly"]
-                                  << opts["bburx"] << opts["bbury"] << '\n';
+                                  << opts["bbllx"] << " " << opts["bblly"] << " "
+                                  << opts["bburx"] << " " << opts["bbury"] << '\n';
                        else if (numberOfbbOptions > 0)
                                cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
                        numberOfbbOptions = 0;
@@ -1593,7 +1702,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                numberOfbbOptions++;
                        if (numberOfbbOptions == 2)
                                os << "\tBoundingBox 0bp 0bp "
-                                  << opts["natwidth"] << opts["natheight"] << '\n';
+                                  << opts["natwidth"] << " " << opts["natheight"] << '\n';
                        else if (numberOfbbOptions > 0)
                                cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
                        ostringstream special;
@@ -1647,7 +1756,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        context.check_layout(os);
                        string const s = p.verbatim_item();
-                       if (s == "±" || s == "³" || s == "²" || s == "µ")
+                       if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
                                os << s;
                        else
                                handle_ert(os, "\\ensuremath{" + s + "}",
@@ -1752,7 +1861,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
                        context.check_layout(os);
-                       Font oldFont = context.font;
+                       TeXFont oldFont = context.font;
                        context.font.init();
                        context.font.size = oldFont.size;
                        os << "\n\\family " << context.font.family << "\n";
@@ -1766,6 +1875,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                eat_whitespace(p, os, context, false);
                }
 
+               else if (t.cs() == "textcolor") {
+                       // scheme is \textcolor{color name}{text}
+                       string const color = p.verbatim_item();
+                       // we only support the predefined colors of the color package
+                       if (color == "black" || color == "blue" || color == "cyan"
+                               || color == "green" || color == "magenta" || color == "red"
+                               || color == "white" || color == "yellow") {
+                                       context.check_layout(os);
+                                       os << "\n\\color " << color << "\n";
+                                       parse_text_snippet(p, os, FLAG_ITEM, outer, context);
+                                       context.check_layout(os);
+                                       os << "\n\\color inherit\n";
+                       } else
+                               // for custom defined colors
+                               handle_ert(os, t.asInput() + "{" + color + "}", context);
+               }
+
                else if (t.cs() == "underbar") {
                        // Do NOT handle \underline.
                        // \underbar cuts through y, g, q, p etc.,
@@ -1785,6 +1911,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        os << "\n\\" << t.cs() << " default\n";
                }
 
+               else if (t.cs() == "lyxline") {
+                       context.check_layout(os);
+                       os << "\\lyxline";
+               }
+
                else if (use_natbib &&
                         is_known(t.cs(), known_natbib_commands) &&
                         ((t.cs() != "citefullauthor" &&
@@ -1810,8 +1941,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string before;
                        // text after the citation
                        string after;
+                       get_cite_arguments(p, true, before, after);
 
-                       boost::tie(before, after) = getCiteArguments(p, true);
                        if (command == "\\cite") {
                                // \cite without optional argument means
                                // \citet, \cite with at least one optional
@@ -1841,10 +1972,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string const command = '\\' + t.cs();
                        char argumentOrder = '\0';
                        vector<string> const & options = used_packages["jurabib"];
-                       if (std::find(options.begin(), options.end(),
+                       if (find(options.begin(), options.end(),
                                      "natbiborder") != options.end())
                                argumentOrder = 'n';
-                       else if (std::find(options.begin(), options.end(),
+                       else if (find(options.begin(), options.end(),
                                           "jurabiborder") != options.end())
                                argumentOrder = 'j';
 
@@ -1852,9 +1983,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string before;
                        // text after the citation
                        string after;
+                       get_cite_arguments(p, argumentOrder != 'j', before, after);
 
-                       boost::tie(before, after) =
-                               getCiteArguments(p, argumentOrder != 'j');
                        string const citation = p.verbatim_item();
                        if (!before.empty() && argumentOrder == '\0') {
                                cerr << "Warning: Assuming argument order "
@@ -1903,8 +2033,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                         context.new_layout_allowed) {
                        char const * const * where = is_known(t.cs(), known_sizes);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
-                       context.font.size = known_coded_sizes[where - known_sizes];
+                       TeXFont const oldFont = context.font;
+                       // the font size index differs by 1, because the known_coded_sizes
+                       // has additionally a "default" entry
+                       context.font.size = known_coded_sizes[where - known_sizes + 1];
                        output_font_change(os, oldFont, context.font);
                        eat_whitespace(p, os, context, false);
                }
@@ -1914,7 +2046,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where =
                                is_known(t.cs(), known_font_families);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        context.font.family =
                                known_coded_font_families[where - known_font_families];
                        output_font_change(os, oldFont, context.font);
@@ -1926,7 +2058,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where =
                                is_known(t.cs(), known_font_series);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        context.font.series =
                                known_coded_font_series[where - known_font_series];
                        output_font_change(os, oldFont, context.font);
@@ -1938,7 +2070,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where =
                                is_known(t.cs(), known_font_shapes);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        context.font.shape =
                                known_coded_font_shapes[where - known_font_shapes];
                        output_font_change(os, oldFont, context.font);
@@ -1949,7 +2081,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where =
                                is_known(t.cs(), known_old_font_families);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        context.font.init();
                        context.font.size = oldFont.size;
                        context.font.family =
@@ -1963,7 +2095,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where =
                                is_known(t.cs(), known_old_font_series);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        context.font.init();
                        context.font.size = oldFont.size;
                        context.font.series =
@@ -1977,7 +2109,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where =
                                is_known(t.cs(), known_old_font_shapes);
                        context.check_layout(os);
-                       Font const oldFont = context.font;
+                       TeXFont const oldFont = context.font;
                        context.font.init();
                        context.font.size = oldFont.size;
                        context.font.shape =
@@ -1986,6 +2118,27 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        eat_whitespace(p, os, context, false);
                }
 
+               else if (t.cs() == "selectlanguage") {
+                       context.check_layout(os);
+                       // save the language for the case that a \foreignlanguage is used 
+                       selectlang = subst(p.verbatim_item(), "\n", " ");
+                       os << "\\lang " << selectlang << "\n";
+                       
+               }
+
+               else if (t.cs() == "foreignlanguage") {
+                       context.check_layout(os);
+                       os << "\n\\lang " << subst(p.verbatim_item(), "\n", " ") << "\n";
+                       os << subst(p.verbatim_item(), "\n", " ");
+                       // set back to last selectlanguage
+                       os << "\n\\lang " << selectlang << "\n";
+               }
+
+               else if (t.cs() == "inputencoding")
+                       // write nothing because this is done by LyX using the "\lang"
+                       // information given by selectlanguage and foreignlanguage
+                       subst(p.verbatim_item(), "\n", " ");
+               
                else if (t.cs() == "LyX" || t.cs() == "TeX"
                         || t.cs() == "LaTeX") {
                        context.check_layout(os);
@@ -2081,12 +2234,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "\"") {
                        context.check_layout(os);
                        string const name = p.verbatim_item();
-                            if (name == "a") os << 'ä';
-                       else if (name == "o") os << 'ö';
-                       else if (name == "u") os << 'ü';
-                       else if (name == "A") os << 'Ä';
-                       else if (name == "O") os << 'Ö';
-                       else if (name == "U") os << 'Ü';
+                            if (name == "a") os << '\xe4';
+                       else if (name == "o") os << '\xf6';
+                       else if (name == "u") os << '\xfc';
+                       else if (name == "A") os << '\xc4';
+                       else if (name == "O") os << '\xd6';
+                       else if (name == "U") os << '\xdc';
                        else handle_ert(os, "\"{" + name + "}", context);
                }
 
@@ -2104,7 +2257,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // subset of LaTeX, so don't parse anything here,
                        // but use the raw argument.
                        // Otherwise we would convert \~{\i} wrongly.
-                       // This will of course not translate \~{\ss} to \~{ß},
+                       // This will of course not translate \~{\ss} to \~{ß},
                        // but that does at least compile and does only look
                        // strange on screen.
                        context.check_layout(os);
@@ -2115,12 +2268,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "ss") {
                        context.check_layout(os);
-                       os << "ß";
+                       os << "\xdf";
                        skip_braces(p); // eat {}
                }
 
                else if (t.cs() == "i" || t.cs() == "j" || t.cs() == "l" ||
-                        t.cs() == "L") {
+                        t.cs() == "L") {
                        context.check_layout(os);
                        os << "\\i \\" << t.cs() << "{}\n";
                        skip_braces(p); // eat {}
@@ -2140,6 +2293,39 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                }
 
+               else if (t.cs() == "newline" ||
+                       t.cs() == "linebreak") {
+                       context.check_layout(os);
+                       os << "\n\\" << t.cs() << "\n";
+                       skip_braces(p); // eat {}
+               }
+
+               else if (t.cs() == "href") {
+                       context.check_layout(os);
+                       begin_inset(os, "CommandInset ");
+                       os << t.cs() << "\n";
+                       os << "LatexCommand " << t.cs() << "\n";
+                       bool erase = false;
+                       size_t pos;
+                       // the first argument is "type:target", "type:" is optional
+                       // the second argument the name
+                       string href_target = subst(p.verbatim_item(), "\n", " ");
+                       string href_name = subst(p.verbatim_item(), "\n", " ");
+                       string href_type;
+                       // serach for the ":" to divide type from target
+                       if ((pos = href_target.find(":", 0)) != string::npos){
+                               href_type = href_target;
+                               href_type.erase(pos + 1, href_type.length());
+                               href_target.erase(0, pos + 1);
+                           erase = true;                                                                                       
+                       }
+                       os << "name " << '"' << href_name << '"' << "\n";
+                       os << "target " << '"' << href_target << '"' << "\n";
+                       if(erase)
+                               os << "type " << '"' << href_type << '"' << "\n";
+                       end_inset(os);
+               }
+
                else if (t.cs() == "input" || t.cs() == "include"
                         || t.cs() == "verbatiminput") {
                        string name = '\\' + t.cs();
@@ -2154,8 +2340,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // therefore path is only used for testing
                        // FIXME UNICODE encoding of filename and path may be
                        // wrong (makeAbsPath expects utf8)
-                       if (t.cs() == "include" &&
-                           !fs::exists(makeAbsPath(filename, path).toFilesystemEncoding())) {
+                       if ((t.cs() == "include" || t.cs() == "input") &&
+                           !makeAbsPath(filename, path).exists()) {
                                // The file extension is probably missing.
                                // Now try to find it out.
                                string const tex_name =
@@ -2166,7 +2352,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                        // FIXME UNICODE encoding of filename and path may be
                        // wrong (makeAbsPath expects utf8)
-                       if (fs::exists(makeAbsPath(filename, path).toFilesystemEncoding())) {
+                       if (makeAbsPath(filename, path).exists()) {
                                string const abstexname =
                                        makeAbsPath(filename, path).absFilename();
                                string const abslyxname =
@@ -2211,6 +2397,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "parbox")
                        parse_box(p, os, FLAG_ITEM, outer, context, true);
+               
+               //\makebox() is part of the picture environment and different from \makebox{}
+               //\makebox{} will be parsed by parse_box when bug 2956 is fixed
+               else if (t.cs() == "makebox") {
+                       string arg = t.asInput();
+                       if (p.next_token().character() == '(')
+                               //the syntax is: \makebox(x,y)[position]{content}
+                               arg += p.getFullParentheseArg();
+                       else
+                               //the syntax is: \makebox[width][position]{content}
+                               arg += p.getFullOpt();
+                       handle_ert(os, arg + p.getFullOpt(), context);
+               }
 
                else if (t.cs() == "smallskip" ||
                         t.cs() == "medskip" ||
@@ -2226,7 +2425,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (is_known(t.cs(), known_spaces)) {
                        char const * const * where = is_known(t.cs(), known_spaces);
                        context.check_layout(os);
-                       begin_inset(os, "InsetSpace ");
+                       os << "\\InsetSpace ";
                        os << '\\' << known_coded_spaces[where - known_spaces]
                           << '\n';
                        // LaTeX swallows whitespace after all spaces except
@@ -2243,17 +2442,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "newpage" ||
-                        t.cs() == "clearpage" ||
-                        t.cs() == "cleardoublepage") {
+                       t.cs() == "pagebreak" ||
+                       t.cs() == "clearpage" ||
+                       t.cs() == "cleardoublepage") {
                        context.check_layout(os);
-                       // FIXME: what about \\pagebreak?
                        os << "\n\\" << t.cs() << "\n";
                        skip_braces(p); // eat {}
                }
 
                else if (t.cs() == "newcommand" ||
                         t.cs() == "providecommand" ||
-                        t.cs() == "renewcommand") {
+                        t.cs() == "renewcommand" ||
+                        t.cs() == "newlyxcommand") {
                        // these could be handled by parse_command(), but
                        // we need to call add_known_command() here.
                        string name = t.asInput();
@@ -2264,12 +2464,120 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                        string const command = p.verbatim_item();
                        string const opt1 = p.getOpt();
-                       string const opt2 = p.getFullOpt();
-                       add_known_command(command, opt1, !opt2.empty());
-                       string const ert = name + '{' + command + '}' +
-                                          opt1 + opt2 +
-                                          '{' + p.verbatim_item() + '}';
-                       handle_ert(os, ert, context);
+                       string optionals;
+                       unsigned optionalsNum = 0;
+                       while (true) {
+                               string const opt = p.getFullOpt();
+                               if (opt.empty())
+                                       break;
+                               optionalsNum++;
+                               optionals += opt;
+                       }
+                       add_known_command(command, opt1, optionalsNum);
+                       string const ert = name + '{' + command + '}' + opt1
+                               + optionals + '{' + p.verbatim_item() + '}';
+
+                       context.check_layout(os);
+                       begin_inset(os, "FormulaMacro");
+                       os << "\n" << ert;
+                       end_inset(os);
+               }
+               
+               else if (t.cs() == "newcommandx" ||
+                        t.cs() == "renewcommandx") {
+                       // \newcommandx{\foo}[2][usedefault, addprefix=\global,1=default]{#1,#2}
+
+                       // get command name
+                       string command;
+                       if (p.next_token().cat() == catBegin)
+                               command = p.verbatim_item();
+                       else 
+                               command = "\\" + p.get_token().cs();
+                       
+                       // get arity, we do not check that it fits to the given
+                       // optional parameters here.
+                       string const opt1 = p.getOpt();
+                       
+                       // get options and default values for optional parameters
+                       std::vector<string> optionalValues;
+                       int optionalsNum = 0;
+                       if (p.next_token().character() == '[') {
+                               // skip '['
+                               p.get_token();
+                               
+                               // handle 'opt=value' options, separated by ','.
+                               eat_whitespace(p, os, context, false);
+                               while (p.next_token().character() != ']' && p.good()) {
+                                       char_type nextc = p.next_token().character();
+                                       if (nextc >= '1' && nextc <= '9') {
+                                               // optional value -> get parameter number
+                                               int n = p.getChar() - '0';
+
+                                               // skip '='
+                                               if (p.next_token().character() != '=') {
+                                                       cerr << "'=' expected after numeral option of \\newcommandx" << std::endl;
+                                                       // try to find ] or ,
+                                                       while (p.next_token().character() != ','
+                                                              && p.next_token().character() != ']')
+                                                               p.get_token();
+                                                       continue;
+                                               } else
+                                                       p.get_token();
+                                               
+                                               // get value
+                                               optionalValues.resize(max(size_t(n), optionalValues.size()));
+                                               optionalValues[n - 1].clear();
+                                               while (p.next_token().character() != ']'
+                                                      && p.next_token().character() != ',')
+                                                       optionalValues[n - 1] += p.verbatim_item();
+                                               optionalsNum = max(n, optionalsNum);
+                                       } else if (p.next_token().cat() == catLetter) {
+                                               // we in fact ignore every non-optional
+                                               // parameters
+                                               
+                                               // get option name
+                                               docstring opt;
+                                               while (p.next_token().cat() == catLetter)
+                                                       opt += p.getChar();
+                                               
+                                               // value?
+                                               eat_whitespace(p, os, context, false);
+                                               if (p.next_token().character() == '=') {
+                                                       p.get_token();
+                                                       while (p.next_token().character() != ']'
+                                                              && p.next_token().character() != ',')
+                                                               p.verbatim_item();
+                                               }
+                                       } else
+                                               return;
+                                       
+                                       // skip komma
+                                       eat_whitespace(p, os, context, false);
+                                       if (p.next_token().character() == ',') {
+                                               p.getChar();
+                                               eat_whitespace(p, os, context, false);
+                                       } else if (p.next_token().character() != ']')
+                                               continue;
+                               }
+                               
+                               // skip ']'
+                               p.get_token();
+                       }
+                       
+                       // concat the default values to the optionals string
+                       string optionals;
+                       for (unsigned i = 0; i < optionalValues.size(); ++i)
+                               optionals += "[" + optionalValues[i] + "]";
+                       
+                       // register and output command
+                       add_known_command(command, opt1, optionalsNum);
+                       string const ert = "\\newcommand{" + command + '}' + opt1
+                       + optionals + '{' + p.verbatim_item() + '}';
+                       
+                       context.check_layout(os);
+                       begin_inset(os, "FormulaMacro");
+                       os << "\n" << ert;
+                       end_inset(os);
                }
 
                else if (t.cs() == "vspace") {
@@ -2324,7 +2632,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        }
                                }
                        }
-
                        if (known_unit || known_vspace) {
                                // Literal length or known variable
                                context.check_layout(os);
@@ -2352,6 +2659,79 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                }
 
+               else if (t.cs() == "hspace") {
+                       bool starred = false;
+                       if (p.next_token().asInput() == "*") {
+                               p.get_token();
+                               starred = true;
+                       }
+                       string const length = p.verbatim_item();
+                       string unit;
+                       string valstring;
+                       bool valid = splitLatexLength(length, valstring, unit);
+                       bool known_unit = false;
+                       bool fill = false;
+                       double value;
+                       if (valid) {
+                               istringstream iss(valstring);
+                               iss >> value;
+                               if (value == 1.0)
+                                       if (unit == "\\fill") {
+                                               known_unit = true;
+                                               fill = true;
+                                       }
+                               switch (unitFromString(unit)) {
+                               case Length::SP:
+                               case Length::PT:
+                               case Length::BP:
+                               case Length::DD:
+                               case Length::MM:
+                               case Length::PC:
+                               case Length::CC:
+                               case Length::CM:
+                               case Length::IN:
+                               case Length::EX:
+                               case Length::EM:
+                               case Length::MU:
+                                       known_unit = true;
+                                       break;
+                               default:
+                                       break;
+                               }
+                       }
+                       if (known_unit) {
+                               // Literal length or known variable
+                               context.check_layout(os);
+                               begin_inset(os, "Space ");
+                               if (known_unit) {
+                                       os << "\\hspace";
+                                       if (starred)
+                                               os << '*';
+                                       if (fill)
+                                               os << "{" + unit + "}";
+                                       else {
+                                               os << "{}\n";
+                                               os << "\\length " << value << unit;
+                                       }
+                               }                               
+                               end_inset(os);
+                       } else {
+                               // LyX can't handle other length variables in Inset HSpace
+                               string name = t.asInput();
+                               if (starred)
+                                       name += '*';
+                               if (valid) {
+                                       if (value == 1.0)
+                                               handle_ert(os, name + '{' + unit + '}', context);
+                                       else if (value == -1.0)
+                                               handle_ert(os, name + "{-" + unit + '}', context);
+                                       else
+                                               handle_ert(os, name + '{' + valstring + unit + '}', context);
+                               } else
+                                       handle_ert(os, name + '{' + length + '}', context);
+                       }
+               }
+
                else {
                        //cerr << "#: " << t << " mode: " << mode << endl;
                        // heuristic: read up to next non-nested space