X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2Ftex2lyx%2Ftext.cpp;h=2c212a425fffbc7afc57b8171e0a591c94baf0fa;hb=9b530e59c2b74828f3a68f3bb7ee3dee0365cdc0;hp=c6916393f08b282a83719502df109aa947db4421;hpb=3b1a026a934e5e20aba4b6e6b9ae5723b153b004;p=lyx.git diff --git a/src/tex2lyx/text.cpp b/src/tex2lyx/text.cpp index c6916393f0..2c212a425f 100644 --- a/src/tex2lyx/text.cpp +++ b/src/tex2lyx/text.cpp @@ -112,26 +112,39 @@ string parse_text_snippet(Parser & p, unsigned flags, const bool outer, char const * const known_ref_commands[] = { "ref", "pageref", "vref", - "vpageref", "prettyref", "eqref", 0 }; + "vpageref", "prettyref", "nameref", "eqref", 0 }; char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref", - "vpageref", "formatted", "eqref", 0 }; + "vpageref", "formatted", "nameref", "eqref", 0 }; + +char const * const known_refstyle_commands[] = { "algref", "chapref", "corref", + "eqref", "enuref", "figref", "fnref", "lemref", "parref", "partref", "propref", + "secref", "subref", "tabref", "thmref", 0 }; + +char const * const known_refstyle_prefixes[] = { "alg", "chap", "cor", + "eq", "enu", "fig", "fn", "lem", "par", "part", "prop", + "sec", "sub", "tab", "thm", 0 }; + /** * supported CJK encodings - * SJIS and Bg5 cannot be supported as they are not - * supported by iconv * JIS does not work with LyX's encoding conversion */ const char * const supported_CJK_encodings[] = { -"EUC-JP", "KS", "GB", "UTF8", 0}; +"EUC-JP", "KS", "GB", "UTF8", +"Bg5", /*"JIS",*/ "SJIS", 0}; /** * the same as supported_CJK_encodings with their corresponding LyX language name + * FIXME: The mapping "UTF8" => "chinese-traditional" is only correct for files + * created by LyX. + * NOTE: "Bg5", "JIS" and "SJIS" are not supported by LyX, on re-export the + * encodings "UTF8", "EUC-JP" and "EUC-JP" will be used. * please keep this in sync with supported_CJK_encodings line by line! */ const char * const supported_CJK_languages[] = { -"japanese-cjk", "korean", "chinese-simplified", "chinese-traditional", 0}; +"japanese-cjk", "korean", "chinese-simplified", "chinese-traditional", +"chinese-traditional", /*"japanese-cjk",*/ "japanese-cjk", 0}; /*! * natbib commands. @@ -220,12 +233,14 @@ char const * const known_coded_font_shapes[] = { "italic", "slanted", "smallcaps", "up", 0}; /// Known special characters which need skip_spaces_braces() afterwards -char const * const known_special_chars[] = {"ldots", "lyxarrow", -"textcompwordmark", "slash", 0}; +char const * const known_special_chars[] = {"ldots", +"lyxarrow", "textcompwordmark", +"slash", "textasciitilde", "textasciicircum", "textbackslash", 0}; /// the same as known_special_chars with .lyx names -char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator", -"textcompwordmark{}", "slash{}", 0}; +char const * const known_coded_special_chars[] = {"\\SpecialChar \\ldots{}\n", +"\\SpecialChar \\menuseparator\n", "\\SpecialChar \\textcompwordmark{}\n", +"\\SpecialChar \\slash{}\n", "~", "^", "\n\\backslash\n", 0}; /*! * Graphics file extensions known by the dvips driver of the graphics package. @@ -272,6 +287,18 @@ char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0}; char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0}; int const known_phrase_lengths[] = {3, 5, 7, 0}; +/// known TIPA combining diacritical marks +char const * const known_tipa_marks[] = {"textsubwedge", "textsubumlaut", +"textsubtilde", "textseagull", "textsubbridge", "textinvsubbridge", +"textsubsquare", "textsubrhalfring", "textsublhalfring", "textsubplus", +"textovercross", "textsubarch", "textsuperimposetilde", "textraising", +"textlowering", "textadvancing", "textretracting", "textdoublegrave", +"texthighrise", "textlowrise", "textrisefall", "textsyllabic", +"textsubring", 0}; + +/// TIPA tones that need special handling +char const * const known_tones[] = {"15", "51", "45", "12", "454", 0}; + // string to store the float type to be able to determine the type of subfloats string float_type = ""; @@ -503,51 +530,33 @@ string convert_command_inset_arg(string s) } -void handle_backslash(ostream & os, string const & s) -{ - for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) { - if (*it == '\\') - os << "\n\\backslash\n"; - else - os << *it; - } -} - - -void handle_ert(ostream & os, string const & s, Context & context) +void output_ert(ostream & os, string const & s, Context & context) { - // We must have a valid layout before outputting the ERT inset. context.check_layout(os); - Context newcontext(true, context.textclass); - begin_inset(os, "ERT"); - os << "\nstatus collapsed\n"; - newcontext.check_layout(os); for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) { if (*it == '\\') os << "\n\\backslash\n"; else if (*it == '\n') { - newcontext.new_paragraph(os); - newcontext.check_layout(os); + context.new_paragraph(os); + context.check_layout(os); } else os << *it; } - newcontext.check_end_layout(os); - end_inset(os); + context.check_end_layout(os); } -void handle_comment(ostream & os, string const & s, Context & context) +void output_ert_inset(ostream & os, string const & s, Context & context) { - // TODO: Handle this better + // We must have a valid layout before outputting the ERT inset. + context.check_layout(os); Context newcontext(true, context.textclass); + InsetLayout const & layout = context.textclass.insetLayout(from_ascii("ERT")); + if (layout.forcePlainLayout()) + newcontext.layout = &context.textclass.plainLayout(); begin_inset(os, "ERT"); os << "\nstatus collapsed\n"; - newcontext.check_layout(os); - handle_backslash(os, s); - // make sure that our comment is the last thing on the line - newcontext.new_paragraph(os); - newcontext.check_layout(os); - newcontext.check_end_layout(os); + output_ert(os, s, newcontext); end_inset(os); } @@ -629,6 +638,8 @@ void output_command_layout(ostream & os, Parser & p, bool outer, context.check_deeper(os); context.check_layout(os); // FIXME: Adjust to format 446! + // Since format 446, layouts do not require anymore all optional + // arguments before the required ones. Needs to be implemented! int optargs = 0; while (optargs < context.layout->optArgs()) { eat_whitespace(p, os, context, false); @@ -636,7 +647,11 @@ void output_command_layout(ostream & os, Parser & p, bool outer, p.next_token().character() != '[') break; p.get_token(); // eat '[' - // FIXME: Just a workaround + // FIXME: Just a workaround. InsetArgument::updateBuffer + // will compute a proper ID for all "999" Arguments + // (which is also what lyx2lyx produces). + // However, tex2lyx should be able to output proper IDs + // itself. begin_inset(os, "Argument 999\n"); os << "status collapsed\n\n"; parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context); @@ -650,7 +665,11 @@ void output_command_layout(ostream & os, Parser & p, bool outer, if (p.next_token().cat() != catBegin) break; p.get_token(); // eat '{' - // FIXME: Just a workaround + // FIXME: Just a workaround. InsetArgument::updateBuffer + // will compute a proper ID for all "999" Arguments + // (which is also what lyx2lyx produces). + // However, tex2lyx should be able to output proper IDs + // itself. begin_inset(os, "Argument 999\n"); os << "status collapsed\n\n"; parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context); @@ -725,7 +744,7 @@ void parse_arguments(string const & command, case required: case req_group: // This argument contains regular LaTeX - handle_ert(os, ert + '{', context); + output_ert_inset(os, ert + '{', context); eat_whitespace(p, os, context, false); if (template_arguments[i] == required) parse_text(p, os, FLAG_ITEM, outer, context); @@ -760,7 +779,7 @@ void parse_arguments(string const & command, break; } } - handle_ert(os, ert, context); + output_ert_inset(os, ert, context); } @@ -895,7 +914,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, // If yes, we need to output ERT. p.pushPosition(); if (inner_flags & FLAG_END) - p.verbatimEnvironment(inner_type); + p.ertEnvironment(inner_type); else p.verbatim_item(); p.skip_spaces(true); @@ -916,7 +935,9 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, width_unit = "in"; width_special = "width"; } else if (latex_width.empty() && outer_type == "framebox") { - use_ert = true; + width_value.clear(); + width_unit.clear(); + width_special = "none"; } if (use_ert) { ostringstream ss; @@ -948,14 +969,14 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, } if (inner_type == "shaded") ss << "\\begin{shaded}"; - handle_ert(os, ss.str(), parent_context); + output_ert_inset(os, ss.str(), parent_context); if (!inner_type.empty()) { parse_text(p, os, inner_flags, outer, parent_context); if (inner_flags & FLAG_END) - handle_ert(os, "\\end{" + inner_type + '}', + output_ert_inset(os, "\\end{" + inner_type + '}', parent_context); else - handle_ert(os, "}", parent_context); + output_ert_inset(os, "}", parent_context); } if (!outer_type.empty()) { // If we already read the inner box we have to pop @@ -971,13 +992,10 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, } parse_text(p, os, outer_flags, outer, parent_context); if (outer_flags & FLAG_END) - handle_ert(os, "\\end{" + outer_type + '}', + output_ert_inset(os, "\\end{" + outer_type + '}', parent_context); - else if (inner_type.empty() && outer_type == "framebox") - // in this case it is already closed later - ; else - handle_ert(os, "}", parent_context); + output_ert_inset(os, "}", parent_context); } } else { // LyX does not like empty positions, so we have @@ -990,7 +1008,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, begin_inset(os, "Box "); if (outer_type == "framed") os << "Framed\n"; - else if (outer_type == "framebox") + else if (outer_type == "framebox" || outer_type == "fbox") os << "Boxed\n"; else if (outer_type == "shadowbox") os << "Shadowbox\n"; @@ -1001,18 +1019,27 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, preamble.registerAutomaticallyLoadedPackage("color"); } else if (outer_type == "doublebox") os << "Doublebox\n"; - else if (outer_type.empty()) + else if (outer_type.empty() || outer_type == "mbox") os << "Frameless\n"; else os << outer_type << '\n'; os << "position \"" << position << "\"\n"; os << "hor_pos \"" << hor_pos << "\"\n"; - os << "has_inner_box " << !inner_type.empty() << "\n"; + if (outer_type == "mbox") + os << "has_inner_box 1\n"; + else + os << "has_inner_box " << !inner_type.empty() << "\n"; os << "inner_pos \"" << inner_pos << "\"\n"; os << "use_parbox " << (inner_type == "parbox" || shadedparbox) << '\n'; - os << "use_makebox " << (inner_type == "makebox") << '\n'; - os << "width \"" << width_value << width_unit << "\"\n"; + if (outer_type == "mbox") + os << "use_makebox 1\n"; + else + os << "use_makebox " << (inner_type == "makebox") << '\n'; + if (outer_type == "fbox" || outer_type == "mbox") + os << "width \"\"\n"; + else + os << "width \"" << width_value << width_unit << "\"\n"; os << "special \"" << width_special << "\"\n"; os << "height \"" << height_value << height_unit << "\"\n"; os << "height_special \"" << height_special << "\"\n"; @@ -1060,13 +1087,13 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags, // LyX puts a % after the end of the minipage if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) { // new paragraph - //handle_comment(os, "%dummy", parent_context); + //output_ert_inset(os, "%dummy", parent_context); p.get_token(); p.skip_spaces(); parent_context.new_paragraph(os); } else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) { - //handle_comment(os, "%dummy", parent_context); + //output_ert_inset(os, "%dummy", parent_context); p.get_token(); p.skip_spaces(); // We add a protected space if something real follows @@ -1112,7 +1139,8 @@ void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer, p.skip_spaces(true); } } - if (outer_type == "shaded") { + if (outer_type == "shaded" || outer_type == "fbox" + || outer_type == "mbox") { // These boxes never have an inner box ; } else if (p.next_token().asInput() == "\\parbox") { @@ -1169,24 +1197,15 @@ void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_ context.layout = &parent_context.textclass.plainLayout(); string s; if (in_line) { - s = p.plainCommand('!', '!', "lstinline"); - context.new_paragraph(os); - context.check_layout(os); + // set catcodes to verbatim early, just in case. + p.setCatcodes(VERBATIM_CATCODES); + string delim = p.get_token().asInput(); + //FIXME: handler error condition + s = p.verbatimStuff(delim).second; +// context.new_paragraph(os); } else - s = p.plainEnvironment("lstlisting"); - for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) { - if (*it == '\\') - os << "\n\\backslash\n"; - else if (*it == '\n') { - // avoid adding an empty paragraph at the end - if (it + 1 != et) { - context.new_paragraph(os); - context.check_layout(os); - } - } else - os << *it; - } - context.check_end_layout(os); + s = p.verbatimEnvironment("lstlisting"); + output_ert(os, s, context); end_inset(os); } @@ -1210,9 +1229,9 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os, bool const new_layout_allowed = parent_context.new_layout_allowed; if (specialfont) parent_context.new_layout_allowed = false; - handle_ert(os, "\\begin{" + name + "}", parent_context); + output_ert_inset(os, "\\begin{" + name + "}", parent_context); parse_text_snippet(p, os, flags, outer, parent_context); - handle_ert(os, "\\end{" + name + "}", parent_context); + output_ert_inset(os, "\\end{" + name + "}", parent_context); if (specialfont) parent_context.new_layout_allowed = new_layout_allowed; } @@ -1406,31 +1425,31 @@ void parse_environment(Parser & p, ostream & os, bool outer, } else if (name == "verbatim") { - os << "\n\\end_layout\n\n\\begin_layout Verbatim\n"; - string const s = p.plainEnvironment("verbatim"); - string::const_iterator it2 = s.begin(); - for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) { - if (*it == '\\') - os << "\\backslash "; - else if (*it == '\n') { - it2 = it + 1; - // avoid adding an empty paragraph at the end - // FIXME: if there are 2 consecutive spaces at the end ignore it - // because LyX will re-add a \n - // This hack must be removed once bug 8049 is fixed! - if ((it + 1 != et) && (it + 2 != et || *it2 != '\n')) - os << "\n\\end_layout\n\\begin_layout Verbatim\n"; - } else - os << *it; - } - os << "\n\\end_layout\n\n"; + // FIXME: this should go in the generic code that + // handles environments defined in layout file that + // have "PassThru 1". However, the code over there is + // already too complicated for my taste. + parent_context.new_paragraph(os); + Context context(true, parent_context.textclass, + &parent_context.textclass[from_ascii("Verbatim")]); + string s = p.verbatimEnvironment("verbatim"); + output_ert(os, s, context); + p.skip_spaces(); + } + + else if (name == "IPA") { + eat_whitespace(p, os, parent_context, false); + parent_context.check_layout(os); + begin_inset(os, "IPA\n"); + parse_text_in_inset(p, os, FLAG_END, outer, parent_context); + end_inset(os); p.skip_spaces(); - // reset to Standard layout - os << "\n\\begin_layout Standard\n"; + preamble.registerAutomaticallyLoadedPackage("tipa"); + preamble.registerAutomaticallyLoadedPackage("tipx"); } else if (name == "CJK") { - // the scheme is \begin{CJK}{encoding}{mapping}{text} + // the scheme is \begin{CJK}{encoding}{mapping}text\end{CJK} // It is impossible to decide if a CJK environment was in its own paragraph or within // a line. We therefore always assume a paragraph since the latter is a rare case. eat_whitespace(p, os, parent_context, false); @@ -1438,33 +1457,47 @@ void parse_environment(Parser & p, ostream & os, bool outer, // store the encoding to be able to reset it string const encoding_old = p.getEncoding(); string const encoding = p.getArg('{', '}'); - // SJIS and Bg5 cannot be handled by iconv - // JIS does not work with LyX's encoding conversion - if (encoding != "Bg5" && encoding != "JIS" && encoding != "SJIS") - p.setEncoding(encoding); - else - p.setEncoding("utf8"); - // LyX doesn't support the second argument so if - // this is used we need to output everything as ERT - string const mapping = p.getArg('{', '}'); + // FIXME: For some reason JIS does not work. Although the text + // in tests/CJK.tex is identical with the SJIS version if you + // convert both snippets using the recode command line utility, + // the resulting .lyx file contains some extra characters if + // you set buggy_encoding to false for JIS. + bool const buggy_encoding = encoding == "JIS"; + if (!buggy_encoding) + p.setEncoding(encoding, Encoding::CJK); + else { + // FIXME: This will read garbage, since the data is not encoded in utf8. + p.setEncoding("UTF-8"); + } + // LyX only supports the same mapping for all CJK + // environments, so we might need to output everything as ERT + string const mapping = trim(p.getArg('{', '}')); char const * const * const where = is_known(encoding, supported_CJK_encodings); - if ((!mapping.empty() && mapping != " ") || !where) { + if (!buggy_encoding && !preamble.fontCJKSet()) + preamble.fontCJK(mapping); + bool knownMapping = mapping == preamble.fontCJK(); + if (buggy_encoding || !knownMapping || !where) { parent_context.check_layout(os); - handle_ert(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}", + output_ert_inset(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}", parent_context); // we must parse the content as verbatim because e.g. JIS can contain // normally invalid characters + // FIXME: This works only for the most simple cases. + // Since TeX control characters are not parsed, + // things like comments are completely wrong. string const s = p.plainEnvironment("CJK"); for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) { if (*it == '\\') - handle_ert(os, "\\", parent_context); + output_ert_inset(os, "\\", parent_context); else if (*it == '$') - handle_ert(os, "$", parent_context); - else + output_ert_inset(os, "$", parent_context); + else if (*it == '\n' && it + 1 != et && s.begin() + 1 != it) + os << "\n "; + else os << *it; } - handle_ert(os, "\\end{" + name + "}", + output_ert_inset(os, "\\end{" + name + "}", parent_context); } else { string const lang = @@ -1492,6 +1525,37 @@ void parse_environment(Parser & p, ostream & os, bool outer, preamble.registerAutomaticallyLoadedPackage("color"); } + else if (name == "btSect") { + eat_whitespace(p, os, parent_context, false); + parent_context.check_layout(os); + begin_command_inset(os, "bibtex", "bibtex"); + string bibstyle = "plain"; + if (p.hasOpt()) { + bibstyle = p.getArg('[', ']'); + p.skip_spaces(true); + } + string const bibfile = p.getArg('{', '}'); + eat_whitespace(p, os, parent_context, false); + Token t = p.get_token(); + if (t.asInput() == "\\btPrintCited") { + p.skip_spaces(true); + os << "btprint " << '"' << "btPrintCited" << '"' << "\n"; + } + if (t.asInput() == "\\btPrintNotCited") { + p.skip_spaces(true); + os << "btprint " << '"' << "btPrintNotCited" << '"' << "\n"; + } + if (t.asInput() == "\\btPrintAll") { + p.skip_spaces(true); + os << "btprint " << '"' << "btPrintAll" << '"' << "\n"; + } + os << "bibfiles " << '"' << bibfile << '"' << "\n"; + os << "options " << '"' << bibstyle << '"' << "\n"; + parse_text_in_inset(p, os, FLAG_END, outer, parent_context); + end_inset(os); + p.skip_spaces(); + } + else if (name == "framed" || name == "shaded") { eat_whitespace(p, os, parent_context, false); parse_outer_box(p, os, FLAG_END, outer, parent_context, name, ""); @@ -1566,24 +1630,13 @@ void parse_environment(Parser & p, ostream & os, bool outer, if (last_env == name) { // we need to output a separator since LyX would export // the two environments as one otherwise (bug 5716) - docstring const sep = from_ascii("--Separator--"); TeX2LyXDocClass const & textclass(parent_context.textclass); - if (textclass.hasLayout(sep)) { - Context newcontext(parent_context); - newcontext.layout = &(textclass[sep]); - newcontext.check_layout(os); - newcontext.check_end_layout(os); - } else { - parent_context.check_layout(os); - begin_inset(os, "Note Note\n"); - os << "status closed\n"; - Context newcontext(true, textclass, - &(textclass.defaultLayout())); - newcontext.check_layout(os); - newcontext.check_end_layout(os); - end_inset(os); - parent_context.check_end_layout(os); - } + Context newcontext(true, textclass, + &(textclass.defaultLayout())); + newcontext.check_layout(os); + begin_inset(os, "Separator plain\n"); + end_inset(os); + newcontext.check_end_layout(os); } switch (context.layout->latextype) { case LATEX_LIST_ENVIRONMENT: @@ -1600,11 +1653,11 @@ void parse_environment(Parser & p, ostream & os, bool outer, } context.check_deeper(os); // handle known optional and required arguments - // layouts require all optional arguments before the required ones + // FIXME: Since format 446, layouts do not require anymore all optional + // arguments before the required ones. Needs to be implemented! // Unfortunately LyX can't handle arguments of list arguments (bug 7468): // It is impossible to place anything after the environment name, // but before the first \\item. - // FIXME: Adjust to format 446! if (context.layout->latextype == LATEX_ENVIRONMENT) { bool need_layout = true; int optargs = 0; @@ -1618,7 +1671,11 @@ void parse_environment(Parser & p, ostream & os, bool outer, context.check_layout(os); need_layout = false; } - // FIXME: Just a workaround + // FIXME: Just a workaround. InsetArgument::updateBuffer + // will compute a proper ID for all "999" Arguments + // (which is also what lyx2lyx produces). + // However, tex2lyx should be able to output proper IDs + // itself. begin_inset(os, "Argument 999\n"); os << "status collapsed\n\n"; parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context); @@ -1636,7 +1693,11 @@ void parse_environment(Parser & p, ostream & os, bool outer, context.check_layout(os); need_layout = false; } - // FIXME: Just a workaround + // FIXME: Just a workaround. InsetArgument::updateBuffer + // will compute a proper ID for all "999" Arguments + // (which is also what lyx2lyx produces). + // However, tex2lyx should be able to output proper IDs + // itself. begin_inset(os, "Argument 999\n"); os << "status collapsed\n\n"; parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context); @@ -1671,7 +1732,14 @@ void parse_environment(Parser & p, ostream & os, bool outer, begin_inset(os, "Flex "); os << to_utf8(newinsetlayout->name()) << '\n' << "status collapsed\n"; - parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout); + if (newinsetlayout->isPassThru()) { + string const arg = p.verbatimEnvironment(name); + Context context(true, parent_context.textclass, + &parent_context.textclass.plainLayout(), + parent_context.layout); + output_ert(os, arg, parent_context); + } else + parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout); end_inset(os); } @@ -1709,12 +1777,12 @@ void parse_environment(Parser & p, ostream & os, bool outer, parse_arguments("\\begin{" + name + "}", arguments, p, os, outer, parent_context); if (contents == verbatim) - handle_ert(os, p.verbatimEnvironment(name), + output_ert_inset(os, p.ertEnvironment(name), parent_context); else parse_text_snippet(p, os, FLAG_END, outer, parent_context); - handle_ert(os, "\\end{" + name + "}", parent_context); + output_ert_inset(os, "\\end{" + name + "}", parent_context); if (specialfont) parent_context.new_layout_allowed = new_layout_allowed; } @@ -1734,7 +1802,7 @@ void parse_comment(Parser & p, ostream & os, Token const & t, Context & context) LASSERT(t.cat() == catComment, return); if (!t.cs().empty()) { context.check_layout(os); - handle_comment(os, '%' + t.cs(), context); + output_ert_inset(os, '%' + t.cs(), context); if (p.next_token().cat() == catNewline) { // A newline after a comment line starts a new // paragraph @@ -1744,7 +1812,7 @@ void parse_comment(Parser & p, ostream & os, Token const & t, Context & context) // done (we might get called recursively) context.new_paragraph(os); } else - handle_ert(os, "\n", context); + output_ert_inset(os, "\n", context); eat_whitespace(p, os, context, true); } } else { @@ -1821,7 +1889,7 @@ void get_cite_arguments(Parser & p, bool natbibOrder, /// can understand string const normalize_filename(string const & name) { - Parser p(trim(name, "\"")); + Parser p(name); ostringstream os; while (p.good()) { Token const & t = p.get_token(); @@ -1835,10 +1903,29 @@ string const normalize_filename(string const & name) } else if (t.cs() == "space") { os << ' '; p.skip_spaces(); + } else if (t.cs() == "string") { + // Convert \string" to " and \string~ to ~ + Token const & n = p.next_token(); + if (n.asInput() != "\"" && n.asInput() != "~") + os << t.asInput(); } else os << t.asInput(); } - return os.str(); + // Strip quotes. This is a bit complicated (see latex_path()). + string full = os.str(); + if (!full.empty() && full[0] == '"') { + string base = removeExtension(full); + string ext = getExtension(full); + if (!base.empty() && base[base.length()-1] == '"') + // "a b" + // "a b".tex + return addExtension(trim(base, "\""), ext); + if (full[full.length()-1] == '"') + // "a b.c" + // "a b.c".tex + return trim(full, "\""); + } + return full; } @@ -1926,78 +2013,58 @@ void copy_file(FileName const & src, string dstname) } -/// Parse a NoWeb Scrap section. The initial "<<" is already parsed. -void parse_noweb(Parser & p, ostream & os, Context & context) +/// Parse a literate Chunk section. The initial "<<" is already parsed. +bool parse_chunk(Parser & p, ostream & os, Context & context) { - // assemble the rest of the keyword - string name("<<"); - bool scrap = false; - while (p.good()) { - Token const & t = p.get_token(); - if (t.asInput() == ">" && p.next_token().asInput() == ">") { - name += ">>"; - p.get_token(); - scrap = (p.good() && p.next_token().asInput() == "="); - if (scrap) - name += p.get_token().asInput(); - break; - } - name += t.asInput(); + // check whether a chunk is possible here. + if (!context.textclass.hasInsetLayout(from_ascii("Flex:Chunk"))) { + return false; } - if (!scrap || !context.new_layout_allowed || - !context.textclass.hasLayout(from_ascii("Scrap"))) { - cerr << "Warning: Could not interpret '" << name - << "'. Ignoring it." << endl; - return; + p.pushPosition(); + + // read the parameters + Parser::Arg const params = p.verbatimStuff(">>=\n", false); + if (!params.first) { + p.popPosition(); + return false; } - // We use new_paragraph instead of check_end_layout because the stuff - // following the noweb chunk needs to start with a \begin_layout. - // This may create a new paragraph even if there was none in the - // noweb file, but the alternative is an invalid LyX file. Since - // noweb code chunks are implemented with a layout style in LyX they - // always must be in an own paragraph. - context.new_paragraph(os); - Context newcontext(true, context.textclass, - &context.textclass[from_ascii("Scrap")]); - newcontext.check_layout(os); - os << name; - while (p.good()) { - Token const & t = p.get_token(); - // We abuse the parser a bit, because this is no TeX syntax - // at all. - if (t.cat() == catEscape) - os << subst(t.asInput(), "\\", "\n\\backslash\n"); - else { - ostringstream oss; - Context tmp(false, context.textclass, - &context.textclass[from_ascii("Scrap")]); - tmp.need_end_layout = true; - tmp.check_layout(oss); - os << subst(t.asInput(), "\n", oss.str()); - } - // The scrap chunk is ended by an @ at the beginning of a line. - // After the @ the line may contain a comment and/or - // whitespace, but nothing else. - if (t.asInput() == "@" && p.prev_token().cat() == catNewline && - (p.next_token().cat() == catSpace || - p.next_token().cat() == catNewline || - p.next_token().cat() == catComment)) { - while (p.good() && p.next_token().cat() == catSpace) - os << p.get_token().asInput(); - if (p.next_token().cat() == catComment) - // The comment includes a final '\n' - os << p.get_token().asInput(); - else { - if (p.next_token().cat() == catNewline) - p.get_token(); - os << '\n'; - } - break; - } + Parser::Arg const code = p.verbatimStuff("\n@"); + if (!code.first) { + p.popPosition(); + return false; } - newcontext.check_end_layout(os); + string const post_chunk = p.verbatimStuff("\n").second + '\n'; + if (post_chunk[0] != ' ' && post_chunk[0] != '\n') { + p.popPosition(); + return false; + } + // The last newline read is important for paragraph handling + p.putback(); + p.deparse(); + + //cerr << "params=[" << params.second << "], code=[" << code.second << "]" < Settings > Language.\n" << "\\end_layout\n"; end_inset(os); is_nonCJKJapanese = false; @@ -2264,19 +2335,27 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, skip_braces(p); } - else if (t.asInput() == "<" && p.next_token().asInput() == "<") { - context.check_layout(os); - begin_inset(os, "Quotes "); - os << "ard"; - end_inset(os); - p.get_token(); - skip_braces(p); - } - else if (t.asInput() == "<" - && p.next_token().asInput() == "<" && noweb_mode) { - p.get_token(); - parse_noweb(p, os, context); + && p.next_token().asInput() == "<") { + bool has_chunk = false; + if (noweb_mode) { + p.pushPosition(); + p.get_token(); + has_chunk = parse_chunk(p, os, context); + if (!has_chunk) + p.popPosition(); + } + + if (!has_chunk) { + context.check_layout(os); + begin_inset(os, "Quotes "); + //FIXME: this is a right danish quote; + // why not a left french quote? + os << "ard"; + end_inset(os); + p.get_token(); + skip_braces(p); + } } else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph())) @@ -2292,7 +2371,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, else cerr << "Warning: Inserting missing ']' in '" << s << "'." << endl; - handle_ert(os, s, context); + output_ert_inset(os, s, context); } else if (t.cat() == catLetter) { @@ -2309,7 +2388,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i) phrase += p.get_token().cs(); if (is_known(phrase, known_coded_phrases)) { - handle_ert(os, phrase, context); + output_ert_inset(os, phrase, context); handled = true; break; } else { @@ -2333,7 +2412,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, if (context.new_layout_allowed) context.new_paragraph(os); else - handle_ert(os, "\\par ", context); + output_ert_inset(os, "\\par ", context); eat_whitespace(p, os, context, true); } @@ -2354,15 +2433,15 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, Token const next = p.next_token(); Token const end = p.next_next_token(); if (next.cat() == catEnd) { - // {} - Token const prev = p.prev_token(); - p.get_token(); - if (p.next_token().character() == '`' || - (prev.character() == '-' && - p.next_token().character() == '-')) - ; // ignore it in {}`` or -{}- - else - handle_ert(os, "{}", context); + // {} + Token const prev = p.prev_token(); + p.get_token(); + if (p.next_token().character() == '`' || + (prev.character() == '-' && + p.next_token().character() == '-')) + ; // ignore it in {}`` or -{}- + else + output_ert_inset(os, "{}", context); } else if (next.cat() == catEscape && is_known(next.cs(), known_quotes) && end.cat() == catEnd) { @@ -2372,6 +2451,32 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // braces here for better readability. parse_text_snippet(p, os, FLAG_BRACE_LAST, outer, context); + } else if (p.next_token().asInput() == "\\ascii") { + // handle the \ascii characters + // (the case without braces is handled later) + // the code is "{\ascii\xxx}" + p.get_token(); // eat \ascii + string name2 = p.get_token().asInput(); + p.get_token(); // eat the final '}' + string const name = "{\\ascii" + name2 + "}"; + bool termination; + docstring rem; + set req; + // get the character from unicodesymbols + docstring s = encodings.fromLaTeXCommand(from_utf8(name), + Encodings::TEXT_CMD, termination, rem, &req); + if (!s.empty()) { + context.check_layout(os); + os << to_utf8(s); + if (!rem.empty()) + output_ert_inset(os, + to_utf8(rem), context); + for (set::const_iterator it = req.begin(); + it != req.end(); ++it) + preamble.registerAutomaticallyLoadedPackage(*it); + } else + // we did not find a non-ert version + output_ert_inset(os, name, context); } else { context.check_layout(os); // special handling of font attribute changes @@ -2386,17 +2491,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, p.get_token(); } else { p.putback(); - handle_ert(os, "{", context); + output_ert_inset(os, "{", context); parse_text_snippet(p, os, FLAG_BRACE_LAST, outer, context); - handle_ert(os, "}", context); + output_ert_inset(os, "}", context); } } else if (! context.new_layout_allowed) { - handle_ert(os, "{", context); + output_ert_inset(os, "{", context); parse_text_snippet(p, os, FLAG_BRACE_LAST, outer, context); - handle_ert(os, "}", context); + output_ert_inset(os, "}", context); } else if (is_known(next.cs(), known_sizes)) { // next will change the size, so we must // reset it here @@ -2444,10 +2549,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, << "\n\\shape " << context.font.shape << "\n"; } else { - handle_ert(os, "{", context); + output_ert_inset(os, "{", context); parse_text_snippet(p, os, FLAG_BRACE_LAST, outer, context); - handle_ert(os, "}", context); + output_ert_inset(os, "}", context); } } } @@ -2457,7 +2562,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, return; } cerr << "stray '}' in text\n"; - handle_ert(os, "}", context); + output_ert_inset(os, "}", context); } else if (t.cat() == catComment) @@ -2467,26 +2572,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // control sequences // - else if (t.cs() == "(") { + else if (t.cs() == "(" || t.cs() == "[") { + bool const simple = t.cs() == "("; context.check_layout(os); begin_inset(os, "Formula"); - os << " \\("; - parse_math(p, os, FLAG_SIMPLE2, MATH_MODE); - os << "\\)"; + os << " \\" << t.cs(); + parse_math(p, os, simple ? FLAG_SIMPLE2 : FLAG_EQUATION, MATH_MODE); + os << '\\' << (simple ? ')' : ']'); end_inset(os); - } - - else if (t.cs() == "[") { - context.check_layout(os); - begin_inset(os, "Formula"); - os << " \\["; - parse_math(p, os, FLAG_EQUATION, MATH_MODE); - os << "\\]"; - end_inset(os); - // Prevent the conversion of a line break to a space - // (bug 7668). This does not change the output, but - // looks ugly in LyX. - eat_whitespace(p, os, context, false); + if (!simple) { + // Prevent the conversion of a line break to a + // space (bug 7668). This does not change the + // output, but looks ugly in LyX. + eat_whitespace(p, os, context, false); + } } else if (t.cs() == "begin") @@ -2523,17 +2622,24 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // FIXME: Do this in check_layout()! context.has_item = false; if (optarg) - handle_ert(os, "\\item", context); + output_ert_inset(os, "\\item", context); else - handle_ert(os, "\\item ", context); + output_ert_inset(os, "\\item ", context); } if (optarg) { if (context.layout->labeltype != LABEL_MANUAL) { - // LyX does not support \item[\mybullet] - // in itemize environments + // handle option of itemize item + begin_inset(os, "Argument item:1\n"); + os << "status open\n"; + os << "\n\\begin_layout Plain Layout\n"; Parser p2(s + ']'); os << parse_text_snippet(p2, FLAG_BRACK_LAST, outer, context); + // we must not use context.check_end_layout(os) + // because that would close the outer itemize layout + os << "\n\\end_layout\n"; + end_inset(os); + eat_whitespace(p, os, context, false); } else if (!s.empty()) { // LyX adds braces around the argument, // so we need to remove them here. @@ -2545,12 +2651,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // would misinterpret the space as // item delimiter (bug 7663) if (contains(s, ' ')) { - handle_ert(os, s, context); + output_ert_inset(os, s, context); } else { Parser p2(s + ']'); os << parse_text_snippet(p2, - FLAG_BRACK_LAST, - outer, context); + FLAG_BRACK_LAST, outer, context); } // The space is needed to separate the // item from the rest of the sentence. @@ -2568,7 +2673,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, string key = convert_command_inset_arg(p.verbatim_item()); if (contains(label, '\\') || contains(key, '\\')) { // LyX can't handle LaTeX commands in labels or keys - handle_ert(os, t.asInput() + '[' + label + + output_ert_inset(os, t.asInput() + '[' + label + "]{" + p.verbatim_item() + '}', context); } else { @@ -2652,7 +2757,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, context.check_layout(os); // FIXME: This is a hack to prevent paragraph // deletion if it is empty. Handle this better! - handle_comment(os, + output_ert_inset(os, "%dummy comment inserted by tex2lyx to " "ensure that this paragraph is not empty", context); @@ -2688,7 +2793,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, for (; it != en; ++it) preamble.registerAutomaticallyLoadedPackage(*it); } else - handle_ert(os, + output_ert_inset(os, "\\date{" + p.verbatim_item() + '}', context); } @@ -2724,16 +2829,26 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, } else if (t.cs() == "caption") { + bool starred = false; + if (p.next_token().asInput() == "*") { + p.get_token(); + starred = true; + } p.skip_spaces(); context.check_layout(os); p.skip_spaces(); - begin_inset(os, "Caption\n"); + if (starred) + begin_inset(os, "Caption LongTableNoNumber\n"); + else + begin_inset(os, "Caption Standard\n"); Context newcontext(true, context.textclass, 0, 0, context.font); newcontext.check_layout(os); + // FIXME InsetArgument is now properly implemented in InsetLayout + // (for captions, but also for others) if (p.next_token().cat() != catEscape && p.next_token().character() == '[') { p.get_token(); // eat '[' - begin_inset(os, "Argument\n"); + begin_inset(os, "Argument 1\n"); os << "status collapsed\n"; parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context); end_inset(os); @@ -2777,7 +2892,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // we must make sure that the caption gets a \begin_layout os << "\n\\begin_layout Plain Layout"; p.skip_spaces(); - begin_inset(os, "Caption\n"); + begin_inset(os, "Caption Standard\n"); Context newcontext(true, context.textclass, 0, 0, context.font); newcontext.check_layout(os); @@ -2804,10 +2919,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // output it as ERT if (p.hasOpt()) { string opt_arg = convert_command_inset_arg(p.getArg('[', ']')); - handle_ert(os, t.asInput() + '[' + opt_arg + + output_ert_inset(os, t.asInput() + '[' + opt_arg + "]{" + p.verbatim_item() + '}', context); } else - handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context); + output_ert_inset(os, t.asInput() + "{" + p.verbatim_item() + '}', context); } } @@ -3005,7 +3120,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5") os << s; else - handle_ert(os, "\\ensuremath{" + s + "}", + output_ert_inset(os, "\\ensuremath{" + s + "}", context); } @@ -3014,7 +3129,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // swallow this skip_spaces_braces(p); } else - handle_ert(os, t.asInput(), context); + output_ert_inset(os, t.asInput(), context); } else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") { @@ -3026,16 +3141,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, preamble.registerAutomaticallyLoadedPackage("listings"); } - else if (t.cs() == "listoffigures") { - context.check_layout(os); - begin_inset(os, "FloatList figure\n"); - end_inset(os); - skip_spaces_braces(p); - } - - else if (t.cs() == "listoftables") { + else if (t.cs() == "listoffigures" || t.cs() == "listoftables") { context.check_layout(os); - begin_inset(os, "FloatList table\n"); + if (t.cs() == "listoffigures") + begin_inset(os, "FloatList figure\n"); + else + begin_inset(os, "FloatList table\n"); end_inset(os); skip_spaces_braces(p); } @@ -3050,7 +3161,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, end_inset(os); p.get_token(); // swallow second arg } else - handle_ert(os, "\\listof{" + name + "}", context); + output_ert_inset(os, "\\listof{" + name + "}", context); } else if ((where = is_known(t.cs(), known_text_font_families))) @@ -3099,7 +3210,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, preamble.registerAutomaticallyLoadedPackage("color"); } else // for custom defined colors - handle_ert(os, t.asInput() + "{" + color + "}", context); + output_ert_inset(os, t.asInput() + "{" + color + "}", context); } else if (t.cs() == "underbar" || t.cs() == "uline") { @@ -3143,11 +3254,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, string localtime = p.getArg('{', '}'); preamble.registerAuthor(name); Author const & author = preamble.getAuthor(name); - // from_ctime() will fail if LyX decides to output the - // time in the text language. It might also use a wrong - // time zone (if the original LyX document was exported - // with a different time zone). - time_t ptime = from_ctime(localtime); + // from_asctime_utc() will fail if LyX decides to output the + // time in the text language. + time_t ptime = from_asctime_utc(localtime); if (ptime == static_cast(-1)) { cerr << "Warning: Could not parse time `" << localtime << "´ for change tracking, using current time instead.\n"; @@ -3181,6 +3290,61 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, } } + else if (t.cs() == "textipa") { + context.check_layout(os); + begin_inset(os, "IPA\n"); + parse_text_in_inset(p, os, FLAG_ITEM, outer, context); + end_inset(os); + preamble.registerAutomaticallyLoadedPackage("tipa"); + preamble.registerAutomaticallyLoadedPackage("tipx"); + } + + else if (t.cs() == "texttoptiebar" || t.cs() == "textbottomtiebar") { + context.check_layout(os); + begin_inset(os, "IPADeco " + t.cs().substr(4) + "\n"); + os << "status open\n"; + parse_text_in_inset(p, os, FLAG_ITEM, outer, context); + end_inset(os); + p.skip_spaces(); + } + + else if (t.cs() == "textvertline") { + // FIXME: This is not correct, \textvertline is higher than | + os << "|"; + skip_braces(p); + continue; + } + + else if (t.cs() == "tone" ) { + context.check_layout(os); + // register the tone package + preamble.registerAutomaticallyLoadedPackage("tone"); + string content = trimSpaceAndEol(p.verbatim_item()); + string command = t.asInput() + "{" + content + "}"; + // some tones can be detected by unicodesymbols, some need special code + if (is_known(content, known_tones)) { + os << "\\IPAChar " << command << "\n"; + continue; + } + // try to see whether the string is in unicodesymbols + bool termination; + docstring rem; + set req; + docstring s = encodings.fromLaTeXCommand(from_utf8(command), + Encodings::TEXT_CMD | Encodings::MATH_CMD, + termination, rem, &req); + if (!s.empty()) { + os << to_utf8(s); + if (!rem.empty()) + output_ert_inset(os, to_utf8(rem), context); + for (set::const_iterator it = req.begin(); + it != req.end(); ++it) + preamble.registerAutomaticallyLoadedPackage(*it); + } else + // we did not find a non-ert version + output_ert_inset(os, command, context); + } + else if (t.cs() == "phantom" || t.cs() == "hphantom" || t.cs() == "vphantom") { context.check_layout(os); @@ -3274,7 +3438,27 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, skip_spaces_braces(p); } - else if ((where = is_known(t.cs(), known_ref_commands))) { + // handle refstyle first to catch \eqref which can also occur + // without refstyle. Only recognize these commands if + // refstyle.sty was found in the preamble (otherwise \eqref + // and user defined ref commands could be misdetected). + else if ((where = is_known(t.cs(), known_refstyle_commands)) && + preamble.refstyle()) { + context.check_layout(os); + begin_command_inset(os, "ref", "formatted"); + os << "reference \""; + os << known_refstyle_prefixes[where - known_refstyle_commands] + << ":"; + os << convert_command_inset_arg(p.verbatim_item()) + << "\"\n"; + end_inset(os); + preamble.registerAutomaticallyLoadedPackage("refstyle"); + } + + // if refstyle is used, we must not convert \prettyref to a + // formatted reference, since that would result in a refstyle command. + else if ((where = is_known(t.cs(), known_ref_commands)) && + (t.cs() != "prettyref" || !preamble.refstyle())) { string const opt = p.getOpt(); if (opt.empty()) { context.check_layout(os); @@ -3286,11 +3470,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, end_inset(os); if (t.cs() == "vref" || t.cs() == "vpageref") preamble.registerAutomaticallyLoadedPackage("varioref"); - + else if (t.cs() == "prettyref") + preamble.registerAutomaticallyLoadedPackage("prettyref"); } else { - // LyX does not support optional arguments of ref commands - handle_ert(os, t.asInput() + '[' + opt + "]{" + - p.verbatim_item() + "}", context); + // LyX does not yet support optional arguments of ref commands + output_ert_inset(os, t.asInput() + '[' + opt + "]{" + + p.verbatim_item() + '}', context); } } @@ -3351,6 +3536,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, << convert_command_inset_arg(p.verbatim_item()) << "\"\n"; end_inset(os); + // Need to set the cite engine if natbib is loaded by + // the document class directly + if (preamble.citeEngine() == "basic") + preamble.citeEngine("natbib"); } else if (use_jurabib && @@ -3401,6 +3590,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, os << "before " << '"' << before << '"' << "\n"; os << "key " << '"' << citation << '"' << "\n"; end_inset(os); + // Need to set the cite engine if jurabib is loaded by + // the document class directly + if (preamble.citeEngine() == "basic") + preamble.citeEngine("jurabib"); } else if (t.cs() == "cite" @@ -3457,10 +3650,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, end_inset(os); } - else if (t.cs() == "printindex") { + else if (t.cs() == "printindex" || t.cs() == "printsubindex") { context.check_layout(os); - begin_command_inset(os, "index_print", "printindex"); - os << "type \"idx\"\n"; + string commandname = t.cs(); + bool star = false; + if (p.next_token().asInput() == "*") { + commandname += "*"; + star = true; + p.get_token(); + } + begin_command_inset(os, "index_print", commandname); + string const indexname = p.getArg('[', ']'); + if (!star) { + if (indexname.empty()) + os << "type \"idx\"\n"; + else + os << "type \"" << indexname << "\"\n"; + } end_inset(os); skip_spaces_braces(p); preamble.registerAutomaticallyLoadedPackage("makeidx"); @@ -3605,8 +3811,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, context, "\\lang", context.font.language, lang); } - - else if (prefixIs(t.cs(), "text") + + else if (prefixIs(t.cs(), "text") && is_known(t.cs().substr(4), preamble.polyglossia_languages)) { // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[] string lang; @@ -3627,7 +3833,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, context, "\\lang", context.font.language, lang); } else - handle_ert(os, t.asInput() + langopts, context); + output_ert_inset(os, t.asInput() + langopts, context); } else { lang = preamble.polyglossia2lyx(t.cs().substr(4, string::npos)); parse_text_attributes(p, os, FLAG_ITEM, outer, @@ -3639,21 +3845,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, else if (t.cs() == "inputencoding") { // nothing to write here string const enc = subst(p.verbatim_item(), "\n", " "); - p.setEncoding(enc); + p.setEncoding(enc, Encoding::inputenc); } else if ((where = is_known(t.cs(), known_special_chars))) { context.check_layout(os); - os << "\\SpecialChar \\" - << known_coded_special_chars[where - known_special_chars] - << '\n'; + os << known_coded_special_chars[where - known_special_chars]; skip_spaces_braces(p); } - else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") { + else if ((t.cs() == "nobreakdash" && p.next_token().asInput() == "-") || + (t.cs() == "@" && p.next_token().asInput() == ".")) { context.check_layout(os); - os << "\\SpecialChar \\nobreakdash-\n"; - p.get_token(); + os << "\\SpecialChar \\" << t.cs() + << p.get_token().asInput() << '\n'; } else if (t.cs() == "textquotedbl") { @@ -3662,40 +3867,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, skip_braces(p); } - else if (t.cs() == "@" && p.next_token().asInput() == ".") { - context.check_layout(os); - os << "\\SpecialChar \\@.\n"; - p.get_token(); - } - - else if (t.cs() == "-") { - context.check_layout(os); - os << "\\SpecialChar \\-\n"; - } - - else if (t.cs() == "textasciitilde") { - context.check_layout(os); - os << '~'; - skip_spaces_braces(p); - } - - else if (t.cs() == "textasciicircum") { - context.check_layout(os); - os << '^'; - skip_spaces_braces(p); - } - - else if (t.cs() == "textbackslash") { - context.check_layout(os); - os << "\n\\backslash\n"; - skip_spaces_braces(p); - } - else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#" || t.cs() == "$" || t.cs() == "{" || t.cs() == "}" - || t.cs() == "%") { + || t.cs() == "%" || t.cs() == "-") { context.check_layout(os); - os << t.cs(); + if (t.cs() == "-") + os << "\\SpecialChar \\-\n"; + else + os << t.cs(); } else if (t.cs() == "char") { @@ -3707,69 +3886,42 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, os << '"'; skip_braces(p); } else { - handle_ert(os, "\\char`", context); + output_ert_inset(os, "\\char`", context); } } else { - handle_ert(os, "\\char", context); + output_ert_inset(os, "\\char", context); } } else if (t.cs() == "verb") { context.check_layout(os); - char const delimiter = p.next_token().character(); - // \verb is special: The usual escaping rules do not - // apply, e.g. "\verb+\+" is valid and denotes a single - // backslash (bug #4468). Therefore we do not allow - // escaping in getArg(). - string const arg = p.getArg(delimiter, delimiter, false); - ostringstream oss; - oss << "\\verb" << delimiter << arg << delimiter; - handle_ert(os, oss.str(), context); + // set catcodes to verbatim early, just in case. + p.setCatcodes(VERBATIM_CATCODES); + string delim = p.get_token().asInput(); + Parser::Arg arg = p.verbatimStuff(delim); + if (arg.first) + output_ert_inset(os, "\\verb" + delim + + arg.second + delim, context); + else + cerr << "invalid \\verb command. Skipping" << endl; } // Problem: \= creates a tabstop inside the tabbing environment // and else an accent. In the latter case we really would want // \={o} instead of \= o. else if (t.cs() == "=" && (flags & FLAG_TABBING)) - handle_ert(os, t.asInput(), context); - - // accents (see Table 6 in Comprehensive LaTeX Symbol List) - else if (t.cs().size() == 1 - && contains("\"'.=^`bcdHkrtuv~", t.cs())) { - context.check_layout(os); - // try to see whether the string is in unicodesymbols - bool termination; - docstring rem; - string command = t.asInput() + "{" - + trimSpaceAndEol(p.verbatim_item()) - + "}"; - set req; - docstring s = encodings.fromLaTeXCommand(from_utf8(command), - Encodings::TEXT_CMD | Encodings::MATH_CMD, - termination, rem, &req); - if (!s.empty()) { - if (!rem.empty()) - cerr << "When parsing " << command - << ", result is " << to_utf8(s) - << "+" << to_utf8(rem) << endl; - os << to_utf8(s); - for (set::const_iterator it = req.begin(); it != req.end(); ++it) - preamble.registerAutomaticallyLoadedPackage(*it); - } else - // we did not find a non-ert version - handle_ert(os, command, context); - } + output_ert_inset(os, t.asInput(), context); else if (t.cs() == "\\") { context.check_layout(os); if (p.hasOpt()) - handle_ert(os, "\\\\" + p.getOpt(), context); + output_ert_inset(os, "\\\\" + p.getOpt(), context); else if (p.next_token().asInput() == "*") { p.get_token(); // getOpt() eats the following space if there // is no optional argument, but that is OK // here since it has no effect in the output. - handle_ert(os, "\\\\*" + p.getOpt(), context); + output_ert_inset(os, "\\\\*" + p.getOpt(), context); } else { begin_inset(os, "Newline newline"); @@ -3815,8 +3967,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, string const absfigname = changeExtension(abstexname, ".fig"); fix_child_filename(filename); - string const lyxname = - changeExtension(filename, ".lyx"); + string const lyxname = changeExtension(filename, + roundtripMode() ? ".lyx.lyx" : ".lyx"); string const abslyxname = makeAbsPath( lyxname, getParentFilePath(false)).absFileName(); bool xfig = false; @@ -3897,6 +4049,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, registerExternalTemplatePackages("XFig"); } else { begin_command_inset(os, "include", name); + outname = subst(outname, "\"", "\\\""); os << "preview false\n" "filename \"" << outname << "\"\n"; if (t.cs() == "verbatiminput") @@ -3908,8 +4061,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, else if (t.cs() == "bibliographystyle") { // store new bibliographystyle bibliographystyle = p.verbatim_item(); - // If any other command than \bibliography and - // \nocite{*} follows, we need to output the style + // If any other command than \bibliography, \addcontentsline + // and \nocite{*} follows, we need to output the style // (because it might be used by that command). // Otherwise, it will automatically be output by LyX. p.pushPosition(); @@ -3924,18 +4077,53 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, continue; } else if (t2.cs() == "bibliography") output = false; + else if (t2.cs() == "phantomsection") { + output = false; + continue; + } + else if (t2.cs() == "addcontentsline") { + // get the 3 arguments of \addcontentsline + p.getArg('{', '}'); + p.getArg('{', '}'); + contentslineContent = p.getArg('{', '}'); + // if the last argument is not \refname we must output + if (contentslineContent == "\\refname") + output = false; + } break; } p.popPosition(); if (output) { - handle_ert(os, + output_ert_inset(os, "\\bibliographystyle{" + bibliographystyle + '}', context); } } + else if (t.cs() == "phantomsection") { + // we only support this if it occurs between + // \bibliographystyle and \bibliography + if (bibliographystyle.empty()) + output_ert_inset(os, "\\phantomsection", context); + } + + else if (t.cs() == "addcontentsline") { + context.check_layout(os); + // get the 3 arguments of \addcontentsline + string const one = p.getArg('{', '}'); + string const two = p.getArg('{', '}'); + string const three = p.getArg('{', '}'); + // only if it is a \refname, we support if for the bibtex inset + if (contentslineContent != "\\refname") { + output_ert_inset(os, + "\\addcontentsline{" + one + "}{" + two + "}{"+ three + '}', + context); + } + } + else if (t.cs() == "bibliography") { context.check_layout(os); + string BibOpts; begin_command_inset(os, "bibtex", "bibtex"); if (!btprint.empty()) { os << "btprint " << '"' << "btPrintAll" << '"' << "\n"; @@ -3944,9 +4132,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, btprint.clear(); } os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n"; + // Do we have addcontentsline? + if (contentslineContent == "\\refname") { + BibOpts = "bibtotoc"; + // clear string because next BibTeX inset can be without addcontentsline + contentslineContent.clear(); + } // Do we have a bibliographystyle set? - if (!bibliographystyle.empty()) - os << "options " << '"' << bibliographystyle << '"' << "\n"; + if (!bibliographystyle.empty()) { + if (BibOpts.empty()) + BibOpts = bibliographystyle; + else + BibOpts = BibOpts + ',' + bibliographystyle; + // clear it because each bibtex entry has its style + // and we need an empty string to handle \phantomsection + bibliographystyle.clear(); + } + os << "options " << '"' << BibOpts << '"' << "\n"; end_inset(os); } @@ -3980,7 +4182,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, "", "", t.cs()); } - else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" || + else if (t.cs() == "fbox" || t.cs() == "mbox" || + t.cs() == "ovalbox" || t.cs() == "Ovalbox" || t.cs() == "shadowbox" || t.cs() == "doublebox") parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), ""); @@ -3991,22 +4194,15 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, arg += p.getFullParentheseArg(); arg += p.getFullOpt(); eat_whitespace(p, os, context, false); - handle_ert(os, arg + '{', context); + output_ert_inset(os, arg + '{', context); parse_text(p, os, FLAG_ITEM, outer, context); - handle_ert(os, "}", context); + output_ert_inset(os, "}", context); } else { + //the syntax is: \framebox[width][position]{content} string special = p.getFullOpt(); special += p.getOpt(); - // LyX does not yet support \framebox without any option - if (!special.empty()) - parse_outer_box(p, os, FLAG_ITEM, outer, - context, t.cs(), special); - else { - eat_whitespace(p, os, context, false); - handle_ert(os, "\\framebox{", context); - parse_text(p, os, FLAG_ITEM, outer, context); - handle_ert(os, "}", context); - } + parse_outer_box(p, os, FLAG_ITEM, outer, + context, t.cs(), special); } } @@ -4019,9 +4215,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, arg += p.getFullParentheseArg(); arg += p.getFullOpt(); eat_whitespace(p, os, context, false); - handle_ert(os, arg + '{', context); + output_ert_inset(os, arg + '{', context); parse_text(p, os, FLAG_ITEM, outer, context); - handle_ert(os, "}", context); + output_ert_inset(os, "}", context); } else //the syntax is: \makebox[width][position]{content} parse_box(p, os, 0, FLAG_ITEM, outer, context, @@ -4100,7 +4296,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, t.cs() == "providecommand" || t.cs() == "providecommandx" || name[name.length()-1] == '*') - handle_ert(os, ert, context); + output_ert_inset(os, ert, context); else { context.check_layout(os); begin_inset(os, "FormulaMacro"); @@ -4137,7 +4333,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, CommandMap::iterator it = known_commands.find(command); if (it != known_commands.end()) known_commands[t.asInput()] = it->second; - handle_ert(os, ert, context); + output_ert_inset(os, ert, context); } else if (t.cs() == "hspace" || t.cs() == "vspace") { @@ -4236,13 +4432,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, name += '*'; if (valid) { if (value == 1.0) - handle_ert(os, name + '{' + unit + '}', context); + output_ert_inset(os, name + '{' + unit + '}', context); else if (value == -1.0) - handle_ert(os, name + "{-" + unit + '}', context); + output_ert_inset(os, name + "{-" + unit + '}', context); else - handle_ert(os, name + '{' + valstring + unit + '}', context); + output_ert_inset(os, name + '{' + valstring + unit + '}', context); } else - handle_ert(os, name + '{' + length + '}', context); + output_ert_inset(os, name + '{' + length + '}', context); } } @@ -4253,7 +4449,21 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, begin_inset(os, "Flex "); os << to_utf8(newinsetlayout->name()) << '\n' << "status collapsed\n"; - parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout); + if (newinsetlayout->isPassThru()) { + // set catcodes to verbatim early, just in case. + p.setCatcodes(VERBATIM_CATCODES); + string delim = p.get_token().asInput(); + if (delim != "{") + cerr << "Warning: bad delimiter for command " << t.asInput() << endl; + //FIXME: handle error condition + string const arg = p.verbatimStuff("}").second; + Context newcontext(true, context.textclass); + if (newinsetlayout->forcePlainLayout()) + newcontext.layout = &context.textclass.plainLayout(); + output_ert(os, arg, newcontext); + } else + + parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout); end_inset(os); } @@ -4368,18 +4578,112 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, // try to see whether the string is in unicodesymbols // Only use text mode commands, since we are in text mode here, // and math commands may be invalid (bug 6797) + string name = t.asInput(); + // handle the dingbats and Cyrillic + if (name == "\\ding" || name == "\\textcyr") + name = name + '{' + p.getArg('{', '}') + '}'; + // handle the ifsym characters + else if (name == "\\textifsymbol") { + string const optif = p.getFullOpt(); + string const argif = p.getArg('{', '}'); + name = name + optif + '{' + argif + '}'; + } + // handle the \ascii characters + // the case of \ascii within braces, as LyX outputs it, is already + // handled for t.cat() == catBegin + else if (name == "\\ascii") { + // the code is "\asci\xxx" + name = "{" + name + p.get_token().asInput() + "}"; + skip_braces(p); + } + // handle some TIPA special characters + else if (preamble.isPackageUsed("tipa")) { + if (name == "\\textglobfall") { + name = "End"; + skip_braces(p); + } else if (name == "\\s") { + // fromLaTeXCommand() does not yet + // recognize tipa short cuts + name = "\\textsyllabic"; + } else if (name == "\\=" && + p.next_token().asInput() == "*") { + // fromLaTeXCommand() does not yet + // recognize tipa short cuts + p.get_token(); + name = "\\b"; + } else if (name == "\\textdoublevertline") { + // FIXME: This is not correct, + // \textvertline is higher than \textbardbl + name = "\\textbardbl"; + skip_braces(p); + } else if (name == "\\!" ) { + if (p.next_token().asInput() == "b") { + p.get_token(); // eat 'b' + name = "\\texthtb"; + skip_braces(p); + } else if (p.next_token().asInput() == "d") { + p.get_token(); + name = "\\texthtd"; + skip_braces(p); + } else if (p.next_token().asInput() == "g") { + p.get_token(); + name = "\\texthtg"; + skip_braces(p); + } else if (p.next_token().asInput() == "G") { + p.get_token(); + name = "\\texthtscg"; + skip_braces(p); + } else if (p.next_token().asInput() == "j") { + p.get_token(); + name = "\\texthtbardotlessj"; + skip_braces(p); + } else if (p.next_token().asInput() == "o") { + p.get_token(); + name = "\\textbullseye"; + skip_braces(p); + } + } else if (name == "\\*" ) { + if (p.next_token().asInput() == "k") { + p.get_token(); + name = "\\textturnk"; + skip_braces(p); + } else if (p.next_token().asInput() == "r") { + p.get_token(); // eat 'b' + name = "\\textturnr"; + skip_braces(p); + } else if (p.next_token().asInput() == "t") { + p.get_token(); + name = "\\textturnt"; + skip_braces(p); + } else if (p.next_token().asInput() == "w") { + p.get_token(); + name = "\\textturnw"; + skip_braces(p); + } + } + } + if ((name.size() == 2 && + contains("\"'.=^`bcdHkrtuv~", name[1]) && + p.next_token().asInput() != "*") || + is_known(name.substr(1), known_tipa_marks)) { + // name is a command that corresponds to a + // combining character in unicodesymbols. + // Append the argument, fromLaTeXCommand() + // will either convert it to a single + // character or a combining sequence. + name += '{' + p.verbatim_item() + '}'; + } + // now get the character from unicodesymbols bool termination; docstring rem; set req; - docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()), + docstring s = encodings.fromLaTeXCommand(from_utf8(name), Encodings::TEXT_CMD, termination, rem, &req); if (!s.empty()) { - if (!rem.empty()) - cerr << "When parsing " << t.cs() - << ", result is " << to_utf8(s) - << "+" << to_utf8(rem) << endl; context.check_layout(os); os << to_utf8(s); + if (!rem.empty()) + output_ert_inset(os, to_utf8(rem), context); if (termination) skip_spaces_braces(p); for (set::const_iterator it = req.begin(); it != req.end(); ++it) @@ -4396,17 +4700,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer, z = p.verbatim_item(); } cerr << "found ERT: " << s << endl; - handle_ert(os, s + ' ', context); + output_ert_inset(os, s + ' ', context); */ else { - string name = t.asInput(); - if (p.next_token().asInput() == "*") { + if (t.asInput() == name && + p.next_token().asInput() == "*") { // Starred commands like \vspace*{} p.get_token(); // Eat '*' name += '*'; } if (!parse_command(name, p, os, outer, context)) - handle_ert(os, name, context); + output_ert_inset(os, name, context); } } @@ -4437,7 +4741,7 @@ string guessLanguage(Parser & p, string const & lang) if (t.cat() == catEscape) { if (t.cs() == "inputencoding") { string const enc = subst(p.verbatim_item(), "\n", " "); - p.setEncoding(enc); + p.setEncoding(enc, Encoding::inputenc); continue; } if (t.cs() != "begin") @@ -4467,10 +4771,10 @@ string guessLanguage(Parser & p, string const & lang) char const * const * const where = is_known(encoding, supported_CJK_encodings); if (where) - p.setEncoding(encoding); + p.setEncoding(encoding, Encoding::CJK); else - p.setEncoding("utf8"); - string const text = p.verbatimEnvironment("CJK"); + p.setEncoding("UTF-8"); + string const text = p.ertEnvironment("CJK"); p.setEncoding(encoding_old); p.skip_spaces(); if (!where) {