X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2FParagraph.cpp;h=a6d61e6000797c2557667c0dd8559df0ca8f6816;hb=7af6575cf6cd764b8b567b358907ac9a7a8e322b;hp=d32c0be2f0ba95d1ddc315f10f4a4ef741c93d11;hpb=53b60462acc6d6e55497aee168be31eef54b6f7b;p=lyx.git diff --git a/src/Paragraph.cpp b/src/Paragraph.cpp index d32c0be2f0..a6d61e6000 100644 --- a/src/Paragraph.cpp +++ b/src/Paragraph.cpp @@ -40,6 +40,7 @@ #include "ParagraphParameters.h" #include "SpellChecker.h" #include "sgml.h" +#include "texstream.h" #include "TextClass.h" #include "TexRow.h" #include "Text.h" @@ -51,6 +52,9 @@ #include "insets/InsetBibitem.h" #include "insets/InsetLabel.h" #include "insets/InsetSpecialChar.h" +#include "insets/InsetText.h" + +#include "mathed/InsetMathHull.h" #include "support/debug.h" #include "support/docstring_list.h" @@ -60,12 +64,22 @@ #include "support/lstrings.h" #include "support/textutils.h" +#include #include #include using namespace std; using namespace lyx::support; +// OSX clang, gcc < 4.8.0, and msvc < 2015 do not support C++11 thread_local +#if defined(__APPLE__) || (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 8) +#define THREAD_LOCAL_STATIC static __thread +#elif defined(_MSC_VER) && (_MSC_VER < 1900) +#define THREAD_LOCAL_STATIC static __declspec(thread) +#else +#define THREAD_LOCAL_STATIC thread_local static +#endif + namespace lyx { namespace { @@ -73,7 +87,7 @@ namespace { /// Inset identifier (above 0x10ffff, for ucs-4) char_type const META_INSET = 0x200001; -} +} // namespace ///////////////////////////////////////////////////////////////////// @@ -96,7 +110,7 @@ public: /// void result(SpellChecker::Result r) { result_ = r; } /// - bool inside(pos_type pos) const { return range_.inside(pos); } + bool contains(pos_type pos) const { return range_.contains(pos); } /// bool covered(FontSpan const & r) const { @@ -104,8 +118,9 @@ public: // 2. last of new range inside current range or // 3. first of current range inside new range or // 4. last of current range inside new range - return range_.inside(r.first) || range_.inside(r.last) || - r.inside(range_.first) || r.inside(range_.last); + //FIXME: is this the same as !range_.intersect(r).empty() ? + return range_.contains(r.first) || range_.contains(r.last) || + r.contains(range_.first) || r.contains(range_.last); } /// void shift(pos_type pos, int offset) @@ -191,7 +206,7 @@ public: RangesIterator et = ranges_.end(); RangesIterator it = ranges_.begin(); for (; it != et; ++it) { - if(it->inside(pos)) { + if(it->contains(pos)) { return it->result(); } } @@ -205,7 +220,7 @@ public: RangesIterator et = ranges_.end(); RangesIterator it = ranges_.begin(); for (; it != et; ++it) { - if(it->inside(pos)) { + if(it->contains(pos)) { return it->range(); } } @@ -281,6 +296,11 @@ private: class Paragraph::Private { + // Enforce our own "copy" constructor + Private(Private const &) = delete; + Private & operator=(Private const &) = delete; + // Unique ID generator + static int make_id(); public: /// Private(Paragraph * owner, Layout const & layout); @@ -294,25 +314,21 @@ public: /// Output the surrogate pair formed by \p c and \p next to \p os. /// \return the number of characters written. - int latexSurrogatePair(otexstream & os, char_type c, char_type next, + int latexSurrogatePair(BufferParams const &, otexstream & os, + char_type c, char_type next, OutputParams const &); /// Output a space in appropriate formatting (or a surrogate pair /// if the next character is a combining character). /// \return whether a surrogate pair was output. - bool simpleTeXBlanks(OutputParams const &, + bool simpleTeXBlanks(BufferParams const &, + OutputParams const &, otexstream &, pos_type i, unsigned int & column, Font const & font, Layout const & style); - /// Output consecutive unicode chars, belonging to the same script as - /// specified by the latex macro \p ltx, to \p os starting from \p i. - /// \return the number of characters written. - int writeScriptChars(otexstream & os, docstring const & ltx, - Change const &, Encoding const &, pos_type & i); - /// This could go to ParagraphParameters if we want to. int startTeXParParams(BufferParams const &, otexstream &, OutputParams const &) const; @@ -340,7 +356,7 @@ public: BufferParams const & bparams, OutputParams const & runparams, Font const & running_font, - Change const & running_change, + string & alien_script, Layout const & style, pos_type & i, pos_type end_pos, @@ -353,24 +369,17 @@ public: pos_type i, unsigned int & column); /// - bool latexSpecialT3( + bool latexSpecialTU( char_type const c, otexstream & os, pos_type i, unsigned int & column); /// - bool latexSpecialTypewriter( + bool latexSpecialT3( char_type const c, otexstream & os, pos_type i, unsigned int & column); - /// - bool latexSpecialPhrase( - otexstream & os, - pos_type & i, - pos_type end_pos, - unsigned int & column, - OutputParams const & runparams); /// void validate(LaTeXFeatures & features) const; @@ -379,15 +388,12 @@ public: bool onlyText(Buffer const & buf, Font const & outerfont, pos_type initial) const; - /// match a string against a particular point in the paragraph - bool isTextAt(string const & str, pos_type pos) const; - /// a vector of speller skip positions typedef vector SkipPositions; typedef SkipPositions::const_iterator SkipPositionsIterator; void appendSkipPosition(SkipPositions & skips, pos_type const pos) const; - + Language * getSpellLanguage(pos_type const from) const; Language * locateSpellRange(pos_type & from, pos_type & to, @@ -402,7 +408,7 @@ public: } bool ignoreWord(docstring const & word) const ; - + void setMisspelled(pos_type from, pos_type to, SpellChecker::Result state) { pos_type textsize = owner_->size(); @@ -439,10 +445,10 @@ public: return; } pos_type endpos = last; - owner_->locateWord(first, endpos, WHOLE_WORD); + owner_->locateWord(first, endpos, WHOLE_WORD, true); if (endpos < last) { endpos = last; - owner_->locateWord(last, endpos, WHOLE_WORD); + owner_->locateWord(last, endpos, WHOLE_WORD, true); } last = endpos; } @@ -499,7 +505,7 @@ public: TextContainer text_; typedef set Words; - typedef map LangWordsMap; + typedef map LangWordsMap; /// LangWordsMap words_; /// @@ -509,60 +515,44 @@ public: }; -namespace { - -struct special_phrase { - string phrase; - docstring macro; - bool builtin; -}; - -special_phrase const special_phrases[] = { - { "LyX", from_ascii("\\LyX{}"), false }, - { "TeX", from_ascii("\\TeX{}"), true }, - { "LaTeX2e", from_ascii("\\LaTeXe{}"), true }, - { "LaTeX", from_ascii("\\LaTeX{}"), true }, -}; - -size_t const phrases_nr = sizeof(special_phrases)/sizeof(special_phrase); - -} // namespace anon - - Paragraph::Private::Private(Paragraph * owner, Layout const & layout) - : owner_(owner), inset_owner_(0), id_(-1), begin_of_body_(0), layout_(&layout) + : owner_(owner), inset_owner_(nullptr), id_(-1), begin_of_body_(0), layout_(&layout) { text_.reserve(100); } -// Initialization of the counter for the paragraph id's, -// -// FIXME: There should be a more intelligent way to generate and use the -// paragraph ids per buffer instead a global static counter for all InsetText -// in the running program. -static int paragraph_id = -1; +//static +int Paragraph::Private::make_id() +{ + // The id is unique per session across buffers because it is used in + // LFUN_PARAGRAPH_GOTO to switch to a different buffer, for instance in the + // outliner. + // (thread-safe) + static atomic_uint next_id(0); + return next_id++; +} + Paragraph::Private::Private(Private const & p, Paragraph * owner) : owner_(owner), inset_owner_(p.inset_owner_), fontlist_(p.fontlist_), + id_(make_id()), params_(p.params_), changes_(p.changes_), insetlist_(p.insetlist_), begin_of_body_(p.begin_of_body_), text_(p.text_), words_(p.words_), layout_(p.layout_) { - id_ = ++paragraph_id; requestSpellCheck(p.text_.size()); } Paragraph::Private::Private(Private const & p, Paragraph * owner, pos_type beg, pos_type end) - : owner_(owner), inset_owner_(p.inset_owner_), + : owner_(owner), inset_owner_(p.inset_owner_), id_(make_id()), params_(p.params_), changes_(p.changes_), insetlist_(p.insetlist_, beg, end), begin_of_body_(p.begin_of_body_), words_(p.words_), layout_(p.layout_) { - id_ = ++paragraph_id; if (beg >= pos_type(p.text_.size())) return; text_ = p.text_.substr(beg, end - beg); @@ -584,10 +574,10 @@ Paragraph::Private::Private(Private const & p, Paragraph * owner, } -void Paragraph::addChangesToToc(DocIterator const & cdit, - Buffer const & buf, bool output_active) const +void Paragraph::addChangesToToc(DocIterator const & cdit, Buffer const & buf, + bool output_active, TocBackend & backend) const { - d->changes_.addToToc(cdit, buf, output_active); + d->changes_.addToToc(cdit, buf, output_active, backend); } @@ -608,6 +598,28 @@ bool Paragraph::isChanged(pos_type start, pos_type end) const return d->changes_.isChanged(start, end); } +// FIXME: Ideally the diverse isChanged() methods should account for that! +bool Paragraph::hasChangedInsets(pos_type start, pos_type end) const +{ + LASSERT(start >= 0 && start <= size(), return false); + LASSERT(end > start && end <= size() + 1, return false); + + for (auto const & icit : d->insetlist_) { + if (icit.pos < start) + continue; + if (icit.pos >= end) + break; + if (icit.inset && icit.inset->isChanged()) + return true; + } + return false; +} + +bool Paragraph::isChanged() const +{ + return d->changes_.isChanged(); +} + bool Paragraph::isMergedOnEndOfParDeletion(bool trackChanges) const { @@ -615,10 +627,15 @@ bool Paragraph::isMergedOnEndOfParDeletion(bool trackChanges) const if (!trackChanges) return true; - Change const change = d->changes_.lookup(size()); + Change const & change = d->changes_.lookup(size()); return change.inserted() && change.currentAuthor(); } +Change Paragraph::parEndChange() const +{ + return d->changes_.lookup(size()); +} + void Paragraph::setChange(Change const & change) { @@ -653,8 +670,8 @@ void Paragraph::setChange(pos_type pos, Change const & change) // see comment in setChange(Change const &) above if (!change.deleted() && pos < size()) - if (Inset * inset = getInset(pos)) - inset->setChange(change); + if (Inset * inset = getInset(pos)) + inset->setChange(change); } @@ -858,9 +875,10 @@ int Paragraph::eraseChars(pos_type start, pos_type end, bool trackChanges) return end - i; } - -int Paragraph::Private::latexSurrogatePair(otexstream & os, char_type c, - char_type next, OutputParams const & runparams) +// Handle combining characters +int Paragraph::Private::latexSurrogatePair(BufferParams const & bparams, + otexstream & os, char_type c, char_type next, + OutputParams const & runparams) { // Writing next here may circumvent a possible font change between // c and next. Since next is only output if it forms a surrogate pair @@ -877,23 +895,22 @@ int Paragraph::Private::latexSurrogatePair(otexstream & os, char_type c, latex1 = from_ascii(tipashortcut); } } - docstring const latex2 = encoding.latexChar(c).first; - if (docstring(1, next) == latex1) { - // the encoding supports the combination + docstring latex2 = encoding.latexChar(c).first; + + if (bparams.useNonTeXFonts || docstring(1, next) == latex1) { + // Encoding supports the combination: + // output as is (combining char after base char). os << latex2 << latex1; return latex1.length() + latex2.length(); - } else if (runparams.local_font && - runparams.local_font->language()->lang() == "polutonikogreek") { - // polutonikogreek only works without the brackets - os << latex1 << latex2; - return latex1.length() + latex2.length(); - } else - os << latex1 << '{' << latex2 << '}'; + } + + os << latex1 << "{" << latex2 << "}"; return latex1.length() + latex2.length() + 2; } -bool Paragraph::Private::simpleTeXBlanks(OutputParams const & runparams, +bool Paragraph::Private::simpleTeXBlanks(BufferParams const & bparams, + OutputParams const & runparams, otexstream & os, pos_type i, unsigned int & column, @@ -907,7 +924,7 @@ bool Paragraph::Private::simpleTeXBlanks(OutputParams const & runparams, char_type next = text_[i + 1]; if (Encodings::isCombiningChar(next)) { // This space has an accent, so we must always output it. - column += latexSurrogatePair(os, ' ', next, runparams) - 1; + column += latexSurrogatePair(bparams, os, ' ', next, runparams) - 1; return true; } } @@ -938,99 +955,6 @@ bool Paragraph::Private::simpleTeXBlanks(OutputParams const & runparams, } -int Paragraph::Private::writeScriptChars(otexstream & os, - docstring const & ltx, - Change const & runningChange, - Encoding const & encoding, - pos_type & i) -{ - // FIXME: modifying i here is not very nice... - - // We only arrive here when a proper language for character text_[i] has - // not been specified (i.e., it could not be translated in the current - // latex encoding) or its latex translation has been forced, and it - // belongs to a known script. - // Parameter ltx contains the latex translation of text_[i] as specified - // in the unicodesymbols file and is something like "\textXXX{}". - // The latex macro name "textXXX" specifies the script to which text_[i] - // belongs and we use it in order to check whether characters from the - // same script immediately follow, such that we can collect them in a - // single "\textXXX" macro. So, we have to retain "\textXXX{" - // for the first char but only "" for all subsequent chars. - docstring::size_type const brace1 = ltx.find_first_of(from_ascii("{")); - docstring::size_type const brace2 = ltx.find_last_of(from_ascii("}")); - string script = to_ascii(ltx.substr(1, brace1 - 1)); - int pos = 0; - int length = brace2; - bool closing_brace = true; - if (script == "textgreek" && encoding.latexName() == "iso-8859-7") { - // Correct encoding is being used, so we can avoid \textgreek. - pos = brace1 + 1; - length -= pos; - closing_brace = false; - } - os << ltx.substr(pos, length); - int size = text_.size(); - while (i + 1 < size) { - char_type const next = text_[i + 1]; - // Stop here if next character belongs to another script - // or there is a change in change tracking status. - if (!Encodings::isKnownScriptChar(next, script) || - runningChange != owner_->lookupChange(i + 1)) - break; - Font prev_font; - bool found = false; - FontList::const_iterator cit = fontlist_.begin(); - FontList::const_iterator end = fontlist_.end(); - for (; cit != end; ++cit) { - if (cit->pos() >= i && !found) { - prev_font = cit->font(); - found = true; - } - if (cit->pos() >= i + 1) - break; - } - // Stop here if there is a font attribute or encoding change. - if (found && cit != end && prev_font != cit->font()) - break; - docstring const latex = encoding.latexChar(next).first; - docstring::size_type const b1 = - latex.find_first_of(from_ascii("{")); - docstring::size_type const b2 = - latex.find_last_of(from_ascii("}")); - int const len = b2 - b1 - 1; - os << latex.substr(b1 + 1, len); - length += len; - ++i; - } - if (closing_brace) { - os << '}'; - ++length; - } - return length; -} - - -bool Paragraph::Private::isTextAt(string const & str, pos_type pos) const -{ - pos_type const len = str.length(); - - // is the paragraph large enough? - if (pos + len > int(text_.size())) - return false; - - // does the wanted text start at point? - for (string::size_type i = 0; i < str.length(); ++i) { - // Caution: direct comparison of characters works only - // because str is pure ASCII. - if (str[i] != text_[pos + i]) - return false; - } - - return fontlist_.hasChangeInRange(pos, len); -} - - void Paragraph::Private::latexInset(BufferParams const & bparams, otexstream & os, OutputParams & runparams, @@ -1054,16 +978,17 @@ void Paragraph::Private::latexInset(BufferParams const & bparams, } // FIXME: move this to InsetNewline::latex - if (inset->lyxCode() == NEWLINE_CODE) { + if (inset->lyxCode() == NEWLINE_CODE || inset->lyxCode() == SEPARATOR_CODE) { // newlines are handled differently here than // the default in simpleTeXSpecialChars(). if (!style.newline_allowed) { os << '\n'; } else { if (open_font) { + bool needPar = false; column += running_font.latexWriteEndChanges( os, bparams, runparams, - basefont, basefont); + basefont, basefont, needPar); open_font = false; } @@ -1095,14 +1020,16 @@ void Paragraph::Private::latexInset(BufferParams const & bparams, bool close = false; odocstream::pos_type const len = os.os().tellp(); - if (inset->forceLTR() - && !runparams.use_polyglossia + if (inset->forceLTR(runparams) && running_font.isRightToLeft() // ERT is an exception, it should be output with no // decorations at all && inset->lyxCode() != ERT_CODE) { - if (running_font.language()->lang() == "farsi") - os << "\\beginL{}"; + if (runparams.use_polyglossia) { + os << "\\LRE{"; + } else if (running_font.language()->lang() == "farsi" + || running_font.language()->lang() == "arabic_arabi") + os << "\\textLR{" << termcmd; else os << "\\L{"; close = true; @@ -1121,10 +1048,12 @@ void Paragraph::Private::latexInset(BufferParams const & bparams, bool arabtex = basefont.language()->lang() == "arabic_arabtex" || running_font.language()->lang() == "arabic_arabtex"; if (open_font && !inset->inheritFont()) { + bool needPar = false; bool closeLanguage = arabtex || basefont.isRightToLeft() == running_font.isRightToLeft(); unsigned int count = running_font.latexWriteEndChanges(os, - bparams, runparams, basefont, basefont, closeLanguage); + bparams, runparams, basefont, basefont, + needPar, closeLanguage); column += count; // if any font properties were closed, update the running_font, // making sure, however, to leave the language as it was @@ -1143,7 +1072,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams, } } - int prev_rows = os.texrow().rows(); + size_t const previous_row_count = os.texrow().rows(); try { runparams.lastid = id_; @@ -1156,14 +1085,10 @@ void Paragraph::Private::latexInset(BufferParams const & bparams, throw(e); } - if (close) { - if (running_font.language()->lang() == "farsi") - os << "\\endL{}"; - else - os << '}'; - } + if (close) + os << '}'; - if (os.texrow().rows() > prev_rows) { + if (os.texrow().rows() > previous_row_count) { os.texrow().start(owner_->id(), i + 1); column = 0; } else { @@ -1179,18 +1104,17 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, BufferParams const & bparams, OutputParams const & runparams, Font const & running_font, - Change const & running_change, + string & alien_script, Layout const & style, pos_type & i, pos_type end_pos, unsigned int & column) { - // With polyglossia, brackets and stuff need not be reversed - // in RTL scripts (see bug #8251) - char_type const c = (runparams.use_polyglossia) ? - owner_->getUChar(bparams, i) : text_[i]; + char_type const c = owner_->getUChar(bparams, runparams, i); - if (style.pass_thru || runparams.pass_thru) { + if (style.pass_thru || runparams.pass_thru + || contains(style.pass_thru_chars, c) + || contains(runparams.pass_thru_chars, c)) { if (c != '\0') { Encoding const * const enc = runparams.encoding; if (enc && !enc->encodable(c)) @@ -1205,42 +1129,50 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, return; // If T1 font encoding is used, use the special // characters it provides. - // NOTE: some languages reset the font encoding - // internally + // NOTE: Some languages reset the font encoding internally to a + // non-standard font encoding. If we are using such a language, + // we do not output special T1 chars. if (!runparams.inIPA && !running_font.language()->internalFontEncoding() - && lyxrc.fontenc == "T1" && latexSpecialT1(c, os, i, column)) - return; - - // \tt font needs special treatment - if (!runparams.inIPA - && running_font.fontInfo().family() == TYPEWRITER_FAMILY - && latexSpecialTypewriter(c, os, i, column)) + && !runparams.isFullUnicode() && bparams.main_font_encoding() == "T1" + && latexSpecialT1(c, os, i, column)) return; + // NOTE: "fontspec" (non-TeX fonts) sets the font encoding to "TU" (untill 2017 "EU1" or "EU2") + else if (!runparams.inIPA && !running_font.language()->internalFontEncoding() + && runparams.isFullUnicode() && latexSpecialTU(c, os, i, column)) + return; // Otherwise, we use what LaTeX provides us. switch (c) { case '\\': - os << "\\textbackslash{}"; + os << "\\textbackslash" << termcmd; column += 15; break; case '<': - os << "\\textless{}"; + os << "\\textless" << termcmd; column += 10; break; case '>': - os << "\\textgreater{}"; + os << "\\textgreater" << termcmd; column += 13; break; case '|': - os << "\\textbar{}"; + os << "\\textbar" << termcmd; column += 9; break; case '-': os << '-'; + if (i + 1 < static_cast(text_.size()) && + (end_pos == -1 || i + 1 < end_pos) && + text_[i+1] == '-') { + // Prevent "--" becoming an en dash and "---" an em dash. + // (Within \ttfamily, "---" is merged to en dash + hyphen.) + os << "{}"; + column += 2; + } break; case '\"': - os << "\\char`\\\"{}"; - column += 9; + os << "\\textquotedbl" << termcmd; + column += 14; break; case '$': case '&': @@ -1252,12 +1184,12 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, break; case '~': - os << "\\textasciitilde{}"; + os << "\\textasciitilde" << termcmd; column += 16; break; case '^': - os << "\\textasciicircum{}"; + os << "\\textasciicircum" << termcmd; column += 17; break; @@ -1279,11 +1211,28 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, // written. (Asger) break; + case 0x2013: + case 0x2014: + // XeTeX's dash behaviour is determined via a global setting + if (bparams.use_dash_ligatures + && owner_->getFontSettings(bparams, i).fontInfo().family() != TYPEWRITER_FAMILY + && !runparams.inIPA + // TODO #10961: && not in inset Flex Code + // TODO #10961: && not in layout LyXCode + && (!bparams.useNonTeXFonts || runparams.flavor != OutputParams::XETEX)) { + if (c == 0x2013) { + // en-dash + os << "--"; + column +=2; + } else { + // em-dash + os << "---"; + column +=3; + } + break; + } + // fall through default: - // LyX, LaTeX etc. - if (latexSpecialPhrase(os, i, end_pos, column, runparams)) - return; - if (c == '\0') return; @@ -1292,17 +1241,16 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, if (i + 1 < int(text_.size())) { next = text_[i + 1]; if (Encodings::isCombiningChar(next)) { - column += latexSurrogatePair(os, c, next, runparams) - 1; + column += latexSurrogatePair(bparams, os, c, next, runparams) - 1; ++i; break; } } - string script; pair latex = encoding.latexChar(c); docstring nextlatex; bool nexttipas = false; string nexttipashortcut; - if (next != '\0' && next != META_INSET && encoding.encodable(next)) { + if (next != '\0' && next != META_INSET && !encoding.encodable(next)) { nextlatex = encoding.latexChar(next).first; if (runparams.inIPA) { nexttipashortcut = Encodings::TIPAShortcut(next); @@ -1318,19 +1266,22 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, tipas = true; } } - if (Encodings::isKnownScriptChar(c, script) - && prefixIs(latex.first, from_ascii("\\" + script))) - column += writeScriptChars(os, latex.first, - running_change, encoding, i) - 1; - else if (latex.second + // eventually close "script wrapper" command (see `Paragraph::latex`) + if (!alien_script.empty() + && alien_script != Encodings::isKnownScriptChar(next)) { + column += latex.first.length(); + alien_script.clear(); + os << latex.first << "}"; + break; + } + if (latex.second && ((!prefixIs(nextlatex, '\\') && !prefixIs(nextlatex, '{') && !prefixIs(nextlatex, '}')) || (nexttipas && !prefixIs(from_ascii(nexttipashortcut), '\\'))) && !tipas) { - // Prevent eating of a following - // space or command corruption by + // Prevent eating of a following space or command corruption by // following characters if (next == ' ' || next == '\0') { column += latex.first.length() + 1; @@ -1359,7 +1310,7 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os, // but we should avoid ligatures if (i + 1 >= int(text_.size()) || text_[i + 1] != c) return true; - os << "\\textcompwordmark{}"; + os << "\\textcompwordmark" << termcmd; column += 19; return true; case '|': @@ -1367,7 +1318,7 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os, return true; case '\"': // soul.sty breaks with \char`\" - os << "\\textquotedbl{}"; + os << "\\textquotedbl" << termcmd; column += 14; return true; default: @@ -1376,6 +1327,14 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os, } +bool Paragraph::Private::latexSpecialTU(char_type const c, otexstream & os, + pos_type i, unsigned int & column) +{ + // TU encoding is currently on par with T1. + return latexSpecialT1(c, os, i, column); +} + + bool Paragraph::Private::latexSpecialT3(char_type const c, otexstream & os, pos_type /*i*/, unsigned int & column) { @@ -1387,7 +1346,7 @@ bool Paragraph::Private::latexSpecialT3(char_type const c, otexstream & os, os.put(c); return true; case '|': - os << "\\textvertline{}"; + os << "\\textvertline" << termcmd; column += 14; return true; default: @@ -1396,97 +1355,22 @@ bool Paragraph::Private::latexSpecialT3(char_type const c, otexstream & os, } -bool Paragraph::Private::latexSpecialTypewriter(char_type const c, otexstream & os, - pos_type i, unsigned int & column) -{ - switch (c) { - case '-': - // within \ttfamily, "--" is merged to "-" (no endash) - // so we avoid this rather irritating ligature - if (i + 1 < int(text_.size()) && text_[i + 1] == '-') { - os << "-{}"; - column += 2; - } else - os << '-'; - return true; - - // everything else has to be checked separately - // (depending on the encoding) - default: - return false; - } -} - - -/// \param end_pos -/// If [start_pos, end_pos) does not include entirely the special phrase, then -/// do not apply the macro transformation. -bool Paragraph::Private::latexSpecialPhrase(otexstream & os, pos_type & i, pos_type end_pos, - unsigned int & column, OutputParams const & runparams) -{ - // FIXME: if we have "LaTeX" with a font - // change in the middle (before the 'T', then - // the "TeX" part is still special cased. - // Really we should only operate this on - // "words" for some definition of word - - for (size_t pnr = 0; pnr < phrases_nr; ++pnr) { - if (!isTextAt(special_phrases[pnr].phrase, i) - || (end_pos != -1 && i + int(special_phrases[pnr].phrase.size()) > end_pos)) - continue; - if (runparams.moving_arg) - os << "\\protect"; - os << special_phrases[pnr].macro; - i += special_phrases[pnr].phrase.length() - 1; - column += special_phrases[pnr].macro.length() - 1; - return true; - } - return false; -} - - void Paragraph::Private::validate(LaTeXFeatures & features) const { if (layout_->inpreamble && inset_owner_) { - bool const is_command = layout_->latextype == LATEX_COMMAND; - Buffer const & buf = inset_owner_->buffer(); - BufferParams const & bp = features.runparams().is_child - ? buf.masterParams() : buf.params(); - Font f; - TexRow texrow; - // Using a string stream here circumvents the encoding + // FIXME: Using a string stream here circumvents the encoding // switching machinery of odocstream. Therefore the // output is wrong if this paragraph contains content // that needs to switch encoding. - odocstringstream ods; - otexstream os(ods, texrow); - if (is_command) { - os << '\\' << from_ascii(layout_->latexname()); - // we have to provide all the optional arguments here, even though - // the last one is the only one we care about. - // Separate handling of optional argument inset. - if (!layout_->latexargs().empty()) { - OutputParams rp = features.runparams(); - rp.local_font = &owner_->getFirstFontSettings(bp); - latexArgInsets(*owner_, os, rp, layout_->latexargs()); - } - os << from_ascii(layout_->latexparam()); - } - docstring::size_type const length = ods.str().length(); - // this will output "{" at the beginning, but not at the end - owner_->latex(bp, f, os, features.runparams(), 0, -1, true); - if (ods.str().length() > length) { - if (is_command) { - ods << '}'; - if (!layout_->postcommandargs().empty()) { - OutputParams rp = features.runparams(); - rp.local_font = &owner_->getFirstFontSettings(bp); - latexArgInsets(*owner_, os, rp, layout_->postcommandargs(), "post:"); - } - } - string const snippet = to_utf8(ods.str()); - features.addPreambleSnippet(snippet); - } + Buffer const & buf = inset_owner_->buffer(); + otexstringstream os; + os << layout_->preamble(); + size_t const length = os.length(); + TeXOnePar(buf, *inset_owner_->getText(int(buf.getParFromID(owner_->id()).idx())), + buf.getParFromID(owner_->id()).pit(), os, + features.runparams(), string(), 0, -1, true); + if (os.length() > length) + features.addPreambleSnippet(os.release(), true); } if (features.runparams().flavor == OutputParams::HTML @@ -1513,7 +1397,19 @@ void Paragraph::Private::validate(LaTeXFeatures & features) const InsetList::const_iterator iend = insetlist_.end(); for (; icit != iend; ++icit) { if (icit->inset) { + features.inDeletedInset(owner_->isDeleted(icit->pos)); + if (icit->inset->lyxCode() == FOOT_CODE) { + // FIXME: an item inset would make things much easier. + if ((layout_->latextype == LATEX_LIST_ENVIRONMENT + || (layout_->latextype == LATEX_ITEM_ENVIRONMENT + && layout_->margintype == MARGIN_FIRST_DYNAMIC)) + && (icit->pos < begin_of_body_ + || (icit->pos == begin_of_body_ + && (icit->pos == 0 || text_[icit->pos - 1] != ' ')))) + features.saveNoteEnv("description"); + } icit->inset->validate(features); + features.inDeletedInset(false); if (layout_->needprotect && icit->inset->lyxCode() == FOOT_CODE) features.require("NeedLyXFootnoteCode"); @@ -1521,15 +1417,61 @@ void Paragraph::Private::validate(LaTeXFeatures & features) const } // then the contents + BufferParams const bp = features.runparams().is_child + ? features.buffer().masterParams() : features.buffer().params(); for (pos_type i = 0; i < int(text_.size()) ; ++i) { - for (size_t pnr = 0; pnr < phrases_nr; ++pnr) { - if (!special_phrases[pnr].builtin - && isTextAt(special_phrases[pnr].phrase, i)) { - features.require(special_phrases[pnr].phrase); - break; + char_type c = text_[i]; + CharInfo const & ci = Encodings::unicodeCharInfo(c); + if (c == 0x0022) { + if (features.runparams().isFullUnicode() && bp.useNonTeXFonts) + features.require("textquotedblp"); + else if (bp.main_font_encoding() != "T1" + || ((&owner_->getFontSettings(bp, i))->language()->internalFontEncoding())) + features.require("textquotedbl"); + } else if (ci.textfeature() && contains(ci.textpreamble(), '=')) { + // features that depend on the font or input encoding + string feats = ci.textpreamble(); + string fontenc = (&owner_->getFontSettings(bp, i))->language()->fontenc(bp); + if (fontenc.empty()) + fontenc = features.runparams().main_fontenc; + while (!feats.empty()) { + string feat; + feats = split(feats, feat, ','); + if (contains(feat, "!=")) { + // a feature that is required except for the spcified + // font or input encodings + string realfeature; + string const contexts = ltrim(split(feat, realfeature, '!'), "="); + // multiple encodings are separated by semicolon + vector context = getVectorFromString(contexts, ";"); + // require feature if the context matches neither current font + // nor input encoding + if (std::find(context.begin(), context.end(), fontenc) == context.end() + && std::find(context.begin(), context.end(), + features.runparams().encoding->name()) == context.end()) + features.require(realfeature); + } else if (contains(feat, '=')) { + // a feature that is required only for the spcified + // font or input encodings + string realfeature; + string const contexts = split(feat, realfeature, '='); + // multiple encodings are separated by semicolon + vector context = getVectorFromString(contexts, ";"); + // require feature if the context matches either current font + // or input encoding + if (std::find(context.begin(), context.end(), fontenc) != context.end() + || std::find(context.begin(), context.end(), + features.runparams().encoding->name()) != context.end()) + features.require(realfeature); + } } - } - BufferEncodings::validate(text_[i], features); + } else if (!bp.use_dash_ligatures + && (c == 0x2013 || c == 0x2014) + && bp.useNonTeXFonts + && features.runparams().flavor == OutputParams::XETEX) + // XeTeX's dash behaviour is determined via a global setting + features.require("xetexdashbreakstate"); + BufferEncodings::validate(c, features); } } @@ -1598,7 +1540,7 @@ void flushString(ostream & os, docstring & s) s.erase(); } -} +} // namespace void Paragraph::write(ostream & os, BufferParams const & bparams, @@ -1635,7 +1577,7 @@ void Paragraph::write(ostream & os, BufferParams const & bparams, int column = 0; for (pos_type i = 0; i <= size(); ++i) { - Change const change = lookupChange(i); + Change const & change = lookupChange(i); if (change != running_change) flushString(os, write_buffer); Changes::lyxMarkChange(os, bparams, column, running_change, change); @@ -1671,6 +1613,9 @@ void Paragraph::write(ostream & os, BufferParams const & bparams, os << "\n\\end_inset\n\n"; column = 0; } + // FIXME This can be removed again once the mystery + // crash has been resolved. + os << flush; } break; case '\\': @@ -1706,12 +1651,21 @@ void Paragraph::write(ostream & os, BufferParams const & bparams, flushString(os, write_buffer); os << "\n\\end_layout\n"; + // FIXME This can be removed again once the mystery + // crash has been resolved. + os << flush; } void Paragraph::validate(LaTeXFeatures & features) const { d->validate(features); + bool fragile = features.runparams().moving_arg; + fragile |= layout().needprotect; + if (inInset().getLayout().isNeedProtect()) + fragile = true; + if (needsCProtection(fragile)) + features.require("cprotect"); } @@ -1810,7 +1764,7 @@ Font const & Paragraph::getFontSettings(BufferParams const & bparams, // Optimisation: avoid a full font instantiation if there is no // language change from previous call. static Font previous_font; - static Language const * previous_lang = 0; + static Language const * previous_lang = nullptr; Language const * lang = getParLanguage(bparams); if (lang != previous_lang) { previous_lang = lang; @@ -1822,7 +1776,10 @@ Font const & Paragraph::getFontSettings(BufferParams const & bparams, FontSpan Paragraph::fontSpan(pos_type pos) const { - LBUFERR(pos < size()); + LBUFERR(pos <= size()); + + if (pos == size()) + return FontSpan(pos, pos); pos_type start = 0; FontList::const_iterator cit = d->fontlist_.begin(); @@ -1855,7 +1812,7 @@ Font const & Paragraph::getFirstFontSettings(BufferParams const & bparams) const // Optimisation: avoid a full font instantiation if there is no // language change from previous call. static Font previous_font; - static Language const * previous_lang = 0; + static Language const * previous_lang = nullptr; if (bparams.language != previous_lang) { previous_lang = bparams.language; previous_font = Font(inherit_font, bparams.language); @@ -1911,40 +1868,59 @@ Font const Paragraph::getLayoutFont } -/// Returns the height of the highest font in range -FontSize Paragraph::highestFontInRange - (pos_type startpos, pos_type endpos, FontSize def_size) const -{ - return d->fontlist_.highestInRange(startpos, endpos, def_size); -} - - -char_type Paragraph::getUChar(BufferParams const & bparams, pos_type pos) const +char_type Paragraph::getUChar(BufferParams const & bparams, + OutputParams const & rp, + pos_type pos) const { char_type c = d->text_[pos]; - if (!lyxrc.rtl_support || !getFontSettings(bparams, pos).isRightToLeft()) + + // Return unchanged character in LTR languages + // or if we use poylglossia/bidi (XeTeX). + if (rp.useBidiPackage() + || !getFontSettings(bparams, pos).isRightToLeft()) return c; - // FIXME: The arabic special casing is due to the difference of arabic - // round brackets input introduced in r18599. Check if this should be - // unified with Hebrew or at least if all bracket types should be - // handled the same (file format change in either case). + // Without polyglossia/bidi, we need to account for some special cases. + // FIXME This needs to be audited! + // Check if: + // * The input is as expected for all delimiters + // => checked for Hebrew! + // * The output matches the display in the LyX workarea + // => checked for Hebrew! + // * The special cases below are really necessary + // => checked for Hebrew! + // * In arabic_arabi, brackets are transformed to Arabic + // Ornate Parentheses. Is this is really wanted? + string const & lang = getFontSettings(bparams, pos).language()->lang(); - bool const arabic = lang == "arabic_arabtex" || lang == "arabic_arabi" - || lang == "farsi"; char_type uc = c; + + // 1. In the following languages, parentheses need to be reversed. + // Also with polyglodia/luabidi + bool const reverseparens = (lang == "hebrew" || rp.use_polyglossia); + + // 2. In the following languages, brackets don't need to be reversed. + bool const reversebrackets = lang != "arabic_arabtex" + && lang != "arabic_arabi" + && lang != "farsi"; + + // Now swap delimiters if needed. switch (c) { case '(': - uc = arabic ? c : ')'; + if (reverseparens) + uc = ')'; break; case ')': - uc = arabic ? c : '('; + if (reverseparens) + uc = '('; break; case '[': - uc = ']'; + if (reversebrackets) + uc = ']'; break; case ']': - uc = '['; + if (reversebrackets) + uc = '['; break; case '{': uc = '}'; @@ -2024,15 +2000,29 @@ depth_type Paragraph::getMaxDepthAfter() const } -char Paragraph::getAlign() const +LyXAlignment Paragraph::getAlign(BufferParams const & bparams) const { if (d->params_.align() == LYX_ALIGN_LAYOUT) - return d->layout_->align; + return getDefaultAlign(bparams); else return d->params_.align(); } +LyXAlignment Paragraph::getDefaultAlign(BufferParams const & bparams) const +{ + LyXAlignment res = layout().align; + if (isRTL(bparams)) { + // Swap sides + if (res == LYX_ALIGN_LEFT) + res = LYX_ALIGN_RIGHT; + else if (res == LYX_ALIGN_RIGHT) + res = LYX_ALIGN_LEFT; + } + return res; +} + + docstring const & Paragraph::labelString() const { return d->params_.labelString(); @@ -2090,7 +2080,7 @@ docstring Paragraph::expandParagraphLabel(Layout const & layout, docstring parent(fmt, i + 1, j - i - 1); docstring label = from_ascii("??"); if (tclass.hasLayout(parent)) - docstring label = expandParagraphLabel(tclass[parent], bparams, + label = expandParagraphLabel(tclass[parent], bparams, process_appendix); fmt = docstring(fmt, 0, i) + label + docstring(fmt, j + 1, docstring::npos); @@ -2132,17 +2122,17 @@ void Paragraph::setBeginOfBody() // remove unnecessary getChar() calls pos_type i = 0; pos_type end = size(); - if (i < end && !isNewline(i)) { + bool prev_char_deleted = false; + if (i < end && (!(isNewline(i) || isEnvSeparator(i)) || isDeleted(i))) { ++i; - char_type previous_char = 0; - char_type temp = 0; if (i < end) { - previous_char = d->text_[i]; - if (!isNewline(i)) { + char_type previous_char = d->text_[i]; + if (!(isNewline(i) || isEnvSeparator(i))) { ++i; - while (i < end && previous_char != ' ') { - temp = d->text_[i]; - if (isNewline(i)) + while (i < end && (previous_char != ' ' || prev_char_deleted)) { + char_type temp = d->text_[i]; + prev_char_deleted = isDeleted(i); + if (!isDeleted(i) && (isNewline(i) || isEnvSeparator(i))) break; ++i; previous_char = temp; @@ -2197,39 +2187,40 @@ string correction(string const & orig) } -string const corrected_env(string const & suffix, string const & env, - InsetCode code, bool const lastpar) +bool corrected_env(otexstream & os, string const & suffix, string const & env, + InsetCode code, bool const lastpar, int & col) { - string output = suffix + "{"; + string macro = suffix + "{"; if (noTrivlistCentering(code)) { if (lastpar) { // the last paragraph in non-trivlist-aligned // context is special (to avoid unwanted whitespace) - if (suffix == "\\begin") - return "\\" + correction(env) + "{}"; - return string(); + if (suffix == "\\begin") { + macro = "\\" + correction(env) + "{}"; + os << from_ascii(macro); + col += macro.size(); + return true; + } + return false; } - output += correction(env); + macro += correction(env); } else - output += env; - output += "}"; - if (suffix == "\\begin") - output += "\n"; - return output; -} - - -void adjust_column(string const & str, int & column) -{ - if (!contains(str, "\n")) - column += str.size(); - else { - string tmp; - column = rsplit(str, tmp, '\n').size(); + macro += env; + macro += "}"; + if (suffix == "\\par\\end") { + os << breakln; + col = 0; + } + os << from_ascii(macro); + col += macro.size(); + if (suffix == "\\begin") { + os << breakln; + col = 0; } + return true; } -} // namespace anon +} // namespace int Paragraph::Private::startTeXParParams(BufferParams const & bparams, @@ -2237,7 +2228,12 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams, { int column = 0; - if (params_.noindent() && !layout_->pass_thru) { + bool canindent = + (bparams.paragraph_separation == BufferParams::ParagraphIndentSeparation) ? + (layout_->toggle_indent != ITOGGLE_NEVER) : + (layout_->toggle_indent == ITOGGLE_ALWAYS); + + if (canindent && params_.noindent() && !layout_->pass_thru) { os << "\\noindent "; column += 10; } @@ -2267,6 +2263,10 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams, string const begin_tag = "\\begin"; InsetCode code = ownerCode(); bool const lastpar = runparams.isLastPar; + // RTL in classic (PDF)LaTeX (without the Bidi package) + // Luabibdi (used by LuaTeX) behaves like classic + bool const rtl_classic = owner_->getParLanguage(bparams)->rightToLeft() + && !runparams.useBidiPackage(); switch (curAlign) { case LYX_ALIGN_NONE: @@ -2276,28 +2276,21 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams, case LYX_ALIGN_DECIMAL: break; case LYX_ALIGN_LEFT: { - string output; - if (owner_->getParLanguage(bparams)->babel() != "hebrew") - output = corrected_env(begin_tag, "flushleft", code, lastpar); + if (rtl_classic) + // Classic (PDF)LaTeX switches the left/right logic in RTL mode + corrected_env(os, begin_tag, "flushright", code, lastpar, column); else - output = corrected_env(begin_tag, "flushright", code, lastpar); - os << from_ascii(output); - adjust_column(output, column); + corrected_env(os, begin_tag, "flushleft", code, lastpar, column); break; } case LYX_ALIGN_RIGHT: { - string output; - if (owner_->getParLanguage(bparams)->babel() != "hebrew") - output = corrected_env(begin_tag, "flushright", code, lastpar); + if (rtl_classic) + // Classic (PDF)LaTeX switches the left/right logic in RTL mode + corrected_env(os, begin_tag, "flushleft", code, lastpar, column); else - output = corrected_env(begin_tag, "flushleft", code, lastpar); - os << from_ascii(output); - adjust_column(output, column); + corrected_env(os, begin_tag, "flushright", code, lastpar, column); break; } case LYX_ALIGN_CENTER: { - string output; - output = corrected_env(begin_tag, "center", code, lastpar); - os << from_ascii(output); - adjust_column(output, column); + corrected_env(os, begin_tag, "center", code, lastpar, column); break; } } @@ -2329,10 +2322,15 @@ bool Paragraph::Private::endTeXParParams(BufferParams const & bparams, break; } - string output; - string const end_tag = "\n\\par\\end"; + bool output = false; + int col = 0; + string const end_tag = "\\par\\end"; InsetCode code = ownerCode(); bool const lastpar = runparams.isLastPar; + // RTL in classic (PDF)LaTeX (without the Bidi package) + // Luabibdi (used by LuaTeX) behaves like classic + bool const rtl_classic = owner_->getParLanguage(bparams)->rightToLeft() + && !runparams.useBidiPackage(); switch (curAlign) { case LYX_ALIGN_NONE: @@ -2342,27 +2340,26 @@ bool Paragraph::Private::endTeXParParams(BufferParams const & bparams, case LYX_ALIGN_DECIMAL: break; case LYX_ALIGN_LEFT: { - if (owner_->getParLanguage(bparams)->babel() != "hebrew") - output = corrected_env(end_tag, "flushleft", code, lastpar); + if (rtl_classic) + // Classic (PDF)LaTeX switches the left/right logic in RTL mode + output = corrected_env(os, end_tag, "flushright", code, lastpar, col); else - output = corrected_env(end_tag, "flushright", code, lastpar); - os << from_ascii(output); + output = corrected_env(os, end_tag, "flushleft", code, lastpar, col); break; } case LYX_ALIGN_RIGHT: { - if (owner_->getParLanguage(bparams)->babel() != "hebrew") - output = corrected_env(end_tag, "flushright", code, lastpar); + if (rtl_classic) + // Classic (PDF)LaTeX switches the left/right logic in RTL mode + output = corrected_env(os, end_tag, "flushleft", code, lastpar, col); else - output = corrected_env(end_tag, "flushleft", code, lastpar); - os << from_ascii(output); + output = corrected_env(os, end_tag, "flushright", code, lastpar, col); break; } case LYX_ALIGN_CENTER: { - output = corrected_env(end_tag, "center", code, lastpar); - os << from_ascii(output); + corrected_env(os, end_tag, "center", code, lastpar, col); break; } } - return !output.empty() || lastpar; + return output || lastpar; } @@ -2393,6 +2390,11 @@ void Paragraph::latex(BufferParams const & bparams, // of the body. Font basefont; + // If there is an open font-encoding changing command (script wrapper), + // alien_script is set to its name + string alien_script; + string script; + // Maybe we have to create a optional argument. pos_type body_pos = beginOfBody(); unsigned int column = 0; @@ -2424,7 +2426,9 @@ void Paragraph::latex(BufferParams const & bparams, // if the paragraph is empty, the loop will not be entered at all if (empty()) { - if (style.isCommand()) { + // For InTitle commands, we have already opened a group + // in output_latex::TeXOnePar. + if (style.isCommand() && !style.intitle) { os << '{'; ++column; } @@ -2436,14 +2440,19 @@ void Paragraph::latex(BufferParams const & bparams, column += d->startTeXParParams(bparams, os, runparams); } + // Whether a \par can be issued for insets typeset inline with text. + // Yes if greater than 0. This has to be static. + THREAD_LOCAL_STATIC int parInline = 0; + for (pos_type i = 0; i < size(); ++i) { // First char in paragraph or after label? if (i == body_pos) { if (body_pos > 0) { if (open_font) { + bool needPar = false; column += running_font.latexWriteEndChanges( os, bparams, runparams, - basefont, basefont); + basefont, basefont, needPar); open_font = false; } basefont = getLayoutFont(bparams, outerfont); @@ -2457,7 +2466,9 @@ void Paragraph::latex(BufferParams const & bparams, os << "}] "; column +=3; } - if (style.isCommand()) { + // For InTitle commands, we have already opened a group + // in output_latex::TeXOnePar. + if (style.isCommand() && !style.intitle) { os << '{'; ++column; } @@ -2472,18 +2483,69 @@ void Paragraph::latex(BufferParams const & bparams, runparams); } + runparams.wasDisplayMath = runparams.inDisplayMath; + runparams.inDisplayMath = false; + bool deleted_display_math = false; Change const & change = runparams.inDeletedInset ? runparams.changeOfDeletedInset : lookupChange(i); - if (bparams.outputChanges && runningChange != change) { + char_type const c = d->text_[i]; + + // Check whether a display math inset follows + if (c == META_INSET + && i >= start_pos && (end_pos == -1 || i < end_pos)) { + if (isDeleted(i)) + runparams.ctObject = getInset(i)->CtObject(runparams); + + InsetMath const * im = getInset(i)->asInsetMath(); + if (im && im->asHullInset() + && im->asHullInset()->outerDisplay()) { + runparams.inDisplayMath = true; + // runparams.inDeletedInset will be set by + // latexInset later, but we need this info + // before it is called. On the other hand, we + // cannot set it here because it is a counter. + deleted_display_math = isDeleted(i); + } + if (bparams.output_changes && deleted_display_math + && runningChange == change + && change.type == Change::DELETED + && !os.afterParbreak()) { + // A display math in the same paragraph follows. + // We have to close and then reopen \lyxdeleted, + // otherwise the math will be shifted up. + OutputParams rp = runparams; + if (open_font) { + bool needPar = false; + column += running_font.latexWriteEndChanges( + os, bparams, rp, basefont, + basefont, needPar); + open_font = false; + } + basefont = (body_pos > i) ? getLabelFont(bparams, outerfont) + : getLayoutFont(bparams, outerfont); + running_font = basefont; + column += Changes::latexMarkChange(os, bparams, + Change(Change::INSERTED), change, rp); + } + } + + if (bparams.output_changes && runningChange != change) { + if (!alien_script.empty()) { + column += 1; + os << "}"; + alien_script.clear(); + } if (open_font) { + bool needPar = false; column += running_font.latexWriteEndChanges( - os, bparams, runparams, basefont, basefont); + os, bparams, runparams, + basefont, basefont, needPar); open_font = false; } - basefont = getLayoutFont(bparams, outerfont); + basefont = (body_pos > i) ? getLabelFont(bparams, outerfont) + : getLayoutFont(bparams, outerfont); running_font = basefont; - column += Changes::latexMarkChange(os, bparams, runningChange, change, runparams); runningChange = change; @@ -2491,68 +2553,110 @@ void Paragraph::latex(BufferParams const & bparams, // do not output text which is marked deleted // if change tracking output is disabled - if (!bparams.outputChanges && change.deleted()) { + if (!bparams.output_changes && change.deleted()) { continue; } ++column; // Fully instantiated font - Font const font = getFont(bparams, i, outerfont); + Font const current_font = getFont(bparams, i, outerfont); Font const last_font = running_font; // Do we need to close the previous font? if (open_font && - (font != running_font || - font.language() != running_font.language())) + (current_font != running_font || + current_font.language() != running_font.language())) { + // ensure there is no open script-wrapper + if (!alien_script.empty()) { + column += 1; + os << "}"; + alien_script.clear(); + } + bool needPar = false; column += running_font.latexWriteEndChanges( - os, bparams, runparams, basefont, - (i == body_pos-1) ? basefont : font); + os, bparams, runparams, basefont, + (i == body_pos-1) ? basefont : current_font, + needPar); running_font = basefont; open_font = false; } - string const running_lang = runparams.use_polyglossia ? - running_font.language()->polyglossia() : running_font.language()->babel(); - // close babel's font environment before opening CJK. - string const lang_end_command = runparams.use_polyglossia ? - "\\end{$$lang}" : lyxrc.language_command_end; - if (!running_lang.empty() && - font.language()->encoding()->package() == Encoding::CJK) { - string end_tag = subst(lang_end_command, - "$$lang", - running_lang); - os << from_ascii(end_tag); - column += end_tag.length(); + // if necessary, close language environment before opening CJK + string const running_lang = running_font.language()->babel(); + string const lang_end_command = lyxrc.language_command_end; + if (!lang_end_command.empty() && !bparams.useNonTeXFonts + && !running_lang.empty() + && running_lang == openLanguageName() + && current_font.language()->encoding()->package() == Encoding::CJK) { + string end_tag = subst(lang_end_command, "$$lang", running_lang); + os << from_ascii(end_tag); + column += end_tag.length(); + popLanguageName(); } // Switch file encoding if necessary (and allowed) if (!runparams.pass_thru && !style.pass_thru && runparams.encoding->package() != Encoding::none && - font.language()->encoding()->package() != Encoding::none) { + current_font.language()->encoding()->package() != Encoding::none) { pair const enc_switch = switchEncoding(os.os(), bparams, runparams, - *(font.language()->encoding())); + *(current_font.language()->encoding())); if (enc_switch.first) { column += enc_switch.second; - runparams.encoding = font.language()->encoding(); + runparams.encoding = current_font.language()->encoding(); } } - char_type const c = d->text_[i]; + // A display math inset inside an ulem command will be output + // as a box of width \linewidth, so we have to either disable + // indentation if the inset starts a paragraph, or start a new + // line to accommodate such box. This has to be done before + // writing any font changing commands. + if (runparams.inDisplayMath && !deleted_display_math + && runparams.inulemcmd) { + if (os.afterParbreak()) + os << "\\noindent"; + else + os << "\\\\\n"; + } // Do we need to change font? - if ((font != running_font || - font.language() != running_font.language()) && + if ((current_font != running_font || + current_font.language() != running_font.language()) && i != body_pos - 1) { + bool const in_ct_deletion = (bparams.output_changes + && runningChange == change + && change.type == Change::DELETED + && !os.afterParbreak()); + if (in_ct_deletion) { + // We have to close and then reopen \lyxdeleted, + // as strikeout needs to be on lowest level. + bool needPar = false; + OutputParams rp = runparams; + column += running_font.latexWriteEndChanges( + os, bparams, rp, basefont, + basefont, needPar); + os << '}'; + column += 1; + } odocstringstream ods; - column += font.latexWriteStartChanges(ods, bparams, + column += current_font.latexWriteStartChanges(ods, bparams, runparams, basefont, last_font); - running_font = font; + // Check again for display math in ulem commands as a + // font change may also occur just before a math inset. + if (runparams.inDisplayMath && !deleted_display_math + && runparams.inulemcmd) { + if (os.afterParbreak()) + os << "\\noindent"; + else + os << "\\\\\n"; + } + running_font = current_font; open_font = true; docstring fontchange = ods.str(); // check whether the fontchange ends with a \\textcolor @@ -2567,6 +2671,13 @@ void Paragraph::latex(BufferParams const & bparams, << from_ascii("{}"); else os << fontchange; + if (in_ct_deletion) { + // We have to close and then reopen \lyxdeleted, + // as strikeout needs to be on lowest level. + OutputParams rp = runparams; + column += Changes::latexMarkChange(os, bparams, + Change(Change::UNCHANGED), change, rp); + } } // FIXME: think about end_pos implementation... @@ -2577,8 +2688,8 @@ void Paragraph::latex(BufferParams const & bparams, // latexSpecialChar ignores spaces if // style.pass_thru is false. if (i != body_pos - 1) { - if (d->simpleTeXBlanks(runparams, os, - i, column, font, style)) { + if (d->simpleTeXBlanks(bparams, runparams, os, + i, column, current_font, style)) { // A surrogate pair was output. We // must not call latexSpecialChar // in this iteration, since it would output @@ -2591,7 +2702,7 @@ void Paragraph::latex(BufferParams const & bparams, OutputParams rp = runparams; rp.free_spacing = style.free_spacing; - rp.local_font = &font; + rp.local_font = ¤t_font; rp.intitle = style.intitle; // Two major modes: LaTeX or plain @@ -2599,16 +2710,71 @@ void Paragraph::latex(BufferParams const & bparams, // and then split to handle the two modes separately. if (c == META_INSET) { if (i >= start_pos && (end_pos == -1 || i < end_pos)) { + // Greyedout notes and, in general, all insets + // with InsetLayout::isDisplay() == false, + // are typeset inline with the text. So, we + // can add a \par to the last paragraph of + // such insets only if nothing else follows. + bool incremented = false; + Inset const * inset = getInset(i); + InsetText const * textinset = inset + ? inset->asInsetText() + : nullptr; + if (i + 1 == size() && textinset + && !inset->getLayout().isDisplay()) { + ParagraphList const & pars = + textinset->text().paragraphs(); + pit_type const pit = pars.size() - 1; + Font const lastfont = + pit < 0 || pars[pit].empty() + ? pars[pit].getLayoutFont( + bparams, + outerfont) + : pars[pit].getFont(bparams, + pars[pit].size() - 1, + outerfont); + if (lastfont.fontInfo().size() != + basefont.fontInfo().size()) { + ++parInline; + incremented = true; + } + } d->latexInset(bparams, os, rp, running_font, basefont, outerfont, open_font, runningChange, style, i, column); + if (incremented) + --parInline; + + if (runparams.ctObject == OutputParams::CT_DISPLAYOBJECT + || runparams.ctObject == OutputParams::CT_UDISPLAYOBJECT) { + // Close \lyx*deleted and force its + // reopening (if needed) + os << '}'; + column++; + runningChange = Change(Change::UNCHANGED); + runparams.ctObject = OutputParams::CT_NORMAL; + } } - } else { - if (i >= start_pos && (end_pos == -1 || i < end_pos)) { - try { - d->latexSpecialChar(os, bparams, rp, running_font, runningChange, - style, i, end_pos, column); - } catch (EncodingException & e) { + } else if (i >= start_pos && (end_pos == -1 || i < end_pos)) { + if (!bparams.useNonTeXFonts) + script = Encodings::isKnownScriptChar(c); + if (script != alien_script) { + if (!alien_script.empty()) { + os << "}"; + alien_script.clear(); + } + string fontenc = running_font.language()->fontenc(bparams); + if (!script.empty() + && !Encodings::fontencSupportsScript(fontenc, script)) { + column += script.length() + 2; + os << "\\" << script << "{"; + alien_script = script; + } + } + try { + d->latexSpecialChar(os, bparams, rp, running_font, + alien_script, style, i, end_pos, column); + } catch (EncodingException & e) { if (runparams.dryrun) { os << "<" << _("LyX Warning: ") << _("uncodable character") << " '"; @@ -2622,31 +2788,84 @@ void Paragraph::latex(BufferParams const & bparams, } } } - } // Set the encoding to that returned from latexSpecialChar (see // comment for encoding member in OutputParams.h) runparams.encoding = rp.encoding; + + // Also carry on the info on a closed ulem command for insets + // such as Note that do not produce any output, so that no + // command is ever executed but its opening was recorded. + runparams.inulemcmd = rp.inulemcmd; + + // And finally, pass the post_macros upstream + runparams.post_macro = rp.post_macro; + } + + // Close wrapper for alien script + if (!alien_script.empty()) { + os << "}"; + alien_script.clear(); } // If we have an open font definition, we have to close it if (open_font) { + // Make sure that \\par is done with the font of the last + // character if this has another size as the default. + // This is necessary because LaTeX (and LyX on the screen) + // calculates the space between the baselines according + // to this font. (Matthias) + // + // We must not change the font for the last paragraph + // of non-multipar insets, tabular cells or commands, + // since this produces unwanted whitespace. + + Font const font = empty() + ? getLayoutFont(bparams, outerfont) + : getFont(bparams, size() - 1, outerfont); + + InsetText const * textinset = inInset().asInsetText(); + + bool const maintext = textinset + ? textinset->text().isMainText() + : false; + + size_t const numpars = textinset + ? textinset->text().paragraphs().size() + : 0; + + bool needPar = false; + + if (style.resfont.size() != font.fontInfo().size() + && (!runparams.isLastPar || maintext + || (numpars > 1 && d->ownerCode() != CELL_CODE + && (inInset().getLayout().isDisplay() + || parInline))) + && !style.isCommand()) { + needPar = true; + } #ifdef FIXED_LANGUAGE_END_DETECTION if (next_) { running_font.latexWriteEndChanges(os, bparams, runparams, basefont, - next_->getFont(bparams, 0, outerfont)); + next_->getFont(bparams, 0, outerfont), + needPar); } else { running_font.latexWriteEndChanges(os, bparams, - runparams, basefont, basefont); + runparams, basefont, basefont, needPar); } #else //FIXME: For now we ALWAYS have to close the foreign font settings if they are //FIXME: there as we start another \selectlanguage with the next paragraph if //FIXME: we are in need of this. This should be fixed sometime (Jug) running_font.latexWriteEndChanges(os, bparams, runparams, - basefont, basefont); + basefont, basefont, needPar); #endif + if (needPar) { + // The \par could not be inserted at the same nesting + // level of the font size change, so do it now. + os << "{\\" << font.latexSize() << "\\par}"; + } } column += Changes::latexMarkChange(os, bparams, runningChange, @@ -2665,8 +2884,7 @@ void Paragraph::latex(BufferParams const & bparams, if (allowcust && d->endTeXParParams(bparams, os, runparams) && runparams.encoding != prev_encoding) { runparams.encoding = prev_encoding; - if (!runparams.isFullUnicode()) - os << setEncoding(prev_encoding->iconvName()); + os << setEncoding(prev_encoding->iconvName()); } LYXERR(Debug::LATEX, "Paragraph::latex... done " << this); @@ -2836,13 +3054,14 @@ void doFontSwitch(vector & tagsToOpen, flag = false; } } -} +} // namespace docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, XHTMLStream & xs, OutputParams const & runparams, Font const & outerfont, + bool start_paragraph, bool close_paragraph, pos_type initial) const { docstring retval; @@ -2854,6 +3073,7 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, bool ubar_flag = false; bool dbar_flag = false; bool sout_flag = false; + bool xout_flag = false; bool wave_flag = false; // shape tags bool shap_flag = false; @@ -2864,21 +3084,22 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, Layout const & style = *d->layout_; - xs.startParagraph(allowEmpty()); + if (start_paragraph) + xs.startDivision(allowEmpty()); FontInfo font_old = style.labeltype == LABEL_MANUAL ? style.labelfont : style.font; FontShape curr_fs = INHERIT_SHAPE; FontFamily curr_fam = INHERIT_FAMILY; - FontSize curr_size = FONT_SIZE_INHERIT; - - string const default_family = - buf.masterBuffer()->params().fonts_default_family; + FontSize curr_size = INHERIT_SIZE; + + string const default_family = + buf.masterBuffer()->params().fonts_default_family; vector tagsToOpen; vector tagsToClose; - + // parsing main loop for (pos_type i = initial; i < size(); ++i) { // let's not show deleted material in the output @@ -2901,12 +3122,17 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, curstate = font.fontInfo().underbar(); if (font_old.underbar() != curstate) doFontSwitch(tagsToOpen, tagsToClose, ubar_flag, curstate, html::FT_UBAR); - + // strikeout curstate = font.fontInfo().strikeout(); if (font_old.strikeout() != curstate) doFontSwitch(tagsToOpen, tagsToClose, sout_flag, curstate, html::FT_SOUT); + // xout + curstate = font.fontInfo().xout(); + if (font_old.xout() != curstate) + doFontSwitch(tagsToOpen, tagsToClose, xout_flag, curstate, html::FT_XOUT); + // double underbar curstate = font.fontInfo().uuline(); if (font_old.uuline() != curstate) @@ -3031,41 +3257,41 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, if (old_size != curr_size) { if (size_flag) { switch (old_size) { - case FONT_SIZE_TINY: + case TINY_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_TINY)); break; - case FONT_SIZE_SCRIPT: + case SCRIPT_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_SCRIPT)); break; - case FONT_SIZE_FOOTNOTE: + case FOOTNOTE_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_FOOTNOTE)); break; - case FONT_SIZE_SMALL: + case SMALL_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_SMALL)); break; - case FONT_SIZE_LARGE: + case LARGE_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_LARGE)); break; - case FONT_SIZE_LARGER: + case LARGER_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_LARGER)); break; - case FONT_SIZE_LARGEST: + case LARGEST_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_LARGEST)); break; - case FONT_SIZE_HUGE: + case HUGE_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_HUGE)); break; - case FONT_SIZE_HUGER: + case HUGER_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_HUGER)); break; - case FONT_SIZE_INCREASE: + case INCREASE_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_INCREASE)); break; - case FONT_SIZE_DECREASE: + case DECREASE_SIZE: tagsToClose.push_back(html::EndFontTag(html::FT_SIZE_DECREASE)); break; - case FONT_SIZE_INHERIT: - case FONT_SIZE_NORMAL: + case INHERIT_SIZE: + case NORMAL_SIZE: break; default: // the other tags are for internal use @@ -3075,52 +3301,52 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, size_flag = false; } switch (curr_size) { - case FONT_SIZE_TINY: + case TINY_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_TINY)); size_flag = true; break; - case FONT_SIZE_SCRIPT: + case SCRIPT_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_SCRIPT)); size_flag = true; break; - case FONT_SIZE_FOOTNOTE: + case FOOTNOTE_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_FOOTNOTE)); size_flag = true; break; - case FONT_SIZE_SMALL: + case SMALL_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_SMALL)); size_flag = true; break; - case FONT_SIZE_LARGE: + case LARGE_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_LARGE)); size_flag = true; break; - case FONT_SIZE_LARGER: + case LARGER_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_LARGER)); size_flag = true; break; - case FONT_SIZE_LARGEST: + case LARGEST_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_LARGEST)); size_flag = true; break; - case FONT_SIZE_HUGE: + case HUGE_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_HUGE)); size_flag = true; break; - case FONT_SIZE_HUGER: + case HUGER_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_HUGER)); size_flag = true; break; - case FONT_SIZE_INCREASE: + case INCREASE_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_INCREASE)); size_flag = true; break; - case FONT_SIZE_DECREASE: + case DECREASE_SIZE: tagsToOpen.push_back(html::FontTag(html::FT_SIZE_DECREASE)); size_flag = true; break; - case FONT_SIZE_NORMAL: - case FONT_SIZE_INHERIT: + case NORMAL_SIZE: + case INHERIT_SIZE: break; default: // the other tags are for internal use @@ -3150,42 +3376,30 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, if (!runparams.for_toc || inset->isInToc()) { OutputParams np = runparams; np.local_font = &font; - if (!inset->getLayout().htmlisblock()) + // If the paragraph has size 1, then we are in the "special + // case" where we do not output the containing paragraph info + if (!inset->getLayout().htmlisblock() && size() != 1) np.html_in_par = true; retval += inset->xhtml(xs, np); } } else { - char_type c = getUChar(buf.masterBuffer()->params(), i); - - if (style.pass_thru || runparams.pass_thru) - xs << c; - else if (c == '-') { - docstring str; - int j = i + 1; - if (j < size() && d->text_[j] == '-') { - j += 1; - if (j < size() && d->text_[j] == '-') { - str += from_ascii("—"); - i += 2; - } else { - str += from_ascii("–"); - i += 1; - } - } - else - str += c; - // We don't want to escape the entities. Note that - // it is safe to do this, since str can otherwise - // only be "-". E.g., it can't be "<". - xs << XHTMLStream::ESCAPE_NONE << str; - } else + char_type c = getUChar(buf.masterBuffer()->params(), + runparams, i); + if (c == ' ' && (style.free_spacing || runparams.free_spacing)) + xs << XHTMLStream::ESCAPE_NONE << " "; + else xs << c; } font_old = font.fontInfo(); } + // FIXME XHTML + // I'm worried about what happens if a branch, say, is itself + // wrapped in some font stuff. I think that will not work. xs.closeFontTags(); - xs.endParagraph(); + if (close_paragraph) + xs.endDivision(); + return retval; } @@ -3193,18 +3407,29 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf, bool Paragraph::isHfill(pos_type pos) const { Inset const * inset = getInset(pos); - return inset && (inset->lyxCode() == SPACE_CODE && - inset->isStretchableSpace()); + return inset && inset->isHfill(); } bool Paragraph::isNewline(pos_type pos) const { + // U+2028 LINE SEPARATOR + // U+2029 PARAGRAPH SEPARATOR + char_type const c = d->text_[pos]; + if (c == 0x2028 || c == 0x2029) + return true; Inset const * inset = getInset(pos); return inset && inset->lyxCode() == NEWLINE_CODE; } +bool Paragraph::isEnvSeparator(pos_type pos) const +{ + Inset const * inset = getInset(pos); + return inset && inset->lyxCode() == SEPARATOR_CODE; +} + + bool Paragraph::isLineSeparator(pos_type pos) const { char_type const c = d->text_[pos]; @@ -3215,10 +3440,12 @@ bool Paragraph::isLineSeparator(pos_type pos) const } -bool Paragraph::isWordSeparator(pos_type pos) const +bool Paragraph::isWordSeparator(pos_type pos, bool const ignore_deleted) const { if (pos == size()) return true; + if (ignore_deleted && isDeleted(pos)) + return false; if (Inset const * inset = getInset(pos)) return !inset->isLetter(); // if we have a hard hyphen (no en- or emdash) or apostrophe @@ -3248,16 +3475,69 @@ bool Paragraph::isHardHyphenOrApostrophe(pos_type pos) const if ((nextpos == psize || isSpace(nextpos)) && (pos == 0 || isSpace(prevpos))) return false; - return c == '\'' - || ((nextpos == psize || d->text_[nextpos] != '-') - && (pos == 0 || d->text_[prevpos] != '-')); + return true; +} + + +bool Paragraph::needsCProtection(bool const fragile) const +{ + // first check the layout of the paragraph, but only in insets + InsetText const * textinset = inInset().asInsetText(); + bool const maintext = textinset + ? textinset->text().isMainText() + : false; + + if (!maintext && layout().needcprotect) { + // Environments need cprotection regardless the content + if (layout().latextype == LATEX_ENVIRONMENT) + return true; + + // Commands need cprotection if they contain specific chars + int const nchars_escape = 9; + static char_type const chars_escape[nchars_escape] = { + '&', '_', '$', '%', '#', '^', '{', '}', '\\'}; + + docstring const pars = asString(); + for (int k = 0; k < nchars_escape; k++) { + if (contains(pars, chars_escape[k])) + return true; + } + } + + // now check whether we have insets that need cprotection + pos_type size = pos_type(d->text_.size()); + for (pos_type i = 0; i < size; ++i) { + if (!isInset(i)) + continue; + Inset const * ins = getInset(i); + if (ins->needsCProtection(maintext, fragile)) + return true; + if (ins->getLayout().latextype() == InsetLayout::ENVIRONMENT) + // Environments need cprotection regardless the content + return true; + // Now check math environments + InsetMath const * im = getInset(i)->asInsetMath(); + if (!im || im->cell(0).empty()) + continue; + switch(im->cell(0)[0]->lyxCode()) { + case MATH_AMSARRAY_CODE: + case MATH_SUBSTACK_CODE: + case MATH_ENV_CODE: + case MATH_XYMATRIX_CODE: + // these need cprotection + return true; + default: + break; + } + } + + return false; } -bool Paragraph::isSameSpellRange(pos_type pos1, pos_type pos2) const +FontSpan const & Paragraph::getSpellRange(pos_type pos) const { - return pos1 == pos2 - || d->speller_state_.getRange(pos1) == d->speller_state_.getRange(pos2); + return d->speller_state_.getRange(pos); } @@ -3291,8 +3571,7 @@ Paragraph::getParLanguage(BufferParams const & bparams) const bool Paragraph::isRTL(BufferParams const & bparams) const { - return lyxrc.rtl_support - && getParLanguage(bparams)->rightToLeft() + return getParLanguage(bparams)->rightToLeft() && !inInset().getLayout().forceLTR(); } @@ -3315,28 +3594,22 @@ void Paragraph::changeLanguage(BufferParams const & bparams, bool Paragraph::isMultiLingual(BufferParams const & bparams) const { Language const * doc_language = bparams.language; - FontList::const_iterator cit = d->fontlist_.begin(); - FontList::const_iterator end = d->fontlist_.end(); - - for (; cit != end; ++cit) - if (cit->font().language() != ignore_language && - cit->font().language() != latex_language && - cit->font().language() != doc_language) + for (auto const & f : d->fontlist_) + if (f.font().language() != ignore_language && + f.font().language() != latex_language && + f.font().language() != doc_language) return true; return false; } -void Paragraph::getLanguages(std::set & languages) const +void Paragraph::getLanguages(std::set & langs) const { - FontList::const_iterator cit = d->fontlist_.begin(); - FontList::const_iterator end = d->fontlist_.end(); - - for (; cit != end; ++cit) { - Language const * lang = cit->font().language(); + for (auto const & f : d->fontlist_) { + Language const * lang = f.font().language(); if (lang != ignore_language && lang != latex_language) - languages.insert(lang); + langs.insert(lang); } } @@ -3347,7 +3620,7 @@ docstring Paragraph::asString(int options) const } -docstring Paragraph::asString(pos_type beg, pos_type end, int options) const +docstring Paragraph::asString(pos_type beg, pos_type end, int options, const OutputParams *runparams) const { odocstringstream os; @@ -3364,9 +3637,12 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options) const || (c == '\n' && (options & AS_STR_NEWLINES))) os.put(c); else if (c == META_INSET && (options & AS_STR_INSETS)) { - getInset(i)->toString(os); - if (getInset(i)->asInsetMath()) - os << " "; + if (c == META_INSET && (options & AS_STR_PLAINTEXT)) { + LASSERT(runparams != nullptr, return docstring()); + getInset(i)->plaintext(os, *runparams); + } else { + getInset(i)->toString(os); + } } } @@ -3374,48 +3650,25 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options) const } -void Paragraph::forToc(docstring & os, size_t maxlen) const +void Paragraph::forOutliner(docstring & os, size_t const maxlen, + bool const shorten, bool const label) const { - if (!d->params_.labelString().empty()) - os += d->params_.labelString() + ' '; - for (pos_type i = 0; i < size() && os.length() < maxlen; ++i) { + size_t tmplen = shorten ? maxlen + 1 : maxlen; + if (label && !labelString().empty()) + os += labelString() + ' '; + if (!layout().isTocCaption()) + return; + for (pos_type i = 0; i < size() && os.length() < tmplen; ++i) { if (isDeleted(i)) continue; char_type const c = d->text_[i]; if (isPrintable(c)) os += c; - else if (c == '\t' || c == '\n') - os += ' '; else if (c == META_INSET) - getInset(i)->forToc(os, maxlen); + getInset(i)->forOutliner(os, tmplen, false); } -} - - -docstring Paragraph::stringify(pos_type beg, pos_type end, int options, - OutputParams const & runparams) const -{ - odocstringstream os; - - if (beg == 0 - && options & AS_STR_LABEL - && !d->params_.labelString().empty()) - os << d->params_.labelString() << ' '; - - OutputParams op = runparams; - op.for_search = true; - - for (pos_type i = beg; i < end; ++i) { - char_type const c = d->text_[i]; - if (isPrintable(c) || c == '\t' - || (c == '\n' && (options & AS_STR_NEWLINES))) - os.put(c); - else if (c == META_INSET && (options & AS_STR_INSETS)) { - getInset(i)->plaintext(os, op); - } - } - - return os.str(); + if (shorten) + Text::shortenForOutliner(os, maxlen); } @@ -3505,74 +3758,45 @@ bool Paragraph::allowEmpty() const } -char_type Paragraph::transformChar(char_type c, pos_type pos) const -{ - if (!Encodings::isArabicChar(c)) - return c; - - char_type prev_char = ' '; - char_type next_char = ' '; - - for (pos_type i = pos - 1; i >= 0; --i) { - char_type const par_char = d->text_[i]; - if (!Encodings::isArabicComposeChar(par_char)) { - prev_char = par_char; - break; - } - } - - for (pos_type i = pos + 1, end = size(); i < end; ++i) { - char_type const par_char = d->text_[i]; - if (!Encodings::isArabicComposeChar(par_char)) { - next_char = par_char; - break; - } - } - - if (Encodings::isArabicChar(next_char)) { - if (Encodings::isArabicChar(prev_char) && - !Encodings::isArabicSpecialChar(prev_char)) - return Encodings::transformChar(c, Encodings::FORM_MEDIAL); - else - return Encodings::transformChar(c, Encodings::FORM_INITIAL); - } else { - if (Encodings::isArabicChar(prev_char) && - !Encodings::isArabicSpecialChar(prev_char)) - return Encodings::transformChar(c, Encodings::FORM_FINAL); - else - return Encodings::transformChar(c, Encodings::FORM_ISOLATED); - } -} - - bool Paragraph::brokenBiblio() const { - // there is a problem if there is no bibitem at position 0 or - // if there is another bibitem in the paragraph. - return d->layout_->labeltype == LABEL_BIBLIO + // There is a problem if there is no bibitem at position 0 in + // paragraphs that need one, if there is another bibitem in the + // paragraph or if this paragraph is not supposed to have + // a bibitem inset at all. + return ((d->layout_->labeltype == LABEL_BIBLIO && (d->insetlist_.find(BIBITEM_CODE) != 0 - || d->insetlist_.find(BIBITEM_CODE, 1) > 0); + || d->insetlist_.find(BIBITEM_CODE, 1) > 0)) + || (d->layout_->labeltype != LABEL_BIBLIO + && d->insetlist_.find(BIBITEM_CODE) != -1)); } int Paragraph::fixBiblio(Buffer const & buffer) { - // FIXME: What about the case where paragraph is not BIBLIO - // but there is an InsetBibitem? // FIXME: when there was already an inset at 0, the return value is 1, // which does not tell whether another inset has been remove; the // cursor cannot be correctly updated. - if (d->layout_->labeltype != LABEL_BIBLIO) - return 0; - - bool const track_changes = buffer.params().trackChanges; + bool const track_changes = buffer.params().track_changes; int bibitem_pos = d->insetlist_.find(BIBITEM_CODE); - bool const hasbibitem0 = bibitem_pos == 0; + // The case where paragraph is not BIBLIO + if (d->layout_->labeltype != LABEL_BIBLIO) { + if (bibitem_pos == -1) + // No InsetBibitem => OK + return 0; + // There is an InsetBibitem: remove it! + d->insetlist_.release(bibitem_pos); + eraseChar(bibitem_pos, track_changes); + return (bibitem_pos == 0) ? -1 : -bibitem_pos; + } + + bool const hasbibitem0 = bibitem_pos == 0; if (hasbibitem0) { bibitem_pos = d->insetlist_.find(BIBITEM_CODE, 1); - // There was an InsetBibitem at pos 0, and no other one => OK + // There was an InsetBibitem at pos 0, + // and no other one => OK if (bibitem_pos == -1) return 0; // there is a bibitem at the 0 position, but since @@ -3587,7 +3811,7 @@ int Paragraph::fixBiblio(Buffer const & buffer) } // We need to create an inset at the beginning - Inset * inset = 0; + Inset * inset = nullptr; if (bibitem_pos > 0) { // there was one somewhere in the paragraph, let's move it inset = d->insetlist_.release(bibitem_pos); @@ -3598,9 +3822,11 @@ int Paragraph::fixBiblio(Buffer const & buffer) InsetCommandParams(BIBITEM_CODE)); Font font(inherit_font, buffer.params().language); - insertInset(0, inset, font, Change(track_changes ? Change::INSERTED + insertInset(0, inset, font, Change(track_changes ? Change::INSERTED : Change::UNCHANGED)); + // This is needed to get the counters right + buffer.updateBuffer(); return 1; } @@ -3635,12 +3861,18 @@ InsetList const & Paragraph::insetList() const } -void Paragraph::setBuffer(Buffer & b) +void Paragraph::setInsetBuffers(Buffer & b) { d->insetlist_.setBuffer(b); } +void Paragraph::resetBuffer() +{ + d->insetlist_.resetBuffer(); +} + + Inset * Paragraph::releaseInset(pos_type pos) { Inset * inset = d->insetlist_.release(pos); @@ -3653,14 +3885,14 @@ Inset * Paragraph::releaseInset(pos_type pos) Inset * Paragraph::getInset(pos_type pos) { return (pos < pos_type(d->text_.size()) && d->text_[pos] == META_INSET) - ? d->insetlist_.get(pos) : 0; + ? d->insetlist_.get(pos) : nullptr; } Inset const * Paragraph::getInset(pos_type pos) const { return (pos < pos_type(d->text_.size()) && d->text_[pos] == META_INSET) - ? d->insetlist_.get(pos) : 0; + ? d->insetlist_.get(pos) : nullptr; } @@ -3670,9 +3902,11 @@ void Paragraph::changeCase(BufferParams const & bparams, pos_type pos, // process sequences of modified characters; in change // tracking mode, this approach results in much better // usability than changing case on a char-by-char basis - docstring changes; + // We also need to track the current font, since font + // changes within sequences can occur. + vector > changes; - bool const trackChanges = bparams.trackChanges; + bool const trackChanges = bparams.track_changes; bool capitalize = true; @@ -3704,7 +3938,7 @@ void Paragraph::changeCase(BufferParams const & bparams, pos_type pos, } if (oldChar != newChar) { - changes += newChar; + changes.push_back(make_pair(newChar, getFontSettings(bparams, pos))); if (pos != right - 1) continue; // step behind the changing area @@ -3713,9 +3947,8 @@ void Paragraph::changeCase(BufferParams const & bparams, pos_type pos, int erasePos = pos - changes.size(); for (size_t i = 0; i < changes.size(); i++) { - insertChar(pos, changes[i], - getFontSettings(bparams, - erasePos), + insertChar(pos, changes[i].first, + changes[i].second, trackChanges); if (!eraseChar(erasePos, trackChanges)) { ++erasePos; @@ -3805,38 +4038,38 @@ void Paragraph::deregisterWords() Private::LangWordsMap::const_iterator itl = d->words_.begin(); Private::LangWordsMap::const_iterator ite = d->words_.end(); for (; itl != ite; ++itl) { - WordList * wl = theWordList(itl->first); + WordList & wl = theWordList(itl->first); Private::Words::const_iterator it = (itl->second).begin(); Private::Words::const_iterator et = (itl->second).end(); for (; it != et; ++it) - wl->remove(*it); + wl.remove(*it); } d->words_.clear(); } void Paragraph::locateWord(pos_type & from, pos_type & to, - word_location const loc) const + word_location const loc, bool const ignore_deleted) const { switch (loc) { case WHOLE_WORD_STRICT: if (from == 0 || from == size() - || isWordSeparator(from) - || isWordSeparator(from - 1)) { + || isWordSeparator(from, ignore_deleted) + || isWordSeparator(from - 1, ignore_deleted)) { to = from; return; } - // no break here, we go to the next + // fall through case WHOLE_WORD: // If we are already at the beginning of a word, do nothing - if (!from || isWordSeparator(from - 1)) + if (!from || isWordSeparator(from - 1, ignore_deleted)) break; - // no break here, we go to the next + // fall through case PREVIOUS_WORD: // always move the cursor to the beginning of previous word - while (from && !isWordSeparator(from - 1)) + while (from && !isWordSeparator(from - 1, ignore_deleted)) --from; break; case NEXT_WORD: @@ -3847,27 +4080,35 @@ void Paragraph::locateWord(pos_type & from, pos_type & to, break; } to = from; - while (to < size() && !isWordSeparator(to)) + while (to < size() && !isWordSeparator(to, ignore_deleted)) ++to; } void Paragraph::collectWords() { - pos_type n = size(); - for (pos_type pos = 0; pos < n; ++pos) { + for (pos_type pos = 0; pos < size(); ++pos) { if (isWordSeparator(pos)) continue; pos_type from = pos; locateWord(from, pos, WHOLE_WORD); - if ((pos - from) >= (int)lyxrc.completion_minlength) { - docstring word = asString(from, pos, AS_STR_NONE); - FontList::const_iterator cit = d->fontlist_.fontIterator(pos); - if (cit == d->fontlist_.end()) - return; - Language const * lang = cit->font().language(); - d->words_[*lang].insert(word); - } + // Work around MSVC warning: The statement + // if (pos < from + lyxrc.completion_minlength) + // triggers a signed vs. unsigned warning. + // I don't know why this happens, it could be a MSVC bug, or + // related to LLP64 (windows) vs. LP64 (unix) programming + // model, or the C++ standard might be ambigous in the section + // defining the "usual arithmetic conversions". However, using + // a temporary variable is safe and works on all compilers. + pos_type const endpos = from + lyxrc.completion_minlength; + if (pos < endpos) + continue; + FontList::const_iterator cit = d->fontlist_.fontIterator(from); + if (cit == d->fontlist_.end()) + return; + Language const * lang = cit->font().language(); + docstring const word = asString(from, pos, AS_STR_NONE); + d->words_[lang->lang()].insert(word); } } @@ -3877,11 +4118,11 @@ void Paragraph::registerWords() Private::LangWordsMap::const_iterator itl = d->words_.begin(); Private::LangWordsMap::const_iterator ite = d->words_.end(); for (; itl != ite; ++itl) { - WordList * wl = theWordList(itl->first); + WordList & wl = theWordList(itl->first); Private::Words::const_iterator it = (itl->second).begin(); Private::Words::const_iterator et = (itl->second).end(); for (; it != et; ++it) - wl->insert(*it); + wl.insert(*it); } } @@ -3918,7 +4159,7 @@ Language * Paragraph::Private::locateSpellRange( ++from; // don't check empty range if (from >= to) - return 0; + return nullptr; // get current language Language * lang = getSpellLanguage(from); pos_type last = from; @@ -4019,13 +4260,16 @@ SpellChecker::Result Paragraph::spellCheck(pos_type & from, pos_type & to, if (!d->layout_->spellcheck || !inInset().allowSpellCheck()) return result; - locateWord(from, to, WHOLE_WORD); + locateWord(from, to, WHOLE_WORD, true); if (from == to || from >= size()) return result; docstring word = asString(from, to, AS_STR_INSETS | AS_STR_SKIPDELETE); Language * lang = d->getSpellLanguage(from); + if (getFontSettings(d->inset_owner_->buffer().params(), from).fontInfo().nospellcheck() == FONT_ON) + return result; + wl = WordLangTuple(word, lang); if (word.empty()) @@ -4075,6 +4319,15 @@ SpellChecker::Result Paragraph::spellCheck(pos_type & from, pos_type & to, } +void Paragraph::anonymize() +{ + // This is a very crude anonymization for now + for (char_type & c : d->text_) + if (isLetterChar(c) || isNumber(c)) + c = 'a'; +} + + void Paragraph::Private::markMisspelledWords( pos_type const & first, pos_type const & last, SpellChecker::Result result,