]> git.lyx.org Git - lyx.git/blobdiff - src/Paragraph.cpp
Introduce NeedCProtect -1 layout option
[lyx.git] / src / Paragraph.cpp
index 6b960cd0c070dd3d9bede5bea624ccc4904f5a21..ec92fa21cfbc4236c135dcb8bc7a4e3793aedc40 100644 (file)
@@ -25,6 +25,7 @@
 #include "BufferEncodings.h"
 #include "Changes.h"
 #include "Counters.h"
+#include "Cursor.h"
 #include "InsetList.h"
 #include "Language.h"
 #include "LaTeXFeatures.h"
@@ -759,7 +760,6 @@ void Paragraph::acceptChanges(pos_type start, pos_type end)
                                }
                                break;
                }
-
        }
 }
 
@@ -1090,22 +1090,45 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
                running_change = Change(Change::UNCHANGED);
        }
 
-       bool close = false;
+       unsigned int close_brace = 0;
+       bool const disp_env = (inset->isEnvironment() && inset->getLayout().isDisplay())
+                       || runparams.inDisplayMath;
+       string close_env;
        odocstream::pos_type const len = os.os().tellp();
 
        if (inset->forceLTR(runparams)
-           && running_font.isRightToLeft()
-           // ERT is an exception, it should be output with no
-           // decorations at all
-           && inset->lyxCode() != ERT_CODE) {
-               if (runparams.use_polyglossia) {
-                       os << "\\LRE{";
+           // babel with LuaTeX does not need a switch
+           // babel with XeTeX needs a switch only if bidi is used
+           // and \L is not defined there.
+           && (!runparams.isFullUnicode() || bparams.useBidiPackage(runparams) || runparams.use_polyglossia)
+           && running_font.isRightToLeft()) {
+               if (bparams.useBidiPackage(runparams) || runparams.use_polyglossia) {
+                       // (lua)bidi
+                       // Displayed environments go in an LTR environment
+                       if (disp_env) {
+                               os << "\\begin{LTR}";
+                               close_env = "LTR";
+                       } else {
+                               if (runparams.flavor == Flavor::LuaTeX) {
+                                       // luabidi's \LRE needs extra grouping
+                                       // (possibly a LuaTeX bug)
+                                       os << '{';
+                                       close_brace = 1;
+                               }
+                               os << "\\LRE{";
+                               close_brace += 1;
+                       }
                } else if (running_font.language()->lang() == "farsi"
-                          || running_font.language()->lang() == "arabic_arabi")
+                        || running_font.language()->lang() == "arabic_arabi") {
                        os << "\\textLR{" << termcmd;
-               else
+                       close_brace = 1;
+               } else {
+                       // babel classic
                        os << "\\L{";
-               close = true;
+                       if (disp_env)
+                               os << safebreakln;
+                       close_brace = 1;
+               }
        }
 
        if (open_font && fontswitch_inset) {
@@ -1149,6 +1172,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
                        ? textinset->hasCProtectContent(runparams.moving_arg)
                          && !textinset->text().isMainText()
                          && inset->lyxCode() != BRANCH_CODE
+                         && !runparams.no_cprotect
                        : false;
                unsigned int count2 = basefont.latexWriteStartChanges(os, bparams,
                                                      rp, running_font,
@@ -1182,8 +1206,15 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
                throw;
        }
 
-       if (close)
-               os << '}';
+       if (!close_env.empty())
+               os << "\\end{" << close_env << "}";
+
+       if (close_brace > 0) {
+               for (unsigned i = 0; i < close_brace; ++i)
+                       os << '}';
+               if (disp_env)
+                       os << safebreakln;
+       }
 
        if (os.texrow().rows() > previous_row_count) {
                os.texrow().start(owner_->id(), i + 1);
@@ -1209,26 +1240,41 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 {
        char_type const c = owner_->getUChar(bparams, runparams, i);
 
-       if (style.pass_thru || runparams.pass_thru || (runparams.for_searchAdv != OutputParams::NoSearch)
+       if (style.pass_thru || runparams.pass_thru || runparams.find_effective()
            || contains(style.pass_thru_chars, c)
            || contains(runparams.pass_thru_chars, c)) {
-               if (runparams.for_searchAdv != OutputParams::NoSearch) {
-                       if (c == '\\')
+               if (runparams.find_effective()) {
+                       switch (c) {
+                       case '\\':
                                os << "\\\\";
-                       else if (c == '{')
+                               return;
+                       case '{':
                                os << "\\braceleft ";
-                       else if (c == '}')
+                               return;
+                       case '}':
                                os << "\\braceright ";
-                       else if (c != '\0')
+                               return;
+                       case '$':
+                               os << "\\lyxdollar ";
+                               return;
+                       case '~':
+                               os << "\\lyxtilde ";
+                               return;
+                       case ' ':
+                       case '\0':
+                               break;
+                       default:
                                os.put(c);
+                               return;
+                       }
                }
                else if (c != '\0') {
                        Encoding const * const enc = runparams.encoding;
                        if (enc && !enc->encodable(c))
                                throw EncodingException(c);
                        os.put(c);
+                       return;
                }
-               return;
        }
 
        // TIPA uses its own T3 encoding
@@ -1535,9 +1581,9 @@ void Paragraph::Private::validate(LaTeXFeatures & features) const
                        else if (features.runparams().main_fontenc != "T1"
                                 || ((&owner_->getFontSettings(bp, i))->language()->internalFontEncoding()))
                                features.require("textquotedbl");
-               } else if (ci.textfeature() && contains(ci.textpreamble(), '=')) {
+               } else if (ci.textFeature() && contains(ci.textPreamble(), '=')) {
                        // features that depend on the font or input encoding
-                       string feats = ci.textpreamble();
+                       string feats = ci.textPreamble();
                        string fontenc = (&owner_->getFontSettings(bp, i))->language()->fontenc(bp);
                        if (fontenc.empty())
                                fontenc = features.runparams().main_fontenc;
@@ -1731,16 +1777,34 @@ void Paragraph::write(ostream & os, BufferParams const & bparams,
                        column = 0;
                        break;
                case '.':
+               case '!':
+               case '?':
+               case ':':
+               case ';':
+               case ',':
+               case 0x061F:  // ؟ U+061F  ARABIC QUESTION MARK
+               case 0x061B:  // ؛ U+061B  ARABIC SEMICOLON
+               case 0x060C:  // ، U+060C  ARABIC COMMA
                        flushString(os, write_buffer);
                        if (i + 1 < size() && d->text_[i + 1] == ' ') {
-                               os << ".\n";
+                               os << to_utf8(docstring(1, c)) << '\n';
                                column = 0;
                        } else
-                               os << '.';
+                               os << to_utf8(docstring(1, c));
+                       break;
+               case 0x2014:  // — U+2014  EM DASH
+               case 0x3002:  // 。 U+3002  IDEOGRAPHIC FULL STOP
+               case 0xFF01:  // ! U+FF01  FULLWIDTH EXCLAMATION MARK
+               case 0xFF1F:  // ? U+FF1F  FULLWIDTH QUESTION MARK
+               case 0xFF1A:  // : U+FF1A  FULLWIDTH COLON
+               case 0xFF1B:  // ; U+FF1B  FULLWIDTH SEMICOLON
+               case 0xFF0C:  // , U+FF0C  FULLWIDTH COMMA
+                       flushString(os, write_buffer);
+                       os << to_utf8(docstring(1, c)) << '\n';
+                       column = 0;
                        break;
                default:
-                       if ((column > 70 && c == ' ')
-                           || column > 79) {
+                       if (column > 500) {
                                flushString(os, write_buffer);
                                os << '\n';
                                column = 0;
@@ -1912,7 +1976,8 @@ FontSpan Paragraph::fontSpan(pos_type pos) const
 
        // This should not happen, but if so, we take no chances.
        LYXERR0("Paragraph::fontSpan: position not found in fontinfo table!");
-       LASSERT(false, return FontSpan(pos, pos));
+       LASSERT(false, /**/);
+       return FontSpan(pos, pos);
 }
 
 
@@ -1936,7 +2001,7 @@ Font const & Paragraph::getFirstFontSettings(BufferParams const & bparams) const
 
 
 // Gets the fully instantiated font at a given position in a paragraph
-// This is basically the same function as Text::GetFont() in text2.cpp.
+// This is basically the same function as TextMetrics::displayFont().
 // The difference is that this one is used for generating the LaTeX file,
 // and thus cosmetic "improvements" are disallowed: This has to deliver
 // the true picture of the buffer. (Asger)
@@ -1987,13 +2052,16 @@ char_type Paragraph::getUChar(BufferParams const & bparams,
 {
        char_type c = d->text_[pos];
 
-       // Return unchanged character in LTR languages
-       // or if we use poylglossia/bidi (XeTeX).
-       if (rp.useBidiPackage()
-           || !getFontSettings(bparams, pos).isRightToLeft())
+       // Return unchanged character
+       // 1. in all LTR languages
+       // 2. if we use XeTeX (both with babel and polyglossia)
+       // 3. if we use LuaTeX with babel
+       if (!getFontSettings(bparams, pos).isRightToLeft()
+           || rp.flavor == Flavor::XeTeX
+           || (rp.use_babel && rp.flavor == Flavor::LuaTeX))
                return c;
 
-       // Without polyglossia/bidi, we need to account for some special cases.
+       // For the remaining cases, we need to account for some special cases.
        // FIXME This needs to be audited!
        // Check if:
        // * The input is as expected for all delimiters
@@ -2004,18 +2072,27 @@ char_type Paragraph::getUChar(BufferParams const & bparams,
        //   => checked for Hebrew!
        // * In arabic_arabi, brackets are transformed to Arabic
        //   Ornate Parentheses. Is this is really wanted?
+       //   => Yes, in file ararabeyes.enc from the arabi bundle
+       //      the slot of the left bracket (slot 91) is encoded as
+       //      "ornaterightparenthesis". This is also the reason
+       //      brackets don't need to be mirrored with arabi
 
        string const & lang = getFontSettings(bparams, pos).language()->lang();
        char_type uc = c;
 
-       // 1. In the following languages, parentheses need to be reversed.
-       //    Also with polyglodia/luabidi
-       bool const reverseparens = (lang == "hebrew" || rp.use_polyglossia);
-
-       // 2. In the following languages, brackets don't need to be reversed.
-       bool const reversebrackets = lang != "arabic_arabtex"
-                       && lang != "arabic_arabi"
-                       && lang != "farsi";
+       // These are the cases where we need to mirror delimiters in RTL context
+       // in the remaining cases (polyglossia + LuaTeX or classic [pdf]latex):
+       // 1. With polyglossia and LuaTeX (luabidi) parentheses and brackets
+       //    need to be mirrored in RTL, regardless of the language, or script.
+       // 2. In the languages that follow, parentheses need to be mirrored
+       //    in classic (pdf)latex
+       bool const reverseparens = (rp.use_polyglossia || lang == "hebrew");
+       // 3. In all RTL languages except for those that follow, brackets
+       //    need to be mirrored in classic (pdf)latex
+       bool const reversebrackets = rp.use_polyglossia
+                       || (lang != "arabic_arabtex"
+                           && lang != "arabic_arabi"
+                           && lang != "farsi");
 
        // Now swap delimiters if needed.
        switch (c) {
@@ -2057,10 +2134,8 @@ void Paragraph::setFont(pos_type pos, Font const & font)
 {
        LASSERT(pos <= size(), return);
 
-       // First, reduce font against layout/label font
-       // Update: The setCharFont() routine in text2.cpp already
-       // reduces font, so we don't need to do that here. (Asger)
-
+       // Text::setCharFont() already reduces font against layout/label
+       // font, so we don't need to do that here. (Asger)
        d->fontlist_.set(pos, font);
 }
 
@@ -2269,6 +2344,90 @@ bool Paragraph::isPassThru() const
 }
 
 
+bool Paragraph::parbreakIsNewline() const
+{
+       return inInset().getLayout().parbreakIsNewline() || d->layout_->parbreak_is_newline;
+}
+
+
+bool Paragraph::allowedInContext(Cursor const & cur, InsetLayout const & il) const
+{
+       set<docstring> const & allowed_insets = il.allowedInInsets();
+       set<docstring> const & allowed_layouts = il.allowedInLayouts();
+
+       bool in_allowed_inset =
+               allowed_insets.find(inInset().getLayout().name()) != allowed_insets.end();
+
+       bool in_allowed_layout =
+               allowed_layouts.find(d->layout_->name()) != allowed_layouts.end();
+
+       if (!in_allowed_inset && inInset().asInsetArgument()) {
+               // check if the argument allows the inset in question
+               if (cur.depth() > 1) {
+                       docstring parlayout = cur[cur.depth() - 2].inset().getLayout().name()
+                                       + from_ascii("@") + from_ascii(inInset().asInsetArgument()->name());
+                       if (allowed_insets.find(parlayout) != allowed_insets.end())
+                               in_allowed_inset = true;
+               }
+       }
+       
+       int have_ins = 0;
+       // check if we exceed the number of allowed insets in this inset
+       if (in_allowed_inset && inInset().asInsetText() && il.allowedOccurrences() != -1) {
+               ParagraphList & pars = cur.text()->paragraphs();
+                       for (Paragraph const & par : pars) {
+                               for (auto const & elem : par.insetList())
+                               if (elem.inset->getLayout().name() == il.name())
+                                       ++have_ins;
+                       }
+               if (have_ins >= il.allowedOccurrences())
+                       return false;
+       }
+       
+       have_ins = 0;
+       // check if we exceed the number of allowed insets in the layout group
+       if (in_allowed_layout && il.allowedOccurrences() != -1) {
+               pit_type pit = cur.pit();
+               pit_type lastpit = cur.pit();
+               ParagraphList & pars = cur.text()->paragraphs();
+               // If we are not on a list-type environment or AllowedOccurrencesPerItem
+               // is false, we check the whole paragraph group
+               if (d->layout_->isEnvironment()
+                   && !(il.allowedOccurrencesPerItem()
+                        && (d->layout_->latextype == LATEX_LIST_ENVIRONMENT
+                            || d->layout_->latextype == LATEX_ITEM_ENVIRONMENT))) {
+                       lastpit = cur.lastpit();
+                       // get the first paragraph in sequence with this layout
+                       depth_type const current_depth = params().depth();
+                       while (true) {
+                               if (pit == 0)
+                                       break;
+                               Paragraph cpar = pars[pit - 1];
+                               if (&cpar.layout() == d->layout_
+                                   && cpar.params().depth() == current_depth)
+                                       --pit;
+                               else
+                                       break;
+                       }
+               }
+               for (; pit <= lastpit; ++pit) {
+                       if (&pars[pit].layout() != d->layout_)
+                               break;
+                       for (auto const & elem : pars[pit].insetList())
+                               if (elem.inset->getLayout().name() == il.name())
+                                       ++have_ins;
+               }
+               if (have_ins >= il.allowedOccurrences())
+                       return false;
+       }
+       
+       if (in_allowed_layout || in_allowed_inset)
+               return true;
+
+       return (allowed_insets.empty() && allowed_layouts.empty());
+}
+
+
 bool Paragraph::isPartOfTextSequence() const
 {
        for (pos_type i = 0; i < size(); ++i) {
@@ -2361,8 +2520,8 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
            && !layout_->pass_thru
            && curAlign != LYX_ALIGN_CENTER) {
                if (!owner_->empty()
-                   && (owner_->isInset(0)
-                       && owner_->getInset(0)->lyxCode() == VSPACE_CODE))
+                   && owner_->getInset(0)
+                   && owner_->getInset(0)->lyxCode() == VSPACE_CODE)
                        // If the paragraph starts with a vspace, the \\noindent
                        // needs to come after that (as it leaves vmode).
                        // If the paragraph consists only of the vspace,
@@ -2400,7 +2559,7 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
        // RTL in classic (PDF)LaTeX (without the Bidi package)
        // Luabibdi (used by LuaTeX) behaves like classic
        bool const rtl_classic = owner_->getParLanguage(bparams)->rightToLeft()
-               && !runparams.useBidiPackage();
+               && !bparams.useBidiPackage(runparams);
 
        switch (curAlign) {
        case LYX_ALIGN_NONE:
@@ -2464,7 +2623,7 @@ bool Paragraph::Private::endTeXParParams(BufferParams const & bparams,
        // RTL in classic (PDF)LaTeX (without the Bidi package)
        // Luabibdi (used by LuaTeX) behaves like classic
        bool const rtl_classic = owner_->getParLanguage(bparams)->rightToLeft()
-               && !runparams.useBidiPackage();
+               && !bparams.useBidiPackage(runparams);
 
        switch (curAlign) {
        case LYX_ALIGN_NONE:
@@ -2504,7 +2663,7 @@ void Paragraph::latex(BufferParams const & bparams,
        OutputParams const & runparams,
        int start_pos, int end_pos, bool force) const
 {
-       LYXERR(Debug::LATEX, "Paragraph::latex...     " << this);
+       LYXERR(Debug::OUTFILE, "Paragraph::latex...     " << this);
 
        // FIXME This check should not be needed. Perhaps issue an
        // error if it triggers.
@@ -2533,10 +2692,10 @@ void Paragraph::latex(BufferParams const & bparams,
        pos_type body_pos = beginOfBody();
        unsigned int column = 0;
 
-       // If we are inside an non inheritFont() inset, the real outerfont is local_font
-       Font const real_outerfont = (!inInset().inheritFont()
-                                    && runparams.local_font != nullptr)
-                       ? Font(runparams.local_font->fontInfo()) : outerfont;
+       // If we are inside an non inheritFont() inset,
+       // the outerfont is the buffer's main font
+       Font const real_outerfont =
+               inInset().inheritFont() ? outerfont : Font(bparams.getFont());
 
        if (body_pos > 0) {
                // the optional argument is kept in curly brackets in
@@ -2604,7 +2763,7 @@ void Paragraph::latex(BufferParams const & bparams,
                                                runparams);
                                runningChange = Change(Change::UNCHANGED);
 
-                               os << (isEnvSeparator(i) ? "}]~" : "}] ");
+                               os << ((isEnvSeparator(i) && !runparams.find_effective()) ? "}]~" : "}] ");
                                column +=3;
                        }
                        // For InTitle commands, we have already opened a group
@@ -2634,10 +2793,10 @@ void Paragraph::latex(BufferParams const & bparams,
 
                // Check whether a display math inset follows
                bool output_changes;
-               if (runparams.for_searchAdv == OutputParams::NoSearch)
+               if (!runparams.find_effective())
                        output_changes = bparams.output_changes;
                else
-                       output_changes = (runparams.for_searchAdv == OutputParams::SearchWithDeleted);
+                       output_changes = runparams.find_with_deleted();
                if (c == META_INSET
                    && i >= start_pos && (end_pos == -1 || i < end_pos)) {
                        if (isDeleted(i))
@@ -2767,9 +2926,11 @@ void Paragraph::latex(BufferParams const & bparams,
                                os << '}';
                                column += 1;
                        }
-                       if (closeLanguage)
+                       if (closeLanguage) {
                                // Force language closing
                                current_font.setLanguage(basefont.language());
+                               langClosed = true;
+                       }
                        Font const nextfont = (i == body_pos-1) ? basefont : current_font;
                        bool needPar = false;
                        column += running_font.latexWriteEndChanges(
@@ -2782,12 +2943,12 @@ void Paragraph::latex(BufferParams const & bparams,
                                column += Changes::latexMarkChange(os, bparams,
                                        Change(Change::UNCHANGED), Change(Change::DELETED), rp);
                        }
-                       open_font = false;
                        // Has the language been closed in the latexWriteEndChanges() call above?
-                       langClosed = running_font.language() != basefont.language()
+                       langClosed |= running_font.language() != basefont.language()
                                        && running_font.language() != nextfont.language()
                                        && (running_font.language()->encoding()->package() != Encoding::CJK);
                        running_font = basefont;
+                       open_font &= !langClosed;
                }
 
                // if necessary, close language environment before opening CJK
@@ -2800,7 +2961,8 @@ void Paragraph::latex(BufferParams const & bparams,
                        string end_tag = subst(lang_end_command, "$$lang", running_lang);
                        os << from_ascii(end_tag);
                        column += end_tag.length();
-                       popLanguageName();
+                       if (!languageStackEmpty())
+                               popLanguageName();
                }
 
                // Switch file encoding if necessary (and allowed)
@@ -2853,6 +3015,7 @@ void Paragraph::latex(BufferParams const & bparams,
                                        ? textinset->hasCProtectContent(runparams.moving_arg)
                                          && !textinset->text().isMainText()
                                          && inInset().lyxCode() != BRANCH_CODE
+                                         && !runparams.no_cprotect
                                        : false;
                                column += current_font.latexWriteStartChanges(ots, bparams,
                                                                              runparams, basefont, last_font, false,
@@ -2886,9 +3049,14 @@ void Paragraph::latex(BufferParams const & bparams,
                                        column += Changes::latexMarkChange(os, bparams,
                                                Change(Change::UNCHANGED), change, rp);
                                }
-                       } else {
+                       } else {// if fontswitch_inset
+                               if (current_font != running_font || !langClosed)
+                                       // font is still open in fontswitch_insets if we have
+                                       // a non-lang font difference or if the language
+                                       // is the only difference but has not been forcedly
+                                       // closed meanwhile
+                                       open_font = true;
                                running_font = current_font;
-                               open_font = !langClosed;
                        }
                }
 
@@ -2957,7 +3125,7 @@ void Paragraph::latex(BufferParams const & bparams,
                                d->latexInset(bparams, os, rp, running_font,
                                                basefont, real_outerfont, open_font,
                                                runningChange, style, i, column, fontswitch_inset,
-                                               closeLanguage, lang_switched_at_inset);
+                                               closeLanguage, (lang_switched_at_inset || langClosed));
                                if (fontswitch_inset) {
                                        if (open_font) {
                                                bool needPar = false;
@@ -2984,8 +3152,9 @@ void Paragraph::latex(BufferParams const & bparams,
                                }
                        }
                } else if (i >= start_pos && (end_pos == -1 || i < end_pos)) {
-                       if (!bparams.useNonTeXFonts)
-                         script = Encodings::isKnownScriptChar(c);
+                       if (!bparams.useNonTeXFonts && !runparams.pass_thru
+                           && !contains(runparams.pass_thru_chars, c))
+                               script = Encodings::isKnownScriptChar(c);
                        if (script != alien_script) {
                                if (!alien_script.empty()) {
                                        os << "}";
@@ -3120,7 +3289,7 @@ void Paragraph::latex(BufferParams const & bparams,
                os << setEncoding(prev_encoding->iconvName());
        }
 
-       LYXERR(Debug::LATEX, "Paragraph::latex... done " << this);
+       LYXERR(Debug::OUTFILE, "Paragraph::latex... done " << this);
 }
 
 
@@ -3469,15 +3638,18 @@ std::tuple<vector<xml::FontTag>, vector<xml::EndFontTag>> computeDocBookFontSwit
 
 std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring>>
     Paragraph::simpleDocBookOnePar(Buffer const & buf,
-                                                      OutputParams const & runparams,
-                                                      Font const & outerfont,
-                                                      pos_type initial,
-                                                      bool is_last_par,
-                                                      bool ignore_fonts) const
-{
-       std::vector<docstring> prependedParagraphs;
-       std::vector<docstring> generatedParagraphs;
-       std::vector<docstring> appendedParagraphs;
+                                   OutputParams const & runparams,
+                                   Font const & outerfont,
+                                   pos_type initial,
+                                   bool is_last_par,
+                                   bool ignore_fonts) const
+{
+       // Return values: segregation of the content of this paragraph.
+       std::vector<docstring> prependedParagraphs; // Anything that must be output before the main tag of this paragraph.
+       std::vector<docstring> generatedParagraphs; // The main content of the paragraph.
+       std::vector<docstring> appendedParagraphs;  // Anything that must be output after the main tag of this paragraph.
+
+       // Internal string stream to store the output before being added to one of the previous lists.
        odocstringstream os;
 
        // If there is an argument that must be output before the main tag, do it before handling the rest of the paragraph.
@@ -3500,20 +3672,22 @@ std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring
             }
         }
     }
+       rp.lastid = id();
 
     // State variables for the main loop.
     auto xs = new XMLStream(os); // XMLStream has no copy constructor: to create a new object, the only solution
     // is to hold a pointer to the XMLStream (xs = XMLStream(os) is not allowed once the first object is built).
-    std::vector<char_type> delayedChars; // When a font tag ends with a space, output it after the closing font tag.
+       xs->startDivision(false);
+    std::vector<docstring> delayedChars; // When a font tag ends with a space, output it after the closing font tag.
     // This requires to store delayed characters at some point.
 
-    DocBookFontState fs; // Track whether we have opened font tags
+       // Track whether we have opened font tags
+    DocBookFontState fs;
     DocBookFontState old_fs = fs;
 
     Layout const & style = *d->layout_;
-    FontInfo font_old = style.labeltype == LABEL_MANUAL ? style.labelfont : style.font;
-    string const default_family = buf.masterBuffer()->params().fonts_default_family;
 
+       // Conversion of the font opening/closing into DocBook tags.
     vector<xml::FontTag> tagsToOpen;
     vector<xml::EndFontTag> tagsToClose;
 
@@ -3527,8 +3701,8 @@ std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring
                if (isDeleted(i))
                        continue;
 
-               // If this is an InsetNewline, generate a new paragraph. Also reset the fonts, so that tags are closed in
-               // this paragraph.
+               // If this is an InsetNewline, generate a new paragraph (this is the reason why generatedParagraphs is a list
+               // of paragraphs). Also reset the fonts, so that tags are closed in this paragraph.
                if (getInset(i) && getInset(i)->lyxCode() == NEWLINE_CODE) {
                        if (!ignore_fonts_i)
                                xs->closeFontTags();
@@ -3536,22 +3710,34 @@ std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring
                        // Output one paragraph (i.e. one string entry in generatedParagraphs).
                        generatedParagraphs.push_back(os.str());
 
-                       // Create a new XMLStream for the new paragraph, completely independent from the previous one. This implies
+                       xs->endDivision();
+
+                       // Create a new XMLStream for the new paragraph, completely independent of the previous one. This implies
                        // that the string stream must be reset.
                        os.str(from_ascii(""));
                        delete xs;
                        xs = new XMLStream(os);
+                       xs->startDivision(false);
 
                        // Restore the fonts for the new paragraph, so that the right tags are opened for the new entry.
                        if (!ignore_fonts_i) {
-                               font_old = outerfont.fontInfo();
                                fs = old_fs;
                        }
                }
 
-               // Determine which tags should be opened or closed regarding fonts.
+               // Determine which tags should be opened or closed regarding fonts. Consider the last output character (i.e. not
+               // deleted).
+               int last_output_char = (i == 0) ? 0 : i - 1;
+               if (i > 0) {
+                       while (last_output_char > 0 && isDeleted(last_output_char))
+                               --last_output_char;
+               }
+               FontInfo const font_old = (i == 0 ?
+                               (style.labeltype == LABEL_MANUAL ? style.labelfont : style.font) :
+                               getFont(buf.masterBuffer()->params(), last_output_char, outerfont).fontInfo());
                Font const font = getFont(buf.masterBuffer()->params(), i, outerfont);
-        tie(tagsToOpen, tagsToClose) = computeDocBookFontSwitch(font_old, font, default_family, fs);
+        tie(tagsToOpen, tagsToClose) = computeDocBookFontSwitch(
+                               font_old, font, buf.masterBuffer()->params().fonts_default_family, fs);
 
                if (!ignore_fonts_i) {
             vector<xml::EndFontTag>::const_iterator cit = tagsToClose.begin();
@@ -3562,8 +3748,8 @@ std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring
 
         // Deal with the delayed characters *after* closing font tags.
         if (!delayedChars.empty()) {
-            for (char_type c: delayedChars)
-                *xs << c;
+            for (const docstring& c: delayedChars)
+                *xs << XMLStream::ESCAPE_NONE << c;
             delayedChars.clear();
         }
 
@@ -3572,11 +3758,12 @@ std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring
                        vector<xml::FontTag>::const_iterator sen = tagsToOpen.end();
                        for (; sit != sen; ++sit)
                                *xs << *sit;
-
-                       tagsToClose.clear();
-                       tagsToOpen.clear();
                }
 
+               // The font tags are no longer useful; free their memory right now.
+               tagsToClose.clear();
+               tagsToOpen.clear();
+
         // Finally, write the next character or inset.
                if (Inset const * inset = getInset(i)) {
                    bool inset_is_argument_elsewhere = getInset(i)->asInsetArgument() &&
@@ -3594,24 +3781,42 @@ std::tuple<std::vector<docstring>, std::vector<docstring>, std::vector<docstring
                        }
                } else {
                        char_type c = getUChar(buf.masterBuffer()->params(), rp, i);
-                       if (lyx::isSpace(c) && !ignore_fonts)
-                               delayedChars.push_back(c);
-                       else
-                               *xs << c;
+                       if (lyx::isSpace(c) && !ignore_fonts) { // Delay spaces *after* the font-tag closure for cleaner output.
+                               if (c == ' ' && (style.free_spacing || rp.free_spacing)) {
+                                       delayedChars.push_back(from_ascii("&#160;"));
+                               } else {
+                                       delayedChars.emplace_back(1, c);
+                               }
+                       } else { // No need to delay the character.
+                               if (c == '\'' && !ignore_fonts)
+                                       *xs << XMLStream::ESCAPE_NONE << "&#8217;";
+                               else
+                                       *xs << c;
+                       }
                }
-               font_old = font.fontInfo();
        }
 
+       // Ensure that the tags are closed at the right place. Otherwise, there might be an open font tag with no content
+       // that no other code cares to close.
+       *xs << xml::NullTag();
+
        // FIXME, this code is just imported from XHTML
        // I'm worried about what happens if a branch, say, is itself
        // wrapped in some font stuff. I think that will not work.
        if (!ignore_fonts)
                xs->closeFontTags();
 
+       // Close the potentially remaining tags, like pending font tags.
+       // There is no need to check for ignore_fonts, as these tags won't be
+       // inserted in the stack in the first place if ignore_fonts is false.
+       xs->endDivision();
+
        // Deal with the delayed characters *after* closing font tags.
-       if (!delayedChars.empty())
-               for (char_type c: delayedChars)
-                       *xs << c;
+       if (!delayedChars.empty()) {
+               for (const docstring &c: delayedChars)
+                       *xs << XMLStream::ESCAPE_NONE << c;
+               delayedChars.clear();
+       }
 
        // In listings, new lines (i.e. \n characters in the output) are very important. Avoid generating one for the
        // last line to get a clean output.
@@ -3961,16 +4166,10 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
                // FIXME XHTML
                // Other such tags? What about the other text ranges?
 
-               vector<xml::EndFontTag>::const_iterator cit = tagsToClose.begin();
-               vector<xml::EndFontTag>::const_iterator cen = tagsToClose.end();
-               for (; cit != cen; ++cit)
-                       xs << *cit;
-
-               vector<xml::FontTag>::const_iterator sit = tagsToOpen.begin();
-               vector<xml::FontTag>::const_iterator sen = tagsToOpen.end();
-               for (; sit != sen; ++sit)
-                       xs << *sit;
-
+               for (auto const & t : tagsToClose)
+                       xs << t;
+               for (auto const & t : tagsToOpen)
+                       xs << t;
                tagsToClose.clear();
                tagsToOpen.clear();
 
@@ -3989,7 +4188,9 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
                        char_type c = getUChar(buf.masterBuffer()->params(),
                                               runparams, i);
                        if (c == ' ' && (style.free_spacing || runparams.free_spacing))
-                               xs << XMLStream::ESCAPE_NONE << "&nbsp;";
+                               xs << XMLStream::ESCAPE_NONE << "&#160;";
+                       else if (c == '\'')
+                               xs << XMLStream::ESCAPE_NONE << "&#8217;";
                        else
                                xs << c;
                }
@@ -4085,9 +4286,10 @@ bool Paragraph::isHardHyphenOrApostrophe(pos_type pos) const
 bool Paragraph::needsCProtection(bool const fragile) const
 {
        // first check the layout of the paragraph, but only in insets
+       // and not in tables
        InsetText const * textinset = inInset().asInsetText();
        bool const maintext = textinset
-               ? textinset->text().isMainText()
+               ? textinset->text().isMainText() || inInset().lyxCode() == CELL_CODE
                : false;
 
        if (!maintext && layout().needcprotect) {
@@ -4114,20 +4316,6 @@ bool Paragraph::needsCProtection(bool const fragile) const
                        continue;
                if (ins->needsCProtection(maintext, fragile))
                        return true;
-               // Now check math environments
-               InsetMath const * im = ins->asInsetMath();
-               if (!im || im->cell(0).empty())
-                       continue;
-               switch(im->cell(0)[0]->lyxCode()) {
-               case MATH_AMSARRAY_CODE:
-               case MATH_SUBSTACK_CODE:
-               case MATH_ENV_CODE:
-               case MATH_XYMATRIX_CODE:
-                       // these need cprotection
-                       return true;
-               default:
-                       break;
-               }
        }
 
        return false;
@@ -4225,6 +4413,7 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options, const Out
 
        if (beg == 0
            && options & AS_STR_LABEL
+           && d->layout_->labeltype != LABEL_MANUAL
            && !d->params_.labelString().empty())
                os << d->params_.labelString() << ' ';
 
@@ -4238,7 +4427,10 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options, const Out
                else if (c == META_INSET && (options & AS_STR_INSETS)) {
                        if (c == META_INSET && (options & AS_STR_PLAINTEXT)) {
                                LASSERT(runparams != nullptr, return docstring());
-                               getInset(i)->plaintext(os, *runparams);
+                               if (runparams->find_effective() && getInset(i)->findUsesToString())
+                                       getInset(i)->toString(os);
+                               else
+                                       getInset(i)->plaintext(os, *runparams);
                        } else if (c == META_INSET && (options & AS_STR_MATHED)
                                   && getInset(i)->lyxCode() == REF_CODE) {
                                Buffer const & buf = getInset(i)->buffer();
@@ -4365,6 +4557,27 @@ bool Paragraph::allowEmpty() const
 }
 
 
+int Paragraph::getInsetPos(InsetCode const code, int startpos,
+                          bool ignore_deleted) const
+{
+       while (startpos != -1) {
+               int found_pos = d->insetlist_.find(code, startpos);
+               if (found_pos == -1)
+                       // nothing found
+                       return -1;
+               if (isDeleted(found_pos) && ignore_deleted) {
+                       // we're not interested in deleted insets
+                       if (found_pos + 1 == size())
+                               return -1;
+                       startpos = found_pos + 1;
+                       continue;
+               } else
+                       return found_pos;
+       }
+       return -1;
+}
+
+
 bool Paragraph::brokenBiblio() const
 {
        // There is a problem if there is no bibitem at position 0 in
@@ -4372,10 +4585,10 @@ bool Paragraph::brokenBiblio() const
        // paragraph or if this paragraph is not supposed to have
        // a bibitem inset at all.
        return ((d->layout_->labeltype == LABEL_BIBLIO
-               && (d->insetlist_.find(BIBITEM_CODE) != 0
-                   || d->insetlist_.find(BIBITEM_CODE, 1) > 0))
+               && (getInsetPos(BIBITEM_CODE, 0, true) != 0
+                   || getInsetPos(BIBITEM_CODE, 1, true) > 0))
                || (d->layout_->labeltype != LABEL_BIBLIO
-                   && d->insetlist_.find(BIBITEM_CODE) != -1));
+                   && getInsetPos(BIBITEM_CODE, 0, true) != -1));
 }
 
 
@@ -4386,7 +4599,7 @@ int Paragraph::fixBiblio(Buffer const & buffer)
        // cursor cannot be correctly updated.
 
        bool const track_changes = buffer.params().track_changes;
-       int bibitem_pos = d->insetlist_.find(BIBITEM_CODE);
+       int bibitem_pos = getInsetPos(BIBITEM_CODE, 0, true);
 
        // The case where paragraph is not BIBLIO
        if (d->layout_->labeltype != LABEL_BIBLIO) {
@@ -4401,7 +4614,7 @@ int Paragraph::fixBiblio(Buffer const & buffer)
 
        bool const hasbibitem0 = bibitem_pos == 0;
        if (hasbibitem0) {
-               bibitem_pos = d->insetlist_.find(BIBITEM_CODE, 1);
+               bibitem_pos = getInsetPos(BIBITEM_CODE, 0, true);
                // There was an InsetBibitem at pos 0,
                // and no other one => OK
                if (bibitem_pos == -1)
@@ -4422,11 +4635,19 @@ int Paragraph::fixBiblio(Buffer const & buffer)
        // We need to create an inset at the beginning
        Inset * inset = nullptr;
        if (bibitem_pos > 0) {
-               // there was one somewhere in the paragraph, let's move it
-               inset = d->insetlist_.release(bibitem_pos);
+               // There was one somewhere in the paragraph, let's move it
+               // * With change tracking, we use a clone
+               //   and leave the old inset at its position
+               //   (marked deleted)
+               // * Without change tracking, we release the inset
+               //   from its previous InsetList position
+               inset = track_changes
+                               ? new InsetBibitem(const_cast<Buffer *>(&buffer),
+                                                  getInset(bibitem_pos)->asInsetCommand()->params())
+                               : d->insetlist_.release(bibitem_pos);
                eraseChar(bibitem_pos, track_changes);
        } else
-               // make a fresh one
+               // No inset found -- make a fresh one
                inset = new InsetBibitem(const_cast<Buffer *>(&buffer),
                                         InsetCommandParams(BIBITEM_CODE));
 
@@ -4567,7 +4788,6 @@ void Paragraph::changeCase(BufferParams const & bparams, pos_type pos,
        }
 }
 
-
 int Paragraph::find(docstring const & str, bool cs, bool mw,
                pos_type start_pos, bool del) const
 {
@@ -4592,7 +4812,14 @@ int Paragraph::find(docstring const & str, bool cs, bool mw,
                        if (!inset->isLetter() && !inset->isChar())
                                break;
                        odocstringstream os;
-                       inset->toString(os);
+                       if (inset->lyxCode() == lyx::QUOTE_CODE || inset->lyxCode() == lyx::SPACE_CODE) {
+                               OutputParams op(0);
+                               op.find_set_feature(OutputParams::SearchQuick);
+                               inset->plaintext(os, op);
+                       }
+                       else {
+                               inset->toString(os);
+                       }
                        docstring const insetstring = os.str();
                        if (!insetstring.empty()) {
                                int const insetstringsize = insetstring.length();
@@ -4608,9 +4835,10 @@ int Paragraph::find(docstring const & str, bool cs, bool mw,
                }
                if (nonmatch || i == strsize)
                        break;
-               if (cs && str[i] != d->text_[pos])
+               char_type dp = d->text_[pos];
+               if (cs && str[i] != dp)
                        break;
-               if (!cs && uppercase(str[i]) != uppercase(d->text_[pos]))
+               if (!cs && uppercase(str[i]) != uppercase(dp))
                        break;
        }