]> git.lyx.org Git - lyx.git/blobdiff - src/Buffer.cpp
Fix bookmarks-goto inside insets.
[lyx.git] / src / Buffer.cpp
index 8504c9e4e5f67116b7bcfbd5f4ff09dd8bcd1b0d..fdce444aefa313e43a41cf6ce1766ae0f72b9819 100644 (file)
@@ -14,7 +14,6 @@
 #include "Buffer.h"
 
 #include "Author.h"
-#include "LayoutFile.h"
 #include "BiblioInfo.h"
 #include "BranchList.h"
 #include "buffer_funcs.h"
@@ -22,6 +21,7 @@
 #include "BufferParams.h"
 #include "Bullet.h"
 #include "Chktex.h"
+#include "ColorSet.h"
 #include "Converter.h"
 #include "Counters.h"
 #include "Cursor.h"
 #include "LyX.h"
 #include "LyXRC.h"
 #include "LyXVC.h"
-#include "output_docbook.h"
 #include "output.h"
 #include "output_latex.h"
-#include "output_xhtml.h"
+#include "output_docbook.h"
 #include "output_plaintext.h"
+#include "output_xhtml.h"
 #include "Paragraph.h"
 #include "ParagraphParameters.h"
 #include "ParIterator.h"
 #include "PDFOptions.h"
 #include "Session.h"
 #include "SpellChecker.h"
-#include "sgml.h"
+#include "xml.h"
 #include "texstream.h"
 #include "TexRow.h"
 #include "Text.h"
 #include "VCBackend.h"
 #include "version.h"
 #include "WordLangTuple.h"
-#include "WordList.h"
 
-#include "insets/InsetBibtex.h"
 #include "insets/InsetBranch.h"
 #include "insets/InsetInclude.h"
-#include "insets/InsetTabular.h"
 #include "insets/InsetText.h"
 
 #include "mathed/InsetMathHull.h"
@@ -80,7 +77,6 @@
 #include "mathed/InsetMathMacroTemplate.h"
 #include "mathed/MathSupport.h"
 
-#include "graphics/GraphicsCache.h"
 #include "graphics/PreviewLoader.h"
 
 #include "frontends/Application.h"
 #include "support/FileName.h"
 #include "support/FileNameList.h"
 #include "support/filetools.h"
-#include "support/ForkedCalls.h"
 #include "support/gettext.h"
 #include "support/gzstream.h"
 #include "support/lstrings.h"
-#include "support/lyxalgo.h"
 #include "support/mutex.h"
 #include "support/os.h"
 #include "support/Package.h"
 #include "support/textutils.h"
 #include "support/types.h"
 
-#include "support/bind.h"
-
 #include <algorithm>
 #include <fstream>
 #include <iomanip>
@@ -297,6 +289,8 @@ public:
        ///
        CloneList_ptr clone_list_;
 
+       ///
+       std::list<Buffer const *> include_list_;
 private:
        /// So we can force access via the accessors.
        mutable Buffer const * parent_buffer;
@@ -360,6 +354,9 @@ public:
        /// whether the bibinfo cache is valid
        mutable bool bibinfo_cache_valid_;
 
+       ///
+       mutable bool need_update;
+
 private:
        int word_count_;
        int char_count_;
@@ -466,7 +463,7 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
          internal_buffer(false), read_only(readonly_), file_fully_loaded(false),
          need_format_backup(false), ignore_parent(false), macro_lock(false),
          externally_modified_(false), bibinfo_cache_valid_(false),
-         word_count_(0), char_count_(0), blank_count_(0)
+         need_update(false), word_count_(0), char_count_(0), blank_count_(0)
 {
        refreshFileMonitor();
        if (!cloned_buffer_) {
@@ -630,10 +627,9 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList_ptr clones) const
        // The clone needs its own DocumentClass, since running updateBuffer() will
        // modify it, and we would otherwise be sharing it with the original Buffer.
        buffer_clone->params().makeDocumentClass(true);
-       ErrorList el;
        cap::switchBetweenClasses(
                        params().documentClassPtr(), buffer_clone->params().documentClassPtr(),
-                       static_cast<InsetText &>(buffer_clone->inset()), el);
+                       static_cast<InsetText &>(buffer_clone->inset()));
 
        bufmap[this] = buffer_clone;
        clones->insert(buffer_clone);
@@ -664,7 +660,6 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList_ptr clones) const
                buffer_clone->setChild(dit, child_clone);
        }
        buffer_clone->d->macro_lock = false;
-       return;
 }
 
 
@@ -676,10 +671,9 @@ Buffer * Buffer::cloneBufferOnly() const {
        // The clone needs its own DocumentClass, since running updateBuffer() will
        // modify it, and we would otherwise be sharing it with the original Buffer.
        buffer_clone->params().makeDocumentClass(true);
-       ErrorList el;
        cap::switchBetweenClasses(
                        params().documentClassPtr(), buffer_clone->params().documentClassPtr(),
-                       static_cast<InsetText &>(buffer_clone->inset()), el);
+                       static_cast<InsetText &>(buffer_clone->inset()));
 
        clones->insert(buffer_clone);
        buffer_clone->d->clone_list_ = clones;
@@ -953,7 +947,9 @@ int Buffer::readHeader(Lexer & lex)
        params().fontcolor = RGBColor(0, 0, 0);
        params().isfontcolor = false;
        params().notefontcolor = RGBColor(0xCC, 0xCC, 0xCC);
+       params().isnotefontcolor = false;
        params().boxbgcolor = RGBColor(0xFF, 0, 0);
+       params().isboxbgcolor = false;
        params().html_latex_start.clear();
        params().html_latex_end.clear();
        params().html_math_img_scale = 1.0;
@@ -965,6 +961,7 @@ int Buffer::readHeader(Lexer & lex)
        params().biblatex_citestyle.erase();
        params().multibib.erase();
        params().lineno_opts.clear();
+       params().spellignore().clear();
 
        for (int i = 0; i < 4; ++i) {
                params().user_defined_bullet(i) = ITEMIZE_DEFAULTS[i];
@@ -993,7 +990,7 @@ int Buffer::readHeader(Lexer & lex)
                                      << token << '\'');
 
                string const result =
-                       params().readToken(lex, token, d->filename.onlyPath());
+                       params().readToken(lex, token, d->filename);
                if (!result.empty()) {
                        if (token == "\\textclass") {
                                d->layout_position = result;
@@ -1014,7 +1011,7 @@ int Buffer::readHeader(Lexer & lex)
 
        params().shell_escape = theSession().shellescapeFiles().find(absFileName());
 
-       params().makeDocumentClass();
+       params().makeDocumentClass(isClone(), isInternal());
 
        return unknown_tokens;
 }
@@ -1093,6 +1090,10 @@ bool Buffer::readDocument(Lexer & lex)
                d->old_position = params().origin;
        else
                d->old_position = filePath();
+
+       if (!parent())
+               clearIncludeList();
+
        bool const res = text().read(lex, errorList, d->inset);
        d->old_position.clear();
 
@@ -1186,9 +1187,8 @@ bool Buffer::readString(string const & s)
 
 Buffer::ReadStatus Buffer::readFile(FileName const & fn)
 {
-       FileName fname(fn);
        Lexer lex;
-       if (!lex.setFile(fname)) {
+       if (!lex.setFile(fn)) {
                Alert::error(_("File Not Found"),
                        bformat(_("Unable to open file `%1$s'."),
                                from_utf8(fn.absFileName())));
@@ -1279,9 +1279,9 @@ void Buffer::updatePreviews() const
        if (!ploader)
                return;
 
-       InsetIterator it = inset_iterator_begin(*d->inset);
-       InsetIterator const end = inset_iterator_end(*d->inset);
-       for (; it != end; ++it)
+       InsetIterator it = begin(*d->inset);
+       InsetIterator const itend = end(*d->inset);
+       for (; it != itend; ++it)
                it->addPreview(it, *ploader);
 
        ploader->startLoading();
@@ -1353,7 +1353,7 @@ Buffer::ReadStatus Buffer::convertLyXFormat(FileName const & fn,
        LYXERR(Debug::INFO, "Running '" << command_str << '\'');
 
        cmd_ret const ret = runCommand(command_str);
-       if (ret.first != 0) {
+       if (!ret.valid) {
                if (from_format < LYX_FORMAT) {
                        Alert::error(_("Conversion script failed"),
                                bformat(_("%1$s is from an older version"
@@ -1476,7 +1476,7 @@ bool Buffer::save() const
        // proper location once that has been done successfully. that
        // way we preserve the original file if something goes wrong.
        string const justname = fileName().onlyFileNameWithoutExt();
-       auto tempfile = make_unique<TempFile>(fileName().onlyPath(),
+       auto tempfile = lyx::make_unique<TempFile>(fileName().onlyPath(),
                                              justname + "-XXXXXX.lyx");
        bool const symlink = fileName().isSymLink();
        if (!symlink)
@@ -1545,9 +1545,8 @@ bool Buffer::save() const
                // time stamp is invalidated by copying/moving
                saveCheckSum();
                markClean();
-               if (d->file_format != LYX_FORMAT)
-                       // the file associated with this buffer is now in the current format
-                       d->file_format = LYX_FORMAT;
+               // the file associated with this buffer is now in the current format
+               d->file_format = LYX_FORMAT;
                return true;
        }
        // else we saved the file, but failed to move it to the right location.
@@ -1614,7 +1613,7 @@ bool Buffer::writeFile(FileName const & fname) const
 }
 
 
-docstring Buffer::emergencyWrite()
+docstring Buffer::emergencyWrite() const
 {
        // No need to save if the buffer has not changed.
        if (isClean())
@@ -1777,7 +1776,7 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
        catch (EncodingException const & e) {
                docstring const failed(1, e.failed_char);
                ostringstream oss;
-               oss << "0x" << hex << e.failed_char << dec;
+               oss << "0x" << hex << static_cast<uint32_t>(e.failed_char) << dec;
                if (getParFromID(e.par_id).paragraph().layout().pass_thru) {
                        docstring msg = bformat(_("Uncodable character '%1$s'"
                                                  " (code point %2$s)"),
@@ -1858,6 +1857,7 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
        // This is only set once per document (in master)
        if (!runparams.is_child) {
                runparams.use_polyglossia = features.usePolyglossia();
+               runparams.use_hyperref = features.isRequired("hyperref");
                runparams.use_CJK = features.mustProvide("CJK");
        }
        LYXERR(Debug::LATEX, "  Buffer validation done.");
@@ -1900,9 +1900,9 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
                        docstring uncodable_glyphs;
                        Encoding const * const enc = runparams.encoding;
                        if (enc) {
-                               for (size_t n = 0; n < inputpath.size(); ++n) {
-                                       if (!enc->encodable(inputpath[n])) {
-                                               docstring const glyph(1, inputpath[n]);
+                               for (char_type n : inputpath) {
+                                       if (!enc->encodable(n)) {
+                                               docstring const glyph(1, n);
                                                LYXERR0("Uncodable character '"
                                                        << glyph
                                                        << "' in input path!");
@@ -2107,7 +2107,7 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
        updateMacroInstances(OutputUpdate);
 
        ExportStatus const retval =
-               writeDocBookSource(ofs, fname.absFileName(), runparams, output);
+               writeDocBookSource(ofs, runparams, output);
        if (retval == ExportKilled)
                return ExportKilled;
 
@@ -2118,86 +2118,56 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
 }
 
 
-Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os, string const & fname,
+Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os,
                             OutputParams const & runparams,
                             OutputWhat output) const
 {
        LaTeXFeatures features(*this, params(), runparams);
        validate(features);
+       d->bibinfo_.makeCitationLabels(*this);
 
        d->texrow.reset();
 
        DocumentClass const & tclass = params().documentClass();
-       string const & top_element = tclass.latexname();
 
        bool const output_preamble =
                output == FullSource || output == OnlyPreamble;
        bool const output_body =
-         output == FullSource || output == OnlyBody;
+         output == FullSource || output == OnlyBody || output == IncludedFile;
 
        if (output_preamble) {
-               if (runparams.flavor == OutputParams::XML)
-                       os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
-
-               // FIXME UNICODE
-               os << "<!DOCTYPE " << from_ascii(top_element) << ' ';
-
-               // FIXME UNICODE
-               if (! tclass.class_header().empty())
-                       os << from_ascii(tclass.class_header());
-               else if (runparams.flavor == OutputParams::XML)
-                       os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
-                           << "\"https://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
-               else
-                       os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
-
-               docstring preamble = params().preamble;
-               if (runparams.flavor != OutputParams::XML ) {
-                       preamble += "<!ENTITY % output.print.png \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.pdf \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.eps \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.bmp \"IGNORE\">\n";
-               }
+               // XML preamble, no doctype needed.
+               // Not using XMLStream for this, as the root tag would be in the tag stack and make troubles with the error
+               // detection mechanisms (these are called before the end tag is output, and thus interact with the canary
+               // parsep in output_docbook.cpp).
+               os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                  << "<!-- This DocBook file was created by LyX " << lyx_version
+                  << "\n  See https://www.lyx.org/ for more information -->\n";
 
-               string const name = runparams.nice
-                       ? changeExtension(absFileName(), ".sgml") : fname;
-               preamble += features.getIncludedFiles(name);
-               preamble += features.getLyXSGMLEntities();
+               // Directly output the root tag, based on the current type of document.
+               string languageCode = params().language->code();
+               string params = "xml:lang=\"" + languageCode + '"'
+                                               + " xmlns=\"http://docbook.org/ns/docbook\""
+                                               + " xmlns:xlink=\"http://www.w3.org/1999/xlink\""
+                                               + " xmlns:m=\"http://www.w3.org/1998/Math/MathML\""
+                                               + " xmlns:xi=\"http://www.w3.org/2001/XInclude\""
+                                               + " version=\"5.2\"";
 
-               if (!preamble.empty()) {
-                       os << "\n [ " << preamble << " ]";
-               }
-               os << ">\n\n";
+               os << "<" << from_ascii(tclass.docbookroot()) << " " << from_ascii(params) << ">\n";
        }
 
        if (output_body) {
-               string top = top_element;
-               top += " lang=\"";
-               if (runparams.flavor == OutputParams::XML)
-                       top += params().language->code();
-               else
-                       top += params().language->code().substr(0, 2);
-               top += '"';
-
-               if (!params().options.empty()) {
-                       top += ' ';
-                       top += params().options;
-               }
-
-               os << "<!-- " << ((runparams.flavor == OutputParams::XML)? "XML" : "SGML")
-                               << " file was created by LyX " << lyx_version
-                               << "\n  See https://www.lyx.org/ for more information -->\n";
-
-               params().documentClass().counters().reset();
+               // Start to output the document.
+               XMLStream xs(os);
+               docbookParagraphs(text(), *this, xs, runparams);
+       }
 
-               sgml::openTag(os, top);
-               os << '\n';
-               try {
-                       docbookParagraphs(text(), *this, os, runparams);
-               }
-               catch (ConversionException const &) { return ExportKilled; }
-               sgml::closeTag(os, top_element);
+       if (output_preamble) {
+               // Close the root element. No need for a line break, as free text is never allowed
+               // in a root element, it must always be wrapped in some container.
+               os << "</" << from_ascii(tclass.docbookroot()) << ">";
        }
+
        return ExportSuccess;
 }
 
@@ -2254,17 +2224,13 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                os << "<title>"
                   << (doctitle.empty() ?
                         from_ascii("LyX Document") :
-                        html::htmlize(doctitle, XHTMLStream::ESCAPE_ALL))
+                        xml::escapeString(doctitle, XMLStream::ESCAPE_ALL))
                   << "</title>\n";
 
                docstring styles = features.getTClassHTMLPreamble();
                if (!styles.empty())
                        os << "\n<!-- Text Class Preamble -->\n" << styles << '\n';
 
-               styles = features.getPreambleSnippets().str;
-               if (!styles.empty())
-                       os << "\n<!-- Preamble Snippets -->\n" << styles << '\n';
-
                // we will collect CSS information in a stream, and then output it
                // either here, as part of the header, or else in a separate file.
                odocstringstream css;
@@ -2324,7 +2290,7 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                bool const output_body_tag = (output != IncludedFile);
                if (output_body_tag)
                        os << "<body dir=\"auto\">\n";
-               XHTMLStream xs(os);
+               XMLStream xs(os);
                if (output != IncludedFile)
                        // if we're an included file, the counters are in the master.
                        params().documentClass().counters().reset();
@@ -2359,7 +2325,7 @@ int Buffer::runChktex()
 
        // Generate the LaTeX file if neccessary
        OutputParams runparams(&params().encoding());
-       runparams.flavor = OutputParams::LATEX;
+       runparams.flavor = Flavor::LaTeX;
        runparams.nice = false;
        runparams.linelen = lyxrc.plaintext_linelen;
        ExportStatus const retval =
@@ -2401,6 +2367,9 @@ void Buffer::validate(LaTeXFeatures & features) const
        if (!features.runparams().is_child)
                params().validate(features);
 
+       if (!parent())
+               clearIncludeList();
+
        for (Paragraph const & p : paragraphs())
                p.validate(features);
 
@@ -2613,7 +2582,10 @@ void Buffer::reloadBibInfoCache(bool const force) const
 
 void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 {
-       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
+       if (!parent())
+               clearIncludeList();
+
+       for (InsetIterator it = begin(inset()); it; ++it) {
                it->collectBibKeys(it, checkedFiles);
                if (it->lyxCode() == BIBITEM_CODE) {
                        if (parent() != nullptr)
@@ -2625,18 +2597,16 @@ void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 }
 
 
-void Buffer::addBiblioInfo(BiblioInfo const & bin) const
+void Buffer::addBiblioInfo(BiblioInfo const & bi_in) const
 {
-       // We add the biblio info to the master buffer,
-       // if there is one, but also to every single buffer,
-       // in case a child is compiled alone.
-       BiblioInfo & bi = d->bibinfo_;
-       bi.mergeBiblioInfo(bin);
+       // We add the biblio info to the parent buffer,
+       // if there is one, but also to this buffer, in case
+       // it is compiled alone.
+       BiblioInfo & our_bi = d->bibinfo_;
+       our_bi.mergeBiblioInfo(bi_in);
 
-       if (parent() != nullptr) {
-               BiblioInfo & masterbi = parent()->d->bibinfo_;
-               masterbi.mergeBiblioInfo(bin);
-       }
+       if (parent())
+               parent()->addBiblioInfo(bi_in);
 }
 
 
@@ -2733,7 +2703,7 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                break;
 
        case LFUN_BUFFER_EXPORT: {
-               docstring const arg = cmd.argument();
+               docstring const arg = cmd.argument();
                if (arg == "custom") {
                        enable = true;
                        break;
@@ -2762,7 +2732,7 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                                     || cmd.action() == LFUN_BRANCH_MASTER_DEACTIVATE);
                BranchList const & branchList = master ? masterBuffer()->params().branchlist()
                        : params().branchlist();
-               docstring const branchName = cmd.argument();
+               docstring const branchName = cmd.argument();
                flag.setEnabled(!branchName.empty() && branchList.find(branchName));
                break;
        }
@@ -2924,7 +2894,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                Buffer * buf = master ? const_cast<Buffer *>(masterBuffer())
                                      : this;
 
-               docstring const branch_name = func.argument();
+               docstring const branch_name = func.argument();
                // the case without a branch name is handled elsewhere
                if (branch_name.empty()) {
                        dispatched = false;
@@ -2952,7 +2922,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
        }
 
        case LFUN_BRANCH_ADD: {
-               docstring branchnames = func.argument();
+               docstring const & branchnames = func.argument();
                if (branchnames.empty()) {
                        dispatched = false;
                        break;
@@ -2973,9 +2943,9 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                                undo().recordUndoBufferParams(CursorData());
                                branch_list.add(branch_name);
                                branch = branch_list.find(branch_name);
-                               string const x11hexname = X11hexname(branch->color());
-                               docstring const str = branch_name + ' ' + from_ascii(x11hexname);
-                               lyx::dispatch(FuncRequest(LFUN_SET_COLOR, str));
+                               if (branch)
+                                       // needed to update the color table for dark mode
+                                       branch->setColors("background", "background");
                                dr.setError(false);
                                dr.screenUpdate(Update::Force);
                        }
@@ -2991,10 +2961,10 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 
                docstring const oldname = from_utf8(func.getArg(0));
                docstring const newname = from_utf8(func.getArg(1));
-               InsetIterator it  = inset_iterator_begin(inset());
-               InsetIterator const end = inset_iterator_end(inset());
+               InsetIterator it  = begin(inset());
+               InsetIterator const itend = end(inset());
                bool success = false;
-               for (; it != end; ++it) {
+               for (; it != itend; ++it) {
                        if (it->lyxCode() == BRANCH_CODE) {
                                InsetBranch & ins = static_cast<InsetBranch &>(*it);
                                if (ins.branch() == oldname) {
@@ -3008,7 +2978,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                                // get buffer of external file
                                InsetInclude const & ins =
                                        static_cast<InsetInclude const &>(*it);
-                               Buffer * child = ins.getChildBuffer();
+                               Buffer * child = ins.loadIfNeeded();
                                if (!child)
                                        continue;
                                child->dispatch(func, dr);
@@ -3086,9 +3056,10 @@ void Buffer::changeLanguage(Language const * from, Language const * to)
        LASSERT(from, return);
        LASSERT(to, return);
 
-       for_each(par_iterator_begin(),
-                par_iterator_end(),
-                bind(&Paragraph::changeLanguage, _1, params(), from, to));
+       ParIterator it = par_iterator_begin();
+       ParIterator eit = par_iterator_end();
+       for (; it != eit; ++it)
+               it->changeLanguage(params(), from, to);
 }
 
 
@@ -3170,6 +3141,11 @@ ParConstIterator Buffer::par_iterator_end() const
        return ParConstIterator(doc_iterator_end(this));
 }
 
+bool Buffer::empty() const
+{
+       return paragraphs().size() == 1 && paragraphs().front().empty();
+}
+
 
 Language const * Buffer::language() const
 {
@@ -3475,8 +3451,37 @@ bool Buffer::isReadonly() const
 
 void Buffer::setParent(Buffer const * buffer)
 {
-       // Avoids recursive include.
-       d->setParent(buffer == this ? nullptr : buffer);
+       // We need to do some work here to avoid recursive parent structures.
+       // This is the easy case.
+       if (buffer == this) {
+               LYXERR0("Ignoring attempt to set self as parent in\n" << fileName());
+               return;
+       }
+       // Now we check parents going upward, to make sure that IF we set the
+       // parent as requested, we would not generate a recursive include.
+       set<Buffer const *> sb;
+       Buffer const * b = buffer;
+       bool found_recursion = false;
+       while (b) {
+               if (sb.find(b) != sb.end()) {
+                       found_recursion = true;
+                       break;
+               }
+               sb.insert(b);
+               b = b->parent();
+       }
+
+       if (found_recursion) {
+               LYXERR0("Ignoring attempt to set parent of\n" <<
+                       fileName() <<
+                       "\nto " <<
+                       buffer->fileName() <<
+                       "\nbecause that would create a recursive inclusion.");
+               return;
+       }
+
+       // We should be safe now.
+       d->setParent(buffer);
        updateMacros();
 }
 
@@ -3497,8 +3502,6 @@ ListOfBuffers Buffer::allRelatives() const
 
 Buffer const * Buffer::masterBuffer() const
 {
-       // FIXME Should be make sure we are not in some kind
-       // of recursive include? A -> B -> A will crash this.
        Buffer const * const pbuf = d->parent();
        if (!pbuf)
                return this;
@@ -3529,19 +3532,22 @@ bool Buffer::hasChildren() const
 }
 
 
-void Buffer::collectChildren(ListOfBuffers & clist, bool grand_children) const
+void Buffer::collectChildren(ListOfBuffers & children, bool grand_children) const
 {
        // loop over children
        for (auto const & p : d->children_positions) {
                Buffer * child = const_cast<Buffer *>(p.first);
+               // This can happen when called during GUI operations
+               if (!theBufferList().isLoaded(child))
+                       continue;
                // No duplicates
-               ListOfBuffers::const_iterator bit = find(clist.begin(), clist.end(), child);
-               if (bit != clist.end())
+               ListOfBuffers::const_iterator bit = find(children.begin(), children.end(), child);
+               if (bit != children.end())
                        continue;
-               clist.push_back(child);
+               children.push_back(child);
                if (grand_children)
                        // there might be grandchildren
-                       child->collectChildren(clist, true);
+                       child->collectChildren(children, true);
        }
 }
 
@@ -3730,7 +3736,6 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
        while (it.pit() <= lastpit) {
                Paragraph & par = it.paragraph();
 
-               // FIXME Can this be done with the new-style iterators?
                // iterate over the insets of the current paragraph
                for (auto const & insit : par.insetList()) {
                        it.pos() = insit.pos;
@@ -3769,16 +3774,14 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                                InsetInclude const & incinset =
                                        static_cast<InsetInclude const &>(*insit.inset);
                                macro_lock = true;
-                               Buffer * child = incinset.getChildBuffer();
+                               Buffer * child = incinset.loadIfNeeded();
                                macro_lock = false;
                                if (!child)
                                        continue;
 
                                // register its position, but only when it is
                                // included first in the buffer
-                               if (children_positions.find(child) ==
-                                       children_positions.end())
-                                               children_positions[child] = it;
+                               children_positions.insert({child, it});
 
                                // register child with its scope
                                position_to_children[it] = Impl::ScopeBuffer(scope, child);
@@ -3848,11 +3851,9 @@ void Buffer::updateMacros() const
 
 void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_master) const
 {
-       InsetIterator it  = inset_iterator_begin(inset());
-       InsetIterator const end = inset_iterator_end(inset());
-       for (; it != end; ++it) {
-               if (it->lyxCode() == BRANCH_CODE) {
-                       InsetBranch & br = static_cast<InsetBranch &>(*it);
+       for (Inset const & it : inset()) {
+               if (it.lyxCode() == BRANCH_CODE) {
+                       InsetBranch const & br = static_cast<InsetBranch const &>(it);
                        docstring const name = br.branch();
                        if (!from_master && !params().branchlist().find(name))
                                result.push_back(name);
@@ -3860,11 +3861,11 @@ void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_mast
                                result.push_back(name);
                        continue;
                }
-               if (it->lyxCode() == INCLUDE_CODE) {
+               if (it.lyxCode() == INCLUDE_CODE) {
                        // get buffer of external file
                        InsetInclude const & ins =
-                               static_cast<InsetInclude const &>(*it);
-                       Buffer * child = ins.getChildBuffer();
+                               static_cast<InsetInclude const &>(it);
+                       Buffer * child = ins.loadIfNeeded();
                        if (!child)
                                continue;
                        child->getUsedBranches(result, true);
@@ -3889,9 +3890,9 @@ void Buffer::updateMacroInstances(UpdateType utype) const
                        continue;
 
                // update macro in all cells of the InsetMathNest
-               DocIterator::idx_type n = minset->nargs();
+               idx_type n = minset->nargs();
                MacroContext mc = MacroContext(this, it);
-               for (DocIterator::idx_type i = 0; i < n; ++i) {
+               for (idx_type i = 0; i < n; ++i) {
                        MathData & data = minset->cell(i);
                        data.updateMacros(nullptr, mc, utype, 0);
                }
@@ -4006,10 +4007,7 @@ InsetLabel const * Buffer::insetLabel(docstring const & label,
 
 bool Buffer::activeLabel(docstring const & label) const
 {
-       if (!insetLabel(label, true))
-               return false;
-
-       return true;
+       return insetLabel(label, true) != nullptr;
 }
 
 
@@ -4037,7 +4035,7 @@ void Buffer::changeRefsIfUnique(docstring const & from, docstring const & to)
 
        string const paramName = "key";
        UndoGroupHelper ugh(this);
-       InsetIterator it = inset_iterator_begin(inset());
+       InsetIterator it = begin(inset());
        for (; it; ++it) {
                if (it->lyxCode() != CITE_CODE)
                        continue;
@@ -4066,39 +4064,51 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
        // Some macros rely on font encoding
        runparams.main_fontenc = params().main_font_encoding();
 
+       // Use the right wrapping for the comment at the beginning of the generated
+       // snippet, so that it is either valid LaTeX or valid XML (including HTML and DocBook).
+       docstring comment_start = from_ascii("% ");
+       docstring comment_end = from_ascii("");
+       if (runparams.flavor == Flavor::Html || runparams.flavor == Flavor::DocBook5) {
+               comment_start = from_ascii("<!-- ");
+               comment_end = from_ascii(" -->");
+       }
+
        if (output == CurrentParagraph) {
                runparams.par_begin = par_begin;
                runparams.par_end = par_end;
                if (par_begin + 1 == par_end) {
-                       os << "% "
+                       os << comment_start
                           << bformat(_("Preview source code for paragraph %1$d"), par_begin)
+                          << comment_end
                           << "\n\n";
                } else {
-                       os << "% "
+                       os << comment_start
                           << bformat(_("Preview source code from paragraph %1$s to %2$s"),
                                        convert<docstring>(par_begin),
                                        convert<docstring>(par_end - 1))
+                          << comment_end
                           << "\n\n";
                }
                // output paragraphs
-               if (runparams.flavor == OutputParams::LYX) {
+               if (runparams.flavor == Flavor::LyX) {
                        Paragraph const & par = text().paragraphs()[par_begin];
                        ostringstream ods;
                        depth_type dt = par.getDepth();
                        par.write(ods, params(), dt);
                        os << from_utf8(ods.str());
-               } else if (runparams.flavor == OutputParams::HTML) {
-                       XHTMLStream xs(os);
+               } else if (runparams.flavor == Flavor::Html) {
+                       XMLStream xs(os);
                        setMathFlavor(runparams);
                        xhtmlParagraphs(text(), *this, xs, runparams);
-               } else if (runparams.flavor == OutputParams::TEXT) {
+               } else if (runparams.flavor == Flavor::Text) {
                        bool dummy = false;
                        // FIXME Handles only one paragraph, unlike the others.
                        // Probably should have some routine with a signature like them.
                        writePlaintextParagraph(*this,
                                text().paragraphs()[par_begin], os, runparams, dummy);
-               } else if (params().isDocBook()) {
-                       docbookParagraphs(text(), *this, os, runparams);
+               } else if (runparams.flavor == Flavor::DocBook5) {
+                       XMLStream xs{os};
+                       docbookParagraphs(text(), *this, xs, runparams);
                } else {
                        // If we are previewing a paragraph, even if this is the
                        // child of some other buffer, let's cut the link here,
@@ -4112,6 +4122,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        LaTeXFeatures features(*this, params(), runparams);
                        validate(features);
                        runparams.use_polyglossia = features.usePolyglossia();
+                       runparams.use_hyperref = features.isRequired("hyperref");
                        // latex or literate
                        otexstream ots(os);
                        // output above
@@ -4125,15 +4136,16 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                                d->ignore_parent = false;
                }
        } else {
-               os << "% ";
+               os << comment_start;
                if (output == FullSource)
                        os << _("Preview source code");
                else if (output == OnlyPreamble)
                        os << _("Preview preamble");
                else if (output == OnlyBody)
                        os << _("Preview body");
+               os << comment_end;
                os << "\n\n";
-               if (runparams.flavor == OutputParams::LYX) {
+               if (runparams.flavor == Flavor::LyX) {
                        ostringstream ods;
                        if (output == FullSource)
                                write(ods);
@@ -4142,15 +4154,15 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        else if (output == OnlyBody)
                                text().write(ods);
                        os << from_utf8(ods.str());
-               } else if (runparams.flavor == OutputParams::HTML) {
+               } else if (runparams.flavor == Flavor::Html) {
                        writeLyXHTMLSource(os, runparams, output);
-               } else if (runparams.flavor == OutputParams::TEXT) {
-                       if (output == OnlyPreamble) {
+               } else if (runparams.flavor == Flavor::Text) {
+                       if (output == OnlyPreamble)
                                os << "% "<< _("Plain text does not have a preamble.");
-                       else
+                       else
                                writePlaintextFile(*this, os, runparams);
-               } else if (params().isDocBook()) {
-                               writeDocBookSource(os, absFileName(), runparams, output);
+               } else if (runparams.flavor == Flavor::DocBook5) {
+                       writeDocBookSource(os, runparams, output);
                } else {
                        // latex or literate
                        otexstream ots(os);
@@ -4331,7 +4343,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string & result_file) const
 {
        bool const update_unincluded =
-                       params().maintain_unincluded_children
+                       params().maintain_unincluded_children != BufferParams::CM_None
                        && !params().getIncludedChildren().empty();
 
        // (1) export with all included children (omit \includeonly)
@@ -4384,7 +4396,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        }
        MarkAsExporting exporting(this);
        string backend_format;
-       runparams.flavor = OutputParams::LATEX;
+       runparams.flavor = Flavor::LaTeX;
        runparams.linelen = lyxrc.plaintext_linelen;
        runparams.includeall = includeall;
        vector<string> backs = params().backends();
@@ -4406,7 +4418,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                                // Only show this alert if this is an export to a non-temporary
                                // file (not for previewing).
                                docstring s = bformat(_("No information for exporting the format %1$s."),
-                                                     theFormats().prettyName(format));
+                                                     translateIfPossible(theFormats().prettyName(format)));
                                if (format == "pdf4")
                                        s += "\n"
                                          + bformat(_("Hint: use non-TeX fonts or set input encoding "
@@ -4428,13 +4440,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                LYXERR(Debug::FILES, "backend_format=" << backend_format);
                // FIXME: Don't hardcode format names here, but use a flag
                if (backend_format == "pdflatex")
-                       runparams.flavor = OutputParams::PDFLATEX;
+                       runparams.flavor = Flavor::PdfLaTeX;
                else if (backend_format == "luatex")
-                       runparams.flavor = OutputParams::LUATEX;
+                       runparams.flavor = Flavor::LuaTeX;
                else if (backend_format == "dviluatex")
-                       runparams.flavor = OutputParams::DVILUATEX;
+                       runparams.flavor = Flavor::DviLuaTeX;
                else if (backend_format == "xetex")
-                       runparams.flavor = OutputParams::XETEX;
+                       runparams.flavor = Flavor::XeTeX;
        }
 
        string filename = latexName(false);
@@ -4445,7 +4457,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 
        // Plain text backend
        if (backend_format == "text") {
-               runparams.flavor = OutputParams::TEXT;
+               runparams.flavor = Flavor::Text;
                try {
                        writePlaintextFile(*this, FileName(filename), runparams);
                }
@@ -4453,15 +4465,16 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        }
        // HTML backend
        else if (backend_format == "xhtml") {
-               runparams.flavor = OutputParams::HTML;
+               runparams.flavor = Flavor::Html;
                setMathFlavor(runparams);
                if (makeLyXHTMLFile(FileName(filename), runparams) == ExportKilled)
                        return ExportKilled;
        } else if (backend_format == "lyx")
                writeFile(FileName(filename));
-       // Docbook backend
-       else if (params().isDocBook()) {
-               runparams.nice = !put_in_tempdir;
+       // DocBook backend
+       else if (backend_format == "docbook5") {
+               runparams.flavor = Flavor::DocBook5;
+               runparams.nice = false;
                if (makeDocBookFile(FileName(filename), runparams) == ExportKilled)
                        return ExportKilled;
        }
@@ -4603,13 +4616,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                } else {
                        message(bformat(_("Document exported as %1$s "
                                "to file `%2$s'"),
-                               theFormats().prettyName(format),
+                               translateIfPossible(theFormats().prettyName(format)),
                                makeDisplayPath(result_file)));
                }
        } else {
                // This must be a dummy converter like fax (bug 1888)
                message(bformat(_("Document exported as %1$s"),
-                       theFormats().prettyName(format)));
+                       translateIfPossible(theFormats().prettyName(format))));
        }
 
        return success ? ExportSuccess : ExportConverterError;
@@ -4619,7 +4632,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 Buffer::ExportStatus Buffer::preview(string const & format) const
 {
        bool const update_unincluded =
-                       params().maintain_unincluded_children
+                       params().maintain_unincluded_children != BufferParams::CM_None
                        && !params().getIncludedChildren().empty();
        return preview(format, update_unincluded);
 }
@@ -4950,6 +4963,8 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 
        // do the real work
        ParIterator parit = cbuf.par_iterator_begin();
+       if (scope == UpdateMaster)
+               clearIncludeList();
        updateBuffer(parit, utype);
 
        // If this document has siblings, then update the TocBackend later. The
@@ -4992,9 +5007,12 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        }
        d->cite_labels_valid_ = true;
        /// FIXME: Perf
+       clearIncludeList();
        cbuf.tocBackend().update(true, utype);
        if (scope == UpdateMaster)
                cbuf.structureChanged();
+
+       d->need_update = false;
 }
 
 
@@ -5156,9 +5174,9 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
                }
 
                if (needEnumCounterReset(it)) {
-                       // Increase the master counter?
-                       if (layout.stepmastercounter)
-                               counters.stepMaster(enumcounter, utype);
+                       // Increase the parent counter?
+                       if (layout.stepparentcounter)
+                               counters.stepParent(enumcounter, utype);
                        // Maybe we have to reset the enumeration counter.
                        if (!layout.resumecounter)
                                counters.reset(enumcounter);
@@ -5222,6 +5240,7 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
 
 void Buffer::updateBuffer(ParIterator & parit, UpdateType utype, bool const deleted) const
 {
+       pushIncludedBuffer(this);
        // LASSERT: Is it safe to continue here, or should we just return?
        LASSERT(parit.pit() == 0, /**/);
 
@@ -5229,11 +5248,6 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype, bool const dele
        // to resolve macros in it.
        parit.text()->setMacrocontextPosition(parit);
 
-       // Reset bibitem counter in master (#8499)
-       Buffer const * const master = masterBuffer();
-       if (master == this && !d->ignore_parent)
-               master->params().documentClass().counters().reset(from_ascii("bibitem"));
-
        depth_type maxdepth = 0;
        pit_type const lastpit = parit.lastpit();
        bool changed = false;
@@ -5277,6 +5291,19 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype, bool const dele
        // set change indicator for the inset (or the cell that the iterator
        // points to, if applicable).
        parit.text()->inset().isChanged(changed);
+       popIncludedBuffer();
+}
+
+
+void Buffer::forceUpdate() const
+{
+       d->need_update = true;
+}
+
+
+bool Buffer::needUpdate() const
+{
+       return d->need_update;
 }
 
 
@@ -5353,10 +5380,19 @@ void Buffer::Impl::updateStatistics(DocIterator & from, DocIterator & to, bool s
                                        ++word_count_;
                                        inword = true;
                                }
-                               if (ins && ins->isLetter())
-                                       ++char_count_;
+                               if (ins && ins->isLetter()) {
+                                       odocstringstream os;
+                                       ins->toString(os);
+                                       char_count_ += os.str().length();
+                               }
                                else if (ins && ins->isSpace())
                                        ++blank_count_;
+                               else if (ins) {
+                                       pair<int, int> words = ins->isWords();
+                                       char_count_ += words.first;
+                                       word_count_ += words.second;
+                                       inword = false;
+                               }
                                else {
                                        char_type const c = par.getChar(pos);
                                        if (isPrintableNonspace(c))
@@ -5593,4 +5629,55 @@ void Buffer::clearExternalModification() const
 }
 
 
+void Buffer::pushIncludedBuffer(Buffer const * buf) const
+{
+       masterBuffer()->d->include_list_.push_back(buf);
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Pushed. Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+}
+
+
+void Buffer::popIncludedBuffer() const
+{
+       masterBuffer()->d->include_list_.pop_back();
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Popped. Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+}
+
+
+bool Buffer::isBufferIncluded(Buffer const * buf) const
+{
+       if (!buf)
+               return false;
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Checking for " << buf->fileName() << ". Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+       list<Buffer const *> const & blist = masterBuffer()->d->include_list_;
+       return find(blist.begin(), blist.end(), buf) != blist.end();
+}
+
+
+void Buffer::clearIncludeList() const
+{
+       LYXERR(Debug::FILES, "Clearing include list for " << fileName());
+       d->include_list_.clear();
+}
+
 } // namespace lyx