]> git.lyx.org Git - lyx.git/blobdiff - src/Buffer.cpp
Improved character count statistics for letter based insets (e.g. the LyX logo).
[lyx.git] / src / Buffer.cpp
index ea04b34a619e7bb2dd559cac380a1c13e2ac1f52..97a6cfbd31afe1e771034326ef982b37fff7a7fb 100644 (file)
 #include "LyX.h"
 #include "LyXRC.h"
 #include "LyXVC.h"
-#include "output_docbook.h"
 #include "output.h"
 #include "output_latex.h"
-#include "output_xhtml.h"
+#include "output_docbook.h"
 #include "output_plaintext.h"
+#include "output_xhtml.h"
 #include "Paragraph.h"
 #include "ParagraphParameters.h"
 #include "ParIterator.h"
 #include "PDFOptions.h"
 #include "Session.h"
 #include "SpellChecker.h"
-#include "sgml.h"
+#include "xml.h"
 #include "texstream.h"
 #include "TexRow.h"
 #include "Text.h"
 #include "support/textutils.h"
 #include "support/types.h"
 
-#include "support/bind.h"
-
 #include <algorithm>
 #include <fstream>
 #include <iomanip>
@@ -153,11 +151,12 @@ typedef vector<LabelInfo> LabelCache;
 
 typedef map<docstring, Buffer::References> RefCache;
 
-} // namespace
+// A storehouse for the cloned buffers.
+typedef list<CloneList_ptr> CloneStore;
+CloneStore cloned_buffers;
 
+} // namespace
 
-// A storehouse for the cloned buffers.
-list<CloneList *> cloned_buffers;
 
 
 class Buffer::Impl
@@ -200,40 +199,9 @@ public:
        /// need to regenerate .tex?
        DepClean dep_clean;
 
-       /// is save needed?
-       mutable bool lyx_clean;
-
-       /// is autosave needed?
-       mutable bool bak_clean;
-
-       /// is this an unnamed file (New...)?
-       bool unnamed;
-
-       /// is this an internal bufffer?
-       bool internal_buffer;
-
-       /// buffer is r/o
-       bool read_only;
-
        /// name of the file the buffer is associated with.
        FileName filename;
 
-       /** Set to true only when the file is fully loaded.
-        *  Used to prevent the premature generation of previews
-        *  and by the citation inset.
-        */
-       bool file_fully_loaded;
-
-       /// original format of loaded file
-       int file_format;
-
-       /// if the file was originally loaded from an older format, do
-       /// we need to back it up still?
-       bool need_format_backup;
-
-       /// Ignore the parent (e.g. when exporting a child standalone)?
-       bool ignore_parent;
-
        ///
        mutable TocBackend toc_backend;
 
@@ -250,13 +218,11 @@ public:
        /// map from the macro name to the position map,
        /// which maps the macro definition position to the scope and the MacroData.
        NamePositionScopeMacroMap macros;
-       /// This seem to change the way Buffer::getMacro() works
-       mutable bool macro_lock;
 
        /// positions of child buffers in the buffer
        typedef map<Buffer const * const, DocIterator> BufferPositionMap;
        struct ScopeBuffer {
-               ScopeBuffer() : buffer(0) {}
+               ScopeBuffer() : buffer(nullptr) {}
                ScopeBuffer(DocIterator const & s, Buffer const * b)
                        : scope(s), buffer(b) {}
                DocIterator scope;
@@ -303,25 +269,13 @@ public:
        // file, and then to construct the Buffer's bibinfo from that.
        /// A cache for bibliography info
        mutable BiblioInfo bibinfo_;
-       /// whether the bibinfo cache is valid
-       mutable bool bibinfo_cache_valid_;
        /// Cache of timestamps of .bib files
        map<FileName, time_t> bibfile_status_;
-       /// Indicates whether the bibinfo has changed since the last time
-       /// we ran updateBuffer(), i.e., whether citation labels may need
-       /// to be updated.
-       mutable bool cite_labels_valid_;
-       /// Do we have a bibliography environment?
-       mutable bool have_bibitems_;
 
        /// These two hold the file name and format, written to by
        /// Buffer::preview and read from by LFUN_BUFFER_VIEW_CACHE.
        FileName preview_file_;
        string preview_format_;
-       /// If there was an error when previewing, on the next preview we do
-       /// a fresh compile (e.g. in case the user installed a package that
-       /// was missing).
-       bool require_fresh_start_;
 
        /// Cache the references associated to a label and their positions
        /// in the buffer.
@@ -335,6 +289,83 @@ public:
        ///
        PreviewLoader * preview_loader_;
 
+       /// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
+       /// This one is useful for preview detached in a thread.
+       Buffer const * cloned_buffer_;
+       ///
+       CloneList_ptr clone_list_;
+
+       ///
+       std::list<Buffer const *> include_list_;
+private:
+       /// So we can force access via the accessors.
+       mutable Buffer const * parent_buffer;
+
+       FileMonitorPtr file_monitor_;
+
+/// ints and bools are all listed last so as to avoid alignment issues
+public:
+       /// original format of loaded file
+       int file_format;
+
+       /// are we in the process of exporting this buffer?
+       mutable bool doing_export;
+
+       /// If there was an error when previewing, on the next preview we do
+       /// a fresh compile (e.g. in case the user installed a package that
+       /// was missing).
+       bool require_fresh_start_;
+
+       /// Indicates whether the bibinfo has changed since the last time
+       /// we ran updateBuffer(), i.e., whether citation labels may need
+       /// to be updated.
+       mutable bool cite_labels_valid_;
+       /// Do we have a bibliography environment?
+       mutable bool have_bibitems_;
+
+       /// is save needed?
+       mutable bool lyx_clean;
+
+       /// is autosave needed?
+       mutable bool bak_clean;
+
+       /// is this an unnamed file (New...)?
+       bool unnamed;
+
+       /// is this an internal bufffer?
+       bool internal_buffer;
+
+       /// buffer is r/o
+       bool read_only;
+
+       /** Set to true only when the file is fully loaded.
+        *  Used to prevent the premature generation of previews
+        *  and by the citation inset.
+        */
+       bool file_fully_loaded;
+
+       /// if the file was originally loaded from an older format, do
+       /// we need to back it up still?
+       bool need_format_backup;
+
+       /// Ignore the parent (e.g. when exporting a child standalone)?
+       bool ignore_parent;
+
+       /// This seem to change the way Buffer::getMacro() works
+       mutable bool macro_lock;
+
+       /// has been externally modified? Can be reset by the user.
+       mutable bool externally_modified_;
+
+       /// whether the bibinfo cache is valid
+       mutable bool bibinfo_cache_valid_;
+
+private:
+       int word_count_;
+       int char_count_;
+       int blank_count_;
+
+public:
        /// This is here to force the test to be done whenever parent_buffer
        /// is accessed.
        Buffer const * parent() const
@@ -342,7 +373,7 @@ public:
                // ignore_parent temporarily "orphans" a buffer
                // (e.g. if a child is compiled standalone)
                if (ignore_parent)
-                       return 0;
+                       return nullptr;
                // if parent_buffer is not loaded, then it has been unloaded,
                // which means that parent_buffer is an invalid pointer. So we
                // set it to null in that case.
@@ -350,7 +381,7 @@ public:
                // they will always be regarded as unloaded. in that case, we hope
                // for the best.
                if (!cloned_buffer_ && !theBufferList().isLoaded(parent_buffer))
-                       parent_buffer = 0;
+                       parent_buffer = nullptr;
                return parent_buffer;
        }
 
@@ -367,14 +398,6 @@ public:
                        parent_buffer->invalidateBibinfoCache();
        }
 
-       /// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
-       /// This one is useful for preview detached in a thread.
-       Buffer const * cloned_buffer_;
-       ///
-       CloneList * clone_list_;
-       /// are we in the process of exporting this buffer?
-       mutable bool doing_export;
-
        /// compute statistics
        /// \p from initial position
        /// \p to points to the end position
@@ -397,24 +420,11 @@ public:
        /// Notify or clear of external modification
        void fileExternallyModified(bool exists);
 
-       /// has been externally modified? Can be reset by the user.
-       mutable bool externally_modified_;
-
        ///Binding LaTeX lines with buffer positions.
        //Common routine for LaTeX and Reference errors listing.
         void traverseErrors(TeXErrors::Errors::const_iterator err,
                TeXErrors::Errors::const_iterator end,
                ErrorList & errorList) const;
-
-private:
-       /// So we can force access via the accessors.
-       mutable Buffer const * parent_buffer;
-
-       int word_count_;
-       int char_count_;
-       int blank_count_;
-
-       FileMonitorPtr file_monitor_;
 };
 
 
@@ -447,15 +457,15 @@ static FileName createBufferTmpDir()
 
 Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
        Buffer const * cloned_buffer)
-       : owner_(owner), lyx_clean(true), bak_clean(true), unnamed(false),
-         internal_buffer(false), read_only(readonly_), filename(file),
-         file_fully_loaded(false), file_format(LYX_FORMAT), need_format_backup(false),
-         ignore_parent(false),  toc_backend(owner), macro_lock(false),
-         checksum_(0), wa_(0),  gui_(0), undo_(*owner), bibinfo_cache_valid_(false),
-         cite_labels_valid_(false), have_bibitems_(false), require_fresh_start_(false),
-         inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
-         clone_list_(0), doing_export(false),
-         externally_modified_(false), parent_buffer(0),
+       : owner_(owner), filename(file), toc_backend(owner), checksum_(0),
+         wa_(nullptr),  gui_(nullptr), undo_(*owner), inset(nullptr),
+         preview_loader_(nullptr), cloned_buffer_(cloned_buffer),
+         clone_list_(nullptr), parent_buffer(nullptr), file_format(LYX_FORMAT),
+         doing_export(false), require_fresh_start_(false), cite_labels_valid_(false),
+         have_bibitems_(false), lyx_clean(true), bak_clean(true), unnamed(false),
+         internal_buffer(false), read_only(readonly_), file_fully_loaded(false),
+         need_format_backup(false), ignore_parent(false), macro_lock(false),
+         externally_modified_(false), bibinfo_cache_valid_(false),
          word_count_(0), char_count_(0), blank_count_(0)
 {
        refreshFileMonitor();
@@ -515,7 +525,7 @@ Buffer::~Buffer()
        // saved properly, before it goes into the void.
 
        // GuiView already destroyed
-       d->gui_ = 0;
+       d->gui_ = nullptr;
 
        if (isInternal()) {
                // No need to do additional cleanups for internal buffer.
@@ -530,8 +540,8 @@ Buffer::~Buffer()
                // loop over children
                for (auto const & p : d->children_positions) {
                        Buffer * child = const_cast<Buffer *>(p.first);
-                               if (d->clone_list_->erase(child))
-                                       delete child;
+                       if (d->clone_list_->erase(child))
+                               delete child;
                }
                // if we're the master buffer, then we should get rid of the list
                // of clones
@@ -540,14 +550,15 @@ Buffer::~Buffer()
                        // children still has a reference to this list. But we will try to
                        // continue, rather than shut down.
                        LATTEST(d->clone_list_->empty());
-                       list<CloneList *>::iterator it =
+                       // The clone list itself is empty, but it's still referenced in our list
+                       // of clones. So let's find it and remove it.
+                       CloneStore::iterator it =
                                find(cloned_buffers.begin(), cloned_buffers.end(), d->clone_list_);
                        if (it == cloned_buffers.end()) {
                                // We will leak in this case, but it is safe to continue.
                                LATTEST(false);
                        } else
                                cloned_buffers.erase(it);
-                       delete d->clone_list_;
                }
                // FIXME Do we really need to do this right before we delete d?
                // clear references to children in macro tables
@@ -559,7 +570,7 @@ Buffer::~Buffer()
                        Buffer * child = const_cast<Buffer *>(p.first);
                        if (theBufferList().isLoaded(child)) {
                                if (theBufferList().isOthersChild(this, child))
-                                       child->setParent(0);
+                                       child->setParent(nullptr);
                                else
                                        theBufferList().release(child);
                        }
@@ -594,21 +605,21 @@ Buffer::~Buffer()
 Buffer * Buffer::cloneWithChildren() const
 {
        BufferMap bufmap;
-       cloned_buffers.push_back(new CloneList);
-       CloneList * clones = cloned_buffers.back();
+       cloned_buffers.emplace_back(new CloneList);
+       CloneList_ptr clones = cloned_buffers.back();
 
        cloneWithChildren(bufmap, clones);
 
        // make sure we got cloned
        BufferMap::const_iterator bit = bufmap.find(this);
-       LASSERT(bit != bufmap.end(), return 0);
+       LASSERT(bit != bufmap.end(), return nullptr);
        Buffer * cloned_buffer = bit->second;
 
        return cloned_buffer;
 }
 
 
-void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
+void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList_ptr clones) const
 {
        // have we already been cloned?
        if (bufmap.find(this) != bufmap.end())
@@ -658,8 +669,8 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
 
 
 Buffer * Buffer::cloneBufferOnly() const {
-       cloned_buffers.push_back(new CloneList);
-       CloneList * clones = cloned_buffers.back();
+       cloned_buffers.emplace_back(new CloneList);
+       CloneList_ptr clones = cloned_buffers.back();
        Buffer * buffer_clone = new Buffer(fileName().absFileName(), false, this);
 
        // The clone needs its own DocumentClass, since running updateBuffer() will
@@ -1028,20 +1039,13 @@ bool Buffer::readDocument(Lexer & lex)
        readHeader(lex);
 
        if (params().output_changes) {
-               bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
                bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
                                  LaTeXFeatures::isAvailable("xcolor");
 
-               if (!dvipost && !xcolorulem) {
+               if (!xcolorulem) {
                        Alert::warning(_("Changes not shown in LaTeX output"),
                                       _("Changes will not be highlighted in LaTeX output, "
-                                        "because neither dvipost nor xcolor/ulem are installed.\n"
-                                        "Please install these packages or redefine "
-                                        "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
-               } else if (!xcolorulem) {
-                       Alert::warning(_("Changes not shown in LaTeX output"),
-                                      _("Changes will not be highlighted in LaTeX output "
-                                        "when using pdflatex, because xcolor and ulem are not installed.\n"
+                                        "because xcolor and ulem are not installed.\n"
                                         "Please install both packages or redefine "
                                         "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
                }
@@ -1089,6 +1093,10 @@ bool Buffer::readDocument(Lexer & lex)
                d->old_position = params().origin;
        else
                d->old_position = filePath();
+
+       if (!parent())
+               clearIncludeList();
+
        bool const res = text().read(lex, errorList, d->inset);
        d->old_position.clear();
 
@@ -1140,7 +1148,7 @@ bool Buffer::importFile(string const & format, FileName const & name, ErrorList
 
        FileName const lyx = tempFileName("Buffer_importFileXXXXXX.lyx");
        Converters::RetVal const retval =
-           theConverters().convert(0, name, lyx, name, format, "lyx", errorList);
+               theConverters().convert(nullptr, name, lyx, name, format, "lyx", errorList);
        if (retval == Converters::SUCCESS) {
                bool const success = readFile(lyx) == ReadSuccess;
                removeTempFile(lyx);
@@ -1255,7 +1263,7 @@ void Buffer::requireFreshStart(bool const b) const
 PreviewLoader * Buffer::loader() const
 {
        if (!isExporting() && lyxrc.preview == LyXRC::PREVIEW_OFF)
-               return 0;
+               return nullptr;
        if (!d->preview_loader_)
                d->preview_loader_ = new PreviewLoader(*this);
        return d->preview_loader_;
@@ -1265,7 +1273,7 @@ PreviewLoader * Buffer::loader() const
 void Buffer::removePreviews() const
 {
        delete d->preview_loader_;
-       d->preview_loader_ = 0;
+       d->preview_loader_ = nullptr;
 }
 
 
@@ -1854,6 +1862,7 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
        // This is only set once per document (in master)
        if (!runparams.is_child) {
                runparams.use_polyglossia = features.usePolyglossia();
+               runparams.use_hyperref = features.isRequired("hyperref");
                runparams.use_CJK = features.mustProvide("CJK");
        }
        LYXERR(Debug::LATEX, "  Buffer validation done.");
@@ -2103,7 +2112,7 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
        updateMacroInstances(OutputUpdate);
 
        ExportStatus const retval =
-               writeDocBookSource(ofs, fname.absFileName(), runparams, output);
+               writeDocBookSource(ofs, runparams, output);
        if (retval == ExportKilled)
                return ExportKilled;
 
@@ -2114,85 +2123,56 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
 }
 
 
-Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os, string const & fname,
+Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os,
                             OutputParams const & runparams,
                             OutputWhat output) const
 {
        LaTeXFeatures features(*this, params(), runparams);
        validate(features);
+       d->bibinfo_.makeCitationLabels(*this);
 
        d->texrow.reset();
 
        DocumentClass const & tclass = params().documentClass();
-       string const & top_element = tclass.latexname();
 
        bool const output_preamble =
                output == FullSource || output == OnlyPreamble;
        bool const output_body =
          output == FullSource || output == OnlyBody;
 
-       if (output_preamble) {
-               if (runparams.flavor == OutputParams::XML)
-                       os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
-
-               // FIXME UNICODE
-               os << "<!DOCTYPE " << from_ascii(top_element) << ' ';
+       XMLStream xs(os);
 
-               // FIXME UNICODE
-               if (! tclass.class_header().empty())
-                       os << from_ascii(tclass.class_header());
-               else if (runparams.flavor == OutputParams::XML)
-                       os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
-                           << "\"https://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
-               else
-                       os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
-
-               docstring preamble = params().preamble;
-               if (runparams.flavor != OutputParams::XML ) {
-                       preamble += "<!ENTITY % output.print.png \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.pdf \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.eps \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.bmp \"IGNORE\">\n";
-               }
+       if (output_preamble) {
+               // XML preamble, no doctype needed.
+               // Not using XMLStream for this, as the root tag would be in the tag stack and make troubles with the error
+               // detection mechanisms (these are called before the end tag is output, and thus interact with the canary
+               // parsep in output_docbook.cpp).
+               os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                  << "<!-- This DocBook file was created by LyX " << lyx_version
+                  << "\n  See http://www.lyx.org/ for more information -->\n";
 
-               string const name = runparams.nice
-                       ? changeExtension(absFileName(), ".sgml") : fname;
-               preamble += features.getIncludedFiles(name);
-               preamble += features.getLyXSGMLEntities();
+               // Directly output the root tag, based on the current type of document.
+               string languageCode = params().language->code();
+               string params = "xml:lang=\"" + languageCode + '"'
+                                               + " xmlns=\"http://docbook.org/ns/docbook\""
+                                               + " xmlns:xlink=\"http://www.w3.org/1999/xlink\""
+                                               + " xmlns:m=\"http://www.w3.org/1998/Math/MathML\""
+                                               + " xmlns:xi=\"http://www.w3.org/2001/XInclude\""
+                                               + " version=\"5.2\"";
 
-               if (!preamble.empty()) {
-                       os << "\n [ " << preamble << " ]";
-               }
-               os << ">\n\n";
+               os << "<" << from_ascii(tclass.docbookroot()) << " " << from_ascii(params) << ">\n";
        }
 
        if (output_body) {
-               string top = top_element;
-               top += " lang=\"";
-               if (runparams.flavor == OutputParams::XML)
-                       top += params().language->code();
-               else
-                       top += params().language->code().substr(0, 2);
-               top += '"';
-
-               if (!params().options.empty()) {
-                       top += ' ';
-                       top += params().options;
-               }
-
-               os << "<!-- " << ((runparams.flavor == OutputParams::XML)? "XML" : "SGML")
-                               << " file was created by LyX " << lyx_version
-                               << "\n  See https://www.lyx.org/ for more information -->\n";
-
                params().documentClass().counters().reset();
 
-               sgml::openTag(os, top);
-               os << '\n';
-               try {
-                       docbookParagraphs(text(), *this, os, runparams);
-               }
-               catch (ConversionException const &) { return ExportKilled; }
-               sgml::closeTag(os, top_element);
+               // Start to output the document.
+               docbookParagraphs(text(), *this, xs, runparams);
+       }
+
+       if (output_preamble) {
+               // Close the root element.
+               os << "\n</" << from_ascii(tclass.docbookroot()) << ">";
        }
        return ExportSuccess;
 }
@@ -2250,17 +2230,13 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                os << "<title>"
                   << (doctitle.empty() ?
                         from_ascii("LyX Document") :
-                        html::htmlize(doctitle, XHTMLStream::ESCAPE_ALL))
+                        xml::escapeString(doctitle, XMLStream::ESCAPE_ALL))
                   << "</title>\n";
 
                docstring styles = features.getTClassHTMLPreamble();
                if (!styles.empty())
                        os << "\n<!-- Text Class Preamble -->\n" << styles << '\n';
 
-               styles = features.getPreambleSnippets().str;
-               if (!styles.empty())
-                       os << "\n<!-- Preamble Snippets -->\n" << styles << '\n';
-
                // we will collect CSS information in a stream, and then output it
                // either here, as part of the header, or else in a separate file.
                odocstringstream css;
@@ -2320,7 +2296,7 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                bool const output_body_tag = (output != IncludedFile);
                if (output_body_tag)
                        os << "<body dir=\"auto\">\n";
-               XHTMLStream xs(os);
+               XMLStream xs(os);
                if (output != IncludedFile)
                        // if we're an included file, the counters are in the master.
                        params().documentClass().counters().reset();
@@ -2397,6 +2373,9 @@ void Buffer::validate(LaTeXFeatures & features) const
        if (!features.runparams().is_child)
                params().validate(features);
 
+       if (!parent())
+               clearIncludeList();
+
        for (Paragraph const & p : paragraphs())
                p.validate(features);
 
@@ -2609,10 +2588,13 @@ void Buffer::reloadBibInfoCache(bool const force) const
 
 void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 {
+       if (!parent())
+               clearIncludeList();
+
        for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
                it->collectBibKeys(it, checkedFiles);
                if (it->lyxCode() == BIBITEM_CODE) {
-                       if (parent() != 0)
+                       if (parent() != nullptr)
                                parent()->d->have_bibitems_ = true;
                        else
                                d->have_bibitems_ = true;
@@ -2629,7 +2611,7 @@ void Buffer::addBiblioInfo(BiblioInfo const & bin) const
        BiblioInfo & bi = d->bibinfo_;
        bi.mergeBiblioInfo(bin);
 
-       if (parent() != 0) {
+       if (parent() != nullptr) {
                BiblioInfo & masterbi = parent()->d->bibinfo_;
                masterbi.mergeBiblioInfo(bin);
        }
@@ -2644,7 +2626,7 @@ void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bin) const
        BiblioInfo & bi = d->bibinfo_;
        bi[key] = bin;
 
-       if (parent() != 0) {
+       if (parent() != nullptr) {
                BiblioInfo & masterbi = masterBuffer()->d->bibinfo_;
                masterbi[key] = bin;
        }
@@ -3004,7 +2986,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                                // get buffer of external file
                                InsetInclude const & ins =
                                        static_cast<InsetInclude const &>(*it);
-                               Buffer * child = ins.getChildBuffer();
+                               Buffer * child = ins.loadIfNeeded();
                                if (!child)
                                        continue;
                                child->dispatch(func, dr);
@@ -3035,20 +3017,13 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                        undo().recordUndoBufferParams(CursorData());
                params().output_changes = !params().output_changes;
                if (params().output_changes) {
-                       bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
                        bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
                                          LaTeXFeatures::isAvailable("xcolor");
 
-                       if (!dvipost && !xcolorulem) {
+                       if (!xcolorulem) {
                                Alert::warning(_("Changes not shown in LaTeX output"),
                                               _("Changes will not be highlighted in LaTeX output, "
-                                                "because neither dvipost nor xcolor/ulem are installed.\n"
-                                                "Please install these packages or redefine "
-                                                "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
-                       } else if (!xcolorulem) {
-                               Alert::warning(_("Changes not shown in LaTeX output"),
-                                              _("Changes will not be highlighted in LaTeX output "
-                                                "when using pdflatex, because xcolor and ulem are not installed.\n"
+                                                "because xcolor and ulem are not installed.\n"
                                                 "Please install both packages or redefine "
                                                 "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
                        }
@@ -3089,9 +3064,10 @@ void Buffer::changeLanguage(Language const * from, Language const * to)
        LASSERT(from, return);
        LASSERT(to, return);
 
-       for_each(par_iterator_begin(),
-                par_iterator_end(),
-                bind(&Paragraph::changeLanguage, _1, params(), from, to));
+       ParIterator it = par_iterator_begin();
+       ParIterator eit = par_iterator_end();
+       for (; it != eit; ++it)
+               it->changeLanguage(params(), from, to);
 }
 
 
@@ -3180,7 +3156,7 @@ Language const * Buffer::language() const
 }
 
 
-docstring const Buffer::B_(string const & l10n) const
+docstring Buffer::B_(string const & l10n) const
 {
        return params().B_(l10n);
 }
@@ -3411,7 +3387,7 @@ vector<pair<docstring, string>> const Buffer::prepareBibFilePaths(OutputParams c
                        enc = params().bibFileEncoding(utf8input);
 
                bool recorded = false;
-               for (pair<docstring, string> pe : res) {
+               for (auto const & pe : res) {
                        if (pe.first == path) {
                                recorded = true;
                                break;
@@ -3478,8 +3454,37 @@ bool Buffer::isReadonly() const
 
 void Buffer::setParent(Buffer const * buffer)
 {
-       // Avoids recursive include.
-       d->setParent(buffer == this ? 0 : buffer);
+       // We need to do some work here to avoid recursive parent structures.
+       // This is the easy case.
+       if (buffer == this) {
+               LYXERR0("Ignoring attempt to set self as parent in\n" << fileName());
+               return;
+       }
+       // Now we check parents going upward, to make sure that IF we set the
+       // parent as requested, we would not generate a recursive include.
+       set<Buffer const *> sb;
+       Buffer const * b = buffer;
+       bool found_recursion = false;
+       while (b) {
+               if (sb.find(b) != sb.end()) {
+                       found_recursion = true;
+                       break;
+               }
+               sb.insert(b);
+               b = b->parent();
+       }
+
+       if (found_recursion) {
+               LYXERR0("Ignoring attempt to set parent of\n" <<
+                       fileName() <<
+                       "\nto " <<
+                       buffer->fileName() <<
+                       "\nbecause that would create a recursive inclusion.");
+               return;
+       }
+
+       // We should be safe now.
+       d->setParent(buffer);
        updateMacros();
 }
 
@@ -3500,8 +3505,6 @@ ListOfBuffers Buffer::allRelatives() const
 
 Buffer const * Buffer::masterBuffer() const
 {
-       // FIXME Should be make sure we are not in some kind
-       // of recursive include? A -> B -> A will crash this.
        Buffer const * const pbuf = d->parent();
        if (!pbuf)
                return this;
@@ -3599,11 +3602,11 @@ MacroData const * Buffer::Impl::getBufferMacro(docstring const & name,
 
        // if paragraphs have no macro context set, pos will be empty
        if (pos.empty())
-               return 0;
+               return nullptr;
 
        // we haven't found anything yet
        DocIterator bestPos = owner_->par_iterator_begin();
-       MacroData const * bestData = 0;
+       MacroData const * bestData = nullptr;
 
        // find macro definitions for name
        NamePositionScopeMacroMap::const_iterator nameIt = macros.find(name);
@@ -3674,11 +3677,11 @@ MacroData const * Buffer::getMacro(docstring const & name,
        DocIterator const & pos, bool global) const
 {
        if (d->macro_lock)
-               return 0;
+               return nullptr;
 
        // query buffer macros
        MacroData const * data = d->getBufferMacro(name, pos);
-       if (data != 0)
+       if (data != nullptr)
                return data;
 
        // If there is a master buffer, query that
@@ -3694,11 +3697,11 @@ MacroData const * Buffer::getMacro(docstring const & name,
 
        if (global) {
                data = MacroTable::globalMacros().get(name);
-               if (data != 0)
+               if (data != nullptr)
                        return data;
        }
 
-       return 0;
+       return nullptr;
 }
 
 
@@ -3718,7 +3721,7 @@ MacroData const * Buffer::getMacro(docstring const & name,
        // look where the child buffer is included first
        Impl::BufferPositionMap::iterator it = d->children_positions.find(&child);
        if (it == d->children_positions.end())
-               return 0;
+               return nullptr;
 
        // check for macros at the inclusion position
        return getMacro(name, it->second, global);
@@ -3772,7 +3775,7 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                                InsetInclude const & incinset =
                                        static_cast<InsetInclude const &>(*insit.inset);
                                macro_lock = true;
-                               Buffer * child = incinset.getChildBuffer();
+                               Buffer * child = incinset.loadIfNeeded();
                                macro_lock = false;
                                if (!child)
                                        continue;
@@ -3867,7 +3870,7 @@ void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_mast
                        // get buffer of external file
                        InsetInclude const & ins =
                                static_cast<InsetInclude const &>(*it);
-                       Buffer * child = ins.getChildBuffer();
+                       Buffer * child = ins.loadIfNeeded();
                        if (!child)
                                continue;
                        child->getUsedBranches(result, true);
@@ -3896,7 +3899,7 @@ void Buffer::updateMacroInstances(UpdateType utype) const
                MacroContext mc = MacroContext(this, it);
                for (DocIterator::idx_type i = 0; i < n; ++i) {
                        MathData & data = minset->cell(i);
-                       data.updateMacros(0, mc, utype, 0);
+                       data.updateMacros(nullptr, mc, utype, 0);
                }
        }
 }
@@ -3999,7 +4002,7 @@ void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il,
 InsetLabel const * Buffer::insetLabel(docstring const & label,
                                      bool const active) const
 {
-       for (auto & rc : masterBuffer()->d->label_cache_) {
+       for (auto const & rc : masterBuffer()->d->label_cache_) {
                if (rc.label == label && (rc.active || !active))
                        return rc.inset;
        }
@@ -4039,14 +4042,17 @@ void Buffer::changeRefsIfUnique(docstring const & from, docstring const & to)
                return;
 
        string const paramName = "key";
+       UndoGroupHelper ugh(this);
        InsetIterator it = inset_iterator_begin(inset());
        for (; it; ++it) {
                if (it->lyxCode() != CITE_CODE)
                        continue;
                InsetCommand * inset = it->asInsetCommand();
                docstring const oldValue = inset->getParam(paramName);
-               if (oldValue == from)
+               if (oldValue == from) {
+                       undo().recordUndo(CursorData(it));
                        inset->setParam(paramName, to);
+               }
        }
 }
 
@@ -4088,7 +4094,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        par.write(ods, params(), dt);
                        os << from_utf8(ods.str());
                } else if (runparams.flavor == OutputParams::HTML) {
-                       XHTMLStream xs(os);
+                       XMLStream xs(os);
                        setMathFlavor(runparams);
                        xhtmlParagraphs(text(), *this, xs, runparams);
                } else if (runparams.flavor == OutputParams::TEXT) {
@@ -4097,8 +4103,9 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        // Probably should have some routine with a signature like them.
                        writePlaintextParagraph(*this,
                                text().paragraphs()[par_begin], os, runparams, dummy);
-               } else if (params().isDocBook()) {
-                       docbookParagraphs(text(), *this, os, runparams);
+               } else if (runparams.flavor == OutputParams::DOCBOOK5) {
+                       XMLStream xs{os};
+                       docbookParagraphs(text(), *this, xs, runparams);
                } else {
                        // If we are previewing a paragraph, even if this is the
                        // child of some other buffer, let's cut the link here,
@@ -4112,6 +4119,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        LaTeXFeatures features(*this, params(), runparams);
                        validate(features);
                        runparams.use_polyglossia = features.usePolyglossia();
+                       runparams.use_hyperref = features.isRequired("hyperref");
                        // latex or literate
                        otexstream ots(os);
                        // output above
@@ -4149,8 +4157,8 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                                os << "% "<< _("Plain text does not have a preamble.");
                        } else
                                writePlaintextFile(*this, os, runparams);
-               } else if (params().isDocBook()) {
-                               writeDocBookSource(os, absFileName(), runparams, output);
+               } else if (runparams.flavor == OutputParams::DOCBOOK5) {
+                       writeDocBookSource(os, runparams, output);
                } else {
                        // latex or literate
                        otexstream ots(os);
@@ -4235,91 +4243,6 @@ void Buffer::setGuiDelegate(frontend::GuiBufferDelegate * gui)
 }
 
 
-
-namespace {
-
-class AutoSaveBuffer : public ForkedProcess {
-public:
-       ///
-       AutoSaveBuffer(Buffer const & buffer, FileName const & fname)
-               : buffer_(buffer), fname_(fname) {}
-       ///
-       virtual shared_ptr<ForkedProcess> clone() const
-       {
-               return make_shared<AutoSaveBuffer>(*this);
-       }
-       ///
-       int start()
-       {
-               command_ = to_utf8(bformat(_("Auto-saving %1$s"),
-                                                from_utf8(fname_.absFileName())));
-               return run(DontWait);
-       }
-private:
-       ///
-       virtual int generateChild();
-       ///
-       Buffer const & buffer_;
-       FileName fname_;
-};
-
-
-int AutoSaveBuffer::generateChild()
-{
-#if defined(__APPLE__)
-       /* FIXME fork() is not usable for autosave on Mac OS X 10.6 (snow leopard)
-        *   We should use something else like threads.
-        *
-        * Since I do not know how to determine at run time what is the OS X
-        * version, I just disable forking altogether for now (JMarc)
-        */
-       pid_t const pid = -1;
-#else
-       // tmp_ret will be located (usually) in /tmp
-       // will that be a problem?
-       // Note that this calls ForkedCalls::fork(), so it's
-       // ok cross-platform.
-       pid_t const pid = fork();
-       // If you want to debug the autosave
-       // you should set pid to -1, and comment out the fork.
-       if (pid != 0 && pid != -1)
-               return pid;
-#endif
-
-       // pid = -1 signifies that lyx was unable
-       // to fork. But we will do the save
-       // anyway.
-       bool failed = false;
-       TempFile tempfile("lyxautoXXXXXX.lyx");
-       tempfile.setAutoRemove(false);
-       FileName const tmp_ret = tempfile.name();
-       if (!tmp_ret.empty()) {
-               if (!buffer_.writeFile(tmp_ret))
-                       failed = true;
-               else if (!tmp_ret.moveTo(fname_))
-                       failed = true;
-       } else
-               failed = true;
-
-       if (failed) {
-               // failed to write/rename tmp_ret so try writing direct
-               if (!buffer_.writeFile(fname_)) {
-                       // It is dangerous to do this in the child,
-                       // but safe in the parent, so...
-                       if (pid == -1) // emit message signal.
-                               buffer_.message(_("Autosave failed!"));
-               }
-       }
-
-       if (pid == 0) // we are the child so...
-               _exit(0);
-
-       return pid;
-}
-
-} // namespace
-
-
 FileName Buffer::getEmergencyFileName() const
 {
        return FileName(d->filename.absFileName() + ".emergency");
@@ -4416,7 +4339,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string & result_file) const
 {
        bool const update_unincluded =
-                       params().maintain_unincluded_children
+                       params().maintain_unincluded_children != BufferParams::CM_None
                        && !params().getIncludedChildren().empty();
 
        // (1) export with all included children (omit \includeonly)
@@ -4544,8 +4467,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                        return ExportKilled;
        } else if (backend_format == "lyx")
                writeFile(FileName(filename));
-       // Docbook backend
-       else if (params().isDocBook()) {
+       // DocBook backend
+       else if (backend_format == "docbook5") {
+               runparams.flavor = OutputParams::DOCBOOK5;
                runparams.nice = !put_in_tempdir;
                if (makeDocBookFile(FileName(filename), runparams) == ExportKilled)
                        return ExportKilled;
@@ -4587,8 +4511,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string const ext = theFormats().extension(format);
        FileName const tmp_result_file(changeExtension(filename, ext));
        Converters::RetVal const retval = 
-               converters.convert(this, FileName(filename), tmp_result_file, 
-               FileName(absFileName()), backend_format, format, error_list);
+               converters.convert(this, FileName(filename), tmp_result_file,
+                                  FileName(absFileName()), backend_format, format,
+                                  error_list, Converters::none, includeall);
        if (retval == Converters::KILLED)
                return ExportCancel;
        bool success = (retval == Converters::SUCCESS);
@@ -4703,7 +4628,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 Buffer::ExportStatus Buffer::preview(string const & format) const
 {
        bool const update_unincluded =
-                       params().maintain_unincluded_children
+                       params().maintain_unincluded_children != BufferParams::CM_None
                        && !params().getIncludedChildren().empty();
        return preview(format, update_unincluded);
 }
@@ -4923,7 +4848,7 @@ void Buffer::Impl::traverseErrors(TeXErrors::Errors::const_iterator err, TeXErro
        for (; err != end; ++err) {
                TexRow::TextEntry start = TexRow::text_none, end = TexRow::text_none;
                int errorRow = err->error_in_line;
-               Buffer const * buf = 0;
+               Buffer const * buf = nullptr;
                Impl const * p = this;
                if (err->child_name.empty())
                        tie(start, end) = p->texrow.getEntriesFromRow(errorRow);
@@ -5034,6 +4959,8 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 
        // do the real work
        ParIterator parit = cbuf.par_iterator_begin();
+       if (scope == UpdateMaster)
+               clearIncludeList();
        updateBuffer(parit, utype);
 
        // If this document has siblings, then update the TocBackend later. The
@@ -5076,6 +5003,7 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        }
        d->cite_labels_valid_ = true;
        /// FIXME: Perf
+       clearIncludeList();
        cbuf.tocBackend().update(true, utype);
        if (scope == UpdateMaster)
                cbuf.structureChanged();
@@ -5304,8 +5232,9 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
 }
 
 
-void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
+void Buffer::updateBuffer(ParIterator & parit, UpdateType utype, bool const deleted) const
 {
+       pushIncludedBuffer(this);
        // LASSERT: Is it safe to continue here, or should we just return?
        LASSERT(parit.pit() == 0, /**/);
 
@@ -5313,13 +5242,9 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
        // to resolve macros in it.
        parit.text()->setMacrocontextPosition(parit);
 
-       // Reset bibitem counter in master (#8499)
-       Buffer const * const master = masterBuffer();
-       if (master == this && !d->ignore_parent)
-               master->params().documentClass().counters().reset(from_ascii("bibitem"));
-
        depth_type maxdepth = 0;
        pit_type const lastpit = parit.lastpit();
+       bool changed = false;
        for ( ; parit.pit() <= lastpit ; ++parit.pit()) {
                // reduce depth if necessary
                if (parit->params().depth() > maxdepth) {
@@ -5349,9 +5274,18 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
                // now the insets
                for (auto const & insit : parit->insetList()) {
                        parit.pos() = insit.pos;
-                       insit.inset->updateBuffer(parit, utype);
+                       insit.inset->updateBuffer(parit, utype, deleted || parit->isDeleted(insit.pos));
+                       changed |= insit.inset->isChanged();
                }
+
+               // are there changes in this paragraph?
+               changed |= parit->isChanged();
        }
+
+       // set change indicator for the inset (or the cell that the iterator
+       // points to, if applicable).
+       parit.text()->inset().isChanged(changed);
+       popIncludedBuffer();
 }
 
 
@@ -5428,8 +5362,11 @@ void Buffer::Impl::updateStatistics(DocIterator & from, DocIterator & to, bool s
                                        ++word_count_;
                                        inword = true;
                                }
-                               if (ins && ins->isLetter())
-                                       ++char_count_;
+                               if (ins && ins->isLetter()) {
+                                       odocstringstream os;
+                                       ins->toString(os);
+                                       char_count_ += os.str().length();
+                               }
                                else if (ins && ins->isSpace())
                                        ++blank_count_;
                                else {
@@ -5464,6 +5401,12 @@ int Buffer::charCount(bool with_blanks) const
 }
 
 
+bool Buffer::areChangesPresent() const
+{
+       return inset().isChanged();
+}
+
+
 Buffer::ReadStatus Buffer::reload()
 {
        setBusy(true);
@@ -5474,7 +5417,7 @@ Buffer::ReadStatus Buffer::reload()
        docstring const disp_fn = makeDisplayPath(d->filename.absFileName());
 
        // clear parent. this will get reset if need be.
-       d->setParent(0);
+       d->setParent(nullptr);
        ReadStatus const status = loadLyXFile();
        if (status == ReadSuccess) {
                updateBuffer();
@@ -5548,8 +5491,8 @@ void Buffer::checkChildBuffers()
                if (oldloc == newloc)
                        continue;
                // the location of the child file is incorrect.
-               cbuf->setParent(0);
-               inset_inc->setChildBuffer(0);
+               cbuf->setParent(nullptr);
+               inset_inc->setChildBuffer(nullptr);
        }
        // invalidate cache of children
        d->children_positions.clear();
@@ -5574,7 +5517,7 @@ void Buffer::checkMasterBuffer()
        if (master->isChild(this))
                setParent(master);
        else
-               setParent(0);
+               setParent(nullptr);
 }
 
 
@@ -5662,4 +5605,55 @@ void Buffer::clearExternalModification() const
 }
 
 
+void Buffer::pushIncludedBuffer(Buffer const * buf) const
+{
+       masterBuffer()->d->include_list_.push_back(buf);
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Pushed. Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+}
+
+
+void Buffer::popIncludedBuffer() const
+{
+       masterBuffer()->d->include_list_.pop_back();
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Popped. Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+}
+
+
+bool Buffer::isBufferIncluded(Buffer const * buf) const
+{
+       if (!buf)
+               return false;
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Checking for " << buf->fileName() << ". Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+       list<Buffer const *> const & blist = masterBuffer()->d->include_list_;
+       return find(blist.begin(), blist.end(), buf) != blist.end();
+}
+
+
+void Buffer::clearIncludeList() const
+{
+       LYXERR(Debug::FILES, "Clearing include list for " << fileName());
+       d->include_list_.clear();
+}
+
 } // namespace lyx