]> git.lyx.org Git - lyx.git/blobdiff - src/Buffer.cpp
Attempt to fix bug #13017.
[lyx.git] / src / Buffer.cpp
index f1de2bafe4a0f33fc274efac527f658fb44445dd..7522dce4343938d433bf26fed31bada34d1be385 100644 (file)
@@ -14,7 +14,6 @@
 #include "Buffer.h"
 
 #include "Author.h"
-#include "LayoutFile.h"
 #include "BiblioInfo.h"
 #include "BranchList.h"
 #include "buffer_funcs.h"
@@ -22,6 +21,7 @@
 #include "BufferParams.h"
 #include "Bullet.h"
 #include "Chktex.h"
+#include "ColorSet.h"
 #include "Converter.h"
 #include "Counters.h"
 #include "Cursor.h"
 #include "LyX.h"
 #include "LyXRC.h"
 #include "LyXVC.h"
-#include "output_docbook.h"
 #include "output.h"
 #include "output_latex.h"
-#include "output_xhtml.h"
+#include "output_docbook.h"
 #include "output_plaintext.h"
+#include "output_xhtml.h"
 #include "Paragraph.h"
 #include "ParagraphParameters.h"
 #include "ParIterator.h"
 #include "PDFOptions.h"
 #include "Session.h"
 #include "SpellChecker.h"
-#include "sgml.h"
+#include "xml.h"
 #include "texstream.h"
 #include "TexRow.h"
 #include "Text.h"
 #include "VCBackend.h"
 #include "version.h"
 #include "WordLangTuple.h"
-#include "WordList.h"
 
-#include "insets/InsetBibtex.h"
 #include "insets/InsetBranch.h"
 #include "insets/InsetInclude.h"
-#include "insets/InsetTabular.h"
 #include "insets/InsetText.h"
 
 #include "mathed/InsetMathHull.h"
@@ -80,9 +77,9 @@
 #include "mathed/InsetMathMacroTemplate.h"
 #include "mathed/MathSupport.h"
 
-#include "graphics/GraphicsCache.h"
 #include "graphics/PreviewLoader.h"
 
+#include "frontends/Application.h"
 #include "frontends/alert.h"
 #include "frontends/Delegates.h"
 #include "frontends/WorkAreaManager.h"
 #include "support/FileName.h"
 #include "support/FileNameList.h"
 #include "support/filetools.h"
-#include "support/ForkedCalls.h"
 #include "support/gettext.h"
 #include "support/gzstream.h"
 #include "support/lstrings.h"
-#include "support/lyxalgo.h"
 #include "support/mutex.h"
 #include "support/os.h"
 #include "support/Package.h"
 #include "support/textutils.h"
 #include "support/types.h"
 
-#include "support/bind.h"
-
 #include <algorithm>
 #include <fstream>
 #include <iomanip>
@@ -152,11 +145,12 @@ typedef vector<LabelInfo> LabelCache;
 
 typedef map<docstring, Buffer::References> RefCache;
 
-} // namespace
+// A storehouse for the cloned buffers.
+typedef list<CloneList_ptr> CloneStore;
+CloneStore cloned_buffers;
 
+} // namespace
 
-// A storehouse for the cloned buffers.
-list<CloneList *> cloned_buffers;
 
 
 class Buffer::Impl
@@ -199,40 +193,9 @@ public:
        /// need to regenerate .tex?
        DepClean dep_clean;
 
-       /// is save needed?
-       mutable bool lyx_clean;
-
-       /// is autosave needed?
-       mutable bool bak_clean;
-
-       /// is this an unnamed file (New...)?
-       bool unnamed;
-
-       /// is this an internal bufffer?
-       bool internal_buffer;
-
-       /// buffer is r/o
-       bool read_only;
-
        /// name of the file the buffer is associated with.
        FileName filename;
 
-       /** Set to true only when the file is fully loaded.
-        *  Used to prevent the premature generation of previews
-        *  and by the citation inset.
-        */
-       bool file_fully_loaded;
-
-       /// original format of loaded file
-       int file_format;
-
-       /// if the file was originally loaded from an older format, do
-       /// we need to back it up still?
-       bool need_format_backup;
-
-       /// Ignore the parent (e.g. when exporting a child standalone)?
-       bool ignore_parent;
-
        ///
        mutable TocBackend toc_backend;
 
@@ -249,13 +212,11 @@ public:
        /// map from the macro name to the position map,
        /// which maps the macro definition position to the scope and the MacroData.
        NamePositionScopeMacroMap macros;
-       /// This seem to change the way Buffer::getMacro() works
-       mutable bool macro_lock;
 
        /// positions of child buffers in the buffer
        typedef map<Buffer const * const, DocIterator> BufferPositionMap;
        struct ScopeBuffer {
-               ScopeBuffer() : buffer(0) {}
+               ScopeBuffer() : buffer(nullptr) {}
                ScopeBuffer(DocIterator const & s, Buffer const * b)
                        : scope(s), buffer(b) {}
                DocIterator scope;
@@ -302,23 +263,13 @@ public:
        // file, and then to construct the Buffer's bibinfo from that.
        /// A cache for bibliography info
        mutable BiblioInfo bibinfo_;
-       /// whether the bibinfo cache is valid
-       mutable bool bibinfo_cache_valid_;
        /// Cache of timestamps of .bib files
        map<FileName, time_t> bibfile_status_;
-       /// Indicates whether the bibinfo has changed since the last time
-       /// we ran updateBuffer(), i.e., whether citation labels may need
-       /// to be updated.
-       mutable bool cite_labels_valid_;
 
        /// These two hold the file name and format, written to by
        /// Buffer::preview and read from by LFUN_BUFFER_VIEW_CACHE.
        FileName preview_file_;
        string preview_format_;
-       /// If there was an error when previewing, on the next preview we do
-       /// a fresh compile (e.g. in case the user installed a package that
-       /// was missing).
-       bool preview_error_;
 
        /// Cache the references associated to a label and their positions
        /// in the buffer.
@@ -332,6 +283,86 @@ public:
        ///
        PreviewLoader * preview_loader_;
 
+       /// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
+       /// This one is useful for preview detached in a thread.
+       Buffer const * cloned_buffer_;
+       ///
+       CloneList_ptr clone_list_;
+
+       ///
+       std::list<Buffer const *> include_list_;
+private:
+       /// So we can force access via the accessors.
+       mutable Buffer const * parent_buffer;
+
+       FileMonitorPtr file_monitor_;
+
+/// ints and bools are all listed last so as to avoid alignment issues
+public:
+       /// original format of loaded file
+       int file_format;
+
+       /// are we in the process of exporting this buffer?
+       mutable bool doing_export;
+
+       /// If there was an error when previewing, on the next preview we do
+       /// a fresh compile (e.g. in case the user installed a package that
+       /// was missing).
+       bool require_fresh_start_;
+
+       /// Indicates whether the bibinfo has changed since the last time
+       /// we ran updateBuffer(), i.e., whether citation labels may need
+       /// to be updated.
+       mutable bool cite_labels_valid_;
+       /// Do we have a bibliography environment?
+       mutable bool have_bibitems_;
+
+       /// is save needed?
+       mutable bool lyx_clean;
+
+       /// is autosave needed?
+       mutable bool bak_clean;
+
+       /// is this an unnamed file (New...)?
+       bool unnamed;
+
+       /// is this an internal bufffer?
+       bool internal_buffer;
+
+       /// buffer is r/o
+       bool read_only;
+
+       /** Set to true only when the file is fully loaded.
+        *  Used to prevent the premature generation of previews
+        *  and by the citation inset.
+        */
+       bool file_fully_loaded;
+
+       /// if the file was originally loaded from an older format, do
+       /// we need to back it up still?
+       bool need_format_backup;
+
+       /// Ignore the parent (e.g. when exporting a child standalone)?
+       bool ignore_parent;
+
+       /// This seem to change the way Buffer::getMacro() works
+       mutable bool macro_lock;
+
+       /// has been externally modified? Can be reset by the user.
+       mutable bool externally_modified_;
+
+       /// whether the bibinfo cache is valid
+       mutable bool bibinfo_cache_valid_;
+
+       ///
+       mutable bool need_update;
+
+private:
+       int word_count_;
+       int char_count_;
+       int blank_count_;
+
+public:
        /// This is here to force the test to be done whenever parent_buffer
        /// is accessed.
        Buffer const * parent() const
@@ -339,7 +370,7 @@ public:
                // ignore_parent temporarily "orphans" a buffer
                // (e.g. if a child is compiled standalone)
                if (ignore_parent)
-                       return 0;
+                       return nullptr;
                // if parent_buffer is not loaded, then it has been unloaded,
                // which means that parent_buffer is an invalid pointer. So we
                // set it to null in that case.
@@ -347,7 +378,7 @@ public:
                // they will always be regarded as unloaded. in that case, we hope
                // for the best.
                if (!cloned_buffer_ && !theBufferList().isLoaded(parent_buffer))
-                       parent_buffer = 0;
+                       parent_buffer = nullptr;
                return parent_buffer;
        }
 
@@ -364,14 +395,6 @@ public:
                        parent_buffer->invalidateBibinfoCache();
        }
 
-       /// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
-       /// This one is useful for preview detached in a thread.
-       Buffer const * cloned_buffer_;
-       ///
-       CloneList * clone_list_;
-       /// are we in the process of exporting this buffer?
-       mutable bool doing_export;
-
        /// compute statistics
        /// \p from initial position
        /// \p to points to the end position
@@ -388,28 +411,17 @@ public:
                + (with_blanks ? blank_count_ : 0);
        }
 
-       // does the buffer contain tracked changes? (if so, we automatically
-       // display the review toolbar, for instance)
-       mutable bool tracked_changes_present_;
-
        // Make sure the file monitor monitors the good file.
        void refreshFileMonitor();
 
        /// Notify or clear of external modification
        void fileExternallyModified(bool exists);
 
-       /// has been externally modified? Can be reset by the user.
-       mutable bool externally_modified_;
-
-private:
-       /// So we can force access via the accessors.
-       mutable Buffer const * parent_buffer;
-
-       int word_count_;
-       int char_count_;
-       int blank_count_;
-
-       FileMonitorPtr file_monitor_;
+       ///Binding LaTeX lines with buffer positions.
+       //Common routine for LaTeX and Reference errors listing.
+        void traverseErrors(TeXErrors::Errors::const_iterator err,
+               TeXErrors::Errors::const_iterator end,
+               ErrorList & errorList) const;
 };
 
 
@@ -442,21 +454,26 @@ static FileName createBufferTmpDir()
 
 Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
        Buffer const * cloned_buffer)
-       : owner_(owner), lyx_clean(true), bak_clean(true), unnamed(false),
-         internal_buffer(false), read_only(readonly_), filename(file),
-         file_fully_loaded(false), file_format(LYX_FORMAT), need_format_backup(false),
-         ignore_parent(false),  toc_backend(owner), macro_lock(false),
-         checksum_(0), wa_(0),  gui_(0), undo_(*owner), bibinfo_cache_valid_(false),
-         cite_labels_valid_(false), preview_error_(false),
-         inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
-         clone_list_(0), doing_export(false),
-         tracked_changes_present_(0), externally_modified_(false), parent_buffer(0),
-         word_count_(0), char_count_(0), blank_count_(0)
+       : owner_(owner), filename(file), toc_backend(owner), checksum_(0),
+         wa_(nullptr),  gui_(nullptr), undo_(*owner), inset(nullptr),
+         preview_loader_(nullptr), cloned_buffer_(cloned_buffer),
+         clone_list_(nullptr), parent_buffer(nullptr), file_format(LYX_FORMAT),
+         doing_export(false), require_fresh_start_(false), cite_labels_valid_(false),
+         have_bibitems_(false), lyx_clean(true), bak_clean(true), unnamed(false),
+         internal_buffer(false), read_only(readonly_), file_fully_loaded(false),
+         need_format_backup(false), ignore_parent(false), macro_lock(false),
+         externally_modified_(false), bibinfo_cache_valid_(false),
+         need_update(false), word_count_(0), char_count_(0), blank_count_(0)
 {
        refreshFileMonitor();
        if (!cloned_buffer_) {
                temppath = createBufferTmpDir();
                lyxvc.setBuffer(owner_);
+               Language const * inplang = theApp() ?
+                                       languages.getFromCode(theApp()->inputLanguageCode())
+                                     : nullptr;
+               if (inplang)
+                       params.language = inplang;
                if (use_gui)
                        wa_ = new frontend::WorkAreaManager;
                return;
@@ -469,13 +486,13 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
        bibinfo_cache_valid_ = cloned_buffer_->d->bibinfo_cache_valid_;
        bibfile_status_ = cloned_buffer_->d->bibfile_status_;
        cite_labels_valid_ = cloned_buffer_->d->cite_labels_valid_;
+       have_bibitems_ = cloned_buffer_->d->have_bibitems_;
        unnamed = cloned_buffer_->d->unnamed;
        internal_buffer = cloned_buffer_->d->internal_buffer;
        layout_position = cloned_buffer_->d->layout_position;
        preview_file_ = cloned_buffer_->d->preview_file_;
        preview_format_ = cloned_buffer_->d->preview_format_;
-       preview_error_ = cloned_buffer_->d->preview_error_;
-       tracked_changes_present_ = cloned_buffer_->d->tracked_changes_present_;
+       require_fresh_start_ = cloned_buffer_->d->require_fresh_start_;
 }
 
 
@@ -505,7 +522,7 @@ Buffer::~Buffer()
        // saved properly, before it goes into the void.
 
        // GuiView already destroyed
-       d->gui_ = 0;
+       d->gui_ = nullptr;
 
        if (isInternal()) {
                // No need to do additional cleanups for internal buffer.
@@ -520,8 +537,8 @@ Buffer::~Buffer()
                // loop over children
                for (auto const & p : d->children_positions) {
                        Buffer * child = const_cast<Buffer *>(p.first);
-                               if (d->clone_list_->erase(child))
-                                       delete child;
+                       if (d->clone_list_->erase(child))
+                               delete child;
                }
                // if we're the master buffer, then we should get rid of the list
                // of clones
@@ -530,14 +547,15 @@ Buffer::~Buffer()
                        // children still has a reference to this list. But we will try to
                        // continue, rather than shut down.
                        LATTEST(d->clone_list_->empty());
-                       list<CloneList *>::iterator it =
+                       // The clone list itself is empty, but it's still referenced in our list
+                       // of clones. So let's find it and remove it.
+                       CloneStore::iterator it =
                                find(cloned_buffers.begin(), cloned_buffers.end(), d->clone_list_);
                        if (it == cloned_buffers.end()) {
                                // We will leak in this case, but it is safe to continue.
                                LATTEST(false);
                        } else
                                cloned_buffers.erase(it);
-                       delete d->clone_list_;
                }
                // FIXME Do we really need to do this right before we delete d?
                // clear references to children in macro tables
@@ -549,7 +567,7 @@ Buffer::~Buffer()
                        Buffer * child = const_cast<Buffer *>(p.first);
                        if (theBufferList().isLoaded(child)) {
                                if (theBufferList().isOthersChild(this, child))
-                                       child->setParent(0);
+                                       child->setParent(nullptr);
                                else
                                        theBufferList().release(child);
                        }
@@ -584,21 +602,21 @@ Buffer::~Buffer()
 Buffer * Buffer::cloneWithChildren() const
 {
        BufferMap bufmap;
-       cloned_buffers.push_back(new CloneList);
-       CloneList * clones = cloned_buffers.back();
+       cloned_buffers.emplace_back(new CloneList);
+       CloneList_ptr clones = cloned_buffers.back();
 
        cloneWithChildren(bufmap, clones);
 
        // make sure we got cloned
        BufferMap::const_iterator bit = bufmap.find(this);
-       LASSERT(bit != bufmap.end(), return 0);
+       LASSERT(bit != bufmap.end(), return nullptr);
        Buffer * cloned_buffer = bit->second;
 
        return cloned_buffer;
 }
 
 
-void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
+void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList_ptr clones) const
 {
        // have we already been cloned?
        if (bufmap.find(this) != bufmap.end())
@@ -609,10 +627,9 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
        // The clone needs its own DocumentClass, since running updateBuffer() will
        // modify it, and we would otherwise be sharing it with the original Buffer.
        buffer_clone->params().makeDocumentClass(true);
-       ErrorList el;
        cap::switchBetweenClasses(
                        params().documentClassPtr(), buffer_clone->params().documentClassPtr(),
-                       static_cast<InsetText &>(buffer_clone->inset()), el);
+                       static_cast<InsetText &>(buffer_clone->inset()));
 
        bufmap[this] = buffer_clone;
        clones->insert(buffer_clone);
@@ -643,22 +660,20 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
                buffer_clone->setChild(dit, child_clone);
        }
        buffer_clone->d->macro_lock = false;
-       return;
 }
 
 
 Buffer * Buffer::cloneBufferOnly() const {
-       cloned_buffers.push_back(new CloneList);
-       CloneList * clones = cloned_buffers.back();
+       cloned_buffers.emplace_back(new CloneList);
+       CloneList_ptr clones = cloned_buffers.back();
        Buffer * buffer_clone = new Buffer(fileName().absFileName(), false, this);
 
        // The clone needs its own DocumentClass, since running updateBuffer() will
        // modify it, and we would otherwise be sharing it with the original Buffer.
        buffer_clone->params().makeDocumentClass(true);
-       ErrorList el;
        cap::switchBetweenClasses(
                        params().documentClassPtr(), buffer_clone->params().documentClassPtr(),
-                       static_cast<InsetText &>(buffer_clone->inset()), el);
+                       static_cast<InsetText &>(buffer_clone->inset()));
 
        clones->insert(buffer_clone);
        buffer_clone->d->clone_list_ = clones;
@@ -787,6 +802,12 @@ Undo & Buffer::undo()
 }
 
 
+Undo const & Buffer::undo() const
+{
+       return d->undo_;
+}
+
+
 void Buffer::setChild(DocIterator const & dit, Buffer * child)
 {
        d->children_positions[child] = dit;
@@ -809,10 +830,9 @@ FileName Buffer::Impl::exportFileName() const
        if (branch_suffix.empty())
                return filename;
 
-       string const name = filename.onlyFileNameWithoutExt()
-               + to_utf8(branch_suffix);
+       string const name = addExtension(filename.onlyFileNameWithoutExt()
+                       + to_utf8(branch_suffix), filename.extension());
        FileName res(filename.onlyPath().absFileName() + "/" + name);
-       res.changeExtension(filename.extension());
 
        return res;
 }
@@ -918,19 +938,25 @@ int Buffer::readHeader(Lexer & lex)
        params().headsep.erase();
        params().footskip.erase();
        params().columnsep.erase();
+       params().font_roman_opts.erase();
+       params().font_sans_opts.erase();
+       params().font_typewriter_opts.erase();
        params().fonts_cjk.erase();
        params().listings_params.clear();
        params().clearLayoutModules();
        params().clearRemovedModules();
        params().clearIncludedChildren();
        params().pdfoptions().clear();
+       params().document_metadata.clear();
        params().indiceslist().clear();
        params().backgroundcolor = lyx::rgbFromHexName("#ffffff");
        params().isbackgroundcolor = false;
        params().fontcolor = RGBColor(0, 0, 0);
        params().isfontcolor = false;
        params().notefontcolor = RGBColor(0xCC, 0xCC, 0xCC);
+       params().isnotefontcolor = false;
        params().boxbgcolor = RGBColor(0xFF, 0, 0);
+       params().isboxbgcolor = false;
        params().html_latex_start.clear();
        params().html_latex_end.clear();
        params().html_math_img_scale = 1.0;
@@ -941,6 +967,8 @@ int Buffer::readHeader(Lexer & lex)
        params().biblatex_bibstyle.erase();
        params().biblatex_citestyle.erase();
        params().multibib.erase();
+       params().lineno_opts.clear();
+       params().spellignore().clear();
 
        for (int i = 0; i < 4; ++i) {
                params().user_defined_bullet(i) = ITEMIZE_DEFAULTS[i];
@@ -969,7 +997,7 @@ int Buffer::readHeader(Lexer & lex)
                                      << token << '\'');
 
                string const result =
-                       params().readToken(lex, token, d->filename.onlyPath());
+                       params().readToken(lex, token, d->filename);
                if (!result.empty()) {
                        if (token == "\\textclass") {
                                d->layout_position = result;
@@ -990,7 +1018,7 @@ int Buffer::readHeader(Lexer & lex)
 
        params().shell_escape = theSession().shellescapeFiles().find(absFileName());
 
-       params().makeDocumentClass();
+       params().makeDocumentClass(isClone(), isInternal());
 
        return unknown_tokens;
 }
@@ -1015,20 +1043,13 @@ bool Buffer::readDocument(Lexer & lex)
        readHeader(lex);
 
        if (params().output_changes) {
-               bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
                bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
                                  LaTeXFeatures::isAvailable("xcolor");
 
-               if (!dvipost && !xcolorulem) {
+               if (!xcolorulem) {
                        Alert::warning(_("Changes not shown in LaTeX output"),
                                       _("Changes will not be highlighted in LaTeX output, "
-                                        "because neither dvipost nor xcolor/ulem are installed.\n"
-                                        "Please install these packages or redefine "
-                                        "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
-               } else if (!xcolorulem) {
-                       Alert::warning(_("Changes not shown in LaTeX output"),
-                                      _("Changes will not be highlighted in LaTeX output "
-                                        "when using pdflatex, because xcolor and ulem are not installed.\n"
+                                        "because xcolor and ulem are not installed.\n"
                                         "Please install both packages or redefine "
                                         "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
                }
@@ -1076,6 +1097,10 @@ bool Buffer::readDocument(Lexer & lex)
                d->old_position = params().origin;
        else
                d->old_position = filePath();
+
+       if (!parent())
+               clearIncludeList();
+
        bool const res = text().read(lex, errorList, d->inset);
        d->old_position.clear();
 
@@ -1092,6 +1117,24 @@ bool Buffer::readDocument(Lexer & lex)
 }
 
 
+bool Buffer::isSyncTeXenabled() const
+{
+       bool enabled = params().output_sync;
+
+       if (!enabled)
+               for (auto const & c : theConverters()) {
+                       const string dest = c.to().substr(0,3);
+                       if (dest == "dvi" || dest == "pdf") {
+                               const string cmd = c.command();
+                               enabled |= cmd.find("-synctex=") != string::npos
+                                       && cmd.find("-synctex=0") == string::npos;
+                               if (enabled)
+                                       break;
+                       }
+               }
+       return enabled;
+}
+
 bool Buffer::importString(string const & format, docstring const & contents, ErrorList & errorList)
 {
        Format const * fmt = theFormats().getFormat(format);
@@ -1127,7 +1170,7 @@ bool Buffer::importFile(string const & format, FileName const & name, ErrorList
 
        FileName const lyx = tempFileName("Buffer_importFileXXXXXX.lyx");
        Converters::RetVal const retval =
-           theConverters().convert(0, name, lyx, name, format, "lyx", errorList);
+               theConverters().convert(this, name, lyx, name, format, "lyx", errorList);
        if (retval == Converters::SUCCESS) {
                bool const success = readFile(lyx) == ReadSuccess;
                removeTempFile(lyx);
@@ -1169,9 +1212,8 @@ bool Buffer::readString(string const & s)
 
 Buffer::ReadStatus Buffer::readFile(FileName const & fn)
 {
-       FileName fname(fn);
        Lexer lex;
-       if (!lex.setFile(fname)) {
+       if (!lex.setFile(fn)) {
                Alert::error(_("File Not Found"),
                        bformat(_("Unable to open file `%1$s'."),
                                from_utf8(fn.absFileName())));
@@ -1228,16 +1270,21 @@ void Buffer::setFullyLoaded(bool value)
 }
 
 
-bool Buffer::lastPreviewError() const
+bool Buffer::freshStartRequired() const
+{
+       return d->require_fresh_start_;
+}
+
+void Buffer::requireFreshStart(bool const b) const
 {
-       return d->preview_error_;
+       d->require_fresh_start_ = b;
 }
 
 
 PreviewLoader * Buffer::loader() const
 {
        if (!isExporting() && lyxrc.preview == LyXRC::PREVIEW_OFF)
-               return 0;
+               return nullptr;
        if (!d->preview_loader_)
                d->preview_loader_ = new PreviewLoader(*this);
        return d->preview_loader_;
@@ -1247,7 +1294,7 @@ PreviewLoader * Buffer::loader() const
 void Buffer::removePreviews() const
 {
        delete d->preview_loader_;
-       d->preview_loader_ = 0;
+       d->preview_loader_ = nullptr;
 }
 
 
@@ -1257,9 +1304,9 @@ void Buffer::updatePreviews() const
        if (!ploader)
                return;
 
-       InsetIterator it = inset_iterator_begin(*d->inset);
-       InsetIterator const end = inset_iterator_end(*d->inset);
-       for (; it != end; ++it)
+       InsetIterator it = begin(*d->inset);
+       InsetIterator const itend = end(*d->inset);
+       for (; it != itend; ++it)
                it->addPreview(it, *ploader);
 
        ploader->startLoading();
@@ -1331,7 +1378,7 @@ Buffer::ReadStatus Buffer::convertLyXFormat(FileName const & fn,
        LYXERR(Debug::INFO, "Running '" << command_str << '\'');
 
        cmd_ret const ret = runCommand(command_str);
-       if (ret.first != 0) {
+       if (!ret.valid) {
                if (from_format < LYX_FORMAT) {
                        Alert::error(_("Conversion script failed"),
                                bformat(_("%1$s is from an older version"
@@ -1454,7 +1501,7 @@ bool Buffer::save() const
        // proper location once that has been done successfully. that
        // way we preserve the original file if something goes wrong.
        string const justname = fileName().onlyFileNameWithoutExt();
-       auto tempfile = make_unique<TempFile>(fileName().onlyPath(),
+       auto tempfile = lyx::make_unique<TempFile>(fileName().onlyPath(),
                                              justname + "-XXXXXX.lyx");
        bool const symlink = fileName().isSymLink();
        if (!symlink)
@@ -1523,9 +1570,8 @@ bool Buffer::save() const
                // time stamp is invalidated by copying/moving
                saveCheckSum();
                markClean();
-               if (d->file_format != LYX_FORMAT)
-                       // the file associated with this buffer is now in the current format
-                       d->file_format = LYX_FORMAT;
+               // the file associated with this buffer is now in the current format
+               d->file_format = LYX_FORMAT;
                return true;
        }
        // else we saved the file, but failed to move it to the right location.
@@ -1592,7 +1638,7 @@ bool Buffer::writeFile(FileName const & fname) const
 }
 
 
-docstring Buffer::emergencyWrite()
+docstring Buffer::emergencyWrite() const
 {
        // No need to save if the buffer has not changed.
        if (isClean())
@@ -1717,23 +1763,18 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
 {
        OutputParams runparams = runparams_in;
 
-       // XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740),
-       // Check here, because the "flavor" is not known in BufferParams::encoding()
-       // (power users can override this safety measure selecting "utf8-plain").
-       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
-           && (runparams.encoding->name() != "utf8-plain"))
-               runparams.encoding = encodings.fromLyXName("ascii");
-
        string const encoding = runparams.encoding->iconvName();
-       LYXERR(Debug::LATEX, "makeLaTeXFile encoding: " << encoding << ", fname=" << fname.realPath());
+       LYXERR(Debug::OUTFILE, "makeLaTeXFile encoding: " << encoding << ", fname=" << fname.realPath());
 
        ofdocstream ofs;
        try { ofs.reset(encoding); }
        catch (iconv_codecvt_facet_exception const & e) {
                lyxerr << "Caught iconv exception: " << e.what() << endl;
-               Alert::error(_("Iconv software exception Detected"), bformat(_("Please "
-                       "verify that the support software for your encoding (%1$s) is "
-                       "properly installed"), from_ascii(encoding)));
+               Alert::error(_("Iconv software exception Detected"),
+                       bformat(_("Please verify that the `iconv' support software is"
+                                         " properly installed and supports the selected encoding"
+                                         " (%1$s), or change the encoding in"
+                                         " Document>Settings>Language."), from_ascii(encoding)));
                return ExportError;
        }
        if (!openFileWrite(ofs, fname))
@@ -1760,7 +1801,7 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
        catch (EncodingException const & e) {
                docstring const failed(1, e.failed_char);
                ostringstream oss;
-               oss << "0x" << hex << e.failed_char << dec;
+               oss << "0x" << hex << static_cast<uint32_t>(e.failed_char) << dec;
                if (getParFromID(e.par_id).paragraph().layout().pass_thru) {
                        docstring msg = bformat(_("Uncodable character '%1$s'"
                                                  " (code point %2$s)"),
@@ -1796,7 +1837,7 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
                lyx_exit(1);
        }
 
-       d->texrow = move(os.texrow());
+       d->texrow = std::move(os.texrow());
 
        ofs.close();
        if (ofs.fail()) {
@@ -1821,16 +1862,6 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 
        OutputParams runparams = runparams_in;
 
-       // XeTeX with TeX fonts is only safe with ASCII encoding,
-       // Check here, because the "flavor" is not known in BufferParams::encoding()
-       // (power users can override this safety measure selecting "utf8-plain").
-       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
-           && (runparams.encoding->name() != "utf8-plain"))
-               runparams.encoding = encodings.fromLyXName("ascii");
-       // FIXME: when only the current paragraph is shown, this is ignored
-       //        (or not reached) and characters encodable in the current
-       //        encoding are not converted to ASCII-representation.
-
        // Some macros rely on font encoding
        runparams.main_fontenc = params().main_font_encoding();
 
@@ -1845,13 +1876,16 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
        BufferEncodings::initUnicodeMath(*this);
 
        // validate the buffer.
-       LYXERR(Debug::LATEX, "  Validating buffer...");
+       LYXERR(Debug::OUTFILE, "  Validating buffer...");
        LaTeXFeatures features(*this, params(), runparams);
        validate(features);
        // This is only set once per document (in master)
-       if (!runparams.is_child)
+       if (!runparams.is_child) {
                runparams.use_polyglossia = features.usePolyglossia();
-       LYXERR(Debug::LATEX, "  Buffer validation done.");
+               runparams.use_hyperref = features.isRequired("hyperref");
+               runparams.use_CJK = features.mustProvide("CJK");
+       }
+       LYXERR(Debug::OUTFILE, "  Buffer validation done.");
 
        bool const output_preamble =
                output == FullSource || output == OnlyPreamble;
@@ -1891,9 +1925,9 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
                        docstring uncodable_glyphs;
                        Encoding const * const enc = runparams.encoding;
                        if (enc) {
-                               for (size_t n = 0; n < inputpath.size(); ++n) {
-                                       if (!enc->encodable(inputpath[n])) {
-                                               docstring const glyph(1, inputpath[n]);
+                               for (char_type n : inputpath) {
+                                       if (!enc->encodable(n)) {
+                                               docstring const glyph(1, n);
                                                LYXERR0("Uncodable character '"
                                                        << glyph
                                                        << "' in input path!");
@@ -1935,8 +1969,15 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
                                        os << "\\catcode`\\%=11"
                                              "\\def\\%{%}\\catcode`\\%=14\n";
                                }
-                               bool const detokenize = !isAscii(from_utf8(docdir));
+                               if (contains(docdir, '~'))
+                                       docdir = subst(docdir, "~", "\\string~");
+                               bool const nonascii = !isAscii(from_utf8(docdir));
+                               // LaTeX 2019/10/01 handles non-ascii path without detokenize
+                               bool const utfpathlatex = features.isAvailableAtLeastFrom("LaTeX", 2019, 10);
+                               bool const detokenize = !utfpathlatex && nonascii;
                                bool const quote = contains(docdir, ' ');
+                               if (utfpathlatex && nonascii)
+                                       os << "\\UseRawInputEncoding\n";
                                os << "\\makeatletter\n"
                                   << "\\def\\input@path{{";
                                if (detokenize)
@@ -1962,12 +2003,19 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
                runparams.use_babel = params().writeLaTeX(os, features,
                                                          d->filename.onlyPath());
 
+               // Active characters
+               runparams.active_chars = features.getActiveChars();
+
                // Biblatex bibliographies are loaded here
                if (params().useBiblatex()) {
-                       vector<docstring> const bibfiles =
+                       vector<pair<docstring, string>> const bibfiles =
                                prepareBibFilePaths(runparams, getBibfiles(), true);
-                       for (docstring const & file: bibfiles)
-                               os << "\\addbibresource{" << file << "}\n";
+                       for (pair<docstring, string> const & file: bibfiles) {
+                               os << "\\addbibresource";
+                               if (!file.second.empty())
+                                       os << "[bibencoding=" << file.second << "]";
+                               os << "{" << file.first << "}\n";
+                       }
                }
 
                if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages()
@@ -2056,9 +2104,9 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 
        if (output_preamble) {
                os << "\\end{document}\n";
-               LYXERR(Debug::LATEX, "makeLaTeXFile...done");
+               LYXERR(Debug::OUTFILE, "makeLaTeXFile...done");
        } else {
-               LYXERR(Debug::LATEX, "LaTeXFile for inclusion made.");
+               LYXERR(Debug::OUTFILE, "LaTeXFile for inclusion made.");
        }
        runparams_in.encoding = runparams.encoding;
 
@@ -2072,7 +2120,7 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
                              OutputParams const & runparams,
                              OutputWhat output) const
 {
-       LYXERR(Debug::LATEX, "makeDocBookFile...");
+       LYXERR(Debug::OUTFILE, "makeDocBookFile...");
 
        ofdocstream ofs;
        if (!openFileWrite(ofs, fname))
@@ -2083,8 +2131,7 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
        updateBuffer();
        updateMacroInstances(OutputUpdate);
 
-       ExportStatus const retval =
-               writeDocBookSource(ofs, fname.absFileName(), runparams, output);
+       ExportStatus const retval = writeDocBookSource(ofs, runparams, output);
        if (retval == ExportKilled)
                return ExportKilled;
 
@@ -2095,86 +2142,73 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
 }
 
 
-Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os, string const & fname,
+Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os,
                             OutputParams const & runparams,
                             OutputWhat output) const
 {
        LaTeXFeatures features(*this, params(), runparams);
        validate(features);
+       d->bibinfo_.makeCitationLabels(*this);
 
        d->texrow.reset();
 
        DocumentClass const & tclass = params().documentClass();
-       string const & top_element = tclass.latexname();
 
        bool const output_preamble =
                output == FullSource || output == OnlyPreamble;
        bool const output_body =
-         output == FullSource || output == OnlyBody;
+               output == FullSource || output == OnlyBody || output == IncludedFile;
 
        if (output_preamble) {
-               if (runparams.flavor == OutputParams::XML)
-                       os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
-
-               // FIXME UNICODE
-               os << "<!DOCTYPE " << from_ascii(top_element) << ' ';
-
-               // FIXME UNICODE
-               if (! tclass.class_header().empty())
-                       os << from_ascii(tclass.class_header());
-               else if (runparams.flavor == OutputParams::XML)
-                       os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
-                           << "\"https://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
-               else
-                       os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
-
-               docstring preamble = params().preamble;
-               if (runparams.flavor != OutputParams::XML ) {
-                       preamble += "<!ENTITY % output.print.png \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.pdf \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.eps \"IGNORE\">\n";
-                       preamble += "<!ENTITY % output.print.bmp \"IGNORE\">\n";
+               // XML preamble, no doctype needed.
+               // Not using XMLStream for this, as the root tag would be in the tag stack and make troubles with the error
+               // detection mechanisms (these are called before the end tag is output, and thus interact with the canary
+               // parsep in output_docbook.cpp).
+               os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                  << "<!-- This DocBook file was created by LyX " << lyx_version
+                  << "\n  See https://www.lyx.org/ for more information -->\n";
+
+               // Prepare the name space declaration for MathML depending on document preferences.
+               string mathmlNamespace;
+               if (params().docbook_mathml_prefix != BufferParams::NoPrefix) {
+                       string mathmlPrefix;
+                       if (params().docbook_mathml_prefix == BufferParams::MPrefix)
+                               mathmlPrefix = "m";
+                       else if (params().docbook_mathml_prefix == BufferParams::MMLPrefix)
+                               mathmlPrefix = "mml";
+                       mathmlNamespace = + " xmlns:" + mathmlPrefix + "=\"http://www.w3.org/1998/Math/MathML\"";
                }
 
-               string const name = runparams.nice
-                       ? changeExtension(absFileName(), ".sgml") : fname;
-               preamble += features.getIncludedFiles(name);
-               preamble += features.getLyXSGMLEntities();
+               // XML-compatible language code: in lib/languages, language codes are
+               // given as dictionary file names; HTML5 expects to follow BCP47. This
+               // function implements a simple heuristic that does the conversion.
+               std::string htmlCode = params().language->code();
+               std::replace(htmlCode.begin(), htmlCode.end(), '_', '-');
 
-               if (!preamble.empty()) {
-                       os << "\n [ " << preamble << " ]";
-               }
-               os << ">\n\n";
+               // Directly output the root tag, based on the current type of document.
+               string attributes = "xml:lang=\"" + htmlCode + '"'
+                                                   + " xmlns=\"http://docbook.org/ns/docbook\""
+                                                   + " xmlns:xlink=\"http://www.w3.org/1999/xlink\""
+                                                   + mathmlNamespace
+                                                   + " xmlns:xi=\"http://www.w3.org/2001/XInclude\""
+                                                   + " version=\"5.2\"";
+               // Version 5.2 is required for formalgroup.
+
+               os << "<" << from_ascii(tclass.docbookroot()) << " " << from_ascii(attributes) << ">\n";
        }
 
        if (output_body) {
-               string top = top_element;
-               top += " lang=\"";
-               if (runparams.flavor == OutputParams::XML)
-                       top += params().language->code();
-               else
-                       top += params().language->code().substr(0, 2);
-               top += '"';
-
-               if (!params().options.empty()) {
-                       top += ' ';
-                       top += params().options;
-               }
-
-               os << "<!-- " << ((runparams.flavor == OutputParams::XML)? "XML" : "SGML")
-                               << " file was created by LyX " << lyx_version
-                               << "\n  See https://www.lyx.org/ for more information -->\n";
-
-               params().documentClass().counters().reset();
+               // Start to output the document.
+               XMLStream xs(os);
+               docbookParagraphs(text(), *this, xs, runparams);
+       }
 
-               sgml::openTag(os, top);
-               os << '\n';
-               try {
-                       docbookParagraphs(text(), *this, os, runparams);
-               }
-               catch (ConversionException const &) { return ExportKilled; }
-               sgml::closeTag(os, top_element);
+       if (output_preamble) {
+               // Close the root element. No need for a line break, as free text is never allowed
+               // in a root element, it must always be wrapped in some container.
+               os << "</" << from_ascii(tclass.docbookroot()) << ">";
        }
+
        return ExportSuccess;
 }
 
@@ -2182,7 +2216,7 @@ Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os, string const &
 Buffer::ExportStatus Buffer::makeLyXHTMLFile(FileName const & fname,
                              OutputParams const & runparams) const
 {
-       LYXERR(Debug::LATEX, "makeLyXHTMLFile...");
+       LYXERR(Debug::OUTFILE, "makeLyXHTMLFile...");
 
        ofdocstream ofs;
        if (!openFileWrite(ofs, fname))
@@ -2218,30 +2252,28 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
          output == FullSource || output == OnlyBody || output == IncludedFile;
 
        if (output_preamble) {
-               os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
-                  << "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN\" \"http://www.w3.org/Math/DTD/mathml2/xhtml-math11-f.dtd\">\n"
-                  // FIXME Language should be set properly.
-                  << "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n"
+               // HTML5-compatible language code: in lib/languages, language codes are
+               // given as dictionary file names; HTML5 expects to follow BCP47. This
+               // function implements a simple heuristic that does the conversion.
+               std::string htmlCode = params().language->code();
+               std::replace(htmlCode.begin(), htmlCode.end(), '_', '-');
+
+               os << "<!DOCTYPE html>\n"
+                  << "<html xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"" << from_ascii(htmlCode) << "\">\n"
                   << "<head>\n"
-                  << "<meta name=\"GENERATOR\" content=\"" << PACKAGE_STRING << "\" />\n"
-                  // FIXME Presumably need to set this right
-                  << "<meta http-equiv=\"Content-type\" content=\"text/html;charset=UTF-8\" />\n";
+                  << "<meta name=\"generator\" content=\"" << PACKAGE_STRING << "\" />\n";
 
                docstring const & doctitle = features.htmlTitle();
                os << "<title>"
                   << (doctitle.empty() ?
                         from_ascii("LyX Document") :
-                        html::htmlize(doctitle, XHTMLStream::ESCAPE_ALL))
+                        xml::escapeString(doctitle, XMLStream::ESCAPE_ALL))
                   << "</title>\n";
 
                docstring styles = features.getTClassHTMLPreamble();
                if (!styles.empty())
                        os << "\n<!-- Text Class Preamble -->\n" << styles << '\n';
 
-               styles = features.getPreambleSnippets().str;
-               if (!styles.empty())
-                       os << "\n<!-- Preamble Snippets -->\n" << styles << '\n';
-
                // we will collect CSS information in a stream, and then output it
                // either here, as part of the header, or else in a separate file.
                odocstringstream css;
@@ -2289,7 +2321,7 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                        // we are here if the CSS is supposed to be written to the header
                        // or if we failed to write it to an external file.
                        if (!written) {
-                               os << "<style type='text/css'>\n"
+                               os << "<style>\n"
                                         << dstyles
                                         << "\n</style>\n";
                        }
@@ -2301,7 +2333,7 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                bool const output_body_tag = (output != IncludedFile);
                if (output_body_tag)
                        os << "<body dir=\"auto\">\n";
-               XHTMLStream xs(os);
+               XMLStream xs(os);
                if (output != IncludedFile)
                        // if we're an included file, the counters are in the master.
                        params().documentClass().counters().reset();
@@ -2336,7 +2368,7 @@ int Buffer::runChktex()
 
        // Generate the LaTeX file if neccessary
        OutputParams runparams(&params().encoding());
-       runparams.flavor = OutputParams::LATEX;
+       runparams.flavor = Flavor::LaTeX;
        runparams.nice = false;
        runparams.linelen = lyxrc.plaintext_linelen;
        ExportStatus const retval =
@@ -2378,16 +2410,19 @@ void Buffer::validate(LaTeXFeatures & features) const
        if (!features.runparams().is_child)
                params().validate(features);
 
+       if (!parent())
+               clearIncludeList();
+
        for (Paragraph const & p : paragraphs())
                p.validate(features);
 
-       if (lyxerr.debugging(Debug::LATEX)) {
+       if (lyxerr.debugging(Debug::OUTFILE)) {
                features.showStruct();
        }
 }
 
 
-void Buffer::getLabelList(vector<docstring> & list) const
+void Buffer::getLabelList(vector<std::tuple<docstring, docstring, docstring>> & list) const
 {
        // If this is a child document, use the master's list instead.
        if (parent()) {
@@ -2398,8 +2433,9 @@ void Buffer::getLabelList(vector<docstring> & list) const
        list.clear();
        shared_ptr<Toc> toc = d->toc_backend.toc("label");
        for (auto const & tocit : *toc) {
-               if (tocit.depth() == 0)
-                       list.push_back(tocit.str());
+               if (tocit.depth() == 0) {
+                       list.push_back(make_tuple(tocit.str(), tocit.asString(), tocit.prettyStr()));
+               }
        }
 }
 
@@ -2474,17 +2510,25 @@ FileName Buffer::getBibfilePath(docstring const & bibid) const
        map<docstring, FileName>::const_iterator it =
                bibfileCache.find(bibid);
        if (it != bibfileCache.end()) {
-               // i.e., bibfileCache[bibid]
+               // i.e., return bibfileCache[bibid];
                return it->second;
        }
 
        LYXERR(Debug::FILES, "Reading file location for " << bibid);
-       string texfile = changeExtension(to_utf8(bibid), "bib");
-       // note that, if the filename can be found directly from the path,
-       // findtexfile will just return a FileName object for that path.
-       FileName file(findtexfile(texfile, "bib"));
-       if (file.empty())
-               file = FileName(makeAbsPath(texfile, filePath()));
+       string const texfile = changeExtension(to_utf8(bibid), "bib");
+       // we need to check first if this file exists where it's said to be.
+       // there's a weird bug that occurs otherwise: if the file is in the
+       // Buffer's directory but has the same name as some file that would be
+       // found by kpsewhich, then we find the latter, not the former.
+       FileName const local_file = makeAbsPath(texfile, filePath());
+       FileName file = local_file;
+       if (!file.exists()) {
+               // there's no need now to check whether the file can be found
+               // locally
+               file = findtexfile(texfile, "bib", true);
+               if (file.empty())
+                       file = local_file;
+       }
        LYXERR(Debug::FILES, "Found at: " << file);
 
        bibfileCache[bibid] = file;
@@ -2513,7 +2557,16 @@ void Buffer::checkIfBibInfoCacheIsValid() const
        if (!d->bibinfo_cache_valid_)
                return;
 
-       // we'll assume it's ok and change this if it's not
+       if (d->have_bibitems_) {
+               // We have a bibliography environment.
+               // Invalidate the bibinfo cache unconditionally.
+               // Cite labels will get invalidated by the inset if needed.
+               d->bibinfo_cache_valid_ = false;
+               return;
+       }
+
+       // OK. This is with Bib(la)tex. We'll assume the cache
+       // is valid and change this if we find changes in the bibs.
        d->bibinfo_cache_valid_ = true;
        d->cite_labels_valid_ = true;
 
@@ -2565,6 +2618,7 @@ void Buffer::reloadBibInfoCache(bool const force) const
        clearBibFileCache();
        d->bibinfo_.clear();
        FileNameList checkedFiles;
+       d->have_bibitems_ = false;
        collectBibKeys(checkedFiles);
        d->bibinfo_cache_valid_ = true;
 }
@@ -2572,23 +2626,31 @@ void Buffer::reloadBibInfoCache(bool const force) const
 
 void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 {
-       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it)
+       if (!parent())
+               clearIncludeList();
+
+       for (InsetIterator it = begin(inset()); it; ++it) {
                it->collectBibKeys(it, checkedFiles);
+               if (it->lyxCode() == BIBITEM_CODE) {
+                       if (parent() != nullptr)
+                               parent()->d->have_bibitems_ = true;
+                       else
+                               d->have_bibitems_ = true;
+               }
+       }
 }
 
 
-void Buffer::addBiblioInfo(BiblioInfo const & bin) const
+void Buffer::addBiblioInfo(BiblioInfo const & bi_in) const
 {
-       // We add the biblio info to the master buffer,
-       // if there is one, but also to every single buffer,
-       // in case a child is compiled alone.
-       BiblioInfo & bi = d->bibinfo_;
-       bi.mergeBiblioInfo(bin);
+       // We add the biblio info to the parent buffer,
+       // if there is one, but also to this buffer, in case
+       // it is compiled alone.
+       BiblioInfo & our_bi = d->bibinfo_;
+       our_bi.mergeBiblioInfo(bi_in);
 
-       if (parent() != 0) {
-               BiblioInfo & masterbi = parent()->d->bibinfo_;
-               masterbi.mergeBiblioInfo(bin);
-       }
+       if (parent())
+               parent()->addBiblioInfo(bi_in);
 }
 
 
@@ -2600,7 +2662,7 @@ void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bin) const
        BiblioInfo & bi = d->bibinfo_;
        bi[key] = bin;
 
-       if (parent() != 0) {
+       if (parent() != nullptr) {
                BiblioInfo & masterbi = masterBuffer()->d->bibinfo_;
                masterbi[key] = bin;
        }
@@ -2627,6 +2689,11 @@ bool Buffer::citeLabelsValid() const
 
 void Buffer::removeBiblioTempFiles() const
 {
+       if (theApp()->isBufferBusy(this)) {
+               removeBiblioTemps = true;
+               return;
+       }
+
        // We remove files that contain LaTeX commands specific to the
        // particular bibliographic style being used, in order to avoid
        // LaTeX errors when we switch style.
@@ -2640,6 +2707,7 @@ void Buffer::removeBiblioTempFiles() const
        Buffer const * const pbuf = parent();
        if (pbuf)
                pbuf->removeBiblioTempFiles();
+       removeBiblioTemps = false;
 }
 
 
@@ -2658,7 +2726,25 @@ void Buffer::markDepClean(string const & name)
 }
 
 
-bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
+bool Buffer::branchActivationEnabled(FuncCode act, docstring const & branch) const
+{
+       bool const master = act == LFUN_BRANCH_MASTER_ACTIVATE
+                           || act == LFUN_BRANCH_MASTER_DEACTIVATE;
+       bool const activate = act == LFUN_BRANCH_ACTIVATE
+                           || act == LFUN_BRANCH_MASTER_ACTIVATE;
+       Buffer const * buf = master ? masterBuffer() : this;
+       Branch const * our_branch = buf->params().branchlist().find(branch);
+       // Can be disabled if
+       // - this is  a _MASTER_ command and there is no master
+       // - the relevant buffer does not know the branch
+       // - the branch is already in the desired state
+       return ((!master || parent() != nullptr)
+               && !branch.empty() && our_branch
+               && our_branch->isSelected() != activate);
+}
+
+
+bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag) const
 {
        if (isInternal()) {
                // FIXME? if there is an Buffer LFUN that can be dispatched even
@@ -2685,7 +2771,7 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                break;
 
        case LFUN_BUFFER_EXPORT: {
-               docstring const arg = cmd.argument();
+               docstring const arg = cmd.argument();
                if (arg == "custom") {
                        enable = true;
                        break;
@@ -2709,15 +2795,12 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
        case LFUN_BRANCH_ACTIVATE:
        case LFUN_BRANCH_DEACTIVATE:
        case LFUN_BRANCH_MASTER_ACTIVATE:
-       case LFUN_BRANCH_MASTER_DEACTIVATE: {
-               bool const master = (cmd.action() == LFUN_BRANCH_MASTER_ACTIVATE
-                                    || cmd.action() == LFUN_BRANCH_MASTER_DEACTIVATE);
-               BranchList const & branchList = master ? masterBuffer()->params().branchlist()
-                       : params().branchlist();
-               docstring const branchName = cmd.argument();
-               flag.setEnabled(!branchName.empty() && branchList.find(branchName));
+       case LFUN_BRANCH_MASTER_DEACTIVATE:
+               // Let a branch inset handle that
+               if (cmd.argument().empty())
+                       return false;
+               flag.setEnabled(branchActivationEnabled(cmd.action(), cmd.argument()));
                break;
-       }
 
        case LFUN_BRANCH_ADD:
        case LFUN_BRANCHES_RENAME:
@@ -2762,6 +2845,53 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
 }
 
 
+bool Buffer::branchActivationDispatch(FuncCode act, docstring const & branch)
+{
+       bool const master = (act == LFUN_BRANCH_MASTER_ACTIVATE
+                            || act == LFUN_BRANCH_MASTER_DEACTIVATE);
+       bool const activate = (act == LFUN_BRANCH_ACTIVATE
+                              || act == LFUN_BRANCH_MASTER_ACTIVATE);
+       Buffer * buf = master ? const_cast<Buffer *>(masterBuffer()) : this;
+       Branch * our_branch = buf->params().branchlist().find(branch);
+
+       // See comment in branchActivationStatus
+       if ((master && parent() == nullptr)
+            || !our_branch
+            || our_branch->isSelected() == activate)
+               return false;
+
+       if (master && !buf->hasGuiDelegate()
+           && (!theApp() || !theApp()->unhide(buf)))
+               // at least issue a warning for now (ugly, but better than dataloss).
+               frontend::Alert::warning(_("Branch state changes in master document"),
+                   lyx::support::bformat(_("The state of the branch '%1$s' "
+                       "was changed in the master file. "
+                       "Please make sure to save the master."), branch), true);
+
+       UndoGroupHelper ugh(buf);
+       buf->undo().recordUndoBufferParams(CursorData());
+       our_branch->setSelected(activate);
+       // cur.forceBufferUpdate() is not enough)
+       buf->updateBuffer();
+
+       // if branch exists in a descendant, update previews.
+       // TODO: only needed if "Show preview" is enabled in the included inset.
+       bool exists_in_desc = false;
+       for (auto const & it : buf->getDescendants()) {
+               if (it->params().branchlist().find(branch))
+                       exists_in_desc = true;
+       }
+       if (exists_in_desc) {
+               // TODO: ideally we would only update the previews of the
+               // specific children that have this branch directly or
+               // in one of their descendants
+               buf->removePreviews();
+               buf->updatePreviews();
+       }
+       return true;
+}
+
+
 void Buffer::dispatch(string const & command, DispatchResult & result)
 {
        return dispatch(lyxaction.lookupFunc(command), result);
@@ -2773,6 +2903,7 @@ void Buffer::dispatch(string const & command, DispatchResult & result)
 // whether we have a GUI or not. The boolean use_gui holds this information.
 void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 {
+       LYXERR(Debug::ACTION, "Buffer::dispatch: cmd: " << func);
        if (isInternal()) {
                // FIXME? if there is an Buffer LFUN that can be dispatched even
                // if internal, put a switch '(cmd.action())' here.
@@ -2871,40 +3002,20 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
        case LFUN_BRANCH_DEACTIVATE:
        case LFUN_BRANCH_MASTER_ACTIVATE:
        case LFUN_BRANCH_MASTER_DEACTIVATE: {
-               bool const master = (func.action() == LFUN_BRANCH_MASTER_ACTIVATE
-                                    || func.action() == LFUN_BRANCH_MASTER_DEACTIVATE);
-               Buffer * buf = master ? const_cast<Buffer *>(masterBuffer())
-                                     : this;
-
-               docstring const branch_name = func.argument();
-               // the case without a branch name is handled elsewhere
-               if (branch_name.empty()) {
-                       dispatched = false;
-                       break;
-               }
-               Branch * branch = buf->params().branchlist().find(branch_name);
-               if (!branch) {
-                       LYXERR0("Branch " << branch_name << " does not exist.");
-                       dr.setError(true);
-                       docstring const msg =
-                               bformat(_("Branch \"%1$s\" does not exist."), branch_name);
-                       dr.setMessage(msg);
-                       break;
+               // Let a branch inset handle that
+               if (func.argument().empty()) {
+                       dr.dispatched(false);
+                       return;
                }
-               bool const activate = (func.action() == LFUN_BRANCH_ACTIVATE
-                                      || func.action() == LFUN_BRANCH_MASTER_ACTIVATE);
-               if (branch->isSelected() != activate) {
-                       buf->undo().recordUndoBufferParams(CursorData());
-                       branch->setSelected(activate);
-                       dr.setError(false);
+               bool const res = branchActivationDispatch(func.action(), func.argument());
+               dr.setError(!res);
+               if (res)
                        dr.screenUpdate(Update::Force);
-                       dr.forceBufferUpdate();
-               }
                break;
        }
 
        case LFUN_BRANCH_ADD: {
-               docstring branchnames = func.argument();
+               docstring const & branchnames = func.argument();
                if (branchnames.empty()) {
                        dispatched = false;
                        break;
@@ -2925,9 +3036,9 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                                undo().recordUndoBufferParams(CursorData());
                                branch_list.add(branch_name);
                                branch = branch_list.find(branch_name);
-                               string const x11hexname = X11hexname(branch->color());
-                               docstring const str = branch_name + ' ' + from_ascii(x11hexname);
-                               lyx::dispatch(FuncRequest(LFUN_SET_COLOR, str));
+                               if (branch)
+                                       // needed to update the color table for dark mode
+                                       branch->setColors("background", "background");
                                dr.setError(false);
                                dr.screenUpdate(Update::Force);
                        }
@@ -2943,10 +3054,10 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 
                docstring const oldname = from_utf8(func.getArg(0));
                docstring const newname = from_utf8(func.getArg(1));
-               InsetIterator it  = inset_iterator_begin(inset());
-               InsetIterator const end = inset_iterator_end(inset());
+               InsetIterator it  = begin(inset());
+               InsetIterator const itend = end(inset());
                bool success = false;
-               for (; it != end; ++it) {
+               for (; it != itend; ++it) {
                        if (it->lyxCode() == BRANCH_CODE) {
                                InsetBranch & ins = static_cast<InsetBranch &>(*it);
                                if (ins.branch() == oldname) {
@@ -2960,7 +3071,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                                // get buffer of external file
                                InsetInclude const & ins =
                                        static_cast<InsetInclude const &>(*it);
-                               Buffer * child = ins.getChildBuffer();
+                               Buffer * child = ins.loadIfNeeded();
                                if (!child)
                                        continue;
                                child->dispatch(func, dr);
@@ -2984,8 +3095,6 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                if (params().save_transient_properties)
                        undo().recordUndoBufferParams(CursorData());
                params().track_changes = !params().track_changes;
-               if (!params().track_changes)
-                       dr.forceChangesUpdate();
                break;
 
        case LFUN_CHANGES_OUTPUT:
@@ -2993,20 +3102,13 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                        undo().recordUndoBufferParams(CursorData());
                params().output_changes = !params().output_changes;
                if (params().output_changes) {
-                       bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
                        bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
                                          LaTeXFeatures::isAvailable("xcolor");
 
-                       if (!dvipost && !xcolorulem) {
+                       if (!xcolorulem) {
                                Alert::warning(_("Changes not shown in LaTeX output"),
                                               _("Changes will not be highlighted in LaTeX output, "
-                                                "because neither dvipost nor xcolor/ulem are installed.\n"
-                                                "Please install these packages or redefine "
-                                                "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
-                       } else if (!xcolorulem) {
-                               Alert::warning(_("Changes not shown in LaTeX output"),
-                                              _("Changes will not be highlighted in LaTeX output "
-                                                "when using pdflatex, because xcolor and ulem are not installed.\n"
+                                                "because xcolor and ulem are not installed.\n"
                                                 "Please install both packages or redefine "
                                                 "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
                        }
@@ -3047,9 +3149,10 @@ void Buffer::changeLanguage(Language const * from, Language const * to)
        LASSERT(from, return);
        LASSERT(to, return);
 
-       for_each(par_iterator_begin(),
-                par_iterator_end(),
-                bind(&Paragraph::changeLanguage, _1, params(), from, to));
+       ParIterator it = par_iterator_begin();
+       ParIterator eit = par_iterator_end();
+       for (; it != eit; ++it)
+               it->changeLanguage(params(), from, to);
 }
 
 
@@ -3081,7 +3184,7 @@ void Buffer::getLanguages(std::set<Language const *> & langs) const
        for (ParConstIterator it = par_iterator_begin(); it != end; ++it)
                it->getLanguages(langs);
        // also children
-       ListOfBuffers clist = getDescendents();
+       ListOfBuffers clist = getDescendants();
        for (auto const & cit : clist)
                cit->getLanguages(langs);
 }
@@ -3131,6 +3234,11 @@ ParConstIterator Buffer::par_iterator_end() const
        return ParConstIterator(doc_iterator_end(this));
 }
 
+bool Buffer::empty() const
+{
+       return paragraphs().size() == 1 && paragraphs().front().empty();
+}
+
 
 Language const * Buffer::language() const
 {
@@ -3138,7 +3246,7 @@ Language const * Buffer::language() const
 }
 
 
-docstring const Buffer::B_(string const & l10n) const
+docstring Buffer::B_(string const & l10n) const
 {
        return params().B_(l10n);
 }
@@ -3278,7 +3386,7 @@ string const Buffer::prepareFileNameForLaTeX(string const & name,
 }
 
 
-vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
+vector<pair<docstring, string>> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
                                                docstring_list const & bibfilelist,
                                                bool const add_extension) const
 {
@@ -3292,7 +3400,7 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
        // Otherwise, store the (maybe absolute) path to the original,
        // unmangled database name.
 
-       vector<docstring> res;
+       vector<pair<docstring, string>> res;
 
        // determine the export format
        string const tex_format = flavor2format(runparams.flavor);
@@ -3364,9 +3472,20 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
 
                if (contains(path, ' '))
                        found_space = true;
+               string enc;
+               if (params().useBiblatex() && !params().bibFileEncoding(utf8input).empty())
+                       enc = params().bibFileEncoding(utf8input);
+
+               bool recorded = false;
+               for (auto const & pe : res) {
+                       if (pe.first == path) {
+                               recorded = true;
+                               break;
+                       }
 
-               if (find(res.begin(), res.end(), path) == res.end())
-                       res.push_back(path);
+               }
+               if (!recorded)
+                       res.push_back(make_pair(path, enc));
        }
 
        // Check if there are spaces in the path and warn BibTeX users, if so.
@@ -3425,8 +3544,37 @@ bool Buffer::isReadonly() const
 
 void Buffer::setParent(Buffer const * buffer)
 {
-       // Avoids recursive include.
-       d->setParent(buffer == this ? 0 : buffer);
+       // We need to do some work here to avoid recursive parent structures.
+       // This is the easy case.
+       if (buffer == this) {
+               LYXERR0("Ignoring attempt to set self as parent in\n" << fileName());
+               return;
+       }
+       // Now we check parents going upward, to make sure that IF we set the
+       // parent as requested, we would not generate a recursive include.
+       set<Buffer const *> sb;
+       Buffer const * b = buffer;
+       bool found_recursion = false;
+       while (b) {
+               if (sb.find(b) != sb.end()) {
+                       found_recursion = true;
+                       break;
+               }
+               sb.insert(b);
+               b = b->parent();
+       }
+
+       if (found_recursion) {
+               LYXERR0("Ignoring attempt to set parent of\n" <<
+                       fileName() <<
+                       "\nto " <<
+                       buffer->fileName() <<
+                       "\nbecause that would create a recursive inclusion.");
+               return;
+       }
+
+       // We should be safe now.
+       d->setParent(buffer);
        updateMacros();
 }
 
@@ -3439,7 +3587,7 @@ Buffer const * Buffer::parent() const
 
 ListOfBuffers Buffer::allRelatives() const
 {
-       ListOfBuffers lb = masterBuffer()->getDescendents();
+       ListOfBuffers lb = masterBuffer()->getDescendants();
        lb.push_front(const_cast<Buffer *>(masterBuffer()));
        return lb;
 }
@@ -3447,8 +3595,6 @@ ListOfBuffers Buffer::allRelatives() const
 
 Buffer const * Buffer::masterBuffer() const
 {
-       // FIXME Should be make sure we are not in some kind
-       // of recursive include? A -> B -> A will crash this.
        Buffer const * const pbuf = d->parent();
        if (!pbuf)
                return this;
@@ -3479,19 +3625,19 @@ bool Buffer::hasChildren() const
 }
 
 
-void Buffer::collectChildren(ListOfBuffers & clist, bool grand_children) const
+void Buffer::collectChildren(ListOfBuffers & children, bool grand_children) const
 {
        // loop over children
        for (auto const & p : d->children_positions) {
                Buffer * child = const_cast<Buffer *>(p.first);
                // No duplicates
-               ListOfBuffers::const_iterator bit = find(clist.begin(), clist.end(), child);
-               if (bit != clist.end())
+               ListOfBuffers::const_iterator bit = find(children.begin(), children.end(), child);
+               if (bit != children.end())
                        continue;
-               clist.push_back(child);
+               children.push_back(child);
                if (grand_children)
                        // there might be grandchildren
-                       child->collectChildren(clist, true);
+                       child->collectChildren(children, true);
        }
 }
 
@@ -3510,7 +3656,7 @@ ListOfBuffers Buffer::getChildren() const
 }
 
 
-ListOfBuffers Buffer::getDescendents() const
+ListOfBuffers Buffer::getDescendants() const
 {
        ListOfBuffers v;
        collectChildren(v, true);
@@ -3534,7 +3680,7 @@ typename M::const_iterator greatest_below(M & m, typename M::key_type const & x)
        if (it == m.begin())
                return m.end();
 
-       it--;
+       --it;
        return it;
 }
 
@@ -3546,11 +3692,11 @@ MacroData const * Buffer::Impl::getBufferMacro(docstring const & name,
 
        // if paragraphs have no macro context set, pos will be empty
        if (pos.empty())
-               return 0;
+               return nullptr;
 
        // we haven't found anything yet
        DocIterator bestPos = owner_->par_iterator_begin();
-       MacroData const * bestData = 0;
+       MacroData const * bestData = nullptr;
 
        // find macro definitions for name
        NamePositionScopeMacroMap::const_iterator nameIt = macros.find(name);
@@ -3621,11 +3767,11 @@ MacroData const * Buffer::getMacro(docstring const & name,
        DocIterator const & pos, bool global) const
 {
        if (d->macro_lock)
-               return 0;
+               return nullptr;
 
        // query buffer macros
        MacroData const * data = d->getBufferMacro(name, pos);
-       if (data != 0)
+       if (data != nullptr)
                return data;
 
        // If there is a master buffer, query that
@@ -3641,11 +3787,11 @@ MacroData const * Buffer::getMacro(docstring const & name,
 
        if (global) {
                data = MacroTable::globalMacros().get(name);
-               if (data != 0)
+               if (data != nullptr)
                        return data;
        }
 
-       return 0;
+       return nullptr;
 }
 
 
@@ -3665,7 +3811,7 @@ MacroData const * Buffer::getMacro(docstring const & name,
        // look where the child buffer is included first
        Impl::BufferPositionMap::iterator it = d->children_positions.find(&child);
        if (it == d->children_positions.end())
-               return 0;
+               return nullptr;
 
        // check for macros at the inclusion position
        return getMacro(name, it->second, global);
@@ -3680,7 +3826,6 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
        while (it.pit() <= lastpit) {
                Paragraph & par = it.paragraph();
 
-               // FIXME Can this be done with the new-style iterators?
                // iterate over the insets of the current paragraph
                for (auto const & insit : par.insetList()) {
                        it.pos() = insit.pos;
@@ -3719,16 +3864,14 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                                InsetInclude const & incinset =
                                        static_cast<InsetInclude const &>(*insit.inset);
                                macro_lock = true;
-                               Buffer * child = incinset.getChildBuffer();
+                               Buffer * child = incinset.loadIfNeeded();
                                macro_lock = false;
                                if (!child)
                                        continue;
 
                                // register its position, but only when it is
                                // included first in the buffer
-                               if (children_positions.find(child) ==
-                                       children_positions.end())
-                                               children_positions[child] = it;
+                               children_positions.insert({child, it});
 
                                // register child with its scope
                                position_to_children[it] = Impl::ScopeBuffer(scope, child);
@@ -3763,7 +3906,7 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                        // FIXME (Abdel), I don't understand why we pass 'it' here
                        // instead of 'macroTemplate' defined above... is this correct?
                        macros[macroTemplate.name()][it] =
-                               Impl::ScopeMacro(scope, MacroData(const_cast<Buffer *>(owner_), it));
+                               Impl::ScopeMacro(scope, MacroData(owner_, it));
                }
 
                // next paragraph
@@ -3798,11 +3941,9 @@ void Buffer::updateMacros() const
 
 void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_master) const
 {
-       InsetIterator it  = inset_iterator_begin(inset());
-       InsetIterator const end = inset_iterator_end(inset());
-       for (; it != end; ++it) {
-               if (it->lyxCode() == BRANCH_CODE) {
-                       InsetBranch & br = static_cast<InsetBranch &>(*it);
+       for (Inset const & it : inset()) {
+               if (it.lyxCode() == BRANCH_CODE) {
+                       InsetBranch const & br = static_cast<InsetBranch const &>(it);
                        docstring const name = br.branch();
                        if (!from_master && !params().branchlist().find(name))
                                result.push_back(name);
@@ -3810,11 +3951,11 @@ void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_mast
                                result.push_back(name);
                        continue;
                }
-               if (it->lyxCode() == INCLUDE_CODE) {
+               if (it.lyxCode() == INCLUDE_CODE) {
                        // get buffer of external file
                        InsetInclude const & ins =
-                               static_cast<InsetInclude const &>(*it);
-                       Buffer * child = ins.getChildBuffer();
+                               static_cast<InsetInclude const &>(it);
+                       Buffer * child = ins.loadIfNeeded();
                        if (!child)
                                continue;
                        child->getUsedBranches(result, true);
@@ -3839,11 +3980,11 @@ void Buffer::updateMacroInstances(UpdateType utype) const
                        continue;
 
                // update macro in all cells of the InsetMathNest
-               DocIterator::idx_type n = minset->nargs();
+               idx_type n = minset->nargs();
                MacroContext mc = MacroContext(this, it);
-               for (DocIterator::idx_type i = 0; i < n; ++i) {
+               for (idx_type i = 0; i < n; ++i) {
                        MathData & data = minset->cell(i);
-                       data.updateMacros(0, mc, utype, 0);
+                       data.updateMacros(nullptr, mc, utype, 0);
                }
        }
 }
@@ -3946,7 +4087,7 @@ void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il,
 InsetLabel const * Buffer::insetLabel(docstring const & label,
                                      bool const active) const
 {
-       for (auto & rc : masterBuffer()->d->label_cache_) {
+       for (auto const & rc : masterBuffer()->d->label_cache_) {
                if (rc.label == label && (rc.active || !active))
                        return rc.inset;
        }
@@ -3956,10 +4097,7 @@ InsetLabel const * Buffer::insetLabel(docstring const & label,
 
 bool Buffer::activeLabel(docstring const & label) const
 {
-       if (!insetLabel(label, true))
-               return false;
-
-       return true;
+       return insetLabel(label, true) != nullptr;
 }
 
 
@@ -3986,14 +4124,17 @@ void Buffer::changeRefsIfUnique(docstring const & from, docstring const & to)
                return;
 
        string const paramName = "key";
-       InsetIterator it = inset_iterator_begin(inset());
+       UndoGroupHelper ugh(this);
+       InsetIterator it = begin(inset());
        for (; it; ++it) {
                if (it->lyxCode() != CITE_CODE)
                        continue;
                InsetCommand * inset = it->asInsetCommand();
                docstring const oldValue = inset->getParam(paramName);
-               if (oldValue == from)
+               if (oldValue == from) {
+                       undo().recordUndo(CursorData(it));
                        inset->setParam(paramName, to);
+               }
        }
 }
 
@@ -4013,39 +4154,51 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
        // Some macros rely on font encoding
        runparams.main_fontenc = params().main_font_encoding();
 
+       // Use the right wrapping for the comment at the beginning of the generated
+       // snippet, so that it is either valid LaTeX or valid XML (including HTML and DocBook).
+       docstring comment_start = from_ascii("% ");
+       docstring comment_end = from_ascii("");
+       if (runparams.flavor == Flavor::Html || runparams.flavor == Flavor::DocBook5) {
+               comment_start = from_ascii("<!-- ");
+               comment_end = from_ascii(" -->");
+       }
+
        if (output == CurrentParagraph) {
                runparams.par_begin = par_begin;
                runparams.par_end = par_end;
                if (par_begin + 1 == par_end) {
-                       os << "% "
+                       os << comment_start
                           << bformat(_("Preview source code for paragraph %1$d"), par_begin)
+                          << comment_end
                           << "\n\n";
                } else {
-                       os << "% "
+                       os << comment_start
                           << bformat(_("Preview source code from paragraph %1$s to %2$s"),
                                        convert<docstring>(par_begin),
                                        convert<docstring>(par_end - 1))
+                          << comment_end
                           << "\n\n";
                }
                // output paragraphs
-               if (runparams.flavor == OutputParams::LYX) {
+               if (runparams.flavor == Flavor::LyX) {
                        Paragraph const & par = text().paragraphs()[par_begin];
                        ostringstream ods;
                        depth_type dt = par.getDepth();
                        par.write(ods, params(), dt);
                        os << from_utf8(ods.str());
-               } else if (runparams.flavor == OutputParams::HTML) {
-                       XHTMLStream xs(os);
+               } else if (runparams.flavor == Flavor::Html) {
+                       XMLStream xs(os);
                        setMathFlavor(runparams);
                        xhtmlParagraphs(text(), *this, xs, runparams);
-               } else if (runparams.flavor == OutputParams::TEXT) {
+               } else if (runparams.flavor == Flavor::Text) {
                        bool dummy = false;
                        // FIXME Handles only one paragraph, unlike the others.
                        // Probably should have some routine with a signature like them.
                        writePlaintextParagraph(*this,
                                text().paragraphs()[par_begin], os, runparams, dummy);
-               } else if (params().isDocBook()) {
-                       docbookParagraphs(text(), *this, os, runparams);
+               } else if (runparams.flavor == Flavor::DocBook5) {
+                       XMLStream xs{os};
+                       docbookParagraphs(text(), *this, xs, runparams);
                } else {
                        // If we are previewing a paragraph, even if this is the
                        // child of some other buffer, let's cut the link here,
@@ -4059,6 +4212,8 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        LaTeXFeatures features(*this, params(), runparams);
                        validate(features);
                        runparams.use_polyglossia = features.usePolyglossia();
+                       runparams.use_babel = features.useBabel();
+                       runparams.use_hyperref = features.isRequired("hyperref");
                        // latex or literate
                        otexstream ots(os);
                        // output above
@@ -4072,15 +4227,16 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                                d->ignore_parent = false;
                }
        } else {
-               os << "% ";
+               os << comment_start;
                if (output == FullSource)
                        os << _("Preview source code");
                else if (output == OnlyPreamble)
                        os << _("Preview preamble");
                else if (output == OnlyBody)
                        os << _("Preview body");
+               os << comment_end;
                os << "\n\n";
-               if (runparams.flavor == OutputParams::LYX) {
+               if (runparams.flavor == Flavor::LyX) {
                        ostringstream ods;
                        if (output == FullSource)
                                write(ods);
@@ -4089,15 +4245,15 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        else if (output == OnlyBody)
                                text().write(ods);
                        os << from_utf8(ods.str());
-               } else if (runparams.flavor == OutputParams::HTML) {
+               } else if (runparams.flavor == Flavor::Html) {
                        writeLyXHTMLSource(os, runparams, output);
-               } else if (runparams.flavor == OutputParams::TEXT) {
-                       if (output == OnlyPreamble) {
+               } else if (runparams.flavor == Flavor::Text) {
+                       if (output == OnlyPreamble)
                                os << "% "<< _("Plain text does not have a preamble.");
-                       else
+                       else
                                writePlaintextFile(*this, os, runparams);
-               } else if (params().isDocBook()) {
-                               writeDocBookSource(os, absFileName(), runparams, output);
+               } else if (runparams.flavor == Flavor::DocBook5) {
+                       writeDocBookSource(os, runparams, output);
                } else {
                        // latex or literate
                        otexstream ots(os);
@@ -4163,6 +4319,13 @@ void Buffer::updateTitles() const
 }
 
 
+void Buffer::scheduleRedrawWorkAreas() const
+{
+       if (d->wa_)
+               d->wa_->scheduleRedraw();
+}
+
+
 void Buffer::resetAutosaveTimers() const
 {
        if (d->gui_)
@@ -4182,91 +4345,6 @@ void Buffer::setGuiDelegate(frontend::GuiBufferDelegate * gui)
 }
 
 
-
-namespace {
-
-class AutoSaveBuffer : public ForkedProcess {
-public:
-       ///
-       AutoSaveBuffer(Buffer const & buffer, FileName const & fname)
-               : buffer_(buffer), fname_(fname) {}
-       ///
-       virtual shared_ptr<ForkedProcess> clone() const
-       {
-               return make_shared<AutoSaveBuffer>(*this);
-       }
-       ///
-       int start()
-       {
-               command_ = to_utf8(bformat(_("Auto-saving %1$s"),
-                                                from_utf8(fname_.absFileName())));
-               return run(DontWait);
-       }
-private:
-       ///
-       virtual int generateChild();
-       ///
-       Buffer const & buffer_;
-       FileName fname_;
-};
-
-
-int AutoSaveBuffer::generateChild()
-{
-#if defined(__APPLE__)
-       /* FIXME fork() is not usable for autosave on Mac OS X 10.6 (snow leopard)
-        *   We should use something else like threads.
-        *
-        * Since I do not know how to determine at run time what is the OS X
-        * version, I just disable forking altogether for now (JMarc)
-        */
-       pid_t const pid = -1;
-#else
-       // tmp_ret will be located (usually) in /tmp
-       // will that be a problem?
-       // Note that this calls ForkedCalls::fork(), so it's
-       // ok cross-platform.
-       pid_t const pid = fork();
-       // If you want to debug the autosave
-       // you should set pid to -1, and comment out the fork.
-       if (pid != 0 && pid != -1)
-               return pid;
-#endif
-
-       // pid = -1 signifies that lyx was unable
-       // to fork. But we will do the save
-       // anyway.
-       bool failed = false;
-       TempFile tempfile("lyxautoXXXXXX.lyx");
-       tempfile.setAutoRemove(false);
-       FileName const tmp_ret = tempfile.name();
-       if (!tmp_ret.empty()) {
-               if (!buffer_.writeFile(tmp_ret))
-                       failed = true;
-               else if (!tmp_ret.moveTo(fname_))
-                       failed = true;
-       } else
-               failed = true;
-
-       if (failed) {
-               // failed to write/rename tmp_ret so try writing direct
-               if (!buffer_.writeFile(fname_)) {
-                       // It is dangerous to do this in the child,
-                       // but safe in the parent, so...
-                       if (pid == -1) // emit message signal.
-                               buffer_.message(_("Autosave failed!"));
-               }
-       }
-
-       if (pid == 0) // we are the child so...
-               _exit(0);
-
-       return pid;
-}
-
-} // namespace
-
-
 FileName Buffer::getEmergencyFileName() const
 {
        return FileName(d->filename.absFileName() + ".emergency");
@@ -4340,7 +4418,7 @@ bool Buffer::autoSave() const
 void Buffer::setExportStatus(bool e) const
 {
        d->doing_export = e;
-       ListOfBuffers clist = getDescendents();
+       ListOfBuffers clist = getDescendants();
        for (auto const & bit : clist)
                bit->d->doing_export = e;
 }
@@ -4363,7 +4441,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string & result_file) const
 {
        bool const update_unincluded =
-                       params().maintain_unincluded_children
+                       params().maintain_unincluded_children != BufferParams::CM_None
                        && !params().getIncludedChildren().empty();
 
        // (1) export with all included children (omit \includeonly)
@@ -4380,6 +4458,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 
 void Buffer::setMathFlavor(OutputParams & op) const
 {
+       // Passes the way to generate formulae to the XHTML output code.
+       // In particular, this function has no impact on the DocBook code, as it
+       // uses another mechanism to handle math flavours.
        switch (params().html_math_output) {
        case BufferParams::MathML:
                op.math_flavor = OutputParams::MathAsMathML;
@@ -4416,14 +4497,14 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        }
        MarkAsExporting exporting(this);
        string backend_format;
-       runparams.flavor = OutputParams::LATEX;
+       runparams.flavor = Flavor::LaTeX;
        runparams.linelen = lyxrc.plaintext_linelen;
        runparams.includeall = includeall;
        vector<string> backs = params().backends();
        Converters converters = theConverters();
        bool need_nice_file = false;
        if (find(backs.begin(), backs.end(), format) == backs.end()) {
-               // Get shortest path to format
+               // Get the shortest path to format
                converters.buildGraph();
                Graph::EdgePath path;
                for (string const & sit : backs) {
@@ -4437,9 +4518,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                        if (!put_in_tempdir) {
                                // Only show this alert if this is an export to a non-temporary
                                // file (not for previewing).
-                               Alert::error(_("Couldn't export file"), bformat(
-                                       _("No information for exporting the format %1$s."),
-                                       theFormats().prettyName(format)));
+                               docstring s = bformat(_("No information for exporting the format %1$s."),
+                                                     translateIfPossible(theFormats().prettyName(format)));
+                               if (format == "pdf4")
+                                       s += "\n"
+                                         + bformat(_("Hint: use non-TeX fonts or set input encoding"
+                                                 " to '%1$s'"), from_utf8(encodings.fromLyXName("ascii")->guiName()));
+                               Alert::error(_("Couldn't export file"), s);
                        }
                        return ExportNoPathToFormat;
                }
@@ -4456,13 +4541,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                LYXERR(Debug::FILES, "backend_format=" << backend_format);
                // FIXME: Don't hardcode format names here, but use a flag
                if (backend_format == "pdflatex")
-                       runparams.flavor = OutputParams::PDFLATEX;
+                       runparams.flavor = Flavor::PdfLaTeX;
                else if (backend_format == "luatex")
-                       runparams.flavor = OutputParams::LUATEX;
+                       runparams.flavor = Flavor::LuaTeX;
                else if (backend_format == "dviluatex")
-                       runparams.flavor = OutputParams::DVILUATEX;
+                       runparams.flavor = Flavor::DviLuaTeX;
                else if (backend_format == "xetex")
-                       runparams.flavor = OutputParams::XETEX;
+                       runparams.flavor = Flavor::XeTeX;
        }
 
        string filename = latexName(false);
@@ -4473,7 +4558,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 
        // Plain text backend
        if (backend_format == "text") {
-               runparams.flavor = OutputParams::TEXT;
+               runparams.flavor = Flavor::Text;
                try {
                        writePlaintextFile(*this, FileName(filename), runparams);
                }
@@ -4481,15 +4566,16 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        }
        // HTML backend
        else if (backend_format == "xhtml") {
-               runparams.flavor = OutputParams::HTML;
+               runparams.flavor = Flavor::Html;
                setMathFlavor(runparams);
                if (makeLyXHTMLFile(FileName(filename), runparams) == ExportKilled)
                        return ExportKilled;
        } else if (backend_format == "lyx")
                writeFile(FileName(filename));
-       // Docbook backend
-       else if (params().isDocBook()) {
-               runparams.nice = !put_in_tempdir;
+       // DocBook backend
+       else if (backend_format == "docbook5") {
+               runparams.flavor = Flavor::DocBook5;
+               runparams.nice = false;
                if (makeDocBookFile(FileName(filename), runparams) == ExportKilled)
                        return ExportKilled;
        }
@@ -4510,7 +4596,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                        bformat(_("The directory path to the document\n%1$s\n"
                            "contains spaces, but your TeX installation does "
                            "not allow them. You should save the file to a directory "
-                                       "whose name does not contain spaces."), from_ascii(filePath())));
+                                       "whose name does not contain spaces."), from_utf8(filePath())));
                return ExportTexPathHasSpaces;
        } else {
                runparams.nice = false;
@@ -4530,8 +4616,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string const ext = theFormats().extension(format);
        FileName const tmp_result_file(changeExtension(filename, ext));
        Converters::RetVal const retval = 
-               converters.convert(this, FileName(filename), tmp_result_file, 
-               FileName(absFileName()), backend_format, format, error_list);
+               converters.convert(this, FileName(filename), tmp_result_file,
+                                  FileName(absFileName()), backend_format, format,
+                                  error_list, Converters::none, includeall);
        if (retval == Converters::KILLED)
                return ExportCancel;
        bool success = (retval == Converters::SUCCESS);
@@ -4547,7 +4634,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                else
                        errors(error_type);
                // also to the children, in case of master-buffer-view
-               ListOfBuffers clist = getDescendents();
+               ListOfBuffers clist = getDescendants();
                for (auto const & bit : clist) {
                        if (runparams.silent)
                                bit->d->errorLists[error_type].clear();
@@ -4583,9 +4670,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                result_file = changeExtension(d->exportFileName().absFileName(), ext);
        else
                result_file = dest_filename;
-       // We need to copy referenced files (e. g. included graphics
+       // We need to copy referenced files (e.g. included graphics
        // if format == "dvi") to the result dir.
-       vector<ExportedFile> const files =
+       vector<ExportedFile> const extfiles =
                runparams.exportdata->externalFiles(format);
        string const dest = runparams.export_folder.empty() ?
                onlyPath(result_file) : runparams.export_folder;
@@ -4593,7 +4680,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                                 : force_overwrite == ALL_FILES;
        CopyStatus status = use_force ? FORCE : SUCCESS;
 
-       for (ExportedFile const & exp : files) {
+       for (ExportedFile const & exp : extfiles) {
                if (status == CANCEL) {
                        message(_("Document export cancelled."));
                        return ExportCancel;
@@ -4630,13 +4717,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                } else {
                        message(bformat(_("Document exported as %1$s "
                                "to file `%2$s'"),
-                               theFormats().prettyName(format),
+                               translateIfPossible(theFormats().prettyName(format)),
                                makeDisplayPath(result_file)));
                }
        } else {
                // This must be a dummy converter like fax (bug 1888)
                message(bformat(_("Document exported as %1$s"),
-                       theFormats().prettyName(format)));
+                       translateIfPossible(theFormats().prettyName(format))));
        }
 
        return success ? ExportSuccess : ExportConverterError;
@@ -4646,7 +4733,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 Buffer::ExportStatus Buffer::preview(string const & format) const
 {
        bool const update_unincluded =
-                       params().maintain_unincluded_children
+                       params().maintain_unincluded_children != BufferParams::CM_None
                        && !params().getIncludedChildren().empty();
        return preview(format, update_unincluded);
 }
@@ -4669,7 +4756,7 @@ Buffer::ExportStatus Buffer::preview(string const & format, bool includeall) con
        Impl * theimpl = isClone() ? d->cloned_buffer_->d : d;
        theimpl->preview_file_ = previewFile;
        theimpl->preview_format_ = format;
-       theimpl->preview_error_ = (status != ExportSuccess);
+       theimpl->require_fresh_start_ = (status != ExportSuccess);
 
        if (status != ExportSuccess)
                return status;
@@ -4708,7 +4795,7 @@ Buffer::ReadStatus Buffer::loadEmergency()
                "%1$s exists.\n\nRecover emergency save?"), file);
 
        int const load_emerg = Alert::prompt(_("Load emergency save?"), text,
-               0, 2, _("&Recover"), _("&Load Original"), _("&Cancel"));
+               0, 3, _("&Recover"), _("&Load Original"), _("&Only show difference"), _("&Cancel"));
 
        switch (load_emerg)
        {
@@ -4750,9 +4837,55 @@ Buffer::ReadStatus Buffer::loadEmergency()
                                _("&Remove"), _("&Keep"));
                if (del_emerg == 0)
                        emergencyFile.removeFile();
+               else {
+                       // See bug #11464
+                       FileName newname;
+                       string const ename = emergencyFile.absFileName();
+                       bool noname = true;
+                       // Surely we can find one in 100 tries?
+                       for (int i = 1; i < 100; ++i) {
+                               newname.set(ename + to_string(i) + ".lyx");
+                               if (!newname.exists()) {
+                                       noname = false;
+                                       break;
+                               }
+                       }
+                       if (!noname) {
+                               // renameTo returns true on success. So inverting that
+                               // will give us true if we fail.
+                               noname = !emergencyFile.renameTo(newname);
+                       }
+                       if (noname) {
+                               Alert::warning(_("Can't rename emergency file!"),
+                                       _("LyX was unable to rename the emergency file. "
+                                         "You should do so manually. Otherwise, you will be "
+                                         "asked about it again the next time you try to load "
+                                         "this file, and may over-write your own work."));
+                       } else {
+                               Alert::warning(_("Emergency File Renamed"),
+                                       bformat(_("Emergency file renamed as:\n %1$s"),
+                                       from_utf8(newname.onlyFileName())));
+                       }
+               }
                return ReadOriginal;
        }
 
+       case 2: {
+               string const f1 = d->filename.absFileName();
+               string const f2 = emergencyFile.absFileName();
+               if (loadThisLyXFile(d->filename) != ReadSuccess)
+                       return ReadCancel;
+               string const par = "compare run-blocking " + quoteName(f1) + " " + quoteName(f2);
+               LYXERR(Debug::FILES, par << "\n");
+               lyx::dispatch(FuncRequest(LFUN_DIALOG_SHOW, par));
+
+               //release the emergency buffer loaded by compare
+               Buffer * emerBuffer = theBufferList().getBuffer(emergencyFile);
+               if (emerBuffer)
+                       theBufferList().release(emerBuffer);
+
+               return ReadCancel; //Release the buffer of Original immediatelly
+       }
        default:
                break;
        }
@@ -4831,26 +4964,26 @@ Buffer::ReadStatus Buffer::loadThisLyXFile(FileName const & fn)
 }
 
 
-void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
+void Buffer::Impl::traverseErrors(TeXErrors::Errors::const_iterator err, TeXErrors::Errors::const_iterator end, ErrorList & errorList) const
 {
-       for (auto const & err : terr) {
+       for (; err != end; ++err) {
                TexRow::TextEntry start = TexRow::text_none, end = TexRow::text_none;
-               int errorRow = err.error_in_line;
-               Buffer const * buf = 0;
-               Impl const * p = d;
-               if (err.child_name.empty())
+               int errorRow = err->error_in_line;
+               Buffer const * buf = nullptr;
+               Impl const * p = this;
+               if (err->child_name.empty())
                        tie(start, end) = p->texrow.getEntriesFromRow(errorRow);
                else {
                        // The error occurred in a child
-                       for (Buffer const * child : getDescendents()) {
+                       for (Buffer const * child : owner_->getDescendants()) {
                                string const child_name =
                                        DocFileName(changeExtension(child->absFileName(), "tex")).
                                        mangledFileName();
-                               if (err.child_name != child_name)
+                               if (err->child_name != child_name)
                                        continue;
                                tie(start, end) = child->d->texrow.getEntriesFromRow(errorRow);
                                if (!TexRow::isNone(start)) {
-                                       buf = d->cloned_buffer_
+                                       buf = this->cloned_buffer_
                                                ? child->d->cloned_buffer_->d->owner_
                                                : child->d->owner_;
                                        p = child->d;
@@ -4858,16 +4991,45 @@ void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
                                }
                        }
                }
-               errorList.push_back(ErrorItem(err.error_desc, err.error_text,
+               errorList.push_back(ErrorItem(err->error_desc, err->error_text,
                                              start, end, buf));
        }
 }
 
 
+void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
+{
+       TeXErrors::Errors::const_iterator err = terr.begin();
+       TeXErrors::Errors::const_iterator end = terr.end();
+
+       d->traverseErrors(err, end, errorList);
+}
+
+
+void Buffer::bufferRefs(TeXErrors const & terr, ErrorList & errorList) const
+{
+       TeXErrors::Errors::const_iterator err = terr.begin_ref();
+       TeXErrors::Errors::const_iterator end = terr.end_ref();
+
+       d->traverseErrors(err, end, errorList);
+}
+
+
+void Buffer::updateBuffer() const
+{
+       updateBuffer(UpdateMaster, InternalUpdate);
+       d->need_update = false;
+}
+
+
 void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 {
        LBUFERR(!text().paragraphs().empty());
 
+       // This can be called when loading a file, so that there be no
+       // open undo group.
+       UndoGroupHelper ugh(const_cast<Buffer *>(this));
+
        // Use the master text class also for child documents
        Buffer const * const master = masterBuffer();
        DocumentClass const & textclass = master->params().documentClass();
@@ -4920,12 +5082,17 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        // update all caches
        clearReferenceCache();
        updateMacros();
-       setChangesPresent(false);
 
        Buffer & cbuf = const_cast<Buffer &>(*this);
+       // if we are reloading, then we could have a dangling TOC,
+       // in effect. so we need to go ahead and reset, even though
+       // we will do so again when we rebuild the TOC later.
+       cbuf.tocBackend().reset();
 
        // do the real work
        ParIterator parit = cbuf.par_iterator_begin();
+       if (scope == UpdateMaster)
+               clearIncludeList();
        updateBuffer(parit, utype);
 
        // If this document has siblings, then update the TocBackend later. The
@@ -4957,7 +5124,6 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
                clearReferenceCache();
                // we should not need to do this again?
                // updateMacros();
-               setChangesPresent(false);
                updateBuffer(parit, utype);
                // this will already have been done by reloadBibInfoCache();
                // d->bibinfo_cache_valid_ = true;
@@ -4969,9 +5135,12 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        }
        d->cite_labels_valid_ = true;
        /// FIXME: Perf
+       clearIncludeList();
        cbuf.tocBackend().update(true, utype);
        if (scope == UpdateMaster)
                cbuf.structureChanged();
+
+       d->need_update = false;
 }
 
 
@@ -5083,27 +5252,11 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
 
        switch(layout.labeltype) {
        case LABEL_ITEMIZE: {
-               // At some point of time we should do something more
-               // clever here, like:
-               //   par.params().labelString(
-               //    bp.user_defined_bullet(par.itemdepth).getText());
-               // for now, use a simple hardcoded label
-               docstring itemlabel;
-               switch (par.itemdepth) {
-               case 0:
-                       itemlabel = char_type(0x2022);
-                       break;
-               case 1:
-                       itemlabel = char_type(0x2013);
-                       break;
-               case 2:
-                       itemlabel = char_type(0x2217);
-                       break;
-               case 3:
-                       itemlabel = char_type(0x2219); // or 0x00b7
-                       break;
-               }
-               par.params().labelString(itemlabel);
+               par.params().labelString(
+                       (par.itemdepth < 4)
+                                       ? bp.user_defined_bullet(par.itemdepth).getLabel()
+                                         // Display fallback for too deeply nested items
+                                       : bformat(from_ascii("[?%1$d]"), int(par.itemdepth + 1)));
                break;
        }
 
@@ -5129,9 +5282,9 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
                }
 
                if (needEnumCounterReset(it)) {
-                       // Increase the master counter?
-                       if (layout.stepmastercounter)
-                               counters.stepMaster(enumcounter, utype);
+                       // Increase the parent counter?
+                       if (layout.stepparentcounter)
+                               counters.stepParent(enumcounter, utype);
                        // Maybe we have to reset the enumeration counter.
                        if (!layout.resumecounter)
                                counters.reset(enumcounter);
@@ -5193,8 +5346,13 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
 }
 
 
-void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
+void Buffer::updateBuffer(ParIterator & parit, UpdateType utype, bool const deleted) const
 {
+       // if fomatted references are shown in workarea update buffer accordingly
+       if (params().use_formatted_ref)
+               utype = OutputUpdate;
+
+       pushIncludedBuffer(this);
        // LASSERT: Is it safe to continue here, or should we just return?
        LASSERT(parit.pit() == 0, /**/);
 
@@ -5204,6 +5362,7 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
 
        depth_type maxdepth = 0;
        pit_type const lastpit = parit.lastpit();
+       bool changed = false;
        for ( ; parit.pit() <= lastpit ; ++parit.pit()) {
                // reduce depth if necessary
                if (parit->params().depth() > maxdepth) {
@@ -5230,15 +5389,33 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
                // set the counter for this paragraph
                d->setLabel(parit, utype);
 
-               // update change-tracking flag
-               parit->addChangesToBuffer(*this);
-
                // now the insets
                for (auto const & insit : parit->insetList()) {
                        parit.pos() = insit.pos;
-                       insit.inset->updateBuffer(parit, utype);
+                       insit.inset->updateBuffer(parit, utype, deleted || parit->isDeleted(insit.pos));
+                       changed |= insit.inset->isChanged();
                }
+
+               // are there changes in this paragraph?
+               changed |= parit->isChanged();
        }
+
+       // set change indicator for the inset (or the cell that the iterator
+       // points to, if applicable).
+       parit.text()->inset().isChanged(changed);
+       popIncludedBuffer();
+}
+
+
+void Buffer::forceUpdate() const
+{
+       d->need_update = true;
+}
+
+
+bool Buffer::needUpdate() const
+{
+       return d->need_update;
 }
 
 
@@ -5280,6 +5457,15 @@ int Buffer::spellCheck(DocIterator & from, DocIterator & to,
 }
 
 
+void Buffer::requestSpellcheck()
+{
+       ParagraphList::iterator pit = paragraphs().begin();
+       ParagraphList::iterator pend = paragraphs().end();
+       for (; pit != pend; ++pit)
+               pit->requestSpellCheck();
+}
+
+
 void Buffer::Impl::updateStatistics(DocIterator & from, DocIterator & to, bool skipNoOutput)
 {
        bool inword = false;
@@ -5315,10 +5501,19 @@ void Buffer::Impl::updateStatistics(DocIterator & from, DocIterator & to, bool s
                                        ++word_count_;
                                        inword = true;
                                }
-                               if (ins && ins->isLetter())
-                                       ++char_count_;
+                               if (ins && ins->isLetter()) {
+                                       odocstringstream os;
+                                       ins->toString(os);
+                                       char_count_ += os.str().length();
+                               }
                                else if (ins && ins->isSpace())
                                        ++blank_count_;
+                               else if (ins) {
+                                       pair<int, int> words = ins->isWords();
+                                       char_count_ += words.first;
+                                       word_count_ += words.second;
+                                       inword = false;
+                               }
                                else {
                                        char_type const c = par.getChar(pos);
                                        if (isPrintableNonspace(c))
@@ -5351,6 +5546,12 @@ int Buffer::charCount(bool with_blanks) const
 }
 
 
+bool Buffer::areChangesPresent() const
+{
+       return inset().isChanged();
+}
+
+
 Buffer::ReadStatus Buffer::reload()
 {
        setBusy(true);
@@ -5361,7 +5562,7 @@ Buffer::ReadStatus Buffer::reload()
        docstring const disp_fn = makeDisplayPath(d->filename.absFileName());
 
        // clear parent. this will get reset if need be.
-       d->setParent(0);
+       d->setParent(nullptr);
        ReadStatus const status = loadLyXFile();
        if (status == ReadSuccess) {
                updateBuffer();
@@ -5435,8 +5636,8 @@ void Buffer::checkChildBuffers()
                if (oldloc == newloc)
                        continue;
                // the location of the child file is incorrect.
-               cbuf->setParent(0);
-               inset_inc->setChildBuffer(0);
+               cbuf->setParent(nullptr);
+               inset_inc->setChildBuffer(nullptr);
        }
        // invalidate cache of children
        d->children_positions.clear();
@@ -5461,7 +5662,7 @@ void Buffer::checkMasterBuffer()
        if (master->isChild(this))
                setParent(master);
        else
-               setParent(0);
+               setParent(nullptr);
 }
 
 
@@ -5494,29 +5695,6 @@ string Buffer::includedFilePath(string const & name, string const & ext) const
 }
 
 
-void Buffer::setChangesPresent(bool b) const
-{
-       d->tracked_changes_present_ = b;
-}
-
-
-bool Buffer::areChangesPresent() const
-{
-       return d->tracked_changes_present_;
-}
-
-
-void Buffer::updateChangesPresent() const
-{
-       LYXERR(Debug::CHANGES, "Buffer::updateChangesPresent");
-       setChangesPresent(false);
-       ParConstIterator it = par_iterator_begin();
-       ParConstIterator const end = par_iterator_end();
-       for (; !areChangesPresent() && it != end; ++it)
-               it->addChangesToBuffer(*this);
-}
-
-
 void Buffer::Impl::refreshFileMonitor()
 {
        if (file_monitor_ && file_monitor_->filename() == filename.absFileName()) {
@@ -5543,15 +5721,24 @@ void Buffer::Impl::fileExternallyModified(bool const exists)
                       "checksum unchanged: " << filename);
                return;
        }
+       lyx_clean = false;
        // If the file has been deleted, only mark the file as dirty since it is
        // pointless to prompt for reloading. If later a file is moved into this
        // location, then the externally modified warning will appear then.
        if (exists)
-                       externally_modified_ = true;
+               externally_modified_ = true;
        // Update external modification notification.
        // Dirty buffers must be visible at all times.
-       if (wa_ && wa_->unhide(owner_))
+       if (wa_ && wa_->unhide(owner_)) {
                wa_->updateTitles();
+               if (!exists) {
+                       frontend::Alert::warning(
+                               _("File deleted from disk"),
+                               bformat(_("The file\n  %1$s\n"
+                                          "has been deleted from disk!"),
+                                       from_utf8(filename.absFileName())));
+               }
+       }
        else
                // Unable to unhide the buffer (e.g. no GUI or not current View)
                lyx_clean = true;
@@ -5572,4 +5759,55 @@ void Buffer::clearExternalModification() const
 }
 
 
+void Buffer::pushIncludedBuffer(Buffer const * buf) const
+{
+       masterBuffer()->d->include_list_.push_back(buf);
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Pushed. Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+}
+
+
+void Buffer::popIncludedBuffer() const
+{
+       masterBuffer()->d->include_list_.pop_back();
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Popped. Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+}
+
+
+bool Buffer::isBufferIncluded(Buffer const * buf) const
+{
+       if (!buf)
+               return false;
+       if (lyxerr.debugging(Debug::FILES)) {
+               LYXERR0("Checking for " << buf->fileName() << ". Stack now:");
+               if (masterBuffer()->d->include_list_.empty())
+                       LYXERR0("EMPTY!");
+               else
+                       for (auto const & b : masterBuffer()->d->include_list_)
+                               LYXERR0(b->fileName());
+       }
+       list<Buffer const *> const & blist = masterBuffer()->d->include_list_;
+       return find(blist.begin(), blist.end(), buf) != blist.end();
+}
+
+
+void Buffer::clearIncludeList() const
+{
+       LYXERR(Debug::FILES, "Clearing include list for " << fileName());
+       d->include_list_.clear();
+}
+
 } // namespace lyx