X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2FBuffer.cpp;h=97a6cfbd31afe1e771034326ef982b37fff7a7fb;hb=2dae4ab4f35458f3caa4f85f8c0e64e5fe9065d1;hp=50cc2f243fd25b8cdd3b43358b87c25357fcfedc;hpb=c457ea90e72b7e9d2f6a0edfbf9ae01378df372c;p=lyx.git

diff --git a/src/Buffer.cpp b/src/Buffer.cpp
index 50cc2f243f..97a6cfbd31 100644
--- a/src/Buffer.cpp
+++ b/src/Buffer.cpp
@@ -46,18 +46,18 @@
 #include "LyX.h"
 #include "LyXRC.h"
 #include "LyXVC.h"
-#include "output_docbook.h"
 #include "output.h"
 #include "output_latex.h"
-#include "output_xhtml.h"
+#include "output_docbook.h"
 #include "output_plaintext.h"
+#include "output_xhtml.h"
 #include "Paragraph.h"
 #include "ParagraphParameters.h"
 #include "ParIterator.h"
 #include "PDFOptions.h"
 #include "Session.h"
 #include "SpellChecker.h"
-#include "sgml.h"
+#include "xml.h"
 #include "texstream.h"
 #include "TexRow.h"
 #include "Text.h"
@@ -83,6 +83,7 @@
 #include "graphics/GraphicsCache.h"
 #include "graphics/PreviewLoader.h"
 
+#include "frontends/Application.h"
 #include "frontends/alert.h"
 #include "frontends/Delegates.h"
 #include "frontends/WorkAreaManager.h"
@@ -110,8 +111,6 @@
 #include "support/textutils.h"
 #include "support/types.h"
 
-#include "support/bind.h"
-
 #include <algorithm>
 #include <fstream>
 #include <iomanip>
@@ -148,13 +147,16 @@ struct LabelInfo {
 	bool active;
 };
 
-typedef vector<LabelInfo> RefCache;
-
-} // namespace
+typedef vector<LabelInfo> LabelCache;
 
+typedef map<docstring, Buffer::References> RefCache;
 
 // A storehouse for the cloned buffers.
-list<CloneList *> cloned_buffers;
+typedef list<CloneList_ptr> CloneStore;
+CloneStore cloned_buffers;
+
+} // namespace
+
 
 
 class Buffer::Impl
@@ -197,40 +199,9 @@ public:
 	/// need to regenerate .tex?
 	DepClean dep_clean;
 
-	/// is save needed?
-	mutable bool lyx_clean;
-
-	/// is autosave needed?
-	mutable bool bak_clean;
-
-	/// is this an unnamed file (New...)?
-	bool unnamed;
-
-	/// is this an internal bufffer?
-	bool internal_buffer;
-
-	/// buffer is r/o
-	bool read_only;
-
 	/// name of the file the buffer is associated with.
 	FileName filename;
 
-	/** Set to true only when the file is fully loaded.
-	 *  Used to prevent the premature generation of previews
-	 *  and by the citation inset.
-	 */
-	bool file_fully_loaded;
-
-	/// original format of loaded file
-	int file_format;
-
-	/// if the file was originally loaded from an older format, do
-	/// we need to back it up still?
-	bool need_format_backup;
-
-	/// Ignore the parent (e.g. when exporting a child standalone)?
-	bool ignore_parent;
-
 	///
 	mutable TocBackend toc_backend;
 
@@ -247,13 +218,11 @@ public:
 	/// map from the macro name to the position map,
 	/// which maps the macro definition position to the scope and the MacroData.
 	NamePositionScopeMacroMap macros;
-	/// This seem to change the way Buffer::getMacro() works
-	mutable bool macro_lock;
 
 	/// positions of child buffers in the buffer
 	typedef map<Buffer const * const, DocIterator> BufferPositionMap;
 	struct ScopeBuffer {
-		ScopeBuffer() : buffer(0) {}
+		ScopeBuffer() : buffer(nullptr) {}
 		ScopeBuffer(DocIterator const & s, Buffer const * b)
 			: scope(s), buffer(b) {}
 		DocIterator scope;
@@ -300,27 +269,19 @@ public:
 	// file, and then to construct the Buffer's bibinfo from that.
 	/// A cache for bibliography info
 	mutable BiblioInfo bibinfo_;
-	/// whether the bibinfo cache is valid
-	mutable bool bibinfo_cache_valid_;
 	/// Cache of timestamps of .bib files
 	map<FileName, time_t> bibfile_status_;
-	/// Indicates whether the bibinfo has changed since the last time
-	/// we ran updateBuffer(), i.e., whether citation labels may need
-	/// to be updated.
-	mutable bool cite_labels_valid_;
 
 	/// These two hold the file name and format, written to by
 	/// Buffer::preview and read from by LFUN_BUFFER_VIEW_CACHE.
 	FileName preview_file_;
 	string preview_format_;
-	/// If there was an error when previewing, on the next preview we do
-	/// a fresh compile (e.g. in case the user installed a package that
-	/// was missing).
-	bool preview_error_;
 
-	/// Cache the label insets, their associated refs (with positions),
-	/// and whether the insets are active.
+	/// Cache the references associated to a label and their positions
+	/// in the buffer.
 	mutable RefCache ref_cache_;
+	/// Cache the label insets and their activity status.
+	mutable LabelCache label_cache_;
 
 	/// our Text that should be wrapped in an InsetText
 	InsetText * inset;
@@ -328,6 +289,83 @@ public:
 	///
 	PreviewLoader * preview_loader_;
 
+	/// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
+	/// This one is useful for preview detached in a thread.
+	Buffer const * cloned_buffer_;
+	///
+	CloneList_ptr clone_list_;
+
+	///
+	std::list<Buffer const *> include_list_;
+private:
+	/// So we can force access via the accessors.
+	mutable Buffer const * parent_buffer;
+
+	FileMonitorPtr file_monitor_;
+
+/// ints and bools are all listed last so as to avoid alignment issues
+public:
+	/// original format of loaded file
+	int file_format;
+
+	/// are we in the process of exporting this buffer?
+	mutable bool doing_export;
+
+	/// If there was an error when previewing, on the next preview we do
+	/// a fresh compile (e.g. in case the user installed a package that
+	/// was missing).
+	bool require_fresh_start_;
+
+	/// Indicates whether the bibinfo has changed since the last time
+	/// we ran updateBuffer(), i.e., whether citation labels may need
+	/// to be updated.
+	mutable bool cite_labels_valid_;
+	/// Do we have a bibliography environment?
+	mutable bool have_bibitems_;
+
+	/// is save needed?
+	mutable bool lyx_clean;
+
+	/// is autosave needed?
+	mutable bool bak_clean;
+
+	/// is this an unnamed file (New...)?
+	bool unnamed;
+
+	/// is this an internal bufffer?
+	bool internal_buffer;
+
+	/// buffer is r/o
+	bool read_only;
+
+	/** Set to true only when the file is fully loaded.
+	 *  Used to prevent the premature generation of previews
+	 *  and by the citation inset.
+	 */
+	bool file_fully_loaded;
+
+	/// if the file was originally loaded from an older format, do
+	/// we need to back it up still?
+	bool need_format_backup;
+
+	/// Ignore the parent (e.g. when exporting a child standalone)?
+	bool ignore_parent;
+
+	/// This seem to change the way Buffer::getMacro() works
+	mutable bool macro_lock;
+
+	/// has been externally modified? Can be reset by the user.
+	mutable bool externally_modified_;
+
+	/// whether the bibinfo cache is valid
+	mutable bool bibinfo_cache_valid_;
+
+private:
+	int word_count_;
+	int char_count_;
+	int blank_count_;
+
+public:
 	/// This is here to force the test to be done whenever parent_buffer
 	/// is accessed.
 	Buffer const * parent() const
@@ -335,7 +373,7 @@ public:
 		// ignore_parent temporarily "orphans" a buffer
 		// (e.g. if a child is compiled standalone)
 		if (ignore_parent)
-			return 0;
+			return nullptr;
 		// if parent_buffer is not loaded, then it has been unloaded,
 		// which means that parent_buffer is an invalid pointer. So we
 		// set it to null in that case.
@@ -343,7 +381,7 @@ public:
 		// they will always be regarded as unloaded. in that case, we hope
 		// for the best.
 		if (!cloned_buffer_ && !theBufferList().isLoaded(parent_buffer))
-			parent_buffer = 0;
+			parent_buffer = nullptr;
 		return parent_buffer;
 	}
 
@@ -360,14 +398,6 @@ public:
 			parent_buffer->invalidateBibinfoCache();
 	}
 
-	/// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
-	/// This one is useful for preview detached in a thread.
-	Buffer const * cloned_buffer_;
-	///
-	CloneList * clone_list_;
-	/// are we in the process of exporting this buffer?
-	mutable bool doing_export;
-
 	/// compute statistics
 	/// \p from initial position
 	/// \p to points to the end position
@@ -384,28 +414,17 @@ public:
 		+ (with_blanks ? blank_count_ : 0);
 	}
 
-	// does the buffer contain tracked changes? (if so, we automatically
-	// display the review toolbar, for instance)
-	mutable bool tracked_changes_present_;
-
 	// Make sure the file monitor monitors the good file.
 	void refreshFileMonitor();
 
 	/// Notify or clear of external modification
 	void fileExternallyModified(bool exists);
 
-	/// has been externally modified? Can be reset by the user.
-	mutable bool externally_modified_;
-
-private:
-	/// So we can force access via the accessors.
-	mutable Buffer const * parent_buffer;
-
-	int word_count_;
-	int char_count_;
-	int blank_count_;
-
-	FileMonitorPtr file_monitor_;
+	///Binding LaTeX lines with buffer positions.
+	//Common routine for LaTeX and Reference errors listing.
+        void traverseErrors(TeXErrors::Errors::const_iterator err,
+		TeXErrors::Errors::const_iterator end,
+		ErrorList & errorList) const;
 };
 
 
@@ -438,21 +457,26 @@ static FileName createBufferTmpDir()
 
 Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
 	Buffer const * cloned_buffer)
-	: owner_(owner), lyx_clean(true), bak_clean(true), unnamed(false),
-	  internal_buffer(false), read_only(readonly_), filename(file),
-	  file_fully_loaded(false), file_format(LYX_FORMAT), need_format_backup(false),
-	  ignore_parent(false),  toc_backend(owner), macro_lock(false),
-	  checksum_(0), wa_(0),  gui_(0), undo_(*owner), bibinfo_cache_valid_(false),
-	  cite_labels_valid_(false), preview_error_(false),
-	  inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
-	  clone_list_(0), doing_export(false),
-	  tracked_changes_present_(0), externally_modified_(false), parent_buffer(0),
+	: owner_(owner), filename(file), toc_backend(owner), checksum_(0),
+	  wa_(nullptr),  gui_(nullptr), undo_(*owner), inset(nullptr),
+	  preview_loader_(nullptr), cloned_buffer_(cloned_buffer),
+	  clone_list_(nullptr), parent_buffer(nullptr), file_format(LYX_FORMAT),
+	  doing_export(false), require_fresh_start_(false), cite_labels_valid_(false),
+	  have_bibitems_(false), lyx_clean(true), bak_clean(true), unnamed(false),
+	  internal_buffer(false), read_only(readonly_), file_fully_loaded(false),
+	  need_format_backup(false), ignore_parent(false), macro_lock(false),
+	  externally_modified_(false), bibinfo_cache_valid_(false),
 	  word_count_(0), char_count_(0), blank_count_(0)
 {
 	refreshFileMonitor();
 	if (!cloned_buffer_) {
 		temppath = createBufferTmpDir();
 		lyxvc.setBuffer(owner_);
+		Language const * inplang = theApp() ?
+					languages.getFromCode(theApp()->inputLanguageCode())
+				      : nullptr;
+		if (inplang)
+			params.language = inplang;
 		if (use_gui)
 			wa_ = new frontend::WorkAreaManager;
 		return;
@@ -465,13 +489,13 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
 	bibinfo_cache_valid_ = cloned_buffer_->d->bibinfo_cache_valid_;
 	bibfile_status_ = cloned_buffer_->d->bibfile_status_;
 	cite_labels_valid_ = cloned_buffer_->d->cite_labels_valid_;
+	have_bibitems_ = cloned_buffer_->d->have_bibitems_;
 	unnamed = cloned_buffer_->d->unnamed;
 	internal_buffer = cloned_buffer_->d->internal_buffer;
 	layout_position = cloned_buffer_->d->layout_position;
 	preview_file_ = cloned_buffer_->d->preview_file_;
 	preview_format_ = cloned_buffer_->d->preview_format_;
-	preview_error_ = cloned_buffer_->d->preview_error_;
-	tracked_changes_present_ = cloned_buffer_->d->tracked_changes_present_;
+	require_fresh_start_ = cloned_buffer_->d->require_fresh_start_;
 }
 
 
@@ -501,7 +525,7 @@ Buffer::~Buffer()
 	// saved properly, before it goes into the void.
 
 	// GuiView already destroyed
-	d->gui_ = 0;
+	d->gui_ = nullptr;
 
 	if (isInternal()) {
 		// No need to do additional cleanups for internal buffer.
@@ -516,8 +540,8 @@ Buffer::~Buffer()
 		// loop over children
 		for (auto const & p : d->children_positions) {
 			Buffer * child = const_cast<Buffer *>(p.first);
-				if (d->clone_list_->erase(child))
-					delete child;
+			if (d->clone_list_->erase(child))
+				delete child;
 		}
 		// if we're the master buffer, then we should get rid of the list
 		// of clones
@@ -526,14 +550,15 @@ Buffer::~Buffer()
 			// children still has a reference to this list. But we will try to
 			// continue, rather than shut down.
 			LATTEST(d->clone_list_->empty());
-			list<CloneList *>::iterator it =
+			// The clone list itself is empty, but it's still referenced in our list
+			// of clones. So let's find it and remove it.
+			CloneStore::iterator it =
 				find(cloned_buffers.begin(), cloned_buffers.end(), d->clone_list_);
 			if (it == cloned_buffers.end()) {
 				// We will leak in this case, but it is safe to continue.
 				LATTEST(false);
 			} else
 				cloned_buffers.erase(it);
-			delete d->clone_list_;
 		}
 		// FIXME Do we really need to do this right before we delete d?
 		// clear references to children in macro tables
@@ -545,7 +570,7 @@ Buffer::~Buffer()
 			Buffer * child = const_cast<Buffer *>(p.first);
 			if (theBufferList().isLoaded(child)) {
 				if (theBufferList().isOthersChild(this, child))
-					child->setParent(0);
+					child->setParent(nullptr);
 				else
 					theBufferList().release(child);
 			}
@@ -580,21 +605,21 @@ Buffer::~Buffer()
 Buffer * Buffer::cloneWithChildren() const
 {
 	BufferMap bufmap;
-	cloned_buffers.push_back(new CloneList);
-	CloneList * clones = cloned_buffers.back();
+	cloned_buffers.emplace_back(new CloneList);
+	CloneList_ptr clones = cloned_buffers.back();
 
 	cloneWithChildren(bufmap, clones);
 
 	// make sure we got cloned
 	BufferMap::const_iterator bit = bufmap.find(this);
-	LASSERT(bit != bufmap.end(), return 0);
+	LASSERT(bit != bufmap.end(), return nullptr);
 	Buffer * cloned_buffer = bit->second;
 
 	return cloned_buffer;
 }
 
 
-void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
+void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList_ptr clones) const
 {
 	// have we already been cloned?
 	if (bufmap.find(this) != bufmap.end())
@@ -644,8 +669,8 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
 
 
 Buffer * Buffer::cloneBufferOnly() const {
-	cloned_buffers.push_back(new CloneList);
-	CloneList * clones = cloned_buffers.back();
+	cloned_buffers.emplace_back(new CloneList);
+	CloneList_ptr clones = cloned_buffers.back();
 	Buffer * buffer_clone = new Buffer(fileName().absFileName(), false, this);
 
 	// The clone needs its own DocumentClass, since running updateBuffer() will
@@ -805,10 +830,9 @@ FileName Buffer::Impl::exportFileName() const
 	if (branch_suffix.empty())
 		return filename;
 
-	string const name = filename.onlyFileNameWithoutExt()
-		+ to_utf8(branch_suffix);
+	string const name = addExtension(filename.onlyFileNameWithoutExt()
+			+ to_utf8(branch_suffix), filename.extension());
 	FileName res(filename.onlyPath().absFileName() + "/" + name);
-	res.changeExtension(filename.extension());
 
 	return res;
 }
@@ -914,6 +938,9 @@ int Buffer::readHeader(Lexer & lex)
 	params().headsep.erase();
 	params().footskip.erase();
 	params().columnsep.erase();
+	params().font_roman_opts.erase();
+	params().font_sans_opts.erase();
+	params().font_typewriter_opts.erase();
 	params().fonts_cjk.erase();
 	params().listings_params.clear();
 	params().clearLayoutModules();
@@ -937,6 +964,7 @@ int Buffer::readHeader(Lexer & lex)
 	params().biblatex_bibstyle.erase();
 	params().biblatex_citestyle.erase();
 	params().multibib.erase();
+	params().lineno_opts.clear();
 
 	for (int i = 0; i < 4; ++i) {
 		params().user_defined_bullet(i) = ITEMIZE_DEFAULTS[i];
@@ -1011,20 +1039,13 @@ bool Buffer::readDocument(Lexer & lex)
 	readHeader(lex);
 
 	if (params().output_changes) {
-		bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
 		bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
 				  LaTeXFeatures::isAvailable("xcolor");
 
-		if (!dvipost && !xcolorulem) {
+		if (!xcolorulem) {
 			Alert::warning(_("Changes not shown in LaTeX output"),
 				       _("Changes will not be highlighted in LaTeX output, "
-					 "because neither dvipost nor xcolor/ulem are installed.\n"
-					 "Please install these packages or redefine "
-					 "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
-		} else if (!xcolorulem) {
-			Alert::warning(_("Changes not shown in LaTeX output"),
-				       _("Changes will not be highlighted in LaTeX output "
-					 "when using pdflatex, because xcolor and ulem are not installed.\n"
+					 "because xcolor and ulem are not installed.\n"
 					 "Please install both packages or redefine "
 					 "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
 		}
@@ -1072,6 +1093,10 @@ bool Buffer::readDocument(Lexer & lex)
 		d->old_position = params().origin;
 	else
 		d->old_position = filePath();
+
+	if (!parent())
+		clearIncludeList();
+
 	bool const res = text().read(lex, errorList, d->inset);
 	d->old_position.clear();
 
@@ -1123,7 +1148,7 @@ bool Buffer::importFile(string const & format, FileName const & name, ErrorList
 
 	FileName const lyx = tempFileName("Buffer_importFileXXXXXX.lyx");
 	Converters::RetVal const retval =
-	    theConverters().convert(0, name, lyx, name, format, "lyx", errorList);
+		theConverters().convert(nullptr, name, lyx, name, format, "lyx", errorList);
 	if (retval == Converters::SUCCESS) {
 		bool const success = readFile(lyx) == ReadSuccess;
 		removeTempFile(lyx);
@@ -1224,16 +1249,21 @@ void Buffer::setFullyLoaded(bool value)
 }
 
 
-bool Buffer::lastPreviewError() const
+bool Buffer::freshStartRequired() const
 {
-	return d->preview_error_;
+	return d->require_fresh_start_;
+}
+
+void Buffer::requireFreshStart(bool const b) const
+{
+	d->require_fresh_start_ = b;
 }
 
 
 PreviewLoader * Buffer::loader() const
 {
 	if (!isExporting() && lyxrc.preview == LyXRC::PREVIEW_OFF)
-		return 0;
+		return nullptr;
 	if (!d->preview_loader_)
 		d->preview_loader_ = new PreviewLoader(*this);
 	return d->preview_loader_;
@@ -1243,7 +1273,7 @@ PreviewLoader * Buffer::loader() const
 void Buffer::removePreviews() const
 {
 	delete d->preview_loader_;
-	d->preview_loader_ = 0;
+	d->preview_loader_ = nullptr;
 }
 
 
@@ -1713,13 +1743,6 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
 {
 	OutputParams runparams = runparams_in;
 
-	// XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740),
-	// Check here, because the "flavor" is not known in BufferParams::encoding()
-	// (power users can override this safety measure selecting "utf8-plain").
-	if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
-	    && (runparams.encoding->name() != "utf8-plain"))
-		runparams.encoding = encodings.fromLyXName("ascii");
-
 	string const encoding = runparams.encoding->iconvName();
 	LYXERR(Debug::LATEX, "makeLaTeXFile encoding: " << encoding << ", fname=" << fname.realPath());
 
@@ -1727,9 +1750,11 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
 	try { ofs.reset(encoding); }
 	catch (iconv_codecvt_facet_exception const & e) {
 		lyxerr << "Caught iconv exception: " << e.what() << endl;
-		Alert::error(_("Iconv software exception Detected"), bformat(_("Please "
-			"verify that the support software for your encoding (%1$s) is "
-			"properly installed"), from_ascii(encoding)));
+		Alert::error(_("Iconv software exception Detected"),
+			bformat(_("Please verify that the `iconv' support software is"
+					  " properly installed and supports the selected encoding"
+					  " (%1$s), or change the encoding in"
+					  " Document>Settings>Language."), from_ascii(encoding)));
 		return ExportError;
 	}
 	if (!openFileWrite(ofs, fname))
@@ -1817,16 +1842,6 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 
 	OutputParams runparams = runparams_in;
 
-	// XeTeX with TeX fonts is only safe with ASCII encoding,
-	// Check here, because the "flavor" is not known in BufferParams::encoding()
-	// (power users can override this safety measure selecting "utf8-plain").
-	if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
-	    && (runparams.encoding->name() != "utf8-plain"))
-		runparams.encoding = encodings.fromLyXName("ascii");
-	// FIXME: when only the current paragraph is shown, this is ignored
-	//        (or not reached) and characters encodable in the current
-	//        encoding are not converted to ASCII-representation.
-
 	// Some macros rely on font encoding
 	runparams.main_fontenc = params().main_font_encoding();
 
@@ -1845,8 +1860,11 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 	LaTeXFeatures features(*this, params(), runparams);
 	validate(features);
 	// This is only set once per document (in master)
-	if (!runparams.is_child)
+	if (!runparams.is_child) {
 		runparams.use_polyglossia = features.usePolyglossia();
+		runparams.use_hyperref = features.isRequired("hyperref");
+		runparams.use_CJK = features.mustProvide("CJK");
+	}
 	LYXERR(Debug::LATEX, "  Buffer validation done.");
 
 	bool const output_preamble =
@@ -1931,8 +1949,15 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 					os << "\\catcode`\\%=11"
 					      "\\def\\%{%}\\catcode`\\%=14\n";
 				}
-				bool const detokenize = !isAscii(from_utf8(docdir));
+				if (contains(docdir, '~'))
+					docdir = subst(docdir, "~", "\\string~");
+				bool const nonascii = !isAscii(from_utf8(docdir));
+				// LaTeX 2019/10/01 handles non-ascii path without detokenize
+				bool const utfpathlatex = features.isAvailable("LaTeX-2019/10/01");
+				bool const detokenize = !utfpathlatex && nonascii;
 				bool const quote = contains(docdir, ' ');
+				if (utfpathlatex && nonascii)
+					os << "\\UseRawInputEncoding\n";
 				os << "\\makeatletter\n"
 				   << "\\def\\input@path{{";
 				if (detokenize)
@@ -1958,12 +1983,19 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
 		runparams.use_babel = params().writeLaTeX(os, features,
 							  d->filename.onlyPath());
 
+		// Active characters
+		runparams.active_chars = features.getActiveChars();
+
 		// Biblatex bibliographies are loaded here
 		if (params().useBiblatex()) {
-			vector<docstring> const bibfiles =
+			vector<pair<docstring, string>> const bibfiles =
 				prepareBibFilePaths(runparams, getBibfiles(), true);
-			for (docstring const & file: bibfiles)
-				os << "\\addbibresource{" << file << "}\n";
+			for (pair<docstring, string> const & file: bibfiles) {
+				os << "\\addbibresource";
+				if (!file.second.empty())
+					os << "[bibencoding=" << file.second << "]";
+				os << "{" << file.first << "}\n";
+			}
 		}
 
 		if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages()
@@ -2080,7 +2112,7 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
 	updateMacroInstances(OutputUpdate);
 
 	ExportStatus const retval =
-		writeDocBookSource(ofs, fname.absFileName(), runparams, output);
+		writeDocBookSource(ofs, runparams, output);
 	if (retval == ExportKilled)
 		return ExportKilled;
 
@@ -2091,85 +2123,56 @@ Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
 }
 
 
-Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os, string const & fname,
+Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os,
 			     OutputParams const & runparams,
 			     OutputWhat output) const
 {
 	LaTeXFeatures features(*this, params(), runparams);
 	validate(features);
+	d->bibinfo_.makeCitationLabels(*this);
 
 	d->texrow.reset();
 
 	DocumentClass const & tclass = params().documentClass();
-	string const & top_element = tclass.latexname();
 
 	bool const output_preamble =
 		output == FullSource || output == OnlyPreamble;
 	bool const output_body =
 	  output == FullSource || output == OnlyBody;
 
-	if (output_preamble) {
-		if (runparams.flavor == OutputParams::XML)
-			os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+	XMLStream xs(os);
 
-		// FIXME UNICODE
-		os << "<!DOCTYPE " << from_ascii(top_element) << ' ';
-
-		// FIXME UNICODE
-		if (! tclass.class_header().empty())
-			os << from_ascii(tclass.class_header());
-		else if (runparams.flavor == OutputParams::XML)
-			os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
-			    << "\"https://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
-		else
-			os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
-
-		docstring preamble = params().preamble;
-		if (runparams.flavor != OutputParams::XML ) {
-			preamble += "<!ENTITY % output.print.png \"IGNORE\">\n";
-			preamble += "<!ENTITY % output.print.pdf \"IGNORE\">\n";
-			preamble += "<!ENTITY % output.print.eps \"IGNORE\">\n";
-			preamble += "<!ENTITY % output.print.bmp \"IGNORE\">\n";
-		}
+	if (output_preamble) {
+		// XML preamble, no doctype needed.
+		// Not using XMLStream for this, as the root tag would be in the tag stack and make troubles with the error
+		// detection mechanisms (these are called before the end tag is output, and thus interact with the canary
+		// parsep in output_docbook.cpp).
+		os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+		   << "<!-- This DocBook file was created by LyX " << lyx_version
+		   << "\n  See http://www.lyx.org/ for more information -->\n";
 
-		string const name = runparams.nice
-			? changeExtension(absFileName(), ".sgml") : fname;
-		preamble += features.getIncludedFiles(name);
-		preamble += features.getLyXSGMLEntities();
+		// Directly output the root tag, based on the current type of document.
+		string languageCode = params().language->code();
+		string params = "xml:lang=\"" + languageCode + '"'
+						+ " xmlns=\"http://docbook.org/ns/docbook\""
+						+ " xmlns:xlink=\"http://www.w3.org/1999/xlink\""
+						+ " xmlns:m=\"http://www.w3.org/1998/Math/MathML\""
+						+ " xmlns:xi=\"http://www.w3.org/2001/XInclude\""
+						+ " version=\"5.2\"";
 
-		if (!preamble.empty()) {
-			os << "\n [ " << preamble << " ]";
-		}
-		os << ">\n\n";
+		os << "<" << from_ascii(tclass.docbookroot()) << " " << from_ascii(params) << ">\n";
 	}
 
 	if (output_body) {
-		string top = top_element;
-		top += " lang=\"";
-		if (runparams.flavor == OutputParams::XML)
-			top += params().language->code();
-		else
-			top += params().language->code().substr(0, 2);
-		top += '"';
-
-		if (!params().options.empty()) {
-			top += ' ';
-			top += params().options;
-		}
-
-		os << "<!-- " << ((runparams.flavor == OutputParams::XML)? "XML" : "SGML")
-				<< " file was created by LyX " << lyx_version
-				<< "\n  See https://www.lyx.org/ for more information -->\n";
-
 		params().documentClass().counters().reset();
 
-		sgml::openTag(os, top);
-		os << '\n';
-		try {
-			docbookParagraphs(text(), *this, os, runparams);
-		}
-		catch (ConversionException const &) { return ExportKilled; }
-		sgml::closeTag(os, top_element);
+		// Start to output the document.
+		docbookParagraphs(text(), *this, xs, runparams);
+	}
+
+	if (output_preamble) {
+		// Close the root element.
+		os << "\n</" << from_ascii(tclass.docbookroot()) << ">";
 	}
 	return ExportSuccess;
 }
@@ -2227,17 +2230,13 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
 		os << "<title>"
 		   << (doctitle.empty() ?
 		         from_ascii("LyX Document") :
-		         html::htmlize(doctitle, XHTMLStream::ESCAPE_ALL))
+		         xml::escapeString(doctitle, XMLStream::ESCAPE_ALL))
 		   << "</title>\n";
 
 		docstring styles = features.getTClassHTMLPreamble();
 		if (!styles.empty())
 			os << "\n<!-- Text Class Preamble -->\n" << styles << '\n';
 
-		styles = features.getPreambleSnippets().str;
-		if (!styles.empty())
-			os << "\n<!-- Preamble Snippets -->\n" << styles << '\n';
-
 		// we will collect CSS information in a stream, and then output it
 		// either here, as part of the header, or else in a separate file.
 		odocstringstream css;
@@ -2297,7 +2296,7 @@ Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
 		bool const output_body_tag = (output != IncludedFile);
 		if (output_body_tag)
 			os << "<body dir=\"auto\">\n";
-		XHTMLStream xs(os);
+		XMLStream xs(os);
 		if (output != IncludedFile)
 			// if we're an included file, the counters are in the master.
 			params().documentClass().counters().reset();
@@ -2374,6 +2373,9 @@ void Buffer::validate(LaTeXFeatures & features) const
 	if (!features.runparams().is_child)
 		params().validate(features);
 
+	if (!parent())
+		clearIncludeList();
+
 	for (Paragraph const & p : paragraphs())
 		p.validate(features);
 
@@ -2470,17 +2472,25 @@ FileName Buffer::getBibfilePath(docstring const & bibid) const
 	map<docstring, FileName>::const_iterator it =
 		bibfileCache.find(bibid);
 	if (it != bibfileCache.end()) {
-		// i.e., bibfileCache[bibid]
+		// i.e., return bibfileCache[bibid];
 		return it->second;
 	}
 
 	LYXERR(Debug::FILES, "Reading file location for " << bibid);
-	string texfile = changeExtension(to_utf8(bibid), "bib");
-	// note that, if the filename can be found directly from the path,
-	// findtexfile will just return a FileName object for that path.
-	FileName file(findtexfile(texfile, "bib"));
-	if (file.empty())
-		file = FileName(makeAbsPath(texfile, filePath()));
+	string const texfile = changeExtension(to_utf8(bibid), "bib");
+	// we need to check first if this file exists where it's said to be.
+	// there's a weird bug that occurs otherwise: if the file is in the
+	// Buffer's directory but has the same name as some file that would be
+	// found by kpsewhich, then we find the latter, not the former.
+	FileName const local_file = makeAbsPath(texfile, filePath());
+	FileName file = local_file;
+	if (!file.exists()) {
+		// there's no need now to check whether the file can be found
+		// locally
+		file = findtexfile(texfile, "bib", true);
+		if (file.empty())
+			file = local_file;
+	}
 	LYXERR(Debug::FILES, "Found at: " << file);
 
 	bibfileCache[bibid] = file;
@@ -2497,12 +2507,28 @@ void Buffer::checkIfBibInfoCacheIsValid() const
 		return;
 	}
 
-	// if we already know the cache is invalid, no need to check
-	// the timestamps
+	// If we already know the cache is invalid, stop here.
+	// This is important in the case when the bibliography
+	// environment (rather than Bib[la]TeX) is used.
+	// In that case, the timestamp check below gives no
+	// sensible result. Rather than that, the cache will
+	// be invalidated explicitly via invalidateBibInfoCache()
+	// by the Bibitem inset.
+	// Same applies for bib encoding changes, which trigger
+	// invalidateBibInfoCache() by InsetBibtex.
 	if (!d->bibinfo_cache_valid_)
 		return;
 
-	// we'll assume it's ok and change this if it's not
+	if (d->have_bibitems_) {
+		// We have a bibliography environment.
+		// Invalidate the bibinfo cache unconditionally.
+		// Cite labels will get invalidated by the inset if needed.
+		d->bibinfo_cache_valid_ = false;
+		return;
+	}
+
+	// OK. This is with Bib(la)tex. We'll assume the cache
+	// is valid and change this if we find changes in the bibs.
 	d->bibinfo_cache_valid_ = true;
 	d->cite_labels_valid_ = true;
 
@@ -2554,6 +2580,7 @@ void Buffer::reloadBibInfoCache(bool const force) const
 	clearBibFileCache();
 	d->bibinfo_.clear();
 	FileNameList checkedFiles;
+	d->have_bibitems_ = false;
 	collectBibKeys(checkedFiles);
 	d->bibinfo_cache_valid_ = true;
 }
@@ -2561,8 +2588,18 @@ void Buffer::reloadBibInfoCache(bool const force) const
 
 void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 {
-	for (InsetIterator it = inset_iterator_begin(inset()); it; ++it)
+	if (!parent())
+		clearIncludeList();
+
+	for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
 		it->collectBibKeys(it, checkedFiles);
+		if (it->lyxCode() == BIBITEM_CODE) {
+			if (parent() != nullptr)
+				parent()->d->have_bibitems_ = true;
+			else
+				d->have_bibitems_ = true;
+		}
+	}
 }
 
 
@@ -2574,7 +2611,7 @@ void Buffer::addBiblioInfo(BiblioInfo const & bin) const
 	BiblioInfo & bi = d->bibinfo_;
 	bi.mergeBiblioInfo(bin);
 
-	if (parent() != 0) {
+	if (parent() != nullptr) {
 		BiblioInfo & masterbi = parent()->d->bibinfo_;
 		masterbi.mergeBiblioInfo(bin);
 	}
@@ -2589,7 +2626,7 @@ void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bin) const
 	BiblioInfo & bi = d->bibinfo_;
 	bi[key] = bin;
 
-	if (parent() != 0) {
+	if (parent() != nullptr) {
 		BiblioInfo & masterbi = masterBuffer()->d->bibinfo_;
 		masterbi[key] = bin;
 	}
@@ -2949,7 +2986,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 				// get buffer of external file
 				InsetInclude const & ins =
 					static_cast<InsetInclude const &>(*it);
-				Buffer * child = ins.getChildBuffer();
+				Buffer * child = ins.loadIfNeeded();
 				if (!child)
 					continue;
 				child->dispatch(func, dr);
@@ -2973,8 +3010,6 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 		if (params().save_transient_properties)
 			undo().recordUndoBufferParams(CursorData());
 		params().track_changes = !params().track_changes;
-		if (!params().track_changes)
-			dr.forceChangesUpdate();
 		break;
 
 	case LFUN_CHANGES_OUTPUT:
@@ -2982,20 +3017,13 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
 			undo().recordUndoBufferParams(CursorData());
 		params().output_changes = !params().output_changes;
 		if (params().output_changes) {
-			bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
 			bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
 					  LaTeXFeatures::isAvailable("xcolor");
 
-			if (!dvipost && !xcolorulem) {
+			if (!xcolorulem) {
 				Alert::warning(_("Changes not shown in LaTeX output"),
 					       _("Changes will not be highlighted in LaTeX output, "
-						 "because neither dvipost nor xcolor/ulem are installed.\n"
-						 "Please install these packages or redefine "
-						 "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
-			} else if (!xcolorulem) {
-				Alert::warning(_("Changes not shown in LaTeX output"),
-					       _("Changes will not be highlighted in LaTeX output "
-						 "when using pdflatex, because xcolor and ulem are not installed.\n"
+						 "because xcolor and ulem are not installed.\n"
 						 "Please install both packages or redefine "
 						 "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
 			}
@@ -3036,9 +3064,10 @@ void Buffer::changeLanguage(Language const * from, Language const * to)
 	LASSERT(from, return);
 	LASSERT(to, return);
 
-	for_each(par_iterator_begin(),
-		 par_iterator_end(),
-		 bind(&Paragraph::changeLanguage, _1, params(), from, to));
+	ParIterator it = par_iterator_begin();
+	ParIterator eit = par_iterator_end();
+	for (; it != eit; ++it)
+		it->changeLanguage(params(), from, to);
 }
 
 
@@ -3070,7 +3099,7 @@ void Buffer::getLanguages(std::set<Language const *> & langs) const
 	for (ParConstIterator it = par_iterator_begin(); it != end; ++it)
 		it->getLanguages(langs);
 	// also children
-	ListOfBuffers clist = getDescendents();
+	ListOfBuffers clist = getDescendants();
 	for (auto const & cit : clist)
 		cit->getLanguages(langs);
 }
@@ -3127,7 +3156,7 @@ Language const * Buffer::language() const
 }
 
 
-docstring const Buffer::B_(string const & l10n) const
+docstring Buffer::B_(string const & l10n) const
 {
 	return params().B_(l10n);
 }
@@ -3267,7 +3296,7 @@ string const Buffer::prepareFileNameForLaTeX(string const & name,
 }
 
 
-vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
+vector<pair<docstring, string>> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
 						docstring_list const & bibfilelist,
 						bool const add_extension) const
 {
@@ -3281,7 +3310,7 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
 	// Otherwise, store the (maybe absolute) path to the original,
 	// unmangled database name.
 
-	vector<docstring> res;
+	vector<pair<docstring, string>> res;
 
 	// determine the export format
 	string const tex_format = flavor2format(runparams.flavor);
@@ -3353,9 +3382,20 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
 
 		if (contains(path, ' '))
 			found_space = true;
+		string enc;
+		if (params().useBiblatex() && !params().bibFileEncoding(utf8input).empty())
+			enc = params().bibFileEncoding(utf8input);
+
+		bool recorded = false;
+		for (auto const & pe : res) {
+			if (pe.first == path) {
+				recorded = true;
+				break;
+			}
 
-		if (find(res.begin(), res.end(), path) == res.end())
-			res.push_back(path);
+		}
+		if (!recorded)
+			res.push_back(make_pair(path, enc));
 	}
 
 	// Check if there are spaces in the path and warn BibTeX users, if so.
@@ -3414,8 +3454,37 @@ bool Buffer::isReadonly() const
 
 void Buffer::setParent(Buffer const * buffer)
 {
-	// Avoids recursive include.
-	d->setParent(buffer == this ? 0 : buffer);
+	// We need to do some work here to avoid recursive parent structures.
+	// This is the easy case.
+	if (buffer == this) {
+		LYXERR0("Ignoring attempt to set self as parent in\n" << fileName());
+		return;
+	}
+	// Now we check parents going upward, to make sure that IF we set the
+	// parent as requested, we would not generate a recursive include.
+	set<Buffer const *> sb;
+	Buffer const * b = buffer;
+	bool found_recursion = false;
+	while (b) {
+		if (sb.find(b) != sb.end()) {
+			found_recursion = true;
+			break;
+		}
+		sb.insert(b);
+		b = b->parent();
+	}
+
+	if (found_recursion) {
+		LYXERR0("Ignoring attempt to set parent of\n" <<
+			fileName() <<
+			"\nto " <<
+			buffer->fileName() <<
+			"\nbecause that would create a recursive inclusion.");
+		return;
+	}
+
+	// We should be safe now.
+	d->setParent(buffer);
 	updateMacros();
 }
 
@@ -3428,7 +3497,7 @@ Buffer const * Buffer::parent() const
 
 ListOfBuffers Buffer::allRelatives() const
 {
-	ListOfBuffers lb = masterBuffer()->getDescendents();
+	ListOfBuffers lb = masterBuffer()->getDescendants();
 	lb.push_front(const_cast<Buffer *>(masterBuffer()));
 	return lb;
 }
@@ -3436,8 +3505,6 @@ ListOfBuffers Buffer::allRelatives() const
 
 Buffer const * Buffer::masterBuffer() const
 {
-	// FIXME Should be make sure we are not in some kind
-	// of recursive include? A -> B -> A will crash this.
 	Buffer const * const pbuf = d->parent();
 	if (!pbuf)
 		return this;
@@ -3499,7 +3566,7 @@ ListOfBuffers Buffer::getChildren() const
 }
 
 
-ListOfBuffers Buffer::getDescendents() const
+ListOfBuffers Buffer::getDescendants() const
 {
 	ListOfBuffers v;
 	collectChildren(v, true);
@@ -3523,7 +3590,7 @@ typename M::const_iterator greatest_below(M & m, typename M::key_type const & x)
 	if (it == m.begin())
 		return m.end();
 
-	it--;
+	--it;
 	return it;
 }
 
@@ -3535,11 +3602,11 @@ MacroData const * Buffer::Impl::getBufferMacro(docstring const & name,
 
 	// if paragraphs have no macro context set, pos will be empty
 	if (pos.empty())
-		return 0;
+		return nullptr;
 
 	// we haven't found anything yet
 	DocIterator bestPos = owner_->par_iterator_begin();
-	MacroData const * bestData = 0;
+	MacroData const * bestData = nullptr;
 
 	// find macro definitions for name
 	NamePositionScopeMacroMap::const_iterator nameIt = macros.find(name);
@@ -3610,11 +3677,11 @@ MacroData const * Buffer::getMacro(docstring const & name,
 	DocIterator const & pos, bool global) const
 {
 	if (d->macro_lock)
-		return 0;
+		return nullptr;
 
 	// query buffer macros
 	MacroData const * data = d->getBufferMacro(name, pos);
-	if (data != 0)
+	if (data != nullptr)
 		return data;
 
 	// If there is a master buffer, query that
@@ -3630,11 +3697,11 @@ MacroData const * Buffer::getMacro(docstring const & name,
 
 	if (global) {
 		data = MacroTable::globalMacros().get(name);
-		if (data != 0)
+		if (data != nullptr)
 			return data;
 	}
 
-	return 0;
+	return nullptr;
 }
 
 
@@ -3654,7 +3721,7 @@ MacroData const * Buffer::getMacro(docstring const & name,
 	// look where the child buffer is included first
 	Impl::BufferPositionMap::iterator it = d->children_positions.find(&child);
 	if (it == d->children_positions.end())
-		return 0;
+		return nullptr;
 
 	// check for macros at the inclusion position
 	return getMacro(name, it->second, global);
@@ -3708,7 +3775,7 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
 				InsetInclude const & incinset =
 					static_cast<InsetInclude const &>(*insit.inset);
 				macro_lock = true;
-				Buffer * child = incinset.getChildBuffer();
+				Buffer * child = incinset.loadIfNeeded();
 				macro_lock = false;
 				if (!child)
 					continue;
@@ -3803,7 +3870,7 @@ void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_mast
 			// get buffer of external file
 			InsetInclude const & ins =
 				static_cast<InsetInclude const &>(*it);
-			Buffer * child = ins.getChildBuffer();
+			Buffer * child = ins.loadIfNeeded();
 			if (!child)
 				continue;
 			child->getUsedBranches(result, true);
@@ -3832,7 +3899,7 @@ void Buffer::updateMacroInstances(UpdateType utype) const
 		MacroContext mc = MacroContext(this, it);
 		for (DocIterator::idx_type i = 0; i < n; ++i) {
 			MathData & data = minset->cell(i);
-			data.updateMacros(0, mc, utype, 0);
+			data.updateMacros(nullptr, mc, utype, 0);
 		}
 	}
 }
@@ -3897,33 +3964,14 @@ Buffer::References & Buffer::getReferenceCache(docstring const & label)
 	if (d->parent())
 		return const_cast<Buffer *>(masterBuffer())->getReferenceCache(label);
 
-	// In what follows, we look whether we find an active label
-	// with the given string in the cache. If so, return its
-	// references cache. If we only find an inactive one, return
-	// that (the last we find, coincidentally). If we find nothing,
-	// return an empty references cache.
-	static LabelInfo linfo;
-	linfo.inset = nullptr;
-	linfo.references = References();
-	linfo.active = false;
-	bool have_inactive = false;
-	for (auto & rc : d->ref_cache_) {
-		if (rc.label == label) {
-			if (rc.active)
-				return rc.references;
-			else {
-				linfo = rc;
-				have_inactive = true;
-			}
-		}
-	}
-
-	if (!have_inactive)
-		// We found nothing, so insert the empty one to the cache
-		// for further processing
-		d->ref_cache_.push_back(linfo);
+	RefCache::iterator it = d->ref_cache_.find(label);
+	if (it != d->ref_cache_.end())
+		return it->second;
 
-	return linfo.references;
+	static References const dummy_refs = References();
+	it = d->ref_cache_.insert(
+		make_pair(label, dummy_refs)).first;
+	return it->second;
 }
 
 
@@ -3947,14 +3995,14 @@ void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il,
 	linfo.label = label;
 	linfo.inset = il;
 	linfo.active = active;
-	masterBuffer()->d->ref_cache_.push_back(linfo);
+	masterBuffer()->d->label_cache_.push_back(linfo);
 }
 
 
 InsetLabel const * Buffer::insetLabel(docstring const & label,
 				      bool const active) const
 {
-	for (auto & rc : masterBuffer()->d->ref_cache_) {
+	for (auto const & rc : masterBuffer()->d->label_cache_) {
 		if (rc.label == label && (rc.active || !active))
 			return rc.inset;
 	}
@@ -3973,8 +4021,10 @@ bool Buffer::activeLabel(docstring const & label) const
 
 void Buffer::clearReferenceCache() const
 {
-	if (!d->parent())
+	if (!d->parent()) {
 		d->ref_cache_.clear();
+		d->label_cache_.clear();
+	}
 }
 
 
@@ -3992,14 +4042,17 @@ void Buffer::changeRefsIfUnique(docstring const & from, docstring const & to)
 		return;
 
 	string const paramName = "key";
+	UndoGroupHelper ugh(this);
 	InsetIterator it = inset_iterator_begin(inset());
 	for (; it; ++it) {
 		if (it->lyxCode() != CITE_CODE)
 			continue;
 		InsetCommand * inset = it->asInsetCommand();
 		docstring const oldValue = inset->getParam(paramName);
-		if (oldValue == from)
+		if (oldValue == from) {
+			undo().recordUndo(CursorData(it));
 			inset->setParam(paramName, to);
+		}
 	}
 }
 
@@ -4041,7 +4094,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
 			par.write(ods, params(), dt);
 			os << from_utf8(ods.str());
 		} else if (runparams.flavor == OutputParams::HTML) {
-			XHTMLStream xs(os);
+			XMLStream xs(os);
 			setMathFlavor(runparams);
 			xhtmlParagraphs(text(), *this, xs, runparams);
 		} else if (runparams.flavor == OutputParams::TEXT) {
@@ -4050,8 +4103,9 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
 			// Probably should have some routine with a signature like them.
 			writePlaintextParagraph(*this,
 				text().paragraphs()[par_begin], os, runparams, dummy);
-		} else if (params().isDocBook()) {
-			docbookParagraphs(text(), *this, os, runparams);
+		} else if (runparams.flavor == OutputParams::DOCBOOK5) {
+			XMLStream xs{os};
+			docbookParagraphs(text(), *this, xs, runparams);
 		} else {
 			// If we are previewing a paragraph, even if this is the
 			// child of some other buffer, let's cut the link here,
@@ -4065,6 +4119,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
 			LaTeXFeatures features(*this, params(), runparams);
 			validate(features);
 			runparams.use_polyglossia = features.usePolyglossia();
+			runparams.use_hyperref = features.isRequired("hyperref");
 			// latex or literate
 			otexstream ots(os);
 			// output above
@@ -4102,8 +4157,8 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
 				os << "% "<< _("Plain text does not have a preamble.");
 			} else
 				writePlaintextFile(*this, os, runparams);
-		} else if (params().isDocBook()) {
-				writeDocBookSource(os, absFileName(), runparams, output);
+		} else if (runparams.flavor == OutputParams::DOCBOOK5) {
+			writeDocBookSource(os, runparams, output);
 		} else {
 			// latex or literate
 			otexstream ots(os);
@@ -4188,91 +4243,6 @@ void Buffer::setGuiDelegate(frontend::GuiBufferDelegate * gui)
 }
 
 
-
-namespace {
-
-class AutoSaveBuffer : public ForkedProcess {
-public:
-	///
-	AutoSaveBuffer(Buffer const & buffer, FileName const & fname)
-		: buffer_(buffer), fname_(fname) {}
-	///
-	virtual shared_ptr<ForkedProcess> clone() const
-	{
-		return make_shared<AutoSaveBuffer>(*this);
-	}
-	///
-	int start()
-	{
-		command_ = to_utf8(bformat(_("Auto-saving %1$s"),
-						 from_utf8(fname_.absFileName())));
-		return run(DontWait);
-	}
-private:
-	///
-	virtual int generateChild();
-	///
-	Buffer const & buffer_;
-	FileName fname_;
-};
-
-
-int AutoSaveBuffer::generateChild()
-{
-#if defined(__APPLE__)
-	/* FIXME fork() is not usable for autosave on Mac OS X 10.6 (snow leopard)
-	 *   We should use something else like threads.
-	 *
-	 * Since I do not know how to determine at run time what is the OS X
-	 * version, I just disable forking altogether for now (JMarc)
-	 */
-	pid_t const pid = -1;
-#else
-	// tmp_ret will be located (usually) in /tmp
-	// will that be a problem?
-	// Note that this calls ForkedCalls::fork(), so it's
-	// ok cross-platform.
-	pid_t const pid = fork();
-	// If you want to debug the autosave
-	// you should set pid to -1, and comment out the fork.
-	if (pid != 0 && pid != -1)
-		return pid;
-#endif
-
-	// pid = -1 signifies that lyx was unable
-	// to fork. But we will do the save
-	// anyway.
-	bool failed = false;
-	TempFile tempfile("lyxautoXXXXXX.lyx");
-	tempfile.setAutoRemove(false);
-	FileName const tmp_ret = tempfile.name();
-	if (!tmp_ret.empty()) {
-		if (!buffer_.writeFile(tmp_ret))
-			failed = true;
-		else if (!tmp_ret.moveTo(fname_))
-			failed = true;
-	} else
-		failed = true;
-
-	if (failed) {
-		// failed to write/rename tmp_ret so try writing direct
-		if (!buffer_.writeFile(fname_)) {
-			// It is dangerous to do this in the child,
-			// but safe in the parent, so...
-			if (pid == -1) // emit message signal.
-				buffer_.message(_("Autosave failed!"));
-		}
-	}
-
-	if (pid == 0) // we are the child so...
-		_exit(0);
-
-	return pid;
-}
-
-} // namespace
-
-
 FileName Buffer::getEmergencyFileName() const
 {
 	return FileName(d->filename.absFileName() + ".emergency");
@@ -4346,7 +4316,7 @@ bool Buffer::autoSave() const
 void Buffer::setExportStatus(bool e) const
 {
 	d->doing_export = e;
-	ListOfBuffers clist = getDescendents();
+	ListOfBuffers clist = getDescendants();
 	for (auto const & bit : clist)
 		bit->d->doing_export = e;
 }
@@ -4369,7 +4339,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 	string & result_file) const
 {
 	bool const update_unincluded =
-			params().maintain_unincluded_children
+			params().maintain_unincluded_children != BufferParams::CM_None
 			&& !params().getIncludedChildren().empty();
 
 	// (1) export with all included children (omit \includeonly)
@@ -4443,9 +4413,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 			if (!put_in_tempdir) {
 				// Only show this alert if this is an export to a non-temporary
 				// file (not for previewing).
-				Alert::error(_("Couldn't export file"), bformat(
-					_("No information for exporting the format %1$s."),
-					theFormats().prettyName(format)));
+				docstring s = bformat(_("No information for exporting the format %1$s."),
+						      theFormats().prettyName(format));
+				if (format == "pdf4")
+					s += "\n"
+					  + bformat(_("Hint: use non-TeX fonts or set input encoding "
+						  " to '%1$s'"), from_utf8(encodings.fromLyXName("ascii")->guiName()));
+				Alert::error(_("Couldn't export file"), s);
 			}
 			return ExportNoPathToFormat;
 		}
@@ -4493,8 +4467,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 			return ExportKilled;
 	} else if (backend_format == "lyx")
 		writeFile(FileName(filename));
-	// Docbook backend
-	else if (params().isDocBook()) {
+	// DocBook backend
+	else if (backend_format == "docbook5") {
+		runparams.flavor = OutputParams::DOCBOOK5;
 		runparams.nice = !put_in_tempdir;
 		if (makeDocBookFile(FileName(filename), runparams) == ExportKilled)
 			return ExportKilled;
@@ -4516,7 +4491,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 			bformat(_("The directory path to the document\n%1$s\n"
 			    "contains spaces, but your TeX installation does "
 			    "not allow them. You should save the file to a directory "
-					"whose name does not contain spaces."), from_ascii(filePath())));
+					"whose name does not contain spaces."), from_utf8(filePath())));
 		return ExportTexPathHasSpaces;
 	} else {
 		runparams.nice = false;
@@ -4536,8 +4511,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 	string const ext = theFormats().extension(format);
 	FileName const tmp_result_file(changeExtension(filename, ext));
 	Converters::RetVal const retval = 
-		converters.convert(this, FileName(filename), tmp_result_file, 
-		FileName(absFileName()), backend_format, format, error_list);
+		converters.convert(this, FileName(filename), tmp_result_file,
+				   FileName(absFileName()), backend_format, format,
+				   error_list, Converters::none, includeall);
 	if (retval == Converters::KILLED)
 		return ExportCancel;
 	bool success = (retval == Converters::SUCCESS);
@@ -4553,7 +4529,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 		else
 			errors(error_type);
 		// also to the children, in case of master-buffer-view
-		ListOfBuffers clist = getDescendents();
+		ListOfBuffers clist = getDescendants();
 		for (auto const & bit : clist) {
 			if (runparams.silent)
 				bit->d->errorLists[error_type].clear();
@@ -4591,7 +4567,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 		result_file = dest_filename;
 	// We need to copy referenced files (e. g. included graphics
 	// if format == "dvi") to the result dir.
-	vector<ExportedFile> const files =
+	vector<ExportedFile> const extfiles =
 		runparams.exportdata->externalFiles(format);
 	string const dest = runparams.export_folder.empty() ?
 		onlyPath(result_file) : runparams.export_folder;
@@ -4599,7 +4575,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 				 : force_overwrite == ALL_FILES;
 	CopyStatus status = use_force ? FORCE : SUCCESS;
 
-	for (ExportedFile const & exp : files) {
+	for (ExportedFile const & exp : extfiles) {
 		if (status == CANCEL) {
 			message(_("Document export cancelled."));
 			return ExportCancel;
@@ -4652,7 +4628,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
 Buffer::ExportStatus Buffer::preview(string const & format) const
 {
 	bool const update_unincluded =
-			params().maintain_unincluded_children
+			params().maintain_unincluded_children != BufferParams::CM_None
 			&& !params().getIncludedChildren().empty();
 	return preview(format, update_unincluded);
 }
@@ -4675,7 +4651,7 @@ Buffer::ExportStatus Buffer::preview(string const & format, bool includeall) con
 	Impl * theimpl = isClone() ? d->cloned_buffer_->d : d;
 	theimpl->preview_file_ = previewFile;
 	theimpl->preview_format_ = format;
-	theimpl->preview_error_ = (status != ExportSuccess);
+	theimpl->require_fresh_start_ = (status != ExportSuccess);
 
 	if (status != ExportSuccess)
 		return status;
@@ -4756,6 +4732,36 @@ Buffer::ReadStatus Buffer::loadEmergency()
 				_("&Remove"), _("&Keep"));
 		if (del_emerg == 0)
 			emergencyFile.removeFile();
+		else {
+			// See bug #11464
+			FileName newname;
+			string const ename = emergencyFile.absFileName();
+			bool noname = true;
+			// Surely we can find one in 100 tries?
+			for (int i = 1; i < 100; ++i) {
+				newname.set(ename + to_string(i) + ".lyx");
+				if (!newname.exists()) {
+					noname = false;
+					break;
+				}
+			}
+			if (!noname) {
+				// renameTo returns true on success. So inverting that
+				// will give us true if we fail.
+				noname = !emergencyFile.renameTo(newname);
+			}
+			if (noname) {
+				Alert::warning(_("Can't rename emergency file!"),
+					_("LyX was unable to rename the emergency file. "
+					  "You should do so manually. Otherwise, you will be "
+					  "asked about it again the next time you try to load "
+					  "this file, and may over-write your own work."));
+			} else {
+				Alert::warning(_("Emergency File Renames"),
+					bformat(_("Emergency file renamed as:\n %1$s"),
+					from_utf8(newname.onlyFileName())));
+			}
+		}
 		return ReadOriginal;
 	}
 
@@ -4837,26 +4843,26 @@ Buffer::ReadStatus Buffer::loadThisLyXFile(FileName const & fn)
 }
 
 
-void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
+void Buffer::Impl::traverseErrors(TeXErrors::Errors::const_iterator err, TeXErrors::Errors::const_iterator end, ErrorList & errorList) const
 {
-	for (auto const & err : terr) {
+	for (; err != end; ++err) {
 		TexRow::TextEntry start = TexRow::text_none, end = TexRow::text_none;
-		int errorRow = err.error_in_line;
-		Buffer const * buf = 0;
-		Impl const * p = d;
-		if (err.child_name.empty())
+		int errorRow = err->error_in_line;
+		Buffer const * buf = nullptr;
+		Impl const * p = this;
+		if (err->child_name.empty())
 			tie(start, end) = p->texrow.getEntriesFromRow(errorRow);
 		else {
 			// The error occurred in a child
-			for (Buffer const * child : getDescendents()) {
+			for (Buffer const * child : owner_->getDescendants()) {
 				string const child_name =
 					DocFileName(changeExtension(child->absFileName(), "tex")).
 					mangledFileName();
-				if (err.child_name != child_name)
+				if (err->child_name != child_name)
 					continue;
 				tie(start, end) = child->d->texrow.getEntriesFromRow(errorRow);
 				if (!TexRow::isNone(start)) {
-					buf = d->cloned_buffer_
+					buf = this->cloned_buffer_
 						? child->d->cloned_buffer_->d->owner_
 						: child->d->owner_;
 					p = child->d;
@@ -4864,12 +4870,30 @@ void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
 				}
 			}
 		}
-		errorList.push_back(ErrorItem(err.error_desc, err.error_text,
+		errorList.push_back(ErrorItem(err->error_desc, err->error_text,
 		                              start, end, buf));
 	}
 }
 
 
+void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
+{
+	TeXErrors::Errors::const_iterator err = terr.begin();
+	TeXErrors::Errors::const_iterator end = terr.end();
+
+	d->traverseErrors(err, end, errorList);
+}
+
+
+void Buffer::bufferRefs(TeXErrors const & terr, ErrorList & errorList) const
+{
+	TeXErrors::Errors::const_iterator err = terr.begin_ref();
+	TeXErrors::Errors::const_iterator end = terr.end_ref();
+
+	d->traverseErrors(err, end, errorList);
+}
+
+
 void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 {
 	LBUFERR(!text().paragraphs().empty());
@@ -4926,12 +4950,17 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 	// update all caches
 	clearReferenceCache();
 	updateMacros();
-	setChangesPresent(false);
 
 	Buffer & cbuf = const_cast<Buffer &>(*this);
+	// if we are reloading, then we could have a dangling TOC,
+	// in effect. so we need to go ahead and reset, even though
+	// we will do so again when we rebuild the TOC later.
+	cbuf.tocBackend().reset();
 
 	// do the real work
 	ParIterator parit = cbuf.par_iterator_begin();
+	if (scope == UpdateMaster)
+		clearIncludeList();
 	updateBuffer(parit, utype);
 
 	// If this document has siblings, then update the TocBackend later. The
@@ -4963,7 +4992,6 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 		clearReferenceCache();
 		// we should not need to do this again?
 		// updateMacros();
-		setChangesPresent(false);
 		updateBuffer(parit, utype);
 		// this will already have been done by reloadBibInfoCache();
 		// d->bibinfo_cache_valid_ = true;
@@ -4975,6 +5003,7 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 	}
 	d->cite_labels_valid_ = true;
 	/// FIXME: Perf
+	clearIncludeList();
 	cbuf.tocBackend().update(true, utype);
 	if (scope == UpdateMaster)
 		cbuf.structureChanged();
@@ -5097,16 +5126,20 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
 		docstring itemlabel;
 		switch (par.itemdepth) {
 		case 0:
+			// • U+2022 BULLET
 			itemlabel = char_type(0x2022);
 			break;
 		case 1:
+			// – U+2013 EN DASH
 			itemlabel = char_type(0x2013);
 			break;
 		case 2:
+			// ∗ U+2217 ASTERISK OPERATOR
 			itemlabel = char_type(0x2217);
 			break;
 		case 3:
-			itemlabel = char_type(0x2219); // or 0x00b7
+			// · U+00B7 MIDDLE DOT
+			itemlabel = char_type(0x00b7);
 			break;
 		}
 		par.params().labelString(itemlabel);
@@ -5199,8 +5232,9 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
 }
 
 
-void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
+void Buffer::updateBuffer(ParIterator & parit, UpdateType utype, bool const deleted) const
 {
+	pushIncludedBuffer(this);
 	// LASSERT: Is it safe to continue here, or should we just return?
 	LASSERT(parit.pit() == 0, /**/);
 
@@ -5210,6 +5244,7 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
 
 	depth_type maxdepth = 0;
 	pit_type const lastpit = parit.lastpit();
+	bool changed = false;
 	for ( ; parit.pit() <= lastpit ; ++parit.pit()) {
 		// reduce depth if necessary
 		if (parit->params().depth() > maxdepth) {
@@ -5236,15 +5271,21 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
 		// set the counter for this paragraph
 		d->setLabel(parit, utype);
 
-		// update change-tracking flag
-		parit->addChangesToBuffer(*this);
-
 		// now the insets
 		for (auto const & insit : parit->insetList()) {
 			parit.pos() = insit.pos;
-			insit.inset->updateBuffer(parit, utype);
+			insit.inset->updateBuffer(parit, utype, deleted || parit->isDeleted(insit.pos));
+			changed |= insit.inset->isChanged();
 		}
+
+		// are there changes in this paragraph?
+		changed |= parit->isChanged();
 	}
+
+	// set change indicator for the inset (or the cell that the iterator
+	// points to, if applicable).
+	parit.text()->inset().isChanged(changed);
+	popIncludedBuffer();
 }
 
 
@@ -5321,8 +5362,11 @@ void Buffer::Impl::updateStatistics(DocIterator & from, DocIterator & to, bool s
 					++word_count_;
 					inword = true;
 				}
-				if (ins && ins->isLetter())
-					++char_count_;
+				if (ins && ins->isLetter()) {
+					odocstringstream os;
+					ins->toString(os);
+					char_count_ += os.str().length();
+				}
 				else if (ins && ins->isSpace())
 					++blank_count_;
 				else {
@@ -5357,6 +5401,12 @@ int Buffer::charCount(bool with_blanks) const
 }
 
 
+bool Buffer::areChangesPresent() const
+{
+	return inset().isChanged();
+}
+
+
 Buffer::ReadStatus Buffer::reload()
 {
 	setBusy(true);
@@ -5367,7 +5417,7 @@ Buffer::ReadStatus Buffer::reload()
 	docstring const disp_fn = makeDisplayPath(d->filename.absFileName());
 
 	// clear parent. this will get reset if need be.
-	d->setParent(0);
+	d->setParent(nullptr);
 	ReadStatus const status = loadLyXFile();
 	if (status == ReadSuccess) {
 		updateBuffer();
@@ -5441,8 +5491,8 @@ void Buffer::checkChildBuffers()
 		if (oldloc == newloc)
 			continue;
 		// the location of the child file is incorrect.
-		cbuf->setParent(0);
-		inset_inc->setChildBuffer(0);
+		cbuf->setParent(nullptr);
+		inset_inc->setChildBuffer(nullptr);
 	}
 	// invalidate cache of children
 	d->children_positions.clear();
@@ -5467,7 +5517,7 @@ void Buffer::checkMasterBuffer()
 	if (master->isChild(this))
 		setParent(master);
 	else
-		setParent(0);
+		setParent(nullptr);
 }
 
 
@@ -5500,29 +5550,6 @@ string Buffer::includedFilePath(string const & name, string const & ext) const
 }
 
 
-void Buffer::setChangesPresent(bool b) const
-{
-	d->tracked_changes_present_ = b;
-}
-
-
-bool Buffer::areChangesPresent() const
-{
-	return d->tracked_changes_present_;
-}
-
-
-void Buffer::updateChangesPresent() const
-{
-	LYXERR(Debug::CHANGES, "Buffer::updateChangesPresent");
-	setChangesPresent(false);
-	ParConstIterator it = par_iterator_begin();
-	ParConstIterator const end = par_iterator_end();
-	for (; !areChangesPresent() && it != end; ++it)
-		it->addChangesToBuffer(*this);
-}
-
-
 void Buffer::Impl::refreshFileMonitor()
 {
 	if (file_monitor_ && file_monitor_->filename() == filename.absFileName()) {
@@ -5578,4 +5605,55 @@ void Buffer::clearExternalModification() const
 }
 
 
+void Buffer::pushIncludedBuffer(Buffer const * buf) const
+{
+	masterBuffer()->d->include_list_.push_back(buf);
+	if (lyxerr.debugging(Debug::FILES)) {
+		LYXERR0("Pushed. Stack now:");
+		if (masterBuffer()->d->include_list_.empty())
+			LYXERR0("EMPTY!");
+		else
+			for (auto const & b : masterBuffer()->d->include_list_)
+				LYXERR0(b->fileName());
+	}
+}
+
+
+void Buffer::popIncludedBuffer() const
+{
+	masterBuffer()->d->include_list_.pop_back();
+	if (lyxerr.debugging(Debug::FILES)) {
+		LYXERR0("Popped. Stack now:");
+		if (masterBuffer()->d->include_list_.empty())
+			LYXERR0("EMPTY!");
+		else
+			for (auto const & b : masterBuffer()->d->include_list_)
+				LYXERR0(b->fileName());
+	}
+}
+
+
+bool Buffer::isBufferIncluded(Buffer const * buf) const
+{
+	if (!buf)
+		return false;
+	if (lyxerr.debugging(Debug::FILES)) {
+		LYXERR0("Checking for " << buf->fileName() << ". Stack now:");
+		if (masterBuffer()->d->include_list_.empty())
+			LYXERR0("EMPTY!");
+		else
+			for (auto const & b : masterBuffer()->d->include_list_)
+				LYXERR0(b->fileName());
+	}
+	list<Buffer const *> const & blist = masterBuffer()->d->include_list_;
+	return find(blist.begin(), blist.end(), buf) != blist.end();
+}
+
+
+void Buffer::clearIncludeList() const
+{
+	LYXERR(Debug::FILES, "Clearing include list for " << fileName());
+	d->include_list_.clear();
+}
+
 } // namespace lyx