]> git.lyx.org Git - lyx.git/blobdiff - src/Buffer.cpp
Improve error message and ctests for input encoding cp858.
[lyx.git] / src / Buffer.cpp
index b220ee33e7959751ec2f7280913309dba9dec4b2..cb08e3c4fb9fdc2acb75a0e73a18862227ce730f 100644 (file)
@@ -55,6 +55,7 @@
 #include "ParagraphParameters.h"
 #include "ParIterator.h"
 #include "PDFOptions.h"
+#include "Session.h"
 #include "SpellChecker.h"
 #include "sgml.h"
 #include "texstream.h"
@@ -76,7 +77,7 @@
 
 #include "mathed/InsetMathHull.h"
 #include "mathed/MacroTable.h"
-#include "mathed/MathMacroTemplate.h"
+#include "mathed/InsetMathMacroTemplate.h"
 #include "mathed/MathSupport.h"
 
 #include "graphics/GraphicsCache.h"
@@ -134,9 +135,24 @@ namespace {
 int const LYX_FORMAT = LYX_FORMAT_LYX;
 
 typedef map<string, bool> DepClean;
-typedef map<docstring, pair<InsetLabel const *, Buffer::References> > RefCache;
 
-} // namespace anon
+// Information about labels and their associated refs
+struct LabelInfo {
+       /// label string
+       docstring label;
+       /// label inset
+       InsetLabel const * inset;
+       /// associated references cache
+       Buffer::References references;
+       /// whether this label is active (i.e., not deleted)
+       bool active;
+};
+
+typedef vector<LabelInfo> LabelCache;
+
+typedef map<docstring, Buffer::References> RefCache;
+
+} // namespace
 
 
 // A storehouse for the cloned buffers.
@@ -171,7 +187,7 @@ public:
        /** If we have branches that use the file suffix
            feature, return the file name with suffix appended.
        */
-       support::FileName exportFileName() const;
+       FileName exportFileName() const;
 
        Buffer * owner_;
 
@@ -278,7 +294,7 @@ public:
 
        /// A cache for the bibfiles (including bibfiles of loaded child
        /// documents), needed for appropriate update of natbib labels.
-       mutable support::FileNamePairList bibfiles_cache_;
+       mutable docstring_list bibfiles_cache_;
 
        // FIXME The caching mechanism could be improved. At present, we have a
        // cache for each Buffer, that caches all the bibliography info for that
@@ -288,14 +304,14 @@ public:
        mutable BiblioInfo bibinfo_;
        /// whether the bibinfo cache is valid
        mutable bool bibinfo_cache_valid_;
-       /// whether the bibfile cache is valid
-       mutable bool bibfile_cache_valid_;
        /// Cache of timestamps of .bib files
        map<FileName, time_t> bibfile_status_;
        /// Indicates whether the bibinfo has changed since the last time
        /// we ran updateBuffer(), i.e., whether citation labels may need
        /// to be updated.
        mutable bool cite_labels_valid_;
+       /// Do we have a bibliography environment?
+       mutable bool have_bibitems_;
 
        /// These two hold the file name and format, written to by
        /// Buffer::preview and read from by LFUN_BUFFER_VIEW_CACHE.
@@ -306,7 +322,11 @@ public:
        /// was missing).
        bool preview_error_;
 
+       /// Cache the references associated to a label and their positions
+       /// in the buffer.
        mutable RefCache ref_cache_;
+       /// Cache the label insets and their activity status.
+       mutable LabelCache label_cache_;
 
        /// our Text that should be wrapped in an InsetText
        InsetText * inset;
@@ -342,10 +362,8 @@ public:
                if (!cloned_buffer_ && parent_buffer && pb)
                        LYXERR0("Warning: a buffer should not have two parents!");
                parent_buffer = pb;
-               if (!cloned_buffer_ && parent_buffer) {
-                       parent_buffer->invalidateBibfileCache();
+               if (!cloned_buffer_ && parent_buffer)
                        parent_buffer->invalidateBibinfoCache();
-               }
        }
 
        /// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
@@ -379,14 +397,11 @@ public:
        // Make sure the file monitor monitors the good file.
        void refreshFileMonitor();
 
-       /// has it been notified of an external modification?
-       bool isExternallyModified() const { return externally_modified_; }
-
        /// Notify or clear of external modification
-       void fileExternallyModified(bool modified) const;
+       void fileExternallyModified(bool exists);
 
-       /// Block notifications of external modifications
-       FileMonitorBlocker blockFileMonitor() { return file_monitor_->block(10); }
+       /// has been externally modified? Can be reset by the user.
+       mutable bool externally_modified_;
 
 private:
        /// So we can force access via the accessors.
@@ -396,9 +411,6 @@ private:
        int char_count_;
        int blank_count_;
 
-       /// has been externally modified? Can be reset by the user.
-       mutable bool externally_modified_;
-
        FileMonitorPtr file_monitor_;
 };
 
@@ -437,12 +449,11 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
          file_fully_loaded(false), file_format(LYX_FORMAT), need_format_backup(false),
          ignore_parent(false),  toc_backend(owner), macro_lock(false),
          checksum_(0), wa_(0),  gui_(0), undo_(*owner), bibinfo_cache_valid_(false),
-         bibfile_cache_valid_(false), cite_labels_valid_(false), preview_error_(false),
+         cite_labels_valid_(false), have_bibitems_(false), preview_error_(false),
          inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
          clone_list_(0), doing_export(false),
-         tracked_changes_present_(0), parent_buffer(0),
-         word_count_(0), char_count_(0), blank_count_(0),
-         externally_modified_(false)
+         tracked_changes_present_(0), externally_modified_(false), parent_buffer(0),
+         word_count_(0), char_count_(0), blank_count_(0)
 {
        refreshFileMonitor();
        if (!cloned_buffer_) {
@@ -458,9 +469,9 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
        bibfiles_cache_ = cloned_buffer_->d->bibfiles_cache_;
        bibinfo_ = cloned_buffer_->d->bibinfo_;
        bibinfo_cache_valid_ = cloned_buffer_->d->bibinfo_cache_valid_;
-       bibfile_cache_valid_ = cloned_buffer_->d->bibfile_cache_valid_;
        bibfile_status_ = cloned_buffer_->d->bibfile_status_;
        cite_labels_valid_ = cloned_buffer_->d->cite_labels_valid_;
+       have_bibitems_ = cloned_buffer_->d->have_bibitems_;
        unnamed = cloned_buffer_->d->unnamed;
        internal_buffer = cloned_buffer_->d->internal_buffer;
        layout_position = cloned_buffer_->d->layout_position;
@@ -510,10 +521,8 @@ Buffer::~Buffer()
                // ourselves as a child.
                d->clone_list_->erase(this);
                // loop over children
-               Impl::BufferPositionMap::iterator it = d->children_positions.begin();
-               Impl::BufferPositionMap::iterator end = d->children_positions.end();
-               for (; it != end; ++it) {
-                       Buffer * child = const_cast<Buffer *>(it->first);
+               for (auto const & p : d->children_positions) {
+                       Buffer * child = const_cast<Buffer *>(p.first);
                                if (d->clone_list_->erase(child))
                                        delete child;
                }
@@ -539,21 +548,23 @@ Buffer::~Buffer()
                d->position_to_children.clear();
        } else {
                // loop over children
-               Impl::BufferPositionMap::iterator it = d->children_positions.begin();
-               Impl::BufferPositionMap::iterator end = d->children_positions.end();
-               for (; it != end; ++it) {
-                       Buffer * child = const_cast<Buffer *>(it->first);
-                       if (theBufferList().isLoaded(child)) { 
-                        if (theBufferList().isOthersChild(this, child))
-                                child->setParent(0);
-                        else
-                               theBufferList().release(child);
+               for (auto const & p : d->children_positions) {
+                       Buffer * child = const_cast<Buffer *>(p.first);
+                       if (theBufferList().isLoaded(child)) {
+                               if (theBufferList().isOthersChild(this, child))
+                                       child->setParent(0);
+                               else
+                                       theBufferList().release(child);
                        }
                }
 
                if (!isClean()) {
                        docstring msg = _("LyX attempted to close a document that had unsaved changes!\n");
-                       msg += emergencyWrite();
+                       try {
+                               msg += emergencyWrite();
+                       } catch (...) {
+                               msg += "  " + _("Save failed! Document is lost.");
+                       }
                        Alert::warning(_("Attempting to close changed document!"), msg);
                }
 
@@ -573,13 +584,13 @@ Buffer::~Buffer()
 }
 
 
-Buffer * Buffer::cloneFromMaster() const
+Buffer * Buffer::cloneWithChildren() const
 {
        BufferMap bufmap;
        cloned_buffers.push_back(new CloneList);
        CloneList * clones = cloned_buffers.back();
 
-       masterBuffer()->cloneWithChildren(bufmap, clones);
+       cloneWithChildren(bufmap, clones);
 
        // make sure we got cloned
        BufferMap::const_iterator bit = bufmap.find(this);
@@ -616,12 +627,10 @@ void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
        // math macro caches need to be rethought and simplified.
        // I am not sure wether we should handle Buffer cloning here or in BufferList.
        // Right now BufferList knows nothing about buffer clones.
-       Impl::PositionScopeBufferMap::iterator it = d->position_to_children.begin();
-       Impl::PositionScopeBufferMap::iterator end = d->position_to_children.end();
-       for (; it != end; ++it) {
-               DocIterator dit = it->first.clone(buffer_clone);
+       for (auto const & p : d->position_to_children) {
+               DocIterator dit = p.first.clone(buffer_clone);
                dit.setBuffer(buffer_clone);
-               Buffer * child = const_cast<Buffer *>(it->second.buffer);
+               Buffer * child = const_cast<Buffer *>(p.second.buffer);
 
                child->cloneWithChildren(bufmap, clones);
                BufferMap::iterator const bit = bufmap.find(child);
@@ -714,10 +723,8 @@ BufferParams const & Buffer::masterParams() const
 
        BufferParams & mparams = const_cast<Buffer *>(masterBuffer())->params();
        // Copy child authors to the params. We need those pointers.
-       AuthorList const & child_authors = params().authors();
-       AuthorList::Authors::const_iterator it = child_authors.begin();
-       for (; it != child_authors.end(); ++it)
-               mparams.authors().record(*it);
+       for (Author const & a : params().authors())
+               mparams.authors().record(a);
        return mparams;
 }
 
@@ -725,7 +732,7 @@ BufferParams const & Buffer::masterParams() const
 double Buffer::fontScalingFactor() const
 {
        return isExporting() ? 75.0 * params().html_math_img_scale
-               : 0.01 * lyxrc.dpi * lyxrc.zoom * lyxrc.preview_scale_factor * params().display_pixel_ratio;
+               : 0.01 * lyxrc.dpi * lyxrc.currentZoom * lyxrc.preview_scale_factor * params().display_pixel_ratio;
 }
 
 
@@ -805,10 +812,9 @@ FileName Buffer::Impl::exportFileName() const
        if (branch_suffix.empty())
                return filename;
 
-       string const name = filename.onlyFileNameWithoutExt()
-               + to_utf8(branch_suffix);
+       string const name = addExtension(filename.onlyFileNameWithoutExt()
+                       + to_utf8(branch_suffix), filename.extension());
        FileName res(filename.onlyPath().absFileName() + "/" + name);
-       res.changeExtension(filename.extension());
 
        return res;
 }
@@ -834,7 +840,7 @@ string Buffer::logName(LogType * type) const
        FileName const bname(
                addName(path, onlyFileName(
                        changeExtension(filename,
-                                       formats.extension(params().bufferFormat()) + ".out"))));
+                                       theFormats().extension(params().bufferFormat()) + ".out"))));
 
        // Also consider the master buffer log file
        FileName masterfname = fname;
@@ -903,6 +909,7 @@ int Buffer::readHeader(Lexer & lex)
        params().options.erase();
        params().master.erase();
        params().float_placement.erase();
+       params().float_alignment.erase();
        params().paperwidth.erase();
        params().paperheight.erase();
        params().leftmargin.erase();
@@ -983,6 +990,8 @@ int Buffer::readHeader(Lexer & lex)
                errorList.push_back(ErrorItem(_("Document header error"), s));
        }
 
+       params().shell_escape = theSession().shellescapeFiles().find(absFileName());
+
        params().makeDocumentClass();
 
        return unknown_tokens;
@@ -1053,6 +1062,10 @@ bool Buffer::readDocument(Lexer & lex)
                                                << absFileName()
                                                << ") does not include "
                                                "this document. Ignoring the master assignment.");
+                               // If the master has just been created, un-hide it (#11162)
+                               if (!master->fileName().exists())
+                                       lyx::dispatch(FuncRequest(LFUN_BUFFER_SWITCH,
+                                                                 master->absFileName()));
                        }
                }
        }
@@ -1071,10 +1084,8 @@ bool Buffer::readDocument(Lexer & lex)
        // inform parent buffer about local macros
        if (parent()) {
                Buffer const * pbuf = parent();
-               UserMacroSet::const_iterator cit = usermacros.begin();
-               UserMacroSet::const_iterator end = usermacros.end();
-               for (; cit != end; ++cit)
-                       pbuf->usermacros.insert(*cit);
+               for (auto const & m : usermacros)
+                       pbuf->usermacros.insert(m);
        }
        usermacros.clear();
        updateMacros();
@@ -1085,13 +1096,12 @@ bool Buffer::readDocument(Lexer & lex)
 
 bool Buffer::importString(string const & format, docstring const & contents, ErrorList & errorList)
 {
-       Format const * fmt = formats.getFormat(format);
+       Format const * fmt = theFormats().getFormat(format);
        if (!fmt)
                return false;
        // It is important to use the correct extension here, since some
        // converters create a wrong output file otherwise (e.g. html2latex)
-       TempFile const tempfile("Buffer_importStringXXXXXX." + fmt->extension());
-       FileName const name(tempfile.name());
+       FileName const name = tempFileName("Buffer_importStringXXXXXX." + fmt->extension());
        ofdocstream os(name.toFilesystemEncoding().c_str());
        // Do not convert os implicitly to bool, since that is forbidden in C++11.
        bool const success = !(os << contents).fail();
@@ -1107,8 +1117,7 @@ bool Buffer::importString(string const & format, docstring const & contents, Err
                converted = importFile(format, name, errorList);
        }
 
-       if (name.exists())
-               name.removeFile();
+       removeTempFile(name);
        return converted;
 }
 
@@ -1118,10 +1127,14 @@ bool Buffer::importFile(string const & format, FileName const & name, ErrorList
        if (!theConverters().isReachable(format, "lyx"))
                return false;
 
-       TempFile const tempfile("Buffer_importFileXXXXXX.lyx");
-       FileName const lyx(tempfile.name());
-       if (theConverters().convert(0, name, lyx, name, format, "lyx", errorList))
-               return readFile(lyx) == ReadSuccess;
+       FileName const lyx = tempFileName("Buffer_importFileXXXXXX.lyx");
+       Converters::RetVal const retval =
+           theConverters().convert(0, name, lyx, name, format, "lyx", errorList);
+       if (retval == Converters::SUCCESS) {
+               bool const success = readFile(lyx) == ReadSuccess;
+               removeTempFile(lyx);
+               return success;
+       }
 
        return false;
 }
@@ -1199,7 +1212,7 @@ Buffer::ReadStatus Buffer::readFile(FileName const & fn)
 
        d->file_fully_loaded = true;
        d->read_only = !d->filename.isWritable();
-       params().compressed = formats.isZippedFile(d->filename);
+       params().compressed = theFormats().isZippedFile(d->filename);
        saveCheckSum();
        return ReadSuccess;
 }
@@ -1313,7 +1326,7 @@ Buffer::ReadStatus Buffer::convertLyXFormat(FileName const & fn,
        command << os::python()
                << ' ' << quoteName(lyx2lyx.toFilesystemEncoding())
                << " -t " << convert<string>(LYX_FORMAT)
-               << " -o " << quoteName(tmpfile.toFilesystemEncoding())
+               << " -o " << quoteName(tmpfile.toSafeFilesystemEncoding())
                << ' ' << quoteName(fn.toSafeFilesystemEncoding());
        string const command_str = command.str();
 
@@ -1342,14 +1355,37 @@ Buffer::ReadStatus Buffer::convertLyXFormat(FileName const & fn,
 
 
 FileName Buffer::getBackupName() const {
+       map<int, string> const file_formats = {
+         {544, "23"},
+         {508, "22"},
+         {474, "21"},
+         {413, "20"},
+         {345, "16"},
+         {276, "15"},
+         {245, "14"},
+         {221, "13"},
+         {220, "12"},
+         {218, "1163"},
+         {217, "116"},
+         {216, "115"},
+         {215, "11"},
+         {210, "010"},
+         {200, "006"}
+       };
        FileName const & fn = fileName();
        string const fname = fn.onlyFileNameWithoutExt();
        string const fext  = fn.extension() + "~";
        string const fpath = lyxrc.backupdir_path.empty() ?
                fn.onlyPath().absFileName() :
                lyxrc.backupdir_path;
-       string const fform = convert<string>(d->file_format);
-       string const backname = fname + "-lyxformat-" + fform;
+       string backup_suffix;
+       // If file format is from a stable series use version instead of file format
+       auto const it = file_formats.find(d->file_format);
+       if (it != file_formats.end())
+               backup_suffix = "-lyx" + it->second;
+       else
+               backup_suffix = "-lyxformat-" + convert<string>(d->file_format);
+       string const backname = fname + backup_suffix;
        FileName backup(addName(fpath, addExtension(backname, fext)));
 
        // limit recursion, just in case
@@ -1379,7 +1415,6 @@ FileName Buffer::getBackupName() const {
 // Should probably be moved to somewhere else: BufferView? GuiView?
 bool Buffer::save() const
 {
-       FileMonitorBlocker block = d->blockFileMonitor();
        docstring const file = makeDisplayPath(absFileName(), 20);
        d->filename.refresh();
 
@@ -1429,6 +1464,9 @@ bool Buffer::save() const
 
        FileName savefile(tempfile->name());
        LYXERR(Debug::FILES, "Saving to " << savefile.absFileName());
+       if (!savefile.clonePermissions(fileName()))
+               LYXERR0("Failed to clone the permission from " << fileName().absFileName() << " to " << savefile.absFileName());
+
        if (!writeFile(savefile))
                return false;
 
@@ -1606,7 +1644,7 @@ docstring Buffer::emergencyWrite()
                return user_message;
        }
 
-       user_message += "  " + _("Save failed! Bummer. Document is lost.");
+       user_message += "  " + _("Save failed! Document is lost.");
        // Don't try again.
        markClean();
        return user_message;
@@ -1626,16 +1664,14 @@ bool Buffer::write(ostream & ofs) const
        // Important: Keep the version formatting in sync with lyx2lyx and
        //            tex2lyx (bug 7951)
        ofs << "#LyX " << lyx_version_major << "." << lyx_version_minor
-           << " created this file. For more info see http://www.lyx.org/\n"
+           << " created this file. For more info see https://www.lyx.org/\n"
            << "\\lyxformat " << LYX_FORMAT << "\n"
            << "\\begin_document\n";
 
        /// For each author, set 'used' to true if there is a change
        /// by this author in the document; otherwise set it to 'false'.
-       AuthorList::Authors::const_iterator a_it = params().authors().begin();
-       AuthorList::Authors::const_iterator a_end = params().authors().end();
-       for (; a_it != a_end; ++a_it)
-               a_it->setUsed(false);
+       for (Author const & a : params().authors())
+               a.setUsed(false);
 
        ParIterator const end = const_cast<Buffer *>(this)->par_iterator_end();
        ParIterator it = const_cast<Buffer *>(this)->par_iterator_begin();
@@ -1676,20 +1712,13 @@ bool Buffer::write(ostream & ofs) const
 }
 
 
-bool Buffer::makeLaTeXFile(FileName const & fname,
+Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname,
                           string const & original_path,
                           OutputParams const & runparams_in,
                           OutputWhat output) const
 {
        OutputParams runparams = runparams_in;
 
-       // XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740),
-       // Check here, because the "flavor" is not known in BufferParams::encoding()
-       // (power users can override this safety measure selecting "utf8-plain").
-       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
-           && (runparams.encoding->name() != "utf8-plain"))
-               runparams.encoding = encodings.fromLyXName("ascii");
-
        string const encoding = runparams.encoding->iconvName();
        LYXERR(Debug::LATEX, "makeLaTeXFile encoding: " << encoding << ", fname=" << fname.realPath());
 
@@ -1697,17 +1726,19 @@ bool Buffer::makeLaTeXFile(FileName const & fname,
        try { ofs.reset(encoding); }
        catch (iconv_codecvt_facet_exception const & e) {
                lyxerr << "Caught iconv exception: " << e.what() << endl;
-               Alert::error(_("Iconv software exception Detected"), bformat(_("Please "
-                       "verify that the support software for your encoding (%1$s) is "
-                       "properly installed"), from_ascii(encoding)));
-               return false;
+               Alert::error(_("Iconv software exception Detected"),
+                       bformat(_("Please verify that the 'iconv' support software is"
+                                         " properly installed and supports the selected encoding"
+                                         " (%1$s), or change the encoding in"
+                                         " Document>Settings>Language."), from_ascii(encoding)));
+               return ExportError;
        }
        if (!openFileWrite(ofs, fname))
-               return false;
+               return ExportError;
 
        ErrorList & errorList = d->errorLists["Export"];
        errorList.clear();
-       bool failed_export = false;
+       ExportStatus status = ExportSuccess;
        otexstream os(ofs);
 
        // make sure we are ready to export
@@ -1717,31 +1748,45 @@ bool Buffer::makeLaTeXFile(FileName const & fname,
        updateBuffer();
        updateMacroInstances(OutputUpdate);
 
+       ExportStatus retval;
        try {
-               writeLaTeXSource(os, original_path, runparams, output);
+               retval = writeLaTeXSource(os, original_path, runparams, output);
+               if (retval == ExportKilled)
+                       return ExportKilled;
        }
        catch (EncodingException const & e) {
                docstring const failed(1, e.failed_char);
                ostringstream oss;
                oss << "0x" << hex << e.failed_char << dec;
-               docstring msg = bformat(_("Could not find LaTeX command for character '%1$s'"
-                                         " (code point %2$s)"),
-                                         failed, from_utf8(oss.str()));
-               errorList.push_back(ErrorItem(msg, _("Some characters of your document are probably not "
-                               "representable in the chosen encoding.\n"
-                               "Changing the document encoding to utf8 could help."),
-                                             {e.par_id, e.pos}, {e.par_id, e.pos + 1}));
-               failed_export = true;
+               if (getParFromID(e.par_id).paragraph().layout().pass_thru) {
+                       docstring msg = bformat(_("Uncodable character '%1$s'"
+                                                 " (code point %2$s)"),
+                                                 failed, from_utf8(oss.str()));
+                       errorList.push_back(ErrorItem(msg, _("Some characters of your document are not "
+                                       "representable in specific verbatim contexts.\n"
+                                       "Changing the document encoding to utf8 could help."),
+                                                     {e.par_id, e.pos}, {e.par_id, e.pos + 1}));
+               } else {
+                       docstring msg = bformat(_("Could not find LaTeX command for character '%1$s'"
+                                                 " (code point %2$s)"),
+                                                 failed, from_utf8(oss.str()));
+                       errorList.push_back(ErrorItem(msg, _("Some characters of your document are probably not "
+                                       "representable in the chosen encoding.\n"
+                                       "Changing the document encoding to utf8 could help."),
+                                                     {e.par_id, e.pos}, {e.par_id, e.pos + 1}));
+               }
+               status = ExportError;
        }
        catch (iconv_codecvt_facet_exception const & e) {
                errorList.push_back(ErrorItem(_("iconv conversion failed"),
                                              _(e.what())));
-               failed_export = true;
+               status = ExportError;
        }
        catch (exception const & e) {
                errorList.push_back(ErrorItem(_("conversion failed"),
                                              _(e.what())));
-               failed_export = true;
+               lyxerr << e.what() << endl;
+               status = ExportError;
        }
        catch (...) {
                lyxerr << "Caught some really weird exception..." << endl;
@@ -1752,7 +1797,7 @@ bool Buffer::makeLaTeXFile(FileName const & fname,
 
        ofs.close();
        if (ofs.fail()) {
-               failed_export = true;
+               status = ExportError;
                lyxerr << "File '" << fname << "' was not closed properly." << endl;
        }
 
@@ -1760,11 +1805,11 @@ bool Buffer::makeLaTeXFile(FileName const & fname,
                errorList.clear();
        else
                errors("Export");
-       return !failed_export;
+       return status;
 }
 
 
-void Buffer::writeLaTeXSource(otexstream & os,
+Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os,
                           string const & original_path,
                           OutputParams const & runparams_in,
                           OutputWhat output) const
@@ -1773,15 +1818,8 @@ void Buffer::writeLaTeXSource(otexstream & os,
 
        OutputParams runparams = runparams_in;
 
-       // XeTeX with TeX fonts is only safe with ASCII encoding,
-       // Check here, because the "flavor" is not known in BufferParams::encoding()
-       // (power users can override this safety measure selecting "utf8-plain").
-       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
-           && (runparams.encoding->name() != "utf8-plain"))
-               runparams.encoding = encodings.fromLyXName("ascii");
-       // FIXME: when only the current paragraph is shown, this is ignored
-       //        (or not reached) and characters encodable in the current
-       //        encoding are not converted to ASCII-representation.
+       // Some macros rely on font encoding
+       runparams.main_fontenc = params().main_font_encoding();
 
        // If we are compiling a file standalone, even if this is the
        // child of some other buffer, let's cut the link here, so the
@@ -1798,8 +1836,10 @@ void Buffer::writeLaTeXSource(otexstream & os,
        LaTeXFeatures features(*this, params(), runparams);
        validate(features);
        // This is only set once per document (in master)
-       if (!runparams.is_child)
+       if (!runparams.is_child) {
                runparams.use_polyglossia = features.usePolyglossia();
+               runparams.use_CJK = features.mustProvide("CJK");
+       }
        LYXERR(Debug::LATEX, "  Buffer validation done.");
 
        bool const output_preamble =
@@ -1811,7 +1851,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
        // first paragraph of the document. (Asger)
        if (output_preamble && runparams.nice) {
                os << "%% LyX " << lyx_version << " created this file.  "
-                       "For more info, see http://www.lyx.org/.\n"
+                       "For more info, see https://www.lyx.org/.\n"
                        "%% Do not edit unless you really know what "
                        "you are doing.\n";
        }
@@ -1855,7 +1895,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
                        if (!uncodable_glyphs.empty()) {
                                frontend::Alert::warning(
                                        _("Uncodable character in file path"),
-                                       support::bformat(
+                                       bformat(
                                          _("The path of your document\n"
                                            "(%1$s)\n"
                                            "contains glyphs that are unknown "
@@ -1873,8 +1913,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
                                            "file path name."),
                                          inputpath, uncodable_glyphs));
                        } else {
-                               string docdir =
-                                       support::latex_path(original_path);
+                               string docdir = os::latex_path(original_path);
                                if (contains(docdir, '#')) {
                                        docdir = subst(docdir, "#", "\\#");
                                        os << "\\catcode`\\#=11"
@@ -1885,9 +1924,20 @@ void Buffer::writeLaTeXSource(otexstream & os,
                                        os << "\\catcode`\\%=11"
                                              "\\def\\%{%}\\catcode`\\%=14\n";
                                }
+                               bool const detokenize = !isAscii(from_utf8(docdir));
+                               bool const quote = contains(docdir, ' ');
                                os << "\\makeatletter\n"
-                                  << "\\def\\input@path{{"
-                                  << docdir << "}}\n"
+                                  << "\\def\\input@path{{";
+                               if (detokenize)
+                                       os << "\\detokenize{";
+                               if (quote)
+                                       os << "\"";
+                               os << docdir;
+                               if (quote)
+                                       os << "\"";
+                               if (detokenize)
+                                       os << "}";
+                               os << "}}\n"
                                   << "\\makeatother\n";
                        }
                }
@@ -1903,10 +1953,14 @@ void Buffer::writeLaTeXSource(otexstream & os,
 
                // Biblatex bibliographies are loaded here
                if (params().useBiblatex()) {
-                       vector<docstring> const bibfiles =
-                               prepareBibFilePaths(runparams, getBibfilesCache(), true);
-                       for (docstring const & file: bibfiles)
-                               os << "\\addbibresource{" << file << "}\n";
+                       vector<pair<docstring, string>> const bibfiles =
+                               prepareBibFilePaths(runparams, getBibfiles(), true);
+                       for (pair<docstring, string> const & file: bibfiles) {
+                               os << "\\addbibresource";
+                               if (!file.second.empty())
+                                       os << "[bibencoding=" << file.second << "]";
+                               os << "{" << file.first << "}\n";
+                       }
                }
 
                if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages()
@@ -1917,32 +1971,32 @@ void Buffer::writeLaTeXSource(otexstream & os,
                        vector<string> pll = features.getPolyglossiaExclusiveLanguages();
                        if (!bll.empty()) {
                                docstring langs;
-                               for (vector<string>::const_iterator it = bll.begin(); it != bll.end(); ++it) {
+                               for (string const & sit : bll) {
                                        if (!langs.empty())
                                                langs += ", ";
-                                       langs += _(*it);
+                                       langs += _(sit);
                                }
                                blangs = bll.size() > 1 ?
-                                           support::bformat(_("The languages %1$s are only supported by Babel."), langs)
-                                         : support::bformat(_("The language %1$s is only supported by Babel."), langs);
+                                           bformat(_("The languages %1$s are only supported by Babel."), langs)
+                                         : bformat(_("The language %1$s is only supported by Babel."), langs);
                        }
                        if (!pll.empty()) {
                                docstring langs;
-                               for (vector<string>::const_iterator it = pll.begin(); it != pll.end(); ++it) {
+                               for (string const & pit : pll) {
                                        if (!langs.empty())
                                                langs += ", ";
-                                       langs += _(*it);
+                                       langs += _(pit);
                                }
                                plangs = pll.size() > 1 ?
-                                           support::bformat(_("The languages %1$s are only supported by Polyglossia."), langs)
-                                         : support::bformat(_("The language %1$s is only supported by Polyglossia."), langs);
+                                           bformat(_("The languages %1$s are only supported by Polyglossia."), langs)
+                                         : bformat(_("The language %1$s is only supported by Polyglossia."), langs);
                                if (!blangs.empty())
-                                       plangs += "\n"; 
+                                       plangs += "\n";
                        }
 
                        frontend::Alert::warning(
                                _("Incompatible Languages!"),
-                               support::bformat(
+                               bformat(
                                  _("You cannot use the following languages "
                                    "together in one LaTeX document because "
                                    "they require conflicting language packages:\n"
@@ -1958,7 +2012,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
                        // Restore the parenthood if needed
                        if (!runparams.is_child)
                                d->ignore_parent = false;
-                       return;
+                       return ExportSuccess;
                }
 
                // make the body.
@@ -1971,10 +2025,8 @@ void Buffer::writeLaTeXSource(otexstream & os,
                os.lastChar('\n');
 
                // output the parent macros
-               MacroSet::iterator it = parentMacros.begin();
-               MacroSet::iterator end = parentMacros.end();
-               for (; it != end; ++it) {
-                       int num_lines = (*it)->write(os.os(), true);
+               for (auto const & mac : parentMacros) {
+                       int num_lines = mac->write(os.os(), true);
                        os.texrow().newlines(num_lines);
                }
 
@@ -1983,7 +2035,10 @@ void Buffer::writeLaTeXSource(otexstream & os,
        LYXERR(Debug::INFO, "preamble finished, now the body.");
 
        // the real stuff
-       latexParagraphs(*this, text(), os, runparams);
+       try {
+               latexParagraphs(*this, text(), os, runparams);
+       }
+       catch (ConversionException const &) { return ExportKilled; }
 
        // Restore the parenthood if needed
        if (!runparams.is_child)
@@ -2002,10 +2057,11 @@ void Buffer::writeLaTeXSource(otexstream & os,
 
        LYXERR(Debug::INFO, "Finished making LaTeX file.");
        LYXERR(Debug::INFO, "Row count was " << os.texrow().rows() - 1 << '.');
+       return ExportSuccess;
 }
 
 
-void Buffer::makeDocBookFile(FileName const & fname,
+Buffer::ExportStatus Buffer::makeDocBookFile(FileName const & fname,
                              OutputParams const & runparams,
                              OutputWhat output) const
 {
@@ -2013,22 +2069,26 @@ void Buffer::makeDocBookFile(FileName const & fname,
 
        ofdocstream ofs;
        if (!openFileWrite(ofs, fname))
-               return;
+               return ExportError;
 
        // make sure we are ready to export
        // this needs to be done before we validate
        updateBuffer();
        updateMacroInstances(OutputUpdate);
 
-       writeDocBookSource(ofs, fname.absFileName(), runparams, output);
+       ExportStatus const retval =
+               writeDocBookSource(ofs, fname.absFileName(), runparams, output);
+       if (retval == ExportKilled)
+               return ExportKilled;
 
        ofs.close();
        if (ofs.fail())
                lyxerr << "File '" << fname << "' was not closed properly." << endl;
+       return ExportSuccess;
 }
 
 
-void Buffer::writeDocBookSource(odocstream & os, string const & fname,
+Buffer::ExportStatus Buffer::writeDocBookSource(odocstream & os, string const & fname,
                             OutputParams const & runparams,
                             OutputWhat output) const
 {
@@ -2057,7 +2117,7 @@ void Buffer::writeDocBookSource(odocstream & os, string const & fname,
                        os << from_ascii(tclass.class_header());
                else if (runparams.flavor == OutputParams::XML)
                        os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
-                           << "\"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
+                           << "\"https://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
                else
                        os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
 
@@ -2096,41 +2156,48 @@ void Buffer::writeDocBookSource(odocstream & os, string const & fname,
 
                os << "<!-- " << ((runparams.flavor == OutputParams::XML)? "XML" : "SGML")
                                << " file was created by LyX " << lyx_version
-                               << "\n  See http://www.lyx.org/ for more information -->\n";
+                               << "\n  See https://www.lyx.org/ for more information -->\n";
 
                params().documentClass().counters().reset();
 
                sgml::openTag(os, top);
                os << '\n';
-               docbookParagraphs(text(), *this, os, runparams);
+               try {
+                       docbookParagraphs(text(), *this, os, runparams);
+               }
+               catch (ConversionException const &) { return ExportKilled; }
                sgml::closeTag(os, top_element);
        }
+       return ExportSuccess;
 }
 
 
-void Buffer::makeLyXHTMLFile(FileName const & fname,
+Buffer::ExportStatus Buffer::makeLyXHTMLFile(FileName const & fname,
                              OutputParams const & runparams) const
 {
        LYXERR(Debug::LATEX, "makeLyXHTMLFile...");
 
        ofdocstream ofs;
        if (!openFileWrite(ofs, fname))
-               return;
+               return ExportError;
 
        // make sure we are ready to export
        // this has to be done before we validate
        updateBuffer(UpdateMaster, OutputUpdate);
        updateMacroInstances(OutputUpdate);
 
-       writeLyXHTMLSource(ofs, runparams, FullSource);
+       ExportStatus const retval = writeLyXHTMLSource(ofs, runparams, FullSource);
+       if (retval == ExportKilled)
+               return retval;
 
        ofs.close();
        if (ofs.fail())
                lyxerr << "File '" << fname << "' was not closed properly." << endl;
+       return retval;
 }
 
 
-void Buffer::writeLyXHTMLSource(odocstream & os,
+Buffer::ExportStatus Buffer::writeLyXHTMLSource(odocstream & os,
                             OutputParams const & runparams,
                             OutputWhat output) const
 {
@@ -2231,13 +2298,18 @@ void Buffer::writeLyXHTMLSource(odocstream & os,
                if (output != IncludedFile)
                        // if we're an included file, the counters are in the master.
                        params().documentClass().counters().reset();
-               xhtmlParagraphs(text(), *this, xs, runparams);
+               try {
+                       xhtmlParagraphs(text(), *this, xs, runparams);
+               }
+               catch (ConversionException const &) { return ExportKilled; }
                if (output_body_tag)
                        os << "</body>\n";
        }
 
        if (output_preamble)
                os << "</html>\n";
+
+       return ExportSuccess;
 }
 
 
@@ -2260,7 +2332,12 @@ int Buffer::runChktex()
        runparams.flavor = OutputParams::LATEX;
        runparams.nice = false;
        runparams.linelen = lyxrc.plaintext_linelen;
-       makeLaTeXFile(FileName(name), org_path, runparams);
+       ExportStatus const retval =
+               makeLaTeXFile(FileName(name), org_path, runparams);
+       if (retval != ExportSuccess) {
+               // error code on failure
+               return -1;
+       }
 
        TeXErrors terr;
        Chktex chktex(lyxrc.chktex_command, onlyFileName(name), filePath());
@@ -2313,56 +2390,18 @@ void Buffer::getLabelList(vector<docstring> & list) const
 
        list.clear();
        shared_ptr<Toc> toc = d->toc_backend.toc("label");
-       Toc::const_iterator toc_it = toc->begin();
-       Toc::const_iterator end = toc->end();
-       for (; toc_it != end; ++toc_it) {
-               if (toc_it->depth() == 0)
-                       list.push_back(toc_it->str());
+       for (auto const & tocit : *toc) {
+               if (tocit.depth() == 0)
+                       list.push_back(tocit.str());
        }
 }
 
 
-void Buffer::updateBibfilesCache(UpdateScope scope) const
-{
-       // FIXME This is probably unnecssary, given where we call this.
-       // If this is a child document, use the parent's cache instead.
-       if (parent() && scope != UpdateChildOnly) {
-               masterBuffer()->updateBibfilesCache();
-               return;
-       }
-
-       d->bibfiles_cache_.clear();
-       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
-               if (it->lyxCode() == BIBTEX_CODE) {
-                       InsetBibtex const & inset = static_cast<InsetBibtex const &>(*it);
-                       support::FileNamePairList const bibfiles = inset.getBibFiles();
-                       d->bibfiles_cache_.insert(d->bibfiles_cache_.end(),
-                               bibfiles.begin(),
-                               bibfiles.end());
-               } else if (it->lyxCode() == INCLUDE_CODE) {
-                       InsetInclude & inset = static_cast<InsetInclude &>(*it);
-                       Buffer const * const incbuf = inset.getChildBuffer();
-                       if (!incbuf)
-                               continue;
-                       support::FileNamePairList const & bibfiles =
-                                       incbuf->getBibfilesCache(UpdateChildOnly);
-                       if (!bibfiles.empty()) {
-                               d->bibfiles_cache_.insert(d->bibfiles_cache_.end(),
-                                       bibfiles.begin(),
-                                       bibfiles.end());
-                       }
-               }
-       }
-       d->bibfile_cache_valid_ = true;
-       d->bibinfo_cache_valid_ = false;
-       d->cite_labels_valid_ = false;
-}
-
-
 void Buffer::invalidateBibinfoCache() const
 {
        d->bibinfo_cache_valid_ = false;
        d->cite_labels_valid_ = false;
+       removeBiblioTempFiles();
        // also invalidate the cache for the parent buffer
        Buffer const * const pbuf = d->parent();
        if (pbuf)
@@ -2370,28 +2409,19 @@ void Buffer::invalidateBibinfoCache() const
 }
 
 
-void Buffer::invalidateBibfileCache() const
-{
-       d->bibfile_cache_valid_ = false;
-       d->bibinfo_cache_valid_ = false;
-       d->cite_labels_valid_ = false;
-       // also invalidate the cache for the parent buffer
-       Buffer const * const pbuf = d->parent();
-       if (pbuf)
-               pbuf->invalidateBibfileCache();
-}
-
-
-support::FileNamePairList const & Buffer::getBibfilesCache(UpdateScope scope) const
+docstring_list const & Buffer::getBibfiles(UpdateScope scope) const
 {
        // FIXME This is probably unnecessary, given where we call this.
-       // If this is a child document, use the master's cache instead.
+       // If this is a child document, use the master instead.
        Buffer const * const pbuf = masterBuffer();
        if (pbuf != this && scope != UpdateChildOnly)
-               return pbuf->getBibfilesCache();
+               return pbuf->getBibfiles();
 
-       if (!d->bibfile_cache_valid_)
-               this->updateBibfilesCache(scope);
+       // In 2.3.x, we have:
+       //if (!d->bibfile_cache_valid_)
+       //      this->updateBibfilesCache(scope);
+       // I think that is a leftover, but there have been so many back-
+       // and-forths with this, due to Windows issues, that I am not sure.
 
        return d->bibfiles_cache_;
 }
@@ -2406,6 +2436,55 @@ BiblioInfo const & Buffer::masterBibInfo() const
 }
 
 
+BiblioInfo const & Buffer::bibInfo() const
+{
+       return d->bibinfo_;
+}
+
+
+void Buffer::registerBibfiles(const docstring_list & bf) const
+{
+       // We register the bib files in the master buffer,
+       // if there is one, but also in every single buffer,
+       // in case a child is compiled alone.
+       Buffer const * const tmp = masterBuffer();
+       if (tmp != this)
+               tmp->registerBibfiles(bf);
+
+       for (auto const & p : bf) {
+               docstring_list::const_iterator temp =
+                       find(d->bibfiles_cache_.begin(), d->bibfiles_cache_.end(), p);
+               if (temp == d->bibfiles_cache_.end())
+                       d->bibfiles_cache_.push_back(p);
+       }
+}
+
+
+static map<docstring, FileName> bibfileCache;
+
+FileName Buffer::getBibfilePath(docstring const & bibid) const
+{
+       map<docstring, FileName>::const_iterator it =
+               bibfileCache.find(bibid);
+       if (it != bibfileCache.end()) {
+               // i.e., bibfileCache[bibid]
+               return it->second;
+       }
+
+       LYXERR(Debug::FILES, "Reading file location for " << bibid);
+       string texfile = changeExtension(to_utf8(bibid), "bib");
+       // note that, if the filename can be found directly from the path,
+       // findtexfile will just return a FileName object for that path.
+       FileName file(findtexfile(texfile, "bib"));
+       if (file.empty())
+               file = FileName(makeAbsPath(texfile, filePath()));
+       LYXERR(Debug::FILES, "Found at: " << file);
+
+       bibfileCache[bibid] = file;
+       return bibfileCache[bibid];
+}
+
+
 void Buffer::checkIfBibInfoCacheIsValid() const
 {
        // use the master's cache
@@ -2415,12 +2494,35 @@ void Buffer::checkIfBibInfoCacheIsValid() const
                return;
        }
 
+       // If we already know the cache is invalid, stop here.
+       // This is important in the case when the bibliography
+       // environment (rather than Bib[la]TeX) is used.
+       // In that case, the timestamp check below gives no
+       // sensible result. Rather than that, the cache will
+       // be invalidated explicitly via invalidateBibInfoCache()
+       // by the Bibitem inset.
+       // Same applies for bib encoding changes, which trigger
+       // invalidateBibInfoCache() by InsetBibtex.
+       if (!d->bibinfo_cache_valid_)
+               return;
+
+       if (d->have_bibitems_) {
+               // We have a bibliography environment.
+               // Invalidate the bibinfo cache unconditionally.
+               // Cite labels will get invalidated by the inset if needed.
+               d->bibinfo_cache_valid_ = false;
+               return;
+       }
+
+       // OK. This is with Bib(la)tex. We'll assume the cache
+       // is valid and change this if we find changes in the bibs.
+       d->bibinfo_cache_valid_ = true;
+       d->cite_labels_valid_ = true;
+
        // compare the cached timestamps with the actual ones.
-       FileNamePairList const & bibfiles_cache = getBibfilesCache();
-       FileNamePairList::const_iterator ei = bibfiles_cache.begin();
-       FileNamePairList::const_iterator en = bibfiles_cache.end();
-       for (; ei != en; ++ ei) {
-               FileName const fn = ei->second;
+       docstring_list const & bibfiles_cache = getBibfiles();
+       for (auto const & bf : bibfiles_cache) {
+               FileName const fn = getBibfilePath(bf);
                time_t lastw = fn.lastModified();
                time_t prevw = d->bibfile_status_[fn];
                if (lastw != prevw) {
@@ -2432,47 +2534,86 @@ void Buffer::checkIfBibInfoCacheIsValid() const
 }
 
 
-void Buffer::reloadBibInfoCache() const
+void Buffer::clearBibFileCache() const
+{
+       bibfileCache.clear();
+}
+
+
+void Buffer::reloadBibInfoCache(bool const force) const
 {
+       // we should not need to do this for internal buffers
+       if (isInternal())
+               return;
+
        // use the master's cache
        Buffer const * const tmp = masterBuffer();
        if (tmp != this) {
-               tmp->reloadBibInfoCache();
+               tmp->reloadBibInfoCache(force);
                return;
        }
 
-       checkIfBibInfoCacheIsValid();
-       if (d->bibinfo_cache_valid_)
-               return;
+       if (!force) {
+               checkIfBibInfoCacheIsValid();
+               if (d->bibinfo_cache_valid_)
+                       return;
+       }
 
+       LYXERR(Debug::FILES, "Bibinfo cache was invalid.");
+       // re-read file locations when this info changes
+       // FIXME Is this sufficient? Or should we also force that
+       // in some other cases? If so, then it is easy enough to
+       // add the following line in some other places.
+       clearBibFileCache();
        d->bibinfo_.clear();
-       collectBibKeys();
+       FileNameList checkedFiles;
+       d->have_bibitems_ = false;
+       collectBibKeys(checkedFiles);
        d->bibinfo_cache_valid_ = true;
 }
 
 
-void Buffer::collectBibKeys() const
+void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 {
-       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it)
-               it->collectBibKeys(it);
+       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
+               it->collectBibKeys(it, checkedFiles);
+               if (it->lyxCode() == BIBITEM_CODE) {
+                       if (parent() != 0)
+                               parent()->d->have_bibitems_ = true;
+                       else
+                               d->have_bibitems_ = true;
+               }
+       }
 }
 
 
-void Buffer::addBiblioInfo(BiblioInfo const & bi) const
+void Buffer::addBiblioInfo(BiblioInfo const & bin) const
 {
-       Buffer const * tmp = masterBuffer();
-       BiblioInfo & masterbi = (tmp == this) ?
-               d->bibinfo_ : tmp->d->bibinfo_;
-       masterbi.mergeBiblioInfo(bi);
+       // We add the biblio info to the master buffer,
+       // if there is one, but also to every single buffer,
+       // in case a child is compiled alone.
+       BiblioInfo & bi = d->bibinfo_;
+       bi.mergeBiblioInfo(bin);
+
+       if (parent() != 0) {
+               BiblioInfo & masterbi = parent()->d->bibinfo_;
+               masterbi.mergeBiblioInfo(bin);
+       }
 }
 
 
-void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bi) const
+void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bin) const
 {
-       Buffer const * tmp = masterBuffer();
-       BiblioInfo & masterbi = (tmp == this) ?
-               d->bibinfo_ : tmp->d->bibinfo_;
-       masterbi[key] = bi;
+       // We add the bibtex info to the master buffer,
+       // if there is one, but also to every single buffer,
+       // in case a child is compiled alone.
+       BiblioInfo & bi = d->bibinfo_;
+       bi[key] = bin;
+
+       if (parent() != 0) {
+               BiblioInfo & masterbi = masterBuffer()->d->bibinfo_;
+               masterbi[key] = bin;
+       }
 }
 
 
@@ -2483,6 +2624,11 @@ void Buffer::makeCitationLabels() const
 }
 
 
+void Buffer::invalidateCiteLabels() const
+{
+       masterBuffer()->d->cite_labels_valid_ = false;
+}
+
 bool Buffer::citeLabelsValid() const
 {
        return masterBuffer()->d->cite_labels_valid_;
@@ -2566,10 +2712,6 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                break;
        }
 
-       case LFUN_BUFFER_CHKTEX:
-               enable = params().isLatex() && !lyxrc.chktex_command.empty();
-               break;
-
        case LFUN_BUILD_PROGRAM:
                enable = params().isExportable("program", false);
                break;
@@ -2611,15 +2753,16 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                flag.setOnOff(params().output_changes);
                break;
 
-       case LFUN_BUFFER_TOGGLE_COMPRESSION: {
+       case LFUN_BUFFER_TOGGLE_COMPRESSION:
                flag.setOnOff(params().compressed);
                break;
-       }
 
-       case LFUN_BUFFER_TOGGLE_OUTPUT_SYNC: {
+       case LFUN_BUFFER_TOGGLE_OUTPUT_SYNC:
                flag.setOnOff(params().output_sync);
                break;
-       }
+
+       case LFUN_BUFFER_ANONYMIZE:
+               break;
 
        default:
                return false;
@@ -2649,7 +2792,8 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
        string const argument = to_utf8(func.argument());
        // We'll set this back to false if need be.
        bool dispatched = true;
-       undo().beginUndoGroup();
+       // This handles undo groups automagically
+       UndoGroupHelper ugh(this);
 
        switch (func.action()) {
        case LFUN_BUFFER_TOGGLE_READ_ONLY:
@@ -2681,14 +2825,10 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                break;
        }
 
-       case LFUN_BUFFER_CHKTEX:
-               runChktex();
-               break;
-
        case LFUN_BUFFER_EXPORT_CUSTOM: {
                string format_name;
                string command = split(argument, format_name, ' ');
-               Format const * format = formats.getFormat(format_name);
+               Format const * format = theFormats().getFormat(format_name);
                if (!format) {
                        lyxerr << "Format \"" << format_name
                                << "\" not recognized!"
@@ -2774,18 +2914,16 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
        }
 
        case LFUN_BRANCH_ADD: {
-               docstring branch_name = func.argument();
-               if (branch_name.empty()) {
+               docstring branchnames = func.argument();
+               if (branchnames.empty()) {
                        dispatched = false;
                        break;
                }
                BranchList & branch_list = params().branchlist();
                vector<docstring> const branches =
-                       getVectorFromString(branch_name, branch_list.separator());
+                       getVectorFromString(branchnames, branch_list.separator());
                docstring msg;
-               for (vector<docstring>::const_iterator it = branches.begin();
-                    it != branches.end(); ++it) {
-                       branch_name = *it;
+               for (docstring const & branch_name : branches) {
                        Branch * branch = branch_list.find(branch_name);
                        if (branch) {
                                LYXERR0("Branch " << branch_name << " already exists.");
@@ -2847,7 +2985,7 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
        }
 
        case LFUN_BUFFER_VIEW_CACHE:
-               if (!formats.view(*this, d->preview_file_,
+               if (!theFormats().view(*this, d->preview_file_,
                                  d->preview_format_))
                        dr.setMessage(_("Error viewing the output file."));
                break;
@@ -2896,12 +3034,21 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                params().output_sync = !params().output_sync;
                break;
 
+       case LFUN_BUFFER_ANONYMIZE: {
+               undo().recordUndoFullBuffer(CursorData());
+               CursorData cur(doc_iterator_begin(this));
+               for ( ; cur ; cur.forwardPar())
+                       cur.paragraph().anonymize();
+               dr.forceBufferUpdate();
+               dr.screenUpdate(Update::Force);
+               break;
+       }
+
        default:
                dispatched = false;
                break;
        }
        dr.dispatched(dispatched);
-       undo().endUndoGroup();
 }
 
 
@@ -2929,26 +3076,24 @@ bool Buffer::isMultiLingual() const
 
 std::set<Language const *> Buffer::getLanguages() const
 {
-       std::set<Language const *> languages;
-       getLanguages(languages);
-       return languages;
+       std::set<Language const *> langs;
+       getLanguages(langs);
+       return langs;
 }
 
 
-void Buffer::getLanguages(std::set<Language const *> & languages) const
+void Buffer::getLanguages(std::set<Language const *> & langs) const
 {
        ParConstIterator end = par_iterator_end();
        // add the buffer language, even if it's not actively used
-       languages.insert(language());
+       langs.insert(language());
        // iterate over the paragraphs
        for (ParConstIterator it = par_iterator_begin(); it != end; ++it)
-               it->getLanguages(languages);
+               it->getLanguages(langs);
        // also children
        ListOfBuffers clist = getDescendents();
-       ListOfBuffers::const_iterator cit = clist.begin();
-       ListOfBuffers::const_iterator const cen = clist.end();
-       for (; cit != cen; ++cit)
-               (*cit)->getLanguages(languages);
+       for (auto const & cit : clist)
+               cit->getLanguages(langs);
 }
 
 
@@ -3082,11 +3227,8 @@ void Buffer::markDirty()
        }
        d->bak_clean = false;
 
-       DepClean::iterator it = d->dep_clean.begin();
-       DepClean::const_iterator const end = d->dep_clean.end();
-
-       for (; it != end; ++it)
-               it->second = false;
+       for (auto & depit : d->dep_clean)
+               depit.second = false;
 }
 
 
@@ -3146,8 +3288,8 @@ string const Buffer::prepareFileNameForLaTeX(string const & name,
 }
 
 
-vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
-                                               FileNamePairList const bibfilelist,
+vector<pair<docstring, string>> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
+                                               docstring_list const & bibfilelist,
                                                bool const add_extension) const
 {
        // If we are processing the LaTeX file in a temp directory then
@@ -3160,7 +3302,7 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
        // Otherwise, store the (maybe absolute) path to the original,
        // unmangled database name.
 
-       vector<docstring> res;
+       vector<pair<docstring, string>> res;
 
        // determine the export format
        string const tex_format = flavor2format(runparams.flavor);
@@ -3168,15 +3310,32 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
        // check for spaces in paths
        bool found_space = false;
 
-       FileNamePairList::const_iterator it = bibfilelist.begin();
-       FileNamePairList::const_iterator en = bibfilelist.end();
-       for (; it != en; ++it) {
-               string utf8input = to_utf8(it->first);
+       for (auto const & bit : bibfilelist) {
+               string utf8input = to_utf8(bit);
                string database =
                        prepareFileNameForLaTeX(utf8input, ".bib", runparams.nice);
-               FileName const try_in_file =
+               FileName try_in_file =
                        makeAbsPath(database + ".bib", filePath());
-               bool const not_from_texmf = try_in_file.isReadableFile();
+               bool not_from_texmf = try_in_file.isReadableFile();
+               // If the file has not been found, try with the real file name
+               // (it might come from a child in a sub-directory)
+               if (!not_from_texmf) {
+                       try_in_file = getBibfilePath(bit);
+                       if (try_in_file.isReadableFile()) {
+                               // Check if the file is in texmf
+                               FileName kpsefile(findtexfile(changeExtension(utf8input, "bib"), "bib", true));
+                               not_from_texmf = kpsefile.empty()
+                                               || kpsefile.absFileName() != try_in_file.absFileName();
+                               if (not_from_texmf)
+                                       // If this exists, make path relative to the master
+                                       // FIXME Unicode
+                                       database =
+                                               removeExtension(prepareFileNameForLaTeX(
+                                                                                       to_utf8(makeRelPath(from_utf8(try_in_file.absFileName()),
+                                                                                                                               from_utf8(filePath()))),
+                                                                                       ".bib", runparams.nice));
+                       }
+               }
 
                if (!runparams.inComment && !runparams.dryrun && !runparams.nice &&
                    not_from_texmf) {
@@ -3215,9 +3374,20 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
 
                if (contains(path, ' '))
                        found_space = true;
+               string enc;
+               if (params().useBiblatex() && !params().bibFileEncoding(utf8input).empty())
+                       enc = params().bibFileEncoding(utf8input);
+
+               bool recorded = false;
+               for (pair<docstring, string> pe : res) {
+                       if (pe.first == path) {
+                               recorded = true;
+                               break;
+                       }
 
-               if (find(res.begin(), res.end(), path) == res.end())
-                       res.push_back(path);
+               }
+               if (!recorded)
+                       res.push_back(make_pair(path, enc));
        }
 
        // Check if there are spaces in the path and warn BibTeX users, if so.
@@ -3333,10 +3503,8 @@ bool Buffer::hasChildren() const
 void Buffer::collectChildren(ListOfBuffers & clist, bool grand_children) const
 {
        // loop over children
-       Impl::BufferPositionMap::iterator it = d->children_positions.begin();
-       Impl::BufferPositionMap::iterator end = d->children_positions.end();
-       for (; it != end; ++it) {
-               Buffer * child = const_cast<Buffer *>(it->first);
+       for (auto const & p : d->children_positions) {
+               Buffer * child = const_cast<Buffer *>(p.first);
                // No duplicates
                ListOfBuffers::const_iterator bit = find(clist.begin(), clist.end(), child);
                if (bit != clist.end())
@@ -3533,17 +3701,15 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
        while (it.pit() <= lastpit) {
                Paragraph & par = it.paragraph();
 
+               // FIXME Can this be done with the new-style iterators?
                // iterate over the insets of the current paragraph
-               InsetList const & insets = par.insetList();
-               InsetList::const_iterator iit = insets.begin();
-               InsetList::const_iterator end = insets.end();
-               for (; iit != end; ++iit) {
-                       it.pos() = iit->pos;
+               for (auto const & insit : par.insetList()) {
+                       it.pos() = insit.pos;
 
                        // is it a nested text inset?
-                       if (iit->inset->asInsetText()) {
+                       if (insit.inset->asInsetText()) {
                                // Inset needs its own scope?
-                               InsetText const * itext = iit->inset->asInsetText();
+                               InsetText const * itext = insit.inset->asInsetText();
                                bool newScope = itext->isMacroScope();
 
                                // scope which ends just behind the inset
@@ -3551,14 +3717,14 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                                ++insetScope.pos();
 
                                // collect macros in inset
-                               it.push_back(CursorSlice(*iit->inset));
+                               it.push_back(CursorSlice(*insit.inset));
                                updateMacros(it, newScope ? insetScope : scope);
                                it.pop_back();
                                continue;
                        }
 
-                       if (iit->inset->asInsetTabular()) {
-                               CursorSlice slice(*iit->inset);
+                       if (insit.inset->asInsetTabular()) {
+                               CursorSlice slice(*insit.inset);
                                size_t const numcells = slice.nargs();
                                for (; slice.idx() < numcells; slice.forwardIdx()) {
                                        it.push_back(slice);
@@ -3569,12 +3735,12 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                        }
 
                        // is it an external file?
-                       if (iit->inset->lyxCode() == INCLUDE_CODE) {
+                       if (insit.inset->lyxCode() == INCLUDE_CODE) {
                                // get buffer of external file
-                               InsetInclude const & inset =
-                                       static_cast<InsetInclude const &>(*iit->inset);
+                               InsetInclude const & incinset =
+                                       static_cast<InsetInclude const &>(*insit.inset);
                                macro_lock = true;
-                               Buffer * child = inset.getChildBuffer();
+                               Buffer * child = incinset.getChildBuffer();
                                macro_lock = false;
                                if (!child)
                                        continue;
@@ -3590,19 +3756,19 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                                continue;
                        }
 
-                       InsetMath * im = iit->inset->asInsetMath();
+                       InsetMath * im = insit.inset->asInsetMath();
                        if (doing_export && im)  {
                                InsetMathHull * hull = im->asHullInset();
                                if (hull)
                                        hull->recordLocation(it);
                        }
 
-                       if (iit->inset->lyxCode() != MATHMACRO_CODE)
+                       if (insit.inset->lyxCode() != MATHMACRO_CODE)
                                continue;
 
                        // get macro data
-                       MathMacroTemplate & macroTemplate =
-                               *iit->inset->asInsetMath()->asMacroTemplate();
+                       InsetMathMacroTemplate & macroTemplate =
+                               *insit.inset->asInsetMath()->asMacroTemplate();
                        MacroContext mc(owner_, it);
                        macroTemplate.updateToContext(mc);
 
@@ -3712,16 +3878,16 @@ void Buffer::listMacroNames(MacroNameSet & macros) const
        d->macro_lock = true;
 
        // loop over macro names
-       Impl::NamePositionScopeMacroMap::iterator nameIt = d->macros.begin();
-       Impl::NamePositionScopeMacroMap::iterator nameEnd = d->macros.end();
-       for (; nameIt != nameEnd; ++nameIt)
-               macros.insert(nameIt->first);
+       for (auto const & nameit : d->macros)
+               macros.insert(nameit.first);
 
        // loop over children
-       Impl::BufferPositionMap::iterator it = d->children_positions.begin();
-       Impl::BufferPositionMap::iterator end = d->children_positions.end();
-       for (; it != end; ++it)
-               it->first->listMacroNames(macros);
+       for (auto const & p : d->children_positions) {
+               Buffer * child = const_cast<Buffer *>(p.first);
+               // The buffer might have been closed (see #10766).
+               if (theBufferList().isLoaded(child))
+                       child->listMacroNames(macros);
+       }
 
        // call parent
        Buffer const * const pbuf = d->parent();
@@ -3742,16 +3908,13 @@ void Buffer::listParentMacros(MacroSet & macros, LaTeXFeatures & features) const
        pbuf->listMacroNames(names);
 
        // resolve macros
-       MacroNameSet::iterator it = names.begin();
-       MacroNameSet::iterator end = names.end();
-       for (; it != end; ++it) {
+       for (auto const & mit : names) {
                // defined?
-               MacroData const * data =
-               pbuf->getMacro(*it, *this, false);
+               MacroData const * data = pbuf->getMacro(mit, *this, false);
                if (data) {
                        macros.insert(data);
 
-                       // we cannot access the original MathMacroTemplate anymore
+                       // we cannot access the original InsetMathMacroTemplate anymore
                        // here to calls validate method. So we do its work here manually.
                        // FIXME: somehow make the template accessible here.
                        if (data->optionals() > 0)
@@ -3768,13 +3931,12 @@ Buffer::References & Buffer::getReferenceCache(docstring const & label)
 
        RefCache::iterator it = d->ref_cache_.find(label);
        if (it != d->ref_cache_.end())
-               return it->second.second;
+               return it->second;
 
-       static InsetLabel const * dummy_il = 0;
        static References const dummy_refs = References();
        it = d->ref_cache_.insert(
-               make_pair(label, make_pair(dummy_il, dummy_refs))).first;
-       return it->second.second;
+               make_pair(label, dummy_refs)).first;
+       return it->second;
 }
 
 
@@ -3791,22 +3953,43 @@ void Buffer::addReference(docstring const & label, Inset * inset, ParIterator it
 }
 
 
-void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il)
+void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il,
+                          bool const active)
+{
+       LabelInfo linfo;
+       linfo.label = label;
+       linfo.inset = il;
+       linfo.active = active;
+       masterBuffer()->d->label_cache_.push_back(linfo);
+}
+
+
+InsetLabel const * Buffer::insetLabel(docstring const & label,
+                                     bool const active) const
 {
-       masterBuffer()->d->ref_cache_[label].first = il;
+       for (auto & rc : masterBuffer()->d->label_cache_) {
+               if (rc.label == label && (rc.active || !active))
+                       return rc.inset;
+       }
+       return nullptr;
 }
 
 
-InsetLabel const * Buffer::insetLabel(docstring const & label) const
+bool Buffer::activeLabel(docstring const & label) const
 {
-       return masterBuffer()->d->ref_cache_[label].first;
+       if (!insetLabel(label, true))
+               return false;
+
+       return true;
 }
 
 
 void Buffer::clearReferenceCache() const
 {
-       if (!d->parent())
+       if (!d->parent()) {
                d->ref_cache_.clear();
+               d->label_cache_.clear();
+       }
 }
 
 
@@ -3816,20 +3999,16 @@ void Buffer::changeRefsIfUnique(docstring const & from, docstring const & to)
        reloadBibInfoCache();
 
        // Check if the label 'from' appears more than once
-       BiblioInfo const & keys = masterBibInfo();
-       BiblioInfo::const_iterator bit  = keys.begin();
-       BiblioInfo::const_iterator bend = keys.end();
        vector<docstring> labels;
-
-       for (; bit != bend; ++bit)
-               // FIXME UNICODE
-               labels.push_back(bit->first);
+       for (auto const & bibit : masterBibInfo())
+               labels.push_back(bibit.first);
 
        if (count(labels.begin(), labels.end(), from) > 1)
                return;
 
        string const paramName = "key";
-       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
+       InsetIterator it = inset_iterator_begin(inset());
+       for (; it; ++it) {
                if (it->lyxCode() != CITE_CODE)
                        continue;
                InsetCommand * inset = it->asInsetCommand();
@@ -3852,6 +4031,9 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
        // No side effect of file copying and image conversion
        runparams.dryrun = true;
 
+       // Some macros rely on font encoding
+       runparams.main_fontenc = params().main_font_encoding();
+
        if (output == CurrentParagraph) {
                runparams.par_begin = par_begin;
                runparams.par_end = par_end;
@@ -3944,6 +4126,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        ots.texrow().newlines(2);
                        if (master)
                                runparams.is_child = true;
+                       updateBuffer();
                        writeLaTeXSource(ots, string(), runparams, output);
                        texrow = ots.releaseTexRow();
                }
@@ -4102,7 +4285,7 @@ int AutoSaveBuffer::generateChild()
        return pid;
 }
 
-} // namespace anon
+} // namespace
 
 
 FileName Buffer::getEmergencyFileName() const
@@ -4137,7 +4320,7 @@ void Buffer::removeAutosaveFile() const
 }
 
 
-void Buffer::moveAutosaveFile(support::FileName const & oldauto) const
+void Buffer::moveAutosaveFile(FileName const & oldauto) const
 {
        FileName const newauto = getAutosaveFileName();
        oldauto.refresh();
@@ -4179,10 +4362,8 @@ void Buffer::setExportStatus(bool e) const
 {
        d->doing_export = e;
        ListOfBuffers clist = getDescendents();
-       ListOfBuffers::const_iterator cit = clist.begin();
-       ListOfBuffers::const_iterator const cen = clist.end();
-       for (; cit != cen; ++cit)
-               (*cit)->d->doing_export = e;
+       for (auto const & bit : clist)
+               bit->d->doing_export = e;
 }
 
 
@@ -4266,11 +4447,10 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                // Get shortest path to format
                converters.buildGraph();
                Graph::EdgePath path;
-               for (vector<string>::const_iterator it = backs.begin();
-                    it != backs.end(); ++it) {
-                       Graph::EdgePath p = converters.getPath(*it, format);
+               for (string const & sit : backs) {
+                       Graph::EdgePath p = converters.getPath(sit, format);
                        if (!p.empty() && (path.empty() || p.size() < path.size())) {
-                               backend_format = *it;
+                               backend_format = sit;
                                path = p;
                        }
                }
@@ -4278,17 +4458,22 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                        if (!put_in_tempdir) {
                                // Only show this alert if this is an export to a non-temporary
                                // file (not for previewing).
-                               Alert::error(_("Couldn't export file"), bformat(
-                                       _("No information for exporting the format %1$s."),
-                                       formats.prettyName(format)));
+                               docstring s = bformat(_("No information for exporting the format %1$s."),
+                                                     theFormats().prettyName(format));
+                               if (format == "pdf4")
+                                       s += "\n"
+                                         + bformat(_("Hint: use non-TeX fonts or set input encoding "
+                                                     " to '%1$s' or '%2$s'"),
+                                                   from_utf8(encodings.fromLyXName("utf8")->guiName()),
+                                                   from_utf8(encodings.fromLyXName("ascii")->guiName()));
+                               Alert::error(_("Couldn't export file"), s);
                        }
                        return ExportNoPathToFormat;
                }
                runparams.flavor = converters.getFlavor(path, this);
-               Graph::EdgePath::const_iterator it = path.begin();
-               Graph::EdgePath::const_iterator en = path.end();
-               for (; it != en; ++it)
-                       if (theConverters().get(*it).nice()) {
+               runparams.hyperref_driver = converters.getHyperrefDriver(path);
+               for (auto const & edge : path)
+                       if (theConverters().get(edge).nice()) {
                                need_nice_file = true;
                                break;
                        }
@@ -4310,57 +4495,73 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string filename = latexName(false);
        filename = addName(temppath(), filename);
        filename = changeExtension(filename,
-                                  formats.extension(backend_format));
+                                  theFormats().extension(backend_format));
        LYXERR(Debug::FILES, "filename=" << filename);
 
        // Plain text backend
        if (backend_format == "text") {
                runparams.flavor = OutputParams::TEXT;
-               writePlaintextFile(*this, FileName(filename), runparams);
+               try {
+                       writePlaintextFile(*this, FileName(filename), runparams);
+               }
+               catch (ConversionException const &) { return ExportCancel; }
        }
        // HTML backend
        else if (backend_format == "xhtml") {
                runparams.flavor = OutputParams::HTML;
                setMathFlavor(runparams);
-               makeLyXHTMLFile(FileName(filename), runparams);
+               if (makeLyXHTMLFile(FileName(filename), runparams) == ExportKilled)
+                       return ExportKilled;
        } else if (backend_format == "lyx")
                writeFile(FileName(filename));
        // Docbook backend
        else if (params().isDocBook()) {
                runparams.nice = !put_in_tempdir;
-               makeDocBookFile(FileName(filename), runparams);
+               if (makeDocBookFile(FileName(filename), runparams) == ExportKilled)
+                       return ExportKilled;
        }
        // LaTeX backend
        else if (backend_format == format || need_nice_file) {
                runparams.nice = true;
-               bool const success = makeLaTeXFile(FileName(filename), string(), runparams);
+               ExportStatus const retval =
+                       makeLaTeXFile(FileName(filename), string(), runparams);
+               if (retval == ExportKilled)
+                       return ExportKilled;
                if (d->cloned_buffer_)
                        d->cloned_buffer_->d->errorLists["Export"] = d->errorLists["Export"];
-               if (!success)
-                       return ExportError;
+               if (retval != ExportSuccess)
+                       return retval;
        } else if (!lyxrc.tex_allows_spaces
                   && contains(filePath(), ' ')) {
                Alert::error(_("File name error"),
-                          _("The directory path to the document cannot contain spaces."));
+                       bformat(_("The directory path to the document\n%1$s\n"
+                           "contains spaces, but your TeX installation does "
+                           "not allow them. You should save the file to a directory "
+                                       "whose name does not contain spaces."), from_utf8(filePath())));
                return ExportTexPathHasSpaces;
        } else {
                runparams.nice = false;
-               bool const success = makeLaTeXFile(
-                       FileName(filename), filePath(), runparams);
+               ExportStatus const retval =
+                       makeLaTeXFile(FileName(filename), filePath(), runparams);
+               if (retval == ExportKilled)
+                       return ExportKilled;
                if (d->cloned_buffer_)
                        d->cloned_buffer_->d->errorLists["Export"] = d->errorLists["Export"];
-               if (!success)
+               if (retval != ExportSuccess)
                        return ExportError;
        }
 
        string const error_type = (format == "program")
                ? "Build" : params().bufferFormat();
        ErrorList & error_list = d->errorLists[error_type];
-       string const ext = formats.extension(format);
+       string const ext = theFormats().extension(format);
        FileName const tmp_result_file(changeExtension(filename, ext));
-       bool const success = converters.convert(this, FileName(filename),
-               tmp_result_file, FileName(absFileName()), backend_format, format,
-               error_list);
+       Converters::RetVal const retval = 
+               converters.convert(this, FileName(filename), tmp_result_file, 
+               FileName(absFileName()), backend_format, format, error_list);
+       if (retval == Converters::KILLED)
+               return ExportCancel;
+       bool success = (retval == Converters::SUCCESS);
 
        // Emit the signal to show the error list or copy it back to the
        // cloned Buffer so that it can be emitted afterwards.
@@ -4374,20 +4575,18 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                        errors(error_type);
                // also to the children, in case of master-buffer-view
                ListOfBuffers clist = getDescendents();
-               ListOfBuffers::const_iterator cit = clist.begin();
-               ListOfBuffers::const_iterator const cen = clist.end();
-               for (; cit != cen; ++cit) {
+               for (auto const & bit : clist) {
                        if (runparams.silent)
-                               (*cit)->d->errorLists[error_type].clear();
+                               bit->d->errorLists[error_type].clear();
                        else if (d->cloned_buffer_) {
                                // Enable reverse search by copying back the
                                // texrow object to the cloned buffer.
                                // FIXME: this is not thread safe.
-                               (*cit)->d->cloned_buffer_->d->texrow = (*cit)->d->texrow;
-                               (*cit)->d->cloned_buffer_->d->errorLists[error_type] =
-                                       (*cit)->d->errorLists[error_type];
+                               bit->d->cloned_buffer_->d->texrow = bit->d->texrow;
+                               bit->d->cloned_buffer_->d->errorLists[error_type] =
+                                       bit->d->errorLists[error_type];
                        } else
-                               (*cit)->errors(error_type, true);
+                               bit->errors(error_type, true);
                }
        }
 
@@ -4397,8 +4596,8 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                // FIXME: There is a possibility of concurrent access to texrow
                // here from the main GUI thread that should be securized.
                d->cloned_buffer_->d->texrow = d->texrow;
-               string const error_type = params().bufferFormat();
-               d->cloned_buffer_->d->errorLists[error_type] = d->errorLists[error_type];
+               string const err_type = params().bufferFormat();
+               d->cloned_buffer_->d->errorLists[error_type] = d->errorLists[err_type];
        }
 
 
@@ -4421,11 +4620,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                                 : force_overwrite == ALL_FILES;
        CopyStatus status = use_force ? FORCE : SUCCESS;
 
-       vector<ExportedFile>::const_iterator it = files.begin();
-       vector<ExportedFile>::const_iterator const en = files.end();
-       for (; it != en && status != CANCEL; ++it) {
-               string const fmt = formats.getFormatFromFile(it->sourceName);
-               string fixedName = it->exportName;
+       for (ExportedFile const & exp : files) {
+               if (status == CANCEL) {
+                       message(_("Document export cancelled."));
+                       return ExportCancel;
+               }
+               string const fmt = theFormats().getFormatFromFile(exp.sourceName);
+               string fixedName = exp.exportName;
                if (!runparams.export_folder.empty()) {
                        // Relative pathnames starting with ../ will be sanitized
                        // if exporting to a different folder
@@ -4434,16 +4635,12 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                }
                FileName fixedFileName = makeAbsPath(fixedName, dest);
                fixedFileName.onlyPath().createPath();
-               status = copyFile(fmt, it->sourceName,
+               status = copyFile(fmt, exp.sourceName,
                        fixedFileName,
-                       it->exportName, status == FORCE,
+                       exp.exportName, status == FORCE,
                        runparams.export_folder.empty());
        }
 
-       if (status == CANCEL) {
-               message(_("Document export cancelled."));
-               return ExportCancel;
-       }
 
        if (tmp_result_file.exists()) {
                // Finally copy the main file
@@ -4460,13 +4657,13 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                } else {
                        message(bformat(_("Document exported as %1$s "
                                "to file `%2$s'"),
-                               formats.prettyName(format),
+                               theFormats().prettyName(format),
                                makeDisplayPath(result_file)));
                }
        } else {
                // This must be a dummy converter like fax (bug 1888)
                message(bformat(_("Document exported as %1$s"),
-                       formats.prettyName(format)));
+                       theFormats().prettyName(format)));
        }
 
        return success ? ExportSuccess : ExportConverterError;
@@ -4505,7 +4702,7 @@ Buffer::ExportStatus Buffer::preview(string const & format, bool includeall) con
                return status;
 
        if (previewFile.exists())
-               return formats.view(*this, previewFile, format) ?
+               return theFormats().view(*this, previewFile, format) ?
                        PreviewSuccess : PreviewError;
 
        // Successful export but no output file?
@@ -4580,6 +4777,32 @@ Buffer::ReadStatus Buffer::loadEmergency()
                                _("&Remove"), _("&Keep"));
                if (del_emerg == 0)
                        emergencyFile.removeFile();
+               else {
+                       // See bug #11464
+                       FileName newname;
+                       string const ename = emergencyFile.absFileName();
+                       bool noname = true;
+                       // Surely we can find one in 100 tries?
+                       for (int i = 1; i < 100; ++i) {
+                               newname.set(ename + to_string(i) + ".lyx");
+                               if (!newname.exists()) {
+                                       noname = false;
+                                       break;
+                               }
+                       }
+                       if (!noname) {
+                               // renameTo returns true on success. So inverting that
+                               // will give us true if we fail.
+                               noname = !emergencyFile.renameTo(newname);
+                       }
+                       if (noname) {
+                               Alert::warning(_("Can't rename emergency file!"),
+                                       _("LyX was unable to rename the emergency file. "
+                                         "You should do so manually. Otherwise, you will be "
+                                         "asked about it again the next time you try to load "
+                                         "this file, and may over-write your own work."));
+                       }
+               }
                return ReadOriginal;
        }
 
@@ -4694,12 +4917,6 @@ void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
 }
 
 
-void Buffer::setBuffersForInsets() const
-{
-       inset().setBuffer(const_cast<Buffer &>(*this));
-}
-
-
 void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
 {
        LBUFERR(!text().paragraphs().empty());
@@ -4708,10 +4925,17 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        Buffer const * const master = masterBuffer();
        DocumentClass const & textclass = master->params().documentClass();
 
-       // do this only if we are the top-level Buffer
-       if (master == this) {
+       docstring_list old_bibfiles;
+       // Do this only if we are the top-level Buffer. We also need to account
+       // for the case of a previewed child with ignored parent here.
+       if (master == this && !d->ignore_parent) {
                textclass.counters().reset(from_ascii("bibitem"));
                reloadBibInfoCache();
+               // we will re-read this cache as we go through, but we need
+               // to know whether it's changed to know whether we need to
+               // update the bibinfo cache.
+               old_bibfiles = d->bibfiles_cache_;
+               d->bibfiles_cache_.clear();
        }
 
        // keep the buffers to be children in this set. If the call from the
@@ -4727,7 +4951,7 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
                        // not updated during the updateBuffer call and TocModel::toc_ is invalid
                        // (bug 5699). The same happens if the master buffer is open in a different
                        // window. This test catches both possibilities.
-                       // See: http://marc.info/?l=lyx-devel&m=138590578911716&w=2
+                       // See: https://marc.info/?l=lyx-devel&m=138590578911716&w=2
                        // There remains a problem here: If there is another child open in yet a third
                        // window, that TOC is not updated. So some more general solution is needed at
                        // some point.
@@ -4757,16 +4981,48 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        ParIterator parit = cbuf.par_iterator_begin();
        updateBuffer(parit, utype);
 
-       /// FIXME: Perf
-       /// Update the tocBackend for any buffer. The outliner uses the master's,
-       /// and the navigation menu uses the child's.
-       cbuf.tocBackend().update(true, utype);
-
+       // If this document has siblings, then update the TocBackend later. The
+       // reason is to ensure that later siblings are up to date when e.g. the
+       // broken or not status of references is computed. The update is called
+       // in InsetInclude::addToToc.
        if (master != this)
                return;
 
-       d->bibinfo_cache_valid_ = true;
+       // if the bibfiles changed, the cache of bibinfo is invalid
+       docstring_list new_bibfiles = d->bibfiles_cache_;
+       // this is a trick to determine whether the two vectors have
+       // the same elements.
+       sort(new_bibfiles.begin(), new_bibfiles.end());
+       sort(old_bibfiles.begin(), old_bibfiles.end());
+       if (old_bibfiles != new_bibfiles) {
+               LYXERR(Debug::FILES, "Reloading bibinfo cache.");
+               invalidateBibinfoCache();
+               reloadBibInfoCache();
+               // We relied upon the bibinfo cache when recalculating labels. But that
+               // cache was invalid, although we didn't find that out until now. So we
+               // have to do it all again.
+               // That said, the only thing we really need to do is update the citation
+               // labels. Nothing else will have changed. So we could create a new 
+               // UpdateType that would signal that fact, if we needed to do so.
+               parit = cbuf.par_iterator_begin();
+               // we will be re-doing the counters and references and such.
+               textclass.counters().reset();
+               clearReferenceCache();
+               // we should not need to do this again?
+               // updateMacros();
+               setChangesPresent(false);
+               updateBuffer(parit, utype);
+               // this will already have been done by reloadBibInfoCache();
+               // d->bibinfo_cache_valid_ = true;
+       }
+       else {
+               LYXERR(Debug::FILES, "Bibfiles unchanged.");
+               // this is also set to true on the other path, by reloadBibInfoCache.
+               d->bibinfo_cache_valid_ = true;
+       }
        d->cite_labels_valid_ = true;
+       /// FIXME: Perf
+       cbuf.tocBackend().update(true, utype);
        if (scope == UpdateMaster)
                cbuf.structureChanged();
 }
@@ -4888,16 +5144,20 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
                docstring itemlabel;
                switch (par.itemdepth) {
                case 0:
+                       // • U+2022 BULLET
                        itemlabel = char_type(0x2022);
                        break;
                case 1:
+                       // – U+2013 EN DASH
                        itemlabel = char_type(0x2013);
                        break;
                case 2:
+                       // ∗ U+2217 ASTERISK OPERATOR
                        itemlabel = char_type(0x2217);
                        break;
                case 3:
-                       itemlabel = char_type(0x2219); // or 0x00b7
+                       // · U+00B7 MIDDLE DOT
+                       itemlabel = char_type(0x00b7);
                        break;
                }
                par.params().labelString(itemlabel);
@@ -4910,8 +5170,10 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
                switch (par.itemdepth) {
                case 2:
                        enumcounter += 'i';
+                       // fall through
                case 1:
                        enumcounter += 'i';
+                       // fall through
                case 0:
                        enumcounter += 'i';
                        break;
@@ -4923,13 +5185,14 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
                        break;
                }
 
-               // Increase the master counter?
-               if (layout.stepmastercounter && needEnumCounterReset(it))
-                       counters.stepMaster(enumcounter, utype);
-
-               // Maybe we have to reset the enumeration counter.
-               if (!layout.resumecounter && needEnumCounterReset(it))
-                       counters.reset(enumcounter);
+               if (needEnumCounterReset(it)) {
+                       // Increase the master counter?
+                       if (layout.stepmastercounter)
+                               counters.stepMaster(enumcounter, utype);
+                       // Maybe we have to reset the enumeration counter.
+                       if (!layout.resumecounter)
+                               counters.reset(enumcounter);
+               }
                counters.step(enumcounter, utype);
 
                string const & lang = par.getParLanguage(bp)->code();
@@ -4996,6 +5259,11 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
        // to resolve macros in it.
        parit.text()->setMacrocontextPosition(parit);
 
+       // Reset bibitem counter in master (#8499)
+       Buffer const * const master = masterBuffer();
+       if (master == this && !d->ignore_parent)
+               master->params().documentClass().counters().reset(from_ascii("bibitem"));
+
        depth_type maxdepth = 0;
        pit_type const lastpit = parit.lastpit();
        for ( ; parit.pit() <= lastpit ; ++parit.pit()) {
@@ -5008,7 +5276,7 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
                         * non-const. This would however be costly in
                         * terms of code duplication.
                         */
-                       const_cast<Buffer *>(this)->undo().recordUndo(CursorData(parit));
+                       CursorData(parit).recordUndo();
                        parit->params().depth(maxdepth);
                }
                maxdepth = parit->getMaxDepthAfter();
@@ -5024,15 +5292,13 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
                // set the counter for this paragraph
                d->setLabel(parit, utype);
 
-               // update change-tracking flag 
+               // update change-tracking flag
                parit->addChangesToBuffer(*this);
 
                // now the insets
-               InsetList::const_iterator iit = parit->insetList().begin();
-               InsetList::const_iterator end = parit->insetList().end();
-               for (; iit != end; ++iit) {
-                       parit.pos() = iit->pos;
-                       iit->inset->updateBuffer(parit, utype);
+               for (auto const & insit : parit->insetList()) {
+                       parit.pos() = insit.pos;
+                       insit.inset->updateBuffer(parit, utype);
                }
        }
 }
@@ -5150,7 +5416,7 @@ int Buffer::charCount(bool with_blanks) const
 Buffer::ReadStatus Buffer::reload()
 {
        setBusy(true);
-       // c.f. bug http://www.lyx.org/trac/ticket/6587
+       // c.f. bug https://www.lyx.org/trac/ticket/6587
        removeAutosaveFile();
        // e.g., read-only status could have changed due to version control
        d->filename.refresh();
@@ -5216,11 +5482,9 @@ bool Buffer::saveAs(FileName const & fn)
 
 void Buffer::checkChildBuffers()
 {
-       Impl::BufferPositionMap::iterator it = d->children_positions.begin();
-       Impl::BufferPositionMap::iterator const en = d->children_positions.end();
-       for (; it != en; ++it) {
-               DocIterator dit = it->second;
-               Buffer * cbuf = const_cast<Buffer *>(it->first);
+       for (auto const & bit : d->children_positions) {
+               DocIterator dit = bit.second;
+               Buffer * cbuf = const_cast<Buffer *>(bit.first);
                if (!cbuf || !theBufferList().isLoaded(cbuf))
                        continue;
                Inset * inset = dit.nextInset();
@@ -5317,38 +5581,56 @@ void Buffer::updateChangesPresent() const
 
 void Buffer::Impl::refreshFileMonitor()
 {
-       if (file_monitor_ && file_monitor_->filename() == filename.absFileName())
-               return file_monitor_->refresh();
+       if (file_monitor_ && file_monitor_->filename() == filename.absFileName()) {
+               file_monitor_->refresh();
+               return;
+       }
 
        // The previous file monitor is invalid
        // This also destroys the previous file monitor and all its connections
        file_monitor_ = FileSystemWatcher::monitor(filename);
-       fileExternallyModified(false);
        // file_monitor_ will be destroyed with *this, so it is not going to call a
        // destroyed object method.
-       file_monitor_->connect([this](){ fileExternallyModified(true); });
+       file_monitor_->connect([this](bool exists) {
+                       fileExternallyModified(exists);
+               });
 }
 
 
-void Buffer::Impl::fileExternallyModified(bool modified) const
+void Buffer::Impl::fileExternallyModified(bool const exists)
 {
-       if (modified)
-               lyx_clean = bak_clean = false;
-       externally_modified_ = modified;
-       if (wa_)
+       // ignore notifications after our own saving operations
+       if (checksum_ == filename.checksum()) {
+               LYXERR(Debug::FILES, "External modification but "
+                      "checksum unchanged: " << filename);
+               return;
+       }
+       // If the file has been deleted, only mark the file as dirty since it is
+       // pointless to prompt for reloading. If later a file is moved into this
+       // location, then the externally modified warning will appear then.
+       if (exists)
+                       externally_modified_ = true;
+       // Update external modification notification.
+       // Dirty buffers must be visible at all times.
+       if (wa_ && wa_->unhide(owner_))
                wa_->updateTitles();
+       else
+               // Unable to unhide the buffer (e.g. no GUI or not current View)
+               lyx_clean = true;
 }
 
 
 bool Buffer::notifiesExternalModification() const
 {
-       return d->isExternallyModified();
+       return d->externally_modified_;
 }
 
 
 void Buffer::clearExternalModification() const
 {
-       d->fileExternallyModified(false);
+       d->externally_modified_ = false;
+       if (d->wa_)
+               d->wa_->updateTitles();
 }