]> git.lyx.org Git - lyx.git/blobdiff - src/Buffer.cpp
Revert [9370135c3e/lyxgit] and fix #8499 properly
[lyx.git] / src / Buffer.cpp
index 9eded216f1f32daa5203f5344a3b75ba3cf29bc9..4006a95c91137f79d9a8da79e358214bf7595768 100644 (file)
@@ -55,6 +55,7 @@
 #include "ParagraphParameters.h"
 #include "ParIterator.h"
 #include "PDFOptions.h"
+#include "Session.h"
 #include "SpellChecker.h"
 #include "sgml.h"
 #include "texstream.h"
@@ -76,7 +77,7 @@
 
 #include "mathed/InsetMathHull.h"
 #include "mathed/MacroTable.h"
-#include "mathed/MathMacroTemplate.h"
+#include "mathed/InsetMathMacroTemplate.h"
 #include "mathed/MathSupport.h"
 
 #include "graphics/GraphicsCache.h"
@@ -134,9 +135,24 @@ namespace {
 int const LYX_FORMAT = LYX_FORMAT_LYX;
 
 typedef map<string, bool> DepClean;
-typedef map<docstring, pair<InsetLabel const *, Buffer::References> > RefCache;
 
-} // namespace anon
+// Information about labels and their associated refs
+struct LabelInfo {
+       /// label string
+       docstring label;
+       /// label inset
+       InsetLabel const * inset;
+       /// associated references cache
+       Buffer::References references;
+       /// whether this label is active (i.e., not deleted)
+       bool active;
+};
+
+typedef vector<LabelInfo> LabelCache;
+
+typedef map<docstring, Buffer::References> RefCache;
+
+} // namespace
 
 
 // A storehouse for the cloned buffers.
@@ -278,7 +294,7 @@ public:
 
        /// A cache for the bibfiles (including bibfiles of loaded child
        /// documents), needed for appropriate update of natbib labels.
-       mutable support::FileNamePairList bibfiles_cache_;
+       mutable docstring_list bibfiles_cache_;
 
        // FIXME The caching mechanism could be improved. At present, we have a
        // cache for each Buffer, that caches all the bibliography info for that
@@ -296,6 +312,8 @@ public:
        /// we ran updateBuffer(), i.e., whether citation labels may need
        /// to be updated.
        mutable bool cite_labels_valid_;
+       /// Do we have a bibliography environment?
+       mutable bool have_bibitems_;
 
        /// These two hold the file name and format, written to by
        /// Buffer::preview and read from by LFUN_BUFFER_VIEW_CACHE.
@@ -306,7 +324,11 @@ public:
        /// was missing).
        bool preview_error_;
 
+       /// Cache the references associated to a label and their positions
+       /// in the buffer.
        mutable RefCache ref_cache_;
+       /// Cache the label insets and their activity status.
+       mutable LabelCache label_cache_;
 
        /// our Text that should be wrapped in an InsetText
        InsetText * inset;
@@ -343,7 +365,6 @@ public:
                        LYXERR0("Warning: a buffer should not have two parents!");
                parent_buffer = pb;
                if (!cloned_buffer_ && parent_buffer) {
-                       parent_buffer->invalidateBibfileCache();
                        parent_buffer->invalidateBibinfoCache();
                }
        }
@@ -379,14 +400,11 @@ public:
        // Make sure the file monitor monitors the good file.
        void refreshFileMonitor();
 
-       /// has it been notified of an external modification?
-       bool isExternallyModified() const { return externally_modified_; }
-
        /// Notify or clear of external modification
-       void fileExternallyModified(bool modified) const;
+       void fileExternallyModified(bool exists);
 
-       /// Block notifications of external modifications
-       FileMonitorBlocker blockFileMonitor() { return file_monitor_->block(); }
+       /// has been externally modified? Can be reset by the user.
+       mutable bool externally_modified_;
 
 private:
        /// So we can force access via the accessors.
@@ -396,9 +414,6 @@ private:
        int char_count_;
        int blank_count_;
 
-       /// has been externally modified? Can be reset by the user.
-       mutable bool externally_modified_;
-
        FileMonitorPtr file_monitor_;
 };
 
@@ -437,12 +452,11 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
          file_fully_loaded(false), file_format(LYX_FORMAT), need_format_backup(false),
          ignore_parent(false),  toc_backend(owner), macro_lock(false),
          checksum_(0), wa_(0),  gui_(0), undo_(*owner), bibinfo_cache_valid_(false),
-         bibfile_cache_valid_(false), cite_labels_valid_(false), preview_error_(false),
-         inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
+         bibfile_cache_valid_(false), cite_labels_valid_(false), have_bibitems_(false),
+         preview_error_(false), inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
          clone_list_(0), doing_export(false),
-         tracked_changes_present_(0), parent_buffer(0),
-         word_count_(0), char_count_(0), blank_count_(0),
-         externally_modified_(false)
+         tracked_changes_present_(0), externally_modified_(false), parent_buffer(0),
+         word_count_(0), char_count_(0), blank_count_(0)
 {
        refreshFileMonitor();
        if (!cloned_buffer_) {
@@ -461,6 +475,7 @@ Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
        bibfile_cache_valid_ = cloned_buffer_->d->bibfile_cache_valid_;
        bibfile_status_ = cloned_buffer_->d->bibfile_status_;
        cite_labels_valid_ = cloned_buffer_->d->cite_labels_valid_;
+       have_bibitems_ = cloned_buffer_->d->have_bibitems_;
        unnamed = cloned_buffer_->d->unnamed;
        internal_buffer = cloned_buffer_->d->internal_buffer;
        layout_position = cloned_buffer_->d->layout_position;
@@ -543,11 +558,11 @@ Buffer::~Buffer()
                Impl::BufferPositionMap::iterator end = d->children_positions.end();
                for (; it != end; ++it) {
                        Buffer * child = const_cast<Buffer *>(it->first);
-                       if (theBufferList().isLoaded(child)) { 
-                        if (theBufferList().isOthersChild(this, child))
-                                child->setParent(0);
-                        else
-                               theBufferList().release(child);
+                       if (theBufferList().isLoaded(child)) {
+                               if (theBufferList().isOthersChild(this, child))
+                                       child->setParent(0);
+                               else
+                                       theBufferList().release(child);
                        }
                }
 
@@ -577,13 +592,13 @@ Buffer::~Buffer()
 }
 
 
-Buffer * Buffer::cloneFromMaster() const
+Buffer * Buffer::cloneWithChildren() const
 {
        BufferMap bufmap;
        cloned_buffers.push_back(new CloneList);
        CloneList * clones = cloned_buffers.back();
 
-       masterBuffer()->cloneWithChildren(bufmap, clones);
+       cloneWithChildren(bufmap, clones);
 
        // make sure we got cloned
        BufferMap::const_iterator bit = bufmap.find(this);
@@ -809,10 +824,9 @@ FileName Buffer::Impl::exportFileName() const
        if (branch_suffix.empty())
                return filename;
 
-       string const name = filename.onlyFileNameWithoutExt()
-               + to_utf8(branch_suffix);
+       string const name = addExtension(filename.onlyFileNameWithoutExt()
+                       + to_utf8(branch_suffix), filename.extension());
        FileName res(filename.onlyPath().absFileName() + "/" + name);
-       res.changeExtension(filename.extension());
 
        return res;
 }
@@ -987,6 +1001,8 @@ int Buffer::readHeader(Lexer & lex)
                errorList.push_back(ErrorItem(_("Document header error"), s));
        }
 
+       params().shell_escape = theSession().shellescapeFiles().find(absFileName());
+
        params().makeDocumentClass();
 
        return unknown_tokens;
@@ -1057,6 +1073,10 @@ bool Buffer::readDocument(Lexer & lex)
                                                << absFileName()
                                                << ") does not include "
                                                "this document. Ignoring the master assignment.");
+                               // If the master has just been created, un-hide it (#11162)
+                               if (!master->fileName().exists())
+                                       lyx::dispatch(FuncRequest(LFUN_BUFFER_SWITCH,
+                                                                 master->absFileName()));
                        }
                }
        }
@@ -1094,8 +1114,7 @@ bool Buffer::importString(string const & format, docstring const & contents, Err
                return false;
        // It is important to use the correct extension here, since some
        // converters create a wrong output file otherwise (e.g. html2latex)
-       TempFile const tempfile("Buffer_importStringXXXXXX." + fmt->extension());
-       FileName const name(tempfile.name());
+       FileName const name = tempFileName("Buffer_importStringXXXXXX." + fmt->extension());
        ofdocstream os(name.toFilesystemEncoding().c_str());
        // Do not convert os implicitly to bool, since that is forbidden in C++11.
        bool const success = !(os << contents).fail();
@@ -1111,8 +1130,7 @@ bool Buffer::importString(string const & format, docstring const & contents, Err
                converted = importFile(format, name, errorList);
        }
 
-       if (name.exists())
-               name.removeFile();
+       removeTempFile(name);
        return converted;
 }
 
@@ -1122,10 +1140,12 @@ bool Buffer::importFile(string const & format, FileName const & name, ErrorList
        if (!theConverters().isReachable(format, "lyx"))
                return false;
 
-       TempFile const tempfile("Buffer_importFileXXXXXX.lyx");
-       FileName const lyx(tempfile.name());
-       if (theConverters().convert(0, name, lyx, name, format, "lyx", errorList))
-               return readFile(lyx) == ReadSuccess;
+       FileName const lyx = tempFileName("Buffer_importFileXXXXXX.lyx");
+       if (theConverters().convert(0, name, lyx, name, format, "lyx", errorList)) {
+               bool const success = readFile(lyx) == ReadSuccess;
+               removeTempFile(lyx);
+               return success;
+       }
 
        return false;
 }
@@ -1383,7 +1403,6 @@ FileName Buffer::getBackupName() const {
 // Should probably be moved to somewhere else: BufferView? GuiView?
 bool Buffer::save() const
 {
-       FileMonitorBlocker block = d->blockFileMonitor();
        docstring const file = makeDisplayPath(absFileName(), 20);
        d->filename.refresh();
 
@@ -1433,6 +1452,9 @@ bool Buffer::save() const
 
        FileName savefile(tempfile->name());
        LYXERR(Debug::FILES, "Saving to " << savefile.absFileName());
+       if (!savefile.clonePermissions(fileName()))
+               LYXERR0("Failed to clone the permission from " << fileName().absFileName() << " to " << savefile.absFileName());
+
        if (!writeFile(savefile))
                return false;
 
@@ -1803,8 +1825,10 @@ void Buffer::writeLaTeXSource(otexstream & os,
        LaTeXFeatures features(*this, params(), runparams);
        validate(features);
        // This is only set once per document (in master)
-       if (!runparams.is_child)
+       if (!runparams.is_child) {
                runparams.use_polyglossia = features.usePolyglossia();
+               runparams.use_hyperref = features.isRequired("hyperref");
+       }
        LYXERR(Debug::LATEX, "  Buffer validation done.");
 
        bool const output_preamble =
@@ -1878,8 +1902,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
                                            "file path name."),
                                          inputpath, uncodable_glyphs));
                        } else {
-                               string docdir =
-                                       support::latex_path(original_path);
+                               string docdir = os::latex_path(original_path);
                                if (contains(docdir, '#')) {
                                        docdir = subst(docdir, "#", "\\#");
                                        os << "\\catcode`\\#=11"
@@ -1890,9 +1913,27 @@ void Buffer::writeLaTeXSource(otexstream & os,
                                        os << "\\catcode`\\%=11"
                                              "\\def\\%{%}\\catcode`\\%=14\n";
                                }
+                               if (contains(docdir, '~'))
+                                       docdir = subst(docdir, "~", "\\string~");
+                               bool const nonascii = !isAscii(from_utf8(docdir));
+                               // LaTeX 2019/10/01 handles non-ascii path without detokenize
+                               bool const utfpathlatex = features.isAvailable("LaTeX-2019/10/01");
+                               bool const detokenize = !utfpathlatex && nonascii;
+                               bool const quote = contains(docdir, ' ');
+                               if (utfpathlatex && nonascii)
+                                       os << "\\UseRawInputEncoding\n";
                                os << "\\makeatletter\n"
-                                  << "\\def\\input@path{{"
-                                  << docdir << "}}\n"
+                                  << "\\def\\input@path{{";
+                               if (detokenize)
+                                       os << "\\detokenize{";
+                               if (quote)
+                                       os << "\"";
+                               os << docdir;
+                               if (quote)
+                                       os << "\"";
+                               if (detokenize)
+                                       os << "}";
+                               os << "}}\n"
                                   << "\\makeatother\n";
                        }
                }
@@ -1909,7 +1950,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
                // Biblatex bibliographies are loaded here
                if (params().useBiblatex()) {
                        vector<docstring> const bibfiles =
-                               prepareBibFilePaths(runparams, getBibfilesCache(), true);
+                           prepareBibFilePaths(runparams, getBibfiles(), true);
                        for (docstring const & file: bibfiles)
                                os << "\\addbibresource{" << file << "}\n";
                }
@@ -1942,7 +1983,7 @@ void Buffer::writeLaTeXSource(otexstream & os,
                                            support::bformat(_("The languages %1$s are only supported by Polyglossia."), langs)
                                          : support::bformat(_("The language %1$s is only supported by Polyglossia."), langs);
                                if (!blangs.empty())
-                                       plangs += "\n"; 
+                                       plangs += "\n";
                        }
 
                        frontend::Alert::warning(
@@ -2062,7 +2103,7 @@ void Buffer::writeDocBookSource(odocstream & os, string const & fname,
                        os << from_ascii(tclass.class_header());
                else if (runparams.flavor == OutputParams::XML)
                        os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
-                           << "\"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
+                           << "\"https://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
                else
                        os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
 
@@ -2169,10 +2210,6 @@ void Buffer::writeLyXHTMLSource(odocstream & os,
                if (!styles.empty())
                        os << "\n<!-- Text Class Preamble -->\n" << styles << '\n';
 
-               styles = features.getPreambleSnippets().str;
-               if (!styles.empty())
-                       os << "\n<!-- Preamble Snippets -->\n" << styles << '\n';
-
                // we will collect CSS information in a stream, and then output it
                // either here, as part of the header, or else in a separate file.
                odocstringstream css;
@@ -2327,6 +2364,10 @@ void Buffer::getLabelList(vector<docstring> & list) const
 }
 
 
+// This is not in master, so it has been removed at 2126d5a3, on
+// 14 December 2018, so as to test whether it's needed. If there
+// aren't any complaints, it can be fully removed.
+#if 0
 void Buffer::updateBibfilesCache(UpdateScope scope) const
 {
        // FIXME This is probably unnecssary, given where we call this.
@@ -2336,11 +2377,12 @@ void Buffer::updateBibfilesCache(UpdateScope scope) const
                return;
        }
 
+       docstring_list old_cache = d->bibfiles_cache_;
        d->bibfiles_cache_.clear();
        for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
                if (it->lyxCode() == BIBTEX_CODE) {
                        InsetBibtex const & inset = static_cast<InsetBibtex const &>(*it);
-                       support::FileNamePairList const bibfiles = inset.getBibFiles();
+                       docstring_list const bibfiles = inset.getBibFiles();
                        d->bibfiles_cache_.insert(d->bibfiles_cache_.end(),
                                bibfiles.begin(),
                                bibfiles.end());
@@ -2349,8 +2391,8 @@ void Buffer::updateBibfilesCache(UpdateScope scope) const
                        Buffer const * const incbuf = inset.getChildBuffer();
                        if (!incbuf)
                                continue;
-                       support::FileNamePairList const & bibfiles =
-                                       incbuf->getBibfilesCache(UpdateChildOnly);
+                       docstring_list const & bibfiles =
+                               incbuf->getBibfiles(UpdateChildOnly);
                        if (!bibfiles.empty()) {
                                d->bibfiles_cache_.insert(d->bibfiles_cache_.end(),
                                        bibfiles.begin(),
@@ -2359,15 +2401,18 @@ void Buffer::updateBibfilesCache(UpdateScope scope) const
                }
        }
        d->bibfile_cache_valid_ = true;
-       d->bibinfo_cache_valid_ = false;
        d->cite_labels_valid_ = false;
+       if (d->bibfiles_cache_ != old_cache)
+               d->bibinfo_cache_valid_ = false;
 }
+#endif
 
 
 void Buffer::invalidateBibinfoCache() const
 {
        d->bibinfo_cache_valid_ = false;
        d->cite_labels_valid_ = false;
+       removeBiblioTempFiles();
        // also invalidate the cache for the parent buffer
        Buffer const * const pbuf = d->parent();
        if (pbuf)
@@ -2375,28 +2420,13 @@ void Buffer::invalidateBibinfoCache() const
 }
 
 
-void Buffer::invalidateBibfileCache() const
-{
-       d->bibfile_cache_valid_ = false;
-       d->bibinfo_cache_valid_ = false;
-       d->cite_labels_valid_ = false;
-       // also invalidate the cache for the parent buffer
-       Buffer const * const pbuf = d->parent();
-       if (pbuf)
-               pbuf->invalidateBibfileCache();
-}
-
-
-support::FileNamePairList const & Buffer::getBibfilesCache(UpdateScope scope) const
+docstring_list const & Buffer::getBibfiles(UpdateScope scope) const
 {
        // FIXME This is probably unnecessary, given where we call this.
        // If this is a child document, use the master's cache instead.
        Buffer const * const pbuf = masterBuffer();
        if (pbuf != this && scope != UpdateChildOnly)
-               return pbuf->getBibfilesCache();
-
-       if (!d->bibfile_cache_valid_)
-               this->updateBibfilesCache(scope);
+               return pbuf->getBibfiles();
 
        return d->bibfiles_cache_;
 }
@@ -2411,6 +2441,63 @@ BiblioInfo const & Buffer::masterBibInfo() const
 }
 
 
+BiblioInfo const & Buffer::bibInfo() const
+{
+       return d->bibinfo_;
+}
+
+
+void Buffer::registerBibfiles(const docstring_list & bf) const
+{
+       // We register the bib files in the master buffer,
+       // if there is one, but also in every single buffer,
+       // in case a child is compiled alone.
+       Buffer const * const tmp = masterBuffer();
+       if (tmp != this)
+               tmp->registerBibfiles(bf);
+
+       for (auto const & p : bf) {
+               docstring_list::const_iterator temp =
+                       find(d->bibfiles_cache_.begin(), d->bibfiles_cache_.end(), p);
+               if (temp == d->bibfiles_cache_.end())
+                       d->bibfiles_cache_.push_back(p);
+       }
+}
+
+
+static map<docstring, FileName> bibfileCache;
+
+FileName Buffer::getBibfilePath(docstring const & bibid) const
+{
+       map<docstring, FileName>::const_iterator it =
+               bibfileCache.find(bibid);
+       if (it != bibfileCache.end()) {
+               // i.e., return bibfileCache[bibid];
+               return it->second;
+       }
+
+       LYXERR(Debug::FILES, "Reading file location for " << bibid);
+       string const texfile = changeExtension(to_utf8(bibid), "bib");
+       // we need to check first if this file exists where it's said to be.
+       // there's a weird bug that occurs otherwise: if the file is in the
+       // Buffer's directory but has the same name as some file that would be
+       // found by kpsewhich, then we find the latter, not the former.
+       FileName const local_file = makeAbsPath(texfile, filePath());
+       FileName file = local_file;
+       if (!file.exists()) {
+               // there's no need now to check whether the file can be found
+               // locally
+               file = findtexfile(texfile, "bib", true);
+               if (file.empty())
+                       file = local_file;
+       }
+       LYXERR(Debug::FILES, "Found at: " << file);
+
+       bibfileCache[bibid] = file;
+       return bibfileCache[bibid];
+}
+
+
 void Buffer::checkIfBibInfoCacheIsValid() const
 {
        // use the master's cache
@@ -2420,12 +2507,34 @@ void Buffer::checkIfBibInfoCacheIsValid() const
                return;
        }
 
+       // If we already know the cache is invalid, stop here.
+       // This is important in the case when the bibliography
+       // environment (rather than Bib[la]TeX) is used.
+       // In that case, the timestamp check below gives no
+       // sensible result. Rather than that, the cache will
+       // be invalidated explicitly via invalidateBibInfoCache()
+       // by the Bibitem inset.
+       // Same applies for bib encoding changes, which trigger
+       // invalidateBibInfoCache() by InsetBibtex.
+       if (!d->bibinfo_cache_valid_)
+               return;
+
+       if (d->have_bibitems_) {
+               // We have a bibliography environment.
+               // Invalidate the bibinfo cache unconditionally.
+               // Cite labels will get invalidated by the inset if needed.
+               d->bibinfo_cache_valid_ = false;
+               return;
+       }
+
+       // OK. This is with Bib(la)tex. We'll assume the cache
+       // is valid and change this if we find changes in the bibs.
+       d->bibinfo_cache_valid_ = true;
+       d->cite_labels_valid_ = true;
        // compare the cached timestamps with the actual ones.
-       FileNamePairList const & bibfiles_cache = getBibfilesCache();
-       FileNamePairList::const_iterator ei = bibfiles_cache.begin();
-       FileNamePairList::const_iterator en = bibfiles_cache.end();
-       for (; ei != en; ++ ei) {
-               FileName const fn = ei->second;
+       docstring_list const & bibfiles_cache = getBibfiles();
+       for (auto const & bf : bibfiles_cache) {
+               FileName const fn = getBibfilePath(bf);
                time_t lastw = fn.lastModified();
                time_t prevw = d->bibfile_status_[fn];
                if (lastw != prevw) {
@@ -2437,8 +2546,17 @@ void Buffer::checkIfBibInfoCacheIsValid() const
 }
 
 
+void Buffer::clearBibFileCache() const
+{
+       bibfileCache.clear();
+}
+
+
 void Buffer::reloadBibInfoCache() const
 {
+       if (isInternal())
+               return;
+
        // use the master's cache
        Buffer const * const tmp = masterBuffer();
        if (tmp != this) {
@@ -2449,35 +2567,62 @@ void Buffer::reloadBibInfoCache() const
        checkIfBibInfoCacheIsValid();
        if (d->bibinfo_cache_valid_)
                return;
+       LYXERR(Debug::FILES, "Bibinfo cache was invalid.");
 
+       // re-read file locations when this info changes
+       // FIXME Is this sufficient? Or should we also force that
+       // in some other cases? If so, then it is easy enough to
+       // add the following line in some other places.
+       clearBibFileCache();
        d->bibinfo_.clear();
-       collectBibKeys();
+       FileNameList checkedFiles;
+       d->have_bibitems_ = false;
+       collectBibKeys(checkedFiles);
        d->bibinfo_cache_valid_ = true;
 }
 
 
-void Buffer::collectBibKeys() const
+void Buffer::collectBibKeys(FileNameList & checkedFiles) const
 {
-       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it)
-               it->collectBibKeys(it);
+       for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
+               it->collectBibKeys(it, checkedFiles);
+               if (it->lyxCode() == BIBITEM_CODE) {
+                       if (parent() != 0)
+                               parent()->d->have_bibitems_ = true;
+                       else
+                               d->have_bibitems_ = true;
+               }
+       }
 }
 
 
-void Buffer::addBiblioInfo(BiblioInfo const & bi) const
+void Buffer::addBiblioInfo(BiblioInfo const & bin) const
 {
-       Buffer const * tmp = masterBuffer();
-       BiblioInfo & masterbi = (tmp == this) ?
-               d->bibinfo_ : tmp->d->bibinfo_;
-       masterbi.mergeBiblioInfo(bi);
+       // We add the biblio info to the master buffer,
+       // if there is one, but also to every single buffer,
+       // in case a child is compiled alone.
+       BiblioInfo & bi = d->bibinfo_;
+       bi.mergeBiblioInfo(bin);
+
+       if (parent() != 0) {
+               BiblioInfo & masterbi = parent()->d->bibinfo_;
+               masterbi.mergeBiblioInfo(bin);
+       }
 }
 
 
-void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bi) const
+void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bin) const
 {
-       Buffer const * tmp = masterBuffer();
-       BiblioInfo & masterbi = (tmp == this) ?
-               d->bibinfo_ : tmp->d->bibinfo_;
-       masterbi[key] = bi;
+       // We add the bibtex info to the master buffer,
+       // if there is one, but also to every single buffer,
+       // in case a child is compiled alone.
+       BiblioInfo & bi = d->bibinfo_;
+       bi[key] = bin;
+
+       if (parent() != 0) {
+               BiblioInfo & masterbi = parent()->d->bibinfo_;
+               masterbi[key] = bin;
+       }
 }
 
 
@@ -2488,6 +2633,11 @@ void Buffer::makeCitationLabels() const
 }
 
 
+void Buffer::invalidateCiteLabels() const
+{
+       masterBuffer()->d->cite_labels_valid_ = false;
+}
+
 bool Buffer::citeLabelsValid() const
 {
        return masterBuffer()->d->cite_labels_valid_;
@@ -2571,10 +2721,6 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                break;
        }
 
-       case LFUN_BUFFER_CHKTEX:
-               enable = params().isLatex() && !lyxrc.chktex_command.empty();
-               break;
-
        case LFUN_BUILD_PROGRAM:
                enable = params().isExportable("program", false);
                break;
@@ -2616,15 +2762,16 @@ bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
                flag.setOnOff(params().output_changes);
                break;
 
-       case LFUN_BUFFER_TOGGLE_COMPRESSION: {
+       case LFUN_BUFFER_TOGGLE_COMPRESSION:
                flag.setOnOff(params().compressed);
                break;
-       }
 
-       case LFUN_BUFFER_TOGGLE_OUTPUT_SYNC: {
+       case LFUN_BUFFER_TOGGLE_OUTPUT_SYNC:
                flag.setOnOff(params().output_sync);
                break;
-       }
+
+       case LFUN_BUFFER_ANONYMIZE:
+               break;
 
        default:
                return false;
@@ -2654,7 +2801,8 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
        string const argument = to_utf8(func.argument());
        // We'll set this back to false if need be.
        bool dispatched = true;
-       undo().beginUndoGroup();
+       // This handles undo groups automagically
+       UndoGroupHelper ugh(this);
 
        switch (func.action()) {
        case LFUN_BUFFER_TOGGLE_READ_ONLY:
@@ -2686,10 +2834,6 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                break;
        }
 
-       case LFUN_BUFFER_CHKTEX:
-               runChktex();
-               break;
-
        case LFUN_BUFFER_EXPORT_CUSTOM: {
                string format_name;
                string command = split(argument, format_name, ' ');
@@ -2901,12 +3045,21 @@ void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
                params().output_sync = !params().output_sync;
                break;
 
+       case LFUN_BUFFER_ANONYMIZE: {
+               undo().recordUndoFullBuffer(CursorData());
+               CursorData cur(doc_iterator_begin(this));
+               for ( ; cur ; cur.forwardPar())
+                       cur.paragraph().anonymize();
+               dr.forceBufferUpdate();
+               dr.screenUpdate(Update::Force);
+               break;
+       }
+
        default:
                dispatched = false;
                break;
        }
        dr.dispatched(dispatched);
-       undo().endUndoGroup();
 }
 
 
@@ -3152,7 +3305,7 @@ string const Buffer::prepareFileNameForLaTeX(string const & name,
 
 
 vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
-                                               FileNamePairList const bibfilelist,
+                                               docstring_list const & bibfilelist,
                                                bool const add_extension) const
 {
        // If we are processing the LaTeX file in a temp directory then
@@ -3173,15 +3326,31 @@ vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runpara
        // check for spaces in paths
        bool found_space = false;
 
-       FileNamePairList::const_iterator it = bibfilelist.begin();
-       FileNamePairList::const_iterator en = bibfilelist.end();
-       for (; it != en; ++it) {
-               string utf8input = to_utf8(it->first);
+       for (auto const & bit : bibfilelist) {
+               string utf8input = to_utf8(bit);
                string database =
                        prepareFileNameForLaTeX(utf8input, ".bib", runparams.nice);
-               FileName const try_in_file =
+               FileName try_in_file =
                        makeAbsPath(database + ".bib", filePath());
-               bool const not_from_texmf = try_in_file.isReadableFile();
+               bool not_from_texmf = try_in_file.isReadableFile();
+               // If the file has not been found, try with the real file name
+               // (it might come from a child in a sub-directory)
+               if (!not_from_texmf) {
+                       try_in_file = getBibfilePath(bit);
+                       if (try_in_file.isReadableFile()) {
+                               // Check if the file is in texmf
+                               FileName kpsefile(findtexfile(changeExtension(utf8input, "bib"), "bib", true));
+                               not_from_texmf = kpsefile.empty()
+                                               || kpsefile.absFileName() != try_in_file.absFileName();
+                               if (not_from_texmf)
+                                       // If this exists, make path relative to the master
+                                       // FIXME Unicode
+                                       database = removeExtension(
+                                                       prepareFileNameForLaTeX(to_utf8(makeRelPath(from_utf8(try_in_file.absFileName()),
+                                                                                                   from_utf8(filePath()))),
+                                                                               ".bib", runparams.nice));
+                       }
+               }
 
                if (!runparams.inComment && !runparams.dryrun && !runparams.nice &&
                    not_from_texmf) {
@@ -3606,7 +3775,7 @@ void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
                                continue;
 
                        // get macro data
-                       MathMacroTemplate & macroTemplate =
+                       InsetMathMacroTemplate & macroTemplate =
                                *iit->inset->asInsetMath()->asMacroTemplate();
                        MacroContext mc(owner_, it);
                        macroTemplate.updateToContext(mc);
@@ -3725,8 +3894,12 @@ void Buffer::listMacroNames(MacroNameSet & macros) const
        // loop over children
        Impl::BufferPositionMap::iterator it = d->children_positions.begin();
        Impl::BufferPositionMap::iterator end = d->children_positions.end();
-       for (; it != end; ++it)
-               it->first->listMacroNames(macros);
+       for (; it != end; ++it) {
+               Buffer * child = const_cast<Buffer *>(it->first);
+               // The buffer might have been closed (see #10766).
+               if (theBufferList().isLoaded(child))
+                       child->listMacroNames(macros);
+       }
 
        // call parent
        Buffer const * const pbuf = d->parent();
@@ -3756,7 +3929,7 @@ void Buffer::listParentMacros(MacroSet & macros, LaTeXFeatures & features) const
                if (data) {
                        macros.insert(data);
 
-                       // we cannot access the original MathMacroTemplate anymore
+                       // we cannot access the original InsetMathMacroTemplate anymore
                        // here to calls validate method. So we do its work here manually.
                        // FIXME: somehow make the template accessible here.
                        if (data->optionals() > 0)
@@ -3773,13 +3946,12 @@ Buffer::References & Buffer::getReferenceCache(docstring const & label)
 
        RefCache::iterator it = d->ref_cache_.find(label);
        if (it != d->ref_cache_.end())
-               return it->second.second;
+               return it->second;
 
-       static InsetLabel const * dummy_il = 0;
        static References const dummy_refs = References();
        it = d->ref_cache_.insert(
-               make_pair(label, make_pair(dummy_il, dummy_refs))).first;
-       return it->second.second;
+               make_pair(label, dummy_refs)).first;
+       return it->second;
 }
 
 
@@ -3796,22 +3968,43 @@ void Buffer::addReference(docstring const & label, Inset * inset, ParIterator it
 }
 
 
-void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il)
+void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il,
+                          bool const active)
 {
-       masterBuffer()->d->ref_cache_[label].first = il;
+       LabelInfo linfo;
+       linfo.label = label;
+       linfo.inset = il;
+       linfo.active = active;
+       masterBuffer()->d->label_cache_.push_back(linfo);
 }
 
 
-InsetLabel const * Buffer::insetLabel(docstring const & label) const
+InsetLabel const * Buffer::insetLabel(docstring const & label,
+                                     bool const active) const
 {
-       return masterBuffer()->d->ref_cache_[label].first;
+       for (auto & rc : masterBuffer()->d->label_cache_) {
+               if (rc.label == label && (rc.active || !active))
+                       return rc.inset;
+       }
+       return nullptr;
+}
+
+
+bool Buffer::activeLabel(docstring const & label) const
+{
+       if (!insetLabel(label, true))
+               return false;
+
+       return true;
 }
 
 
 void Buffer::clearReferenceCache() const
 {
-       if (!d->parent())
+       if (!d->parent()) {
                d->ref_cache_.clear();
+               d->label_cache_.clear();
+       }
 }
 
 
@@ -3903,6 +4096,7 @@ unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
                        LaTeXFeatures features(*this, params(), runparams);
                        validate(features);
                        runparams.use_polyglossia = features.usePolyglossia();
+                       runparams.use_hyperref = features.isRequired("hyperref");
                        // latex or literate
                        otexstream ots(os);
                        // output above
@@ -4108,7 +4302,7 @@ int AutoSaveBuffer::generateChild()
        return pid;
 }
 
-} // namespace anon
+} // namespace
 
 
 FileName Buffer::getEmergencyFileName() const
@@ -4291,6 +4485,7 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
                        return ExportNoPathToFormat;
                }
                runparams.flavor = converters.getFlavor(path, this);
+               runparams.hyperref_driver = converters.getHyperrefDriver(path);
                Graph::EdgePath::const_iterator it = path.begin();
                Graph::EdgePath::const_iterator en = path.end();
                for (; it != en; ++it)
@@ -4347,7 +4542,10 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        } else if (!lyxrc.tex_allows_spaces
                   && contains(filePath(), ' ')) {
                Alert::error(_("File name error"),
-                          _("The directory path to the document cannot contain spaces."));
+                       bformat(_("The directory path to the document\n%1$s\n"
+                           "contains spaces, but your TeX installation does "
+                           "not allow them. You should save the file to a directory "
+                                       "whose name does not contain spaces."), from_utf8(filePath())));
                return ExportTexPathHasSpaces;
        } else {
                runparams.nice = false;
@@ -4586,6 +4784,32 @@ Buffer::ReadStatus Buffer::loadEmergency()
                                _("&Remove"), _("&Keep"));
                if (del_emerg == 0)
                        emergencyFile.removeFile();
+               else {
+                       // See bug #11464
+                       FileName newname;
+                       string const ename = emergencyFile.absFileName();
+                       bool noname = true;
+                       // Surely we can find one in 100 tries?
+                       for (int i = 1; i < 100; ++i) {
+                               newname.set(ename + to_string(i) + ".lyx");
+                               if (!newname.exists()) {
+                                       noname = false;
+                                       break;
+                               }
+                       }
+                       if (!noname) {
+                               // renameTo returns true on success. So inverting that
+                               // will give us true if we fail.
+                               noname = !emergencyFile.renameTo(newname);
+                       }
+                       if (noname) {
+                               Alert::warning(_("Can't rename emergency file!"),
+                                       _("LyX was unable to rename the emergency file. "
+                                         "You should do so manually. Otherwise, you will be"
+                                         "asked about it again the next time you try to load"
+                                         "this file, and may over-write your own work."));
+                       }
+               }
                return ReadOriginal;
        }
 
@@ -4714,10 +4938,17 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
        Buffer const * const master = masterBuffer();
        DocumentClass const & textclass = master->params().documentClass();
 
-       // do this only if we are the top-level Buffer
-       if (master == this) {
+       docstring_list old_bibfiles;
+       // Do this only if we are the top-level Buffer. We also need to account
+       // for the case of a previewed child with ignored parent here.
+       if (master == this && !d->ignore_parent) {
                textclass.counters().reset(from_ascii("bibitem"));
                reloadBibInfoCache();
+               // we will re-read this cache as we go through, but we need
+               // to know whether it's changed to know whether we need to
+               // update the bibinfo cache.
+               old_bibfiles = d->bibfiles_cache_;
+               d->bibfiles_cache_.clear();
        }
 
        // keep the buffers to be children in this set. If the call from the
@@ -4733,7 +4964,7 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
                        // not updated during the updateBuffer call and TocModel::toc_ is invalid
                        // (bug 5699). The same happens if the master buffer is open in a different
                        // window. This test catches both possibilities.
-                       // See: http://marc.info/?l=lyx-devel&m=138590578911716&w=2
+                       // See: https://marc.info/?l=lyx-devel&m=138590578911716&w=2
                        // There remains a problem here: If there is another child open in yet a third
                        // window, that TOC is not updated. So some more general solution is needed at
                        // some point.
@@ -4770,7 +5001,38 @@ void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
                // in InsetInclude::addToToc.
                return;
 
-       d->bibinfo_cache_valid_ = true;
+       // if the bibfiles changed, the cache of bibinfo is invalid
+       docstring_list new_bibfiles = d->bibfiles_cache_;
+       // this is a trick to determine whether the two vectors have
+       // the same elements.
+       sort(new_bibfiles.begin(), new_bibfiles.end());
+       sort(old_bibfiles.begin(), old_bibfiles.end());
+       if (old_bibfiles != new_bibfiles) {
+               LYXERR(Debug::FILES, "Reloading bibinfo cache.");
+               invalidateBibinfoCache();
+               reloadBibInfoCache();
+               // We relied upon the bibinfo cache when recalculating labels. But that
+               // cache was invalid, although we didn't find that out until now. So we
+               // have to do it all again.
+               // That said, the only thing we really need to do is update the citation
+               // labels. Nothing else will have changed. So we could create a new 
+               // UpdateType that would signal that fact, if we needed to do so.
+               parit = cbuf.par_iterator_begin();
+               // we will be re-doing the counters and references and such.
+               textclass.counters().reset();
+               clearReferenceCache();
+               // we should not need to do this again?
+               // updateMacros();
+               setChangesPresent(false);
+               updateBuffer(parit, utype);
+               // this will already have been done by reloadBibInfoCache();
+               // d->bibinfo_cache_valid_ = true;
+       }
+       else {
+               LYXERR(Debug::FILES, "Bibfiles unchanged.");
+               // this is also set to true on the other path, by reloadBibInfoCache.
+               d->bibinfo_cache_valid_ = true;
+       }
        d->cite_labels_valid_ = true;
        /// FIXME: Perf
        cbuf.tocBackend().update(true, utype);
@@ -4917,8 +5179,10 @@ void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
                switch (par.itemdepth) {
                case 2:
                        enumcounter += 'i';
+                       // fall through
                case 1:
                        enumcounter += 'i';
+                       // fall through
                case 0:
                        enumcounter += 'i';
                        break;
@@ -5031,7 +5295,7 @@ void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
                // set the counter for this paragraph
                d->setLabel(parit, utype);
 
-               // update change-tracking flag 
+               // update change-tracking flag
                parit->addChangesToBuffer(*this);
 
                // now the insets
@@ -5157,7 +5421,7 @@ int Buffer::charCount(bool with_blanks) const
 Buffer::ReadStatus Buffer::reload()
 {
        setBusy(true);
-       // c.f. bug http://www.lyx.org/trac/ticket/6587
+       // c.f. bug https://www.lyx.org/trac/ticket/6587
        removeAutosaveFile();
        // e.g., read-only status could have changed due to version control
        d->filename.refresh();
@@ -5332,40 +5596,48 @@ void Buffer::Impl::refreshFileMonitor()
        // The previous file monitor is invalid
        // This also destroys the previous file monitor and all its connections
        file_monitor_ = FileSystemWatcher::monitor(filename);
-       fileExternallyModified(false);
        // file_monitor_ will be destroyed with *this, so it is not going to call a
        // destroyed object method.
-       file_monitor_->connect([this](){ fileExternallyModified(true); });
+       file_monitor_->connect([this](bool exists) {
+                       fileExternallyModified(exists);
+               });
 }
 
 
-void Buffer::Impl::fileExternallyModified(bool modified) const
+void Buffer::Impl::fileExternallyModified(bool const exists)
 {
-       if (modified) {
-               // prevent false positives, because FileMonitorBlocker is not enough on
-               // OSX.
-               if (filename.exists() && checksum_ == filename.checksum()) {
-                       LYXERR(Debug::FILES, "External modification but "
-                              "checksum unchanged: " << filename);
-                       return;
-               }
-               lyx_clean = bak_clean = false;
+       // ignore notifications after our own saving operations
+       if (checksum_ == filename.checksum()) {
+               LYXERR(Debug::FILES, "External modification but "
+                      "checksum unchanged: " << filename);
+               return;
        }
-       externally_modified_ = modified;
-       if (wa_)
+       // If the file has been deleted, only mark the file as dirty since it is
+       // pointless to prompt for reloading. If later a file is moved into this
+       // location, then the externally modified warning will appear then.
+       if (exists)
+                       externally_modified_ = true;
+       // Update external modification notification.
+       // Dirty buffers must be visible at all times.
+       if (wa_ && wa_->unhide(owner_))
                wa_->updateTitles();
+       else
+               // Unable to unhide the buffer (e.g. no GUI or not current View)
+               lyx_clean = true;
 }
 
 
 bool Buffer::notifiesExternalModification() const
 {
-       return d->isExternallyModified();
+       return d->externally_modified_;
 }
 
 
 void Buffer::clearExternalModification() const
 {
-       d->fileExternallyModified(false);
+       d->externally_modified_ = false;
+       if (d->wa_)
+               d->wa_->updateTitles();
 }