Improve performance of includeonly with "maintain counters and references"

First, we do not need to run bibtex/biber on the maintenance run, as
the necessary references will be generated on the includeonly run.

Second, exclude the master from DepTable in maintenance run, as the
master is re-checked in any case in the includeonly run, and as it will
always be detected as changed due to the \includeonly statement, which
will trigger a complete build.

More improvements to follow.

diff --git a/src/Buffer.cpp b/src/Buffer.cpp
index 11241afa4e962b923287935688eb86d9f34db4d8..8504c9e4e5f67116b7bcfbd5f4ff09dd8bcd1b0d 100644 (file)
--- a/src/Buffer.cpp
+++ b/src/Buffer.cpp
@@ -4502,8 +4502,9 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir
        string const ext = theFormats().extension(format);
        FileName const tmp_result_file(changeExtension(filename, ext));
        Converters::RetVal const retval = 
-               converters.convert(this, FileName(filename), tmp_result_file, 
-               FileName(absFileName()), backend_format, format, error_list);
+               converters.convert(this, FileName(filename), tmp_result_file,
+                                  FileName(absFileName()), backend_format, format,
+                                  error_list, Converters::none, includeall);
        if (retval == Converters::KILLED)
                return ExportCancel;
        bool success = (retval == Converters::SUCCESS);

diff --git a/src/Converter.cpp b/src/Converter.cpp
index 49aa95b5346ed48fcdce55387a629300b6b617e6..5ea8866ce29c8c22d6db1d0d89ef8fa600b23e34 100644 (file)
--- a/src/Converter.cpp
+++ b/src/Converter.cpp
@@ -400,7 +400,7 @@ Converters::RetVal Converters::convert(Buffer const * buffer,
                         FileName const & from_file, FileName const & to_file,
                         FileName const & orig_from,
                         string const & from_format, string const & to_format,
-                        ErrorList & errorList, int conversionflags)
+                        ErrorList & errorList, int conversionflags, bool includeall)
 {
        if (from_format == to_format)
                return move(from_format, from_file, to_file, false) ?
@@ -488,6 +488,7 @@ Converters::RetVal Converters::convert(Buffer const * buffer,
                runparams.only_childbibs = !bp.useBiblatex()
                                && !bp.useBibtopic()
                                && bp.multibib == "child";
+               runparams.includeall = includeall;
        }
 
        // Some converters (e.g. lilypond) can only output files to the

diff --git a/src/Converter.h b/src/Converter.h
index 9a2e2678963a62107f9047a5219678be67b3304b..684895e984b8e57de914823b09b1a4615b6589db 100644 (file)
--- a/src/Converter.h
+++ b/src/Converter.h
@@ -193,7 +193,7 @@ public:
                     support::FileName const & from_file, support::FileName const & to_file,
                     support::FileName const & orig_from,
                     std::string const & from_format, std::string const & to_format,
-                    ErrorList & errorList, int conversionflags = none);
+                    ErrorList & errorList, int conversionflags = none, bool includeall = false);
        ///
        void update(Formats const & formats);
        ///

diff --git a/src/LaTeX.cpp b/src/LaTeX.cpp
index 9e08ffec47420acdd75d448eba6bfcf3d6106095..c62535c56ea03dc8dee346e2518737c9d1360619 100644 (file)
--- a/src/LaTeX.cpp
+++ b/src/LaTeX.cpp
@@ -15,6 +15,7 @@
 
 #include <config.h>
 
+#include "Buffer.h"
 #include "BufferList.h"
 #include "LaTeX.h"
 #include "LyXRC.h"
@@ -226,6 +227,14 @@ int LaTeX::run(TeXErrors & terr)
        }
 
        if (had_depfile) {
+               if (runparams.includeall) {
+                       // On an "includeall" call (whose purpose is to set up/maintain counters and references
+                       // for includeonly), we remove the master from the dependency list since
+                       // (1) it will be checked anyway on the subsequent includeonly run
+                       // (2) the master is always changed (due to the \includeonly line), and this alone would
+                       //     trigger a complete, expensive run each time
+                       head.remove_file(file);
+               }
                // Update the checksums
                head.update();
                // Can't just check if anything has changed because it might
@@ -339,7 +348,12 @@ int LaTeX::run(TeXErrors & terr)
 
        // run bibtex
        // if (scanres & UNDEF_CIT || scanres & RERUN || run_bibtex)
-       if (scanres & UNDEF_CIT || run_bibtex) {
+       // We do not run bibtex/biber on an "includeall" call (whose purpose is
+       // to set up/maintain counters and references for includeonly) since
+       // (1) bibliographic references will be updated on the subsequent includeonly run
+       // (2) this would trigger a complete run each time (as references in non-included
+       //     children are removed on subsequent includeonly runs)
+       if (!runparams.includeall && (scanres & UNDEF_CIT || run_bibtex)) {
                // Here we must scan the .aux file and look for
                // "\bibdata" and/or "\bibstyle". If one of those
                // tags is found -> run bibtex and set rerun = true;
@@ -395,7 +409,12 @@ int LaTeX::run(TeXErrors & terr)
        // rerun bibtex?
        // Complex bibliography packages such as Biblatex require
        // an additional bibtex cycle sometimes.
-       if (scanres & UNDEF_CIT) {
+       // We do not run bibtex/biber on an "includeall" call (whose purpose is
+       // to set up/maintain counters and references for includeonly) since
+       // (1) bibliographic references will be updated on the subsequent includeonly run
+       // (2) this would trigger a complete run each time (as references in non-included
+       //     children are removed on subsequent includeonly runs)
+       if (!runparams.includeall && scanres & UNDEF_CIT) {
                // Here we must scan the .aux file and look for
                // "\bibdata" and/or "\bibstyle". If one of those
                // tags is found -> run bibtex and set rerun = true;
@@ -826,8 +845,8 @@ int LaTeX::scanLogFile(TeXErrors & terr)
                //TODO: TL 2020 engines will contain new commandline switch --cnf-line which we  
                //can use to set max_print_line variable for appropriate length and detect all
                //errors correctly.
-               if (contains(token, "There were undefined citations.") ||
-                   prefixIs(token, "Package biblatex Warning: The following entry could not be found"))
+               if (!runparams.includeall && (contains(token, "There were undefined citations.") ||
+                   prefixIs(token, "Package biblatex Warning: The following entry could not be found")))
                        retval |= UNDEF_CIT;
 
                if (prefixIs(token, "LaTeX Warning:") ||
@@ -854,7 +873,7 @@ int LaTeX::scanLogFile(TeXErrors & terr)
                                retval |= RERUN;
                                LYXERR(Debug::LATEX, "We should rerun.");
                        //"Citation `cit' on page X undefined on input line X."
-                       } else if (contains(token, "Citation")
+                       } else if (!runparams.includeall && contains(token, "Citation")
                                   //&& contains(token, "on input line") //often split to newline
                                   && contains(token, "undefined")) {
                                retval |= UNDEF_CIT;
@@ -870,7 +889,7 @@ int LaTeX::scanLogFile(TeXErrors & terr)
 
                        //If label is too long pdlaftex log line splitting will make the above fail
                        //so we catch at least this generic statement occuring for both CIT & REF.
-                       } else if (contains(token, "There were undefined references.")) {
+                       } else if (!runparams.includeall && contains(token, "There were undefined references.")) {
                                if (!(retval & UNDEF_CIT)) //if not handled already
                                         retval |= UNDEF_REF;
                        }
@@ -880,7 +899,8 @@ int LaTeX::scanLogFile(TeXErrors & terr)
                        retval |= PACKAGE_WARNING;
                        if (contains(token, "natbib Warning:")) {
                                // Natbib warnings
-                               if (contains(token, "Citation")
+                               if (!runparams.includeall
+                                   && contains(token, "Citation")
                                    && contains(token, "on page")
                                    && contains(token, "undefined")) {
                                        retval |= UNDEF_CIT;
@@ -888,9 +908,9 @@ int LaTeX::scanLogFile(TeXErrors & terr)
                                        terr.insertRef(getLineNumber(token), from_ascii("Citation undefined"),
                                                from_utf8(token), child_name);
                                }
-                       } else if (contains(token, "run BibTeX")) {
+                       } else if (!runparams.includeall && contains(token, "run BibTeX")) {
                                retval |= UNDEF_CIT;
-                       } else if (contains(token, "run Biber")) {
+                       } else if (!runparams.includeall && contains(token, "run Biber")) {
                                retval |= UNDEF_CIT;
                                biber = true;
                        } else if (contains(token, "Rerun LaTeX") ||
@@ -1045,7 +1065,7 @@ int LaTeX::scanLogFile(TeXErrors & terr)
                                retval |= TEX_WARNING;
                        } else if (prefixIs(token, "Underfull ")) {
                                retval |= TEX_WARNING;
-                       } else if (contains(token, "Rerun to get citations")) {
+                       } else if (!runparams.includeall && contains(token, "Rerun to get citations")) {
                                // Natbib seems to use this.
                                retval |= UNDEF_CIT;
                        } else if (contains(token, "No pages of output")