src/Buffer.cpp

   1 /**
   2  * \file Buffer.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  * \author Stefan Schimanski
   8  *
   9  * Full author contact details are available in file CREDITS.
  10  */
  11
  12 #include <config.h>
  13
  14 #include "Buffer.h"
  15
  16 #include "Author.h"
  17 #include "LayoutFile.h"
  18 #include "BiblioInfo.h"
  19 #include "BranchList.h"
  20 #include "buffer_funcs.h"
  21 #include "BufferList.h"
  22 #include "BufferParams.h"
  23 #include "Bullet.h"
  24 #include "Chktex.h"
  25 #include "Converter.h"
  26 #include "Counters.h"
  27 #include "Cursor.h"
  28 #include "CutAndPaste.h"
  29 #include "DispatchResult.h"
  30 #include "DocIterator.h"
  31 #include "BufferEncodings.h"
  32 #include "ErrorList.h"
  33 #include "Exporter.h"
  34 #include "Format.h"
  35 #include "FuncRequest.h"
  36 #include "FuncStatus.h"
  37 #include "IndicesList.h"
  38 #include "InsetIterator.h"
  39 #include "InsetList.h"
  40 #include "Language.h"
  41 #include "LaTeXFeatures.h"
  42 #include "LaTeX.h"
  43 #include "Layout.h"
  44 #include "Lexer.h"
  45 #include "LyXAction.h"
  46 #include "LyX.h"
  47 #include "LyXRC.h"
  48 #include "LyXVC.h"
  49 #include "output_docbook.h"
  50 #include "output.h"
  51 #include "output_latex.h"
  52 #include "output_xhtml.h"
  53 #include "output_plaintext.h"
  54 #include "Paragraph.h"
  55 #include "ParagraphParameters.h"
  56 #include "ParIterator.h"
  57 #include "PDFOptions.h"
  58 #include "SpellChecker.h"
  59 #include "sgml.h"
  60 #include "texstream.h"
  61 #include "TexRow.h"
  62 #include "Text.h"
  63 #include "TextClass.h"
  64 #include "TocBackend.h"
  65 #include "Undo.h"
  66 #include "VCBackend.h"
  67 #include "version.h"
  68 #include "WordLangTuple.h"
  69 #include "WordList.h"
  70
  71 #include "insets/InsetBibtex.h"
  72 #include "insets/InsetBranch.h"
  73 #include "insets/InsetInclude.h"
  74 #include "insets/InsetTabular.h"
  75 #include "insets/InsetText.h"
  76
  77 #include "mathed/InsetMathHull.h"
  78 #include "mathed/MacroTable.h"
  79 #include "mathed/MathMacroTemplate.h"
  80 #include "mathed/MathSupport.h"
  81
  82 #include "graphics/GraphicsCache.h"
  83 #include "graphics/PreviewLoader.h"
  84
  85 #include "frontends/alert.h"
  86 #include "frontends/Delegates.h"
  87 #include "frontends/WorkAreaManager.h"
  88
  89 #include "support/lassert.h"
  90 #include "support/convert.h"
  91 #include "support/debug.h"
  92 #include "support/docstring_list.h"
  93 #include "support/ExceptionMessage.h"
  94 #include "support/FileMonitor.h"
  95 #include "support/FileName.h"
  96 #include "support/FileNameList.h"
  97 #include "support/filetools.h"
  98 #include "support/ForkedCalls.h"
  99 #include "support/gettext.h"
 100 #include "support/gzstream.h"
 101 #include "support/lstrings.h"
 102 #include "support/lyxalgo.h"
 103 #include "support/mutex.h"
 104 #include "support/os.h"
 105 #include "support/Package.h"
 106 #include "support/PathChanger.h"
 107 #include "support/Systemcall.h"
 108 #include "support/TempFile.h"
 109 #include "support/textutils.h"
 110 #include "support/types.h"
 111
 112 #include "support/bind.h"
 113
 114 #include <algorithm>
 115 #include <fstream>
 116 #include <iomanip>
 117 #include <map>
 118 #include <memory>
 119 #include <set>
 120 #include <sstream>
 121 #include <vector>
 122
 123 using namespace std;
 124 using namespace lyx::support;
 125 using namespace lyx::graphics;
 126
 127 namespace lyx {
 128
 129 namespace Alert = frontend::Alert;
 130 namespace os = support::os;
 131
 132 namespace {
 133
 134 int const LYX_FORMAT = LYX_FORMAT_LYX;
 135
 136 typedef map<string, bool> DepClean;
 137 typedef map<docstring, pair<InsetLabel const *, Buffer::References> > RefCache;
 138
 139 } // namespace anon
 140
 141
 142 // A storehouse for the cloned buffers.
 143 list<CloneList *> cloned_buffers;
 144
 145
 146 class Buffer::Impl
 147 {
 148 public:
 149         Impl(Buffer * owner, FileName const & file, bool readonly, Buffer const * cloned_buffer);
 150
 151         ~Impl()
 152         {
 153                 delete preview_loader_;
 154                 if (wa_) {
 155                         wa_->closeAll();
 156                         delete wa_;
 157                 }
 158                 delete inset;
 159         }
 160
 161         /// search for macro in local (buffer) table or in children
 162         MacroData const * getBufferMacro(docstring const & name,
 163                 DocIterator const & pos) const;
 164
 165         /// Update macro table starting with position of it \param it in some
 166         /// text inset.
 167         void updateMacros(DocIterator & it, DocIterator & scope);
 168         ///
 169         void setLabel(ParIterator & it, UpdateType utype) const;
 170
 171         /** If we have branches that use the file suffix
 172             feature, return the file name with suffix appended.
 173         */
 174         support::FileName exportFileName() const;
 175
 176         Buffer * owner_;
 177
 178         BufferParams params;
 179         LyXVC lyxvc;
 180         FileName temppath;
 181         mutable TexRow texrow;
 182
 183         /// need to regenerate .tex?
 184         DepClean dep_clean;
 185
 186         /// is save needed?
 187         mutable bool lyx_clean;
 188
 189         /// is autosave needed?
 190         mutable bool bak_clean;
 191
 192         /// is this an unnamed file (New...)?
 193         bool unnamed;
 194
 195         /// is this an internal bufffer?
 196         bool internal_buffer;
 197
 198         /// buffer is r/o
 199         bool read_only;
 200
 201         /// name of the file the buffer is associated with.
 202         FileName filename;
 203
 204         /** Set to true only when the file is fully loaded.
 205          *  Used to prevent the premature generation of previews
 206          *  and by the citation inset.
 207          */
 208         bool file_fully_loaded;
 209
 210         /// original format of loaded file
 211         int file_format;
 212
 213         /// if the file was originally loaded from an older format, do
 214         /// we need to back it up still?
 215         bool need_format_backup;
 216
 217         /// Ignore the parent (e.g. when exporting a child standalone)?
 218         bool ignore_parent;
 219
 220         ///
 221         mutable TocBackend toc_backend;
 222
 223         /// macro tables
 224         struct ScopeMacro {
 225                 ScopeMacro() {}
 226                 ScopeMacro(DocIterator const & s, MacroData const & m)
 227                         : scope(s), macro(m) {}
 228                 DocIterator scope;
 229                 MacroData macro;
 230         };
 231         typedef map<DocIterator, ScopeMacro> PositionScopeMacroMap;
 232         typedef map<docstring, PositionScopeMacroMap> NamePositionScopeMacroMap;
 233         /// map from the macro name to the position map,
 234         /// which maps the macro definition position to the scope and the MacroData.
 235         NamePositionScopeMacroMap macros;
 236         /// This seem to change the way Buffer::getMacro() works
 237         mutable bool macro_lock;
 238
 239         /// positions of child buffers in the buffer
 240         typedef map<Buffer const * const, DocIterator> BufferPositionMap;
 241         struct ScopeBuffer {
 242                 ScopeBuffer() : buffer(0) {}
 243                 ScopeBuffer(DocIterator const & s, Buffer const * b)
 244                         : scope(s), buffer(b) {}
 245                 DocIterator scope;
 246                 Buffer const * buffer;
 247         };
 248         typedef map<DocIterator, ScopeBuffer> PositionScopeBufferMap;
 249         /// position of children buffers in this buffer
 250         BufferPositionMap children_positions;
 251         /// map from children inclusion positions to their scope and their buffer
 252         PositionScopeBufferMap position_to_children;
 253
 254         /// Contains the old buffer filePath() while saving-as, or the
 255         /// directory where the document was last saved while loading.
 256         string old_position;
 257
 258         /** Keeps track of the path of local layout files.
 259          *  If possible, it is always relative to the buffer path.
 260          *  Empty for layouts in system or user directory.
 261          */
 262         string layout_position;
 263
 264         /// Container for all sort of Buffer dependant errors.
 265         map<string, ErrorList> errorLists;
 266
 267         /// checksum used to test if the file has been externally modified.  Used to
 268         /// double check whether the file had been externally modified when saving.
 269         unsigned long checksum_;
 270
 271         ///
 272         frontend::WorkAreaManager * wa_;
 273         ///
 274         frontend::GuiBufferDelegate * gui_;
 275
 276         ///
 277         Undo undo_;
 278
 279         /// A cache for the bibfiles (including bibfiles of loaded child
 280         /// documents), needed for appropriate update of natbib labels.
 281         mutable support::FileNamePairList bibfiles_cache_;
 282
 283         // FIXME The caching mechanism could be improved. At present, we have a
 284         // cache for each Buffer, that caches all the bibliography info for that
 285         // Buffer. A more efficient solution would be to have a global cache per
 286         // file, and then to construct the Buffer's bibinfo from that.
 287         /// A cache for bibliography info
 288         mutable BiblioInfo bibinfo_;
 289         /// whether the bibinfo cache is valid
 290         mutable bool bibinfo_cache_valid_;
 291         /// whether the bibfile cache is valid
 292         mutable bool bibfile_cache_valid_;
 293         /// Cache of timestamps of .bib files
 294         map<FileName, time_t> bibfile_status_;
 295         /// Indicates whether the bibinfo has changed since the last time
 296         /// we ran updateBuffer(), i.e., whether citation labels may need
 297         /// to be updated.
 298         mutable bool cite_labels_valid_;
 299
 300         /// These two hold the file name and format, written to by
 301         /// Buffer::preview and read from by LFUN_BUFFER_VIEW_CACHE.
 302         FileName preview_file_;
 303         string preview_format_;
 304         /// If there was an error when previewing, on the next preview we do
 305         /// a fresh compile (e.g. in case the user installed a package that
 306         /// was missing).
 307         bool preview_error_;
 308
 309         mutable RefCache ref_cache_;
 310
 311         /// our Text that should be wrapped in an InsetText
 312         InsetText * inset;
 313
 314         ///
 315         PreviewLoader * preview_loader_;
 316
 317         /// This is here to force the test to be done whenever parent_buffer
 318         /// is accessed.
 319         Buffer const * parent() const
 320         {
 321                 // ignore_parent temporarily "orphans" a buffer
 322                 // (e.g. if a child is compiled standalone)
 323                 if (ignore_parent)
 324                         return 0;
 325                 // if parent_buffer is not loaded, then it has been unloaded,
 326                 // which means that parent_buffer is an invalid pointer. So we
 327                 // set it to null in that case.
 328                 // however, the BufferList doesn't know about cloned buffers, so
 329                 // they will always be regarded as unloaded. in that case, we hope
 330                 // for the best.
 331                 if (!cloned_buffer_ && !theBufferList().isLoaded(parent_buffer))
 332                         parent_buffer = 0;
 333                 return parent_buffer;
 334         }
 335
 336         ///
 337         void setParent(Buffer const * pb)
 338         {
 339                 if (parent_buffer == pb)
 340                         // nothing to do
 341                         return;
 342                 if (!cloned_buffer_ && parent_buffer && pb)
 343                         LYXERR0("Warning: a buffer should not have two parents!");
 344                 parent_buffer = pb;
 345                 if (!cloned_buffer_ && parent_buffer) {
 346                         parent_buffer->invalidateBibfileCache();
 347                         parent_buffer->invalidateBibinfoCache();
 348                 }
 349         }
 350
 351         /// If non zero, this buffer is a clone of existing buffer \p cloned_buffer_
 352         /// This one is useful for preview detached in a thread.
 353         Buffer const * cloned_buffer_;
 354         ///
 355         CloneList * clone_list_;
 356         /// are we in the process of exporting this buffer?
 357         mutable bool doing_export;
 358
 359         /// compute statistics
 360         /// \p from initial position
 361         /// \p to points to the end position
 362         void updateStatistics(DocIterator & from, DocIterator & to,
 363                               bool skipNoOutput = true);
 364         /// statistics accessor functions
 365         int wordCount() const
 366         {
 367                 return word_count_;
 368         }
 369         int charCount(bool with_blanks) const
 370         {
 371                 return char_count_
 372                 + (with_blanks ? blank_count_ : 0);
 373         }
 374
 375         // does the buffer contain tracked changes? (if so, we automatically
 376         // display the review toolbar, for instance)
 377         mutable bool tracked_changes_present_;
 378
 379         // Make sure the file monitor monitors the good file.
 380         void refreshFileMonitor();
 381
 382         /// has it been notified of an external modification?
 383         bool isExternallyModified() const { return externally_modified_; }
 384
 385         /// Notify or clear of external modification
 386         void fileExternallyModified(bool modified) const;
 387
 388         /// Block notifications of external modifications
 389         FileMonitorBlocker blockFileMonitor() { return file_monitor_->block(); }
 390
 391 private:
 392         /// So we can force access via the accessors.
 393         mutable Buffer const * parent_buffer;
 394
 395         int word_count_;
 396         int char_count_;
 397         int blank_count_;
 398
 399         /// has been externally modified? Can be reset by the user.
 400         mutable bool externally_modified_;
 401
 402         FileMonitorPtr file_monitor_;
 403 };
 404
 405
 406 /// Creates the per buffer temporary directory
 407 static FileName createBufferTmpDir()
 408 {
 409         // FIXME This would be the ideal application for a TempDir class (like
 410         //       TempFile but for directories)
 411         string counter;
 412         {
 413                 static int count;
 414                 static Mutex mutex;
 415                 Mutex::Locker locker(&mutex);
 416                 counter = convert<string>(count++);
 417         }
 418         // We are in our own directory.  Why bother to mangle name?
 419         // In fact I wrote this code to circumvent a problematic behaviour
 420         // (bug?) of EMX mkstemp().
 421         FileName tmpfl(package().temp_dir().absFileName() + "/lyx_tmpbuf" +
 422                 counter);
 423
 424         if (!tmpfl.createDirectory(0777)) {
 425                 throw ExceptionMessage(WarningException, _("Disk Error: "), bformat(
 426                         _("LyX could not create the temporary directory '%1$s' (Disk is full maybe?)"),
 427                         from_utf8(tmpfl.absFileName())));
 428         }
 429         return tmpfl;
 430 }
 431
 432
 433 Buffer::Impl::Impl(Buffer * owner, FileName const & file, bool readonly_,
 434         Buffer const * cloned_buffer)
 435         : owner_(owner), lyx_clean(true), bak_clean(true), unnamed(false),
 436           internal_buffer(false), read_only(readonly_), filename(file),
 437           file_fully_loaded(false), file_format(LYX_FORMAT), need_format_backup(false),
 438           ignore_parent(false),  toc_backend(owner), macro_lock(false),
 439           checksum_(0), wa_(0),  gui_(0), undo_(*owner), bibinfo_cache_valid_(false),
 440           bibfile_cache_valid_(false), cite_labels_valid_(false), preview_error_(false),
 441           inset(0), preview_loader_(0), cloned_buffer_(cloned_buffer),
 442           clone_list_(0), doing_export(false),
 443           tracked_changes_present_(0), parent_buffer(0),
 444           word_count_(0), char_count_(0), blank_count_(0),
 445           externally_modified_(false)
 446 {
 447         refreshFileMonitor();
 448         if (!cloned_buffer_) {
 449                 temppath = createBufferTmpDir();
 450                 lyxvc.setBuffer(owner_);
 451                 if (use_gui)
 452                         wa_ = new frontend::WorkAreaManager;
 453                 return;
 454         }
 455         temppath = cloned_buffer_->d->temppath;
 456         file_fully_loaded = true;
 457         params = cloned_buffer_->d->params;
 458         bibfiles_cache_ = cloned_buffer_->d->bibfiles_cache_;
 459         bibinfo_ = cloned_buffer_->d->bibinfo_;
 460         bibinfo_cache_valid_ = cloned_buffer_->d->bibinfo_cache_valid_;
 461         bibfile_cache_valid_ = cloned_buffer_->d->bibfile_cache_valid_;
 462         bibfile_status_ = cloned_buffer_->d->bibfile_status_;
 463         cite_labels_valid_ = cloned_buffer_->d->cite_labels_valid_;
 464         unnamed = cloned_buffer_->d->unnamed;
 465         internal_buffer = cloned_buffer_->d->internal_buffer;
 466         layout_position = cloned_buffer_->d->layout_position;
 467         preview_file_ = cloned_buffer_->d->preview_file_;
 468         preview_format_ = cloned_buffer_->d->preview_format_;
 469         preview_error_ = cloned_buffer_->d->preview_error_;
 470         tracked_changes_present_ = cloned_buffer_->d->tracked_changes_present_;
 471 }
 472
 473
 474 Buffer::Buffer(string const & file, bool readonly, Buffer const * cloned_buffer)
 475         : d(new Impl(this, FileName(file), readonly, cloned_buffer))
 476 {
 477         LYXERR(Debug::INFO, "Buffer::Buffer()");
 478         if (cloned_buffer) {
 479                 d->inset = new InsetText(*cloned_buffer->d->inset);
 480                 d->inset->setBuffer(*this);
 481                 // FIXME: optimize this loop somewhat, maybe by creating a new
 482                 // general recursive Inset::setId().
 483                 DocIterator it = doc_iterator_begin(this);
 484                 DocIterator cloned_it = doc_iterator_begin(cloned_buffer);
 485                 for (; !it.atEnd(); it.forwardPar(), cloned_it.forwardPar())
 486                         it.paragraph().setId(cloned_it.paragraph().id());
 487         } else
 488                 d->inset = new InsetText(this);
 489         d->inset->getText(0)->setMacrocontextPosition(par_iterator_begin());
 490 }
 491
 492
 493 Buffer::~Buffer()
 494 {
 495         LYXERR(Debug::INFO, "Buffer::~Buffer()");
 496         // here the buffer should take care that it is
 497         // saved properly, before it goes into the void.
 498
 499         // GuiView already destroyed
 500         d->gui_ = 0;
 501
 502         if (isInternal()) {
 503                 // No need to do additional cleanups for internal buffer.
 504                 delete d;
 505                 return;
 506         }
 507
 508         if (isClone()) {
 509                 // this is in case of recursive includes: we won't try to delete
 510                 // ourselves as a child.
 511                 d->clone_list_->erase(this);
 512                 // loop over children
 513                 Impl::BufferPositionMap::iterator it = d->children_positions.begin();
 514                 Impl::BufferPositionMap::iterator end = d->children_positions.end();
 515                 for (; it != end; ++it) {
 516                         Buffer * child = const_cast<Buffer *>(it->first);
 517                                 if (d->clone_list_->erase(child))
 518                                         delete child;
 519                 }
 520                 // if we're the master buffer, then we should get rid of the list
 521                 // of clones
 522                 if (!parent()) {
 523                         // If this is not empty, we have leaked something. Worse, one of the
 524                         // children still has a reference to this list. But we will try to
 525                         // continue, rather than shut down.
 526                         LATTEST(d->clone_list_->empty());
 527                         list<CloneList *>::iterator it =
 528                                 find(cloned_buffers.begin(), cloned_buffers.end(), d->clone_list_);
 529                         if (it == cloned_buffers.end()) {
 530                                 // We will leak in this case, but it is safe to continue.
 531                                 LATTEST(false);
 532                         } else
 533                                 cloned_buffers.erase(it);
 534                         delete d->clone_list_;
 535                 }
 536                 // FIXME Do we really need to do this right before we delete d?
 537                 // clear references to children in macro tables
 538                 d->children_positions.clear();
 539                 d->position_to_children.clear();
 540         } else {
 541                 // loop over children
 542                 Impl::BufferPositionMap::iterator it = d->children_positions.begin();
 543                 Impl::BufferPositionMap::iterator end = d->children_positions.end();
 544                 for (; it != end; ++it) {
 545                         Buffer * child = const_cast<Buffer *>(it->first);
 546                         if (theBufferList().isLoaded(child)) {
 547                          if (theBufferList().isOthersChild(this, child))
 548                                  child->setParent(0);
 549                          else
 550                                 theBufferList().release(child);
 551                         }
 552                 }
 553
 554                 if (!isClean()) {
 555                         docstring msg = _("LyX attempted to close a document that had unsaved changes!\n");
 556                         try {
 557                                 msg += emergencyWrite();
 558                         } catch (...) {
 559                                 msg += "  " + _("Save failed! Document is lost.");
 560                         }
 561                         Alert::warning(_("Attempting to close changed document!"), msg);
 562                 }
 563
 564                 // FIXME Do we really need to do this right before we delete d?
 565                 // clear references to children in macro tables
 566                 d->children_positions.clear();
 567                 d->position_to_children.clear();
 568
 569                 if (!d->temppath.destroyDirectory()) {
 570                         LYXERR0(bformat(_("Could not remove the temporary directory %1$s"),
 571                                 from_utf8(d->temppath.absFileName())));
 572                 }
 573                 removePreviews();
 574         }
 575
 576         delete d;
 577 }
 578
 579
 580 Buffer * Buffer::cloneFromMaster() const
 581 {
 582         BufferMap bufmap;
 583         cloned_buffers.push_back(new CloneList);
 584         CloneList * clones = cloned_buffers.back();
 585
 586         masterBuffer()->cloneWithChildren(bufmap, clones);
 587
 588         // make sure we got cloned
 589         BufferMap::const_iterator bit = bufmap.find(this);
 590         LASSERT(bit != bufmap.end(), return 0);
 591         Buffer * cloned_buffer = bit->second;
 592
 593         return cloned_buffer;
 594 }
 595
 596
 597 void Buffer::cloneWithChildren(BufferMap & bufmap, CloneList * clones) const
 598 {
 599         // have we already been cloned?
 600         if (bufmap.find(this) != bufmap.end())
 601                 return;
 602
 603         Buffer * buffer_clone = new Buffer(fileName().absFileName(), false, this);
 604
 605         // The clone needs its own DocumentClass, since running updateBuffer() will
 606         // modify it, and we would otherwise be sharing it with the original Buffer.
 607         buffer_clone->params().makeDocumentClass(true);
 608         ErrorList el;
 609         cap::switchBetweenClasses(
 610                         params().documentClassPtr(), buffer_clone->params().documentClassPtr(),
 611                         static_cast<InsetText &>(buffer_clone->inset()), el);
 612
 613         bufmap[this] = buffer_clone;
 614         clones->insert(buffer_clone);
 615         buffer_clone->d->clone_list_ = clones;
 616         buffer_clone->d->macro_lock = true;
 617         buffer_clone->d->children_positions.clear();
 618
 619         // FIXME (Abdel 09/01/2010): this is too complicated. The whole children_positions and
 620         // math macro caches need to be rethought and simplified.
 621         // I am not sure wether we should handle Buffer cloning here or in BufferList.
 622         // Right now BufferList knows nothing about buffer clones.
 623         Impl::PositionScopeBufferMap::iterator it = d->position_to_children.begin();
 624         Impl::PositionScopeBufferMap::iterator end = d->position_to_children.end();
 625         for (; it != end; ++it) {
 626                 DocIterator dit = it->first.clone(buffer_clone);
 627                 dit.setBuffer(buffer_clone);
 628                 Buffer * child = const_cast<Buffer *>(it->second.buffer);
 629
 630                 child->cloneWithChildren(bufmap, clones);
 631                 BufferMap::iterator const bit = bufmap.find(child);
 632                 LASSERT(bit != bufmap.end(), continue);
 633                 Buffer * child_clone = bit->second;
 634
 635                 Inset * inset = dit.nextInset();
 636                 LASSERT(inset && inset->lyxCode() == INCLUDE_CODE, continue);
 637                 InsetInclude * inset_inc = static_cast<InsetInclude *>(inset);
 638                 inset_inc->setChildBuffer(child_clone);
 639                 child_clone->d->setParent(buffer_clone);
 640                 // FIXME Do we need to do this now, or can we wait until we run updateMacros()?
 641                 buffer_clone->setChild(dit, child_clone);
 642         }
 643         buffer_clone->d->macro_lock = false;
 644         return;
 645 }
 646
 647
 648 Buffer * Buffer::cloneBufferOnly() const {
 649         cloned_buffers.push_back(new CloneList);
 650         CloneList * clones = cloned_buffers.back();
 651         Buffer * buffer_clone = new Buffer(fileName().absFileName(), false, this);
 652
 653         // The clone needs its own DocumentClass, since running updateBuffer() will
 654         // modify it, and we would otherwise be sharing it with the original Buffer.
 655         buffer_clone->params().makeDocumentClass(true);
 656         ErrorList el;
 657         cap::switchBetweenClasses(
 658                         params().documentClassPtr(), buffer_clone->params().documentClassPtr(),
 659                         static_cast<InsetText &>(buffer_clone->inset()), el);
 660
 661         clones->insert(buffer_clone);
 662         buffer_clone->d->clone_list_ = clones;
 663
 664         // we won't be cloning the children
 665         buffer_clone->d->children_positions.clear();
 666         return buffer_clone;
 667 }
 668
 669
 670 bool Buffer::isClone() const
 671 {
 672         return d->cloned_buffer_;
 673 }
 674
 675
 676 void Buffer::changed(bool update_metrics) const
 677 {
 678         if (d->wa_)
 679                 d->wa_->redrawAll(update_metrics);
 680 }
 681
 682
 683 frontend::WorkAreaManager & Buffer::workAreaManager() const
 684 {
 685         LBUFERR(d->wa_);
 686         return *d->wa_;
 687 }
 688
 689
 690 Text & Buffer::text() const
 691 {
 692         return d->inset->text();
 693 }
 694
 695
 696 Inset & Buffer::inset() const
 697 {
 698         return *d->inset;
 699 }
 700
 701
 702 BufferParams & Buffer::params()
 703 {
 704         return d->params;
 705 }
 706
 707
 708 BufferParams const & Buffer::params() const
 709 {
 710         return d->params;
 711 }
 712
 713
 714 BufferParams const & Buffer::masterParams() const
 715 {
 716         if (masterBuffer() == this)
 717                 return params();
 718
 719         BufferParams & mparams = const_cast<Buffer *>(masterBuffer())->params();
 720         // Copy child authors to the params. We need those pointers.
 721         AuthorList const & child_authors = params().authors();
 722         AuthorList::Authors::const_iterator it = child_authors.begin();
 723         for (; it != child_authors.end(); ++it)
 724                 mparams.authors().record(*it);
 725         return mparams;
 726 }
 727
 728
 729 double Buffer::fontScalingFactor() const
 730 {
 731         return isExporting() ? 75.0 * params().html_math_img_scale
 732                 : 0.01 * lyxrc.dpi * lyxrc.currentZoom * lyxrc.preview_scale_factor * params().display_pixel_ratio;
 733 }
 734
 735
 736 ParagraphList & Buffer::paragraphs()
 737 {
 738         return text().paragraphs();
 739 }
 740
 741
 742 ParagraphList const & Buffer::paragraphs() const
 743 {
 744         return text().paragraphs();
 745 }
 746
 747
 748 LyXVC & Buffer::lyxvc()
 749 {
 750         return d->lyxvc;
 751 }
 752
 753
 754 LyXVC const & Buffer::lyxvc() const
 755 {
 756         return d->lyxvc;
 757 }
 758
 759
 760 string const Buffer::temppath() const
 761 {
 762         return d->temppath.absFileName();
 763 }
 764
 765
 766 TexRow & Buffer::texrow()
 767 {
 768         return d->texrow;
 769 }
 770
 771
 772 TexRow const & Buffer::texrow() const
 773 {
 774         return d->texrow;
 775 }
 776
 777
 778 TocBackend & Buffer::tocBackend() const
 779 {
 780         return d->toc_backend;
 781 }
 782
 783
 784 Undo & Buffer::undo()
 785 {
 786         return d->undo_;
 787 }
 788
 789
 790 void Buffer::setChild(DocIterator const & dit, Buffer * child)
 791 {
 792         d->children_positions[child] = dit;
 793 }
 794
 795
 796 string Buffer::latexName(bool const no_path) const
 797 {
 798         FileName latex_name =
 799                 makeLatexName(d->exportFileName());
 800         return no_path ? latex_name.onlyFileName()
 801                 : latex_name.absFileName();
 802 }
 803
 804
 805 FileName Buffer::Impl::exportFileName() const
 806 {
 807         docstring const branch_suffix =
 808                 params.branchlist().getFileNameSuffix();
 809         if (branch_suffix.empty())
 810                 return filename;
 811
 812         string const name = filename.onlyFileNameWithoutExt()
 813                 + to_utf8(branch_suffix);
 814         FileName res(filename.onlyPath().absFileName() + "/" + name);
 815         res.changeExtension(filename.extension());
 816
 817         return res;
 818 }
 819
 820
 821 string Buffer::logName(LogType * type) const
 822 {
 823         string const filename = latexName(false);
 824
 825         if (filename.empty()) {
 826                 if (type)
 827                         *type = latexlog;
 828                 return string();
 829         }
 830
 831         string const path = temppath();
 832
 833         FileName const fname(addName(temppath(),
 834                                      onlyFileName(changeExtension(filename,
 835                                                                   ".log"))));
 836
 837         // FIXME: how do we know this is the name of the build log?
 838         FileName const bname(
 839                 addName(path, onlyFileName(
 840                         changeExtension(filename,
 841                                         theFormats().extension(params().bufferFormat()) + ".out"))));
 842
 843         // Also consider the master buffer log file
 844         FileName masterfname = fname;
 845         LogType mtype = latexlog;
 846         if (masterBuffer() != this) {
 847                 string const mlogfile = masterBuffer()->logName(&mtype);
 848                 masterfname = FileName(mlogfile);
 849         }
 850
 851         // If no Latex log or Build log is newer, show Build log
 852         if (bname.exists() &&
 853             ((!fname.exists() && !masterfname.exists())
 854              || (fname.lastModified() < bname.lastModified()
 855                  && masterfname.lastModified() < bname.lastModified()))) {
 856                 LYXERR(Debug::FILES, "Log name calculated as: " << bname);
 857                 if (type)
 858                         *type = buildlog;
 859                 return bname.absFileName();
 860         // If we have a newer master file log or only a master log, show this
 861         } else if (fname != masterfname
 862                    && (!fname.exists() && (masterfname.exists()
 863                    || fname.lastModified() < masterfname.lastModified()))) {
 864                 LYXERR(Debug::FILES, "Log name calculated as: " << masterfname);
 865                 if (type)
 866                         *type = mtype;
 867                 return masterfname.absFileName();
 868         }
 869         LYXERR(Debug::FILES, "Log name calculated as: " << fname);
 870         if (type)
 871                         *type = latexlog;
 872         return fname.absFileName();
 873 }
 874
 875
 876 void Buffer::setReadonly(bool const flag)
 877 {
 878         if (d->read_only != flag) {
 879                 d->read_only = flag;
 880                 changed(false);
 881         }
 882 }
 883
 884
 885 void Buffer::setFileName(FileName const & fname)
 886 {
 887         bool const changed = fname != d->filename;
 888         d->filename = fname;
 889         d->refreshFileMonitor();
 890         if (changed)
 891                 lyxvc().file_found_hook(fname);
 892         setReadonly(d->filename.isReadOnly());
 893         saveCheckSum();
 894         updateTitles();
 895 }
 896
 897
 898 int Buffer::readHeader(Lexer & lex)
 899 {
 900         int unknown_tokens = 0;
 901         int line = -1;
 902         int begin_header_line = -1;
 903
 904         // Initialize parameters that may be/go lacking in header:
 905         params().branchlist().clear();
 906         params().preamble.erase();
 907         params().options.erase();
 908         params().master.erase();
 909         params().float_placement.erase();
 910         params().paperwidth.erase();
 911         params().paperheight.erase();
 912         params().leftmargin.erase();
 913         params().rightmargin.erase();
 914         params().topmargin.erase();
 915         params().bottommargin.erase();
 916         params().headheight.erase();
 917         params().headsep.erase();
 918         params().footskip.erase();
 919         params().columnsep.erase();
 920         params().fonts_cjk.erase();
 921         params().listings_params.clear();
 922         params().clearLayoutModules();
 923         params().clearRemovedModules();
 924         params().clearIncludedChildren();
 925         params().pdfoptions().clear();
 926         params().indiceslist().clear();
 927         params().backgroundcolor = lyx::rgbFromHexName("#ffffff");
 928         params().isbackgroundcolor = false;
 929         params().fontcolor = RGBColor(0, 0, 0);
 930         params().isfontcolor = false;
 931         params().notefontcolor = RGBColor(0xCC, 0xCC, 0xCC);
 932         params().boxbgcolor = RGBColor(0xFF, 0, 0);
 933         params().html_latex_start.clear();
 934         params().html_latex_end.clear();
 935         params().html_math_img_scale = 1.0;
 936         params().output_sync_macro.erase();
 937         params().setLocalLayout(docstring(), false);
 938         params().setLocalLayout(docstring(), true);
 939         params().biblio_opts.erase();
 940         params().biblatex_bibstyle.erase();
 941         params().biblatex_citestyle.erase();
 942         params().multibib.erase();
 943
 944         for (int i = 0; i < 4; ++i) {
 945                 params().user_defined_bullet(i) = ITEMIZE_DEFAULTS[i];
 946                 params().temp_bullet(i) = ITEMIZE_DEFAULTS[i];
 947         }
 948
 949         ErrorList & errorList = d->errorLists["Parse"];
 950
 951         while (lex.isOK()) {
 952                 string token;
 953                 lex >> token;
 954
 955                 if (token.empty())
 956                         continue;
 957
 958                 if (token == "\\end_header")
 959                         break;
 960
 961                 ++line;
 962                 if (token == "\\begin_header") {
 963                         begin_header_line = line;
 964                         continue;
 965                 }
 966
 967                 LYXERR(Debug::PARSER, "Handling document header token: `"
 968                                       << token << '\'');
 969
 970                 string const result =
 971                         params().readToken(lex, token, d->filename.onlyPath());
 972                 if (!result.empty()) {
 973                         if (token == "\\textclass") {
 974                                 d->layout_position = result;
 975                         } else {
 976                                 ++unknown_tokens;
 977                                 docstring const s = bformat(_("Unknown token: "
 978                                                                         "%1$s %2$s\n"),
 979                                                          from_utf8(token),
 980                                                          lex.getDocString());
 981                                 errorList.push_back(ErrorItem(_("Document header error"), s));
 982                         }
 983                 }
 984         }
 985         if (begin_header_line) {
 986                 docstring const s = _("\\begin_header is missing");
 987                 errorList.push_back(ErrorItem(_("Document header error"), s));
 988         }
 989
 990         params().makeDocumentClass();
 991
 992         return unknown_tokens;
 993 }
 994
 995
 996 // Uwe C. Schroeder
 997 // changed to be public and have one parameter
 998 // Returns true if "\end_document" is not read (Asger)
 999 bool Buffer::readDocument(Lexer & lex)
1000 {
1001         ErrorList & errorList = d->errorLists["Parse"];
1002         errorList.clear();
1003
1004         // remove dummy empty par
1005         paragraphs().clear();
1006
1007         if (!lex.checkFor("\\begin_document")) {
1008                 docstring const s = _("\\begin_document is missing");
1009                 errorList.push_back(ErrorItem(_("Document header error"), s));
1010         }
1011
1012         readHeader(lex);
1013
1014         if (params().output_changes) {
1015                 bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
1016                 bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
1017                                   LaTeXFeatures::isAvailable("xcolor");
1018
1019                 if (!dvipost && !xcolorulem) {
1020                         Alert::warning(_("Changes not shown in LaTeX output"),
1021                                        _("Changes will not be highlighted in LaTeX output, "
1022                                          "because neither dvipost nor xcolor/ulem are installed.\n"
1023                                          "Please install these packages or redefine "
1024                                          "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
1025                 } else if (!xcolorulem) {
1026                         Alert::warning(_("Changes not shown in LaTeX output"),
1027                                        _("Changes will not be highlighted in LaTeX output "
1028                                          "when using pdflatex, because xcolor and ulem are not installed.\n"
1029                                          "Please install both packages or redefine "
1030                                          "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
1031                 }
1032         }
1033
1034         if (!parent() && !params().master.empty()) {
1035                 FileName const master_file = makeAbsPath(params().master,
1036                            onlyPath(absFileName()));
1037                 if (isLyXFileName(master_file.absFileName())) {
1038                         Buffer * master =
1039                                 checkAndLoadLyXFile(master_file, true);
1040                         if (master) {
1041                                 // necessary e.g. after a reload
1042                                 // to re-register the child (bug 5873)
1043                                 // FIXME: clean up updateMacros (here, only
1044                                 // child registering is needed).
1045                                 master->updateMacros();
1046                                 // set master as master buffer, but only
1047                                 // if we are a real child
1048                                 if (master->isChild(this))
1049                                         setParent(master);
1050                                 // if the master is not fully loaded
1051                                 // it is probably just loading this
1052                                 // child. No warning needed then.
1053                                 else if (master->isFullyLoaded())
1054                                         LYXERR0("The master '"
1055                                                 << params().master
1056                                                 << "' assigned to this document ("
1057                                                 << absFileName()
1058                                                 << ") does not include "
1059                                                 "this document. Ignoring the master assignment.");
1060                         }
1061                 }
1062         }
1063
1064         // assure we have a default index
1065         params().indiceslist().addDefault(B_("Index"));
1066
1067         // read main text
1068         if (FileName::isAbsolute(params().origin))
1069                 d->old_position = params().origin;
1070         else
1071                 d->old_position = filePath();
1072         bool const res = text().read(lex, errorList, d->inset);
1073         d->old_position.clear();
1074
1075         // inform parent buffer about local macros
1076         if (parent()) {
1077                 Buffer const * pbuf = parent();
1078                 UserMacroSet::const_iterator cit = usermacros.begin();
1079                 UserMacroSet::const_iterator end = usermacros.end();
1080                 for (; cit != end; ++cit)
1081                         pbuf->usermacros.insert(*cit);
1082         }
1083         usermacros.clear();
1084         updateMacros();
1085         updateMacroInstances(InternalUpdate);
1086         return res;
1087 }
1088
1089
1090 bool Buffer::importString(string const & format, docstring const & contents, ErrorList & errorList)
1091 {
1092         Format const * fmt = theFormats().getFormat(format);
1093         if (!fmt)
1094                 return false;
1095         // It is important to use the correct extension here, since some
1096         // converters create a wrong output file otherwise (e.g. html2latex)
1097         TempFile const tempfile("Buffer_importStringXXXXXX." + fmt->extension());
1098         FileName const name(tempfile.name());
1099         ofdocstream os(name.toFilesystemEncoding().c_str());
1100         // Do not convert os implicitly to bool, since that is forbidden in C++11.
1101         bool const success = !(os << contents).fail();
1102         os.close();
1103
1104         bool converted = false;
1105         if (success) {
1106                 params().compressed = false;
1107
1108                 // remove dummy empty par
1109                 paragraphs().clear();
1110
1111                 converted = importFile(format, name, errorList);
1112         }
1113
1114         if (name.exists())
1115                 name.removeFile();
1116         return converted;
1117 }
1118
1119
1120 bool Buffer::importFile(string const & format, FileName const & name, ErrorList & errorList)
1121 {
1122         if (!theConverters().isReachable(format, "lyx"))
1123                 return false;
1124
1125         TempFile const tempfile("Buffer_importFileXXXXXX.lyx");
1126         FileName const lyx(tempfile.name());
1127         if (theConverters().convert(0, name, lyx, name, format, "lyx", errorList))
1128                 return readFile(lyx) == ReadSuccess;
1129
1130         return false;
1131 }
1132
1133
1134 bool Buffer::readString(string const & s)
1135 {
1136         params().compressed = false;
1137
1138         Lexer lex;
1139         istringstream is(s);
1140         lex.setStream(is);
1141         TempFile tempfile("Buffer_readStringXXXXXX.lyx");
1142         FileName const fn = tempfile.name();
1143
1144         int file_format;
1145         bool success = parseLyXFormat(lex, fn, file_format) == ReadSuccess;
1146
1147         if (success && file_format != LYX_FORMAT) {
1148                 // We need to call lyx2lyx, so write the input to a file
1149                 ofstream os(fn.toFilesystemEncoding().c_str());
1150                 os << s;
1151                 os.close();
1152                 // lyxvc in readFile
1153                 if (readFile(fn) != ReadSuccess)
1154                         success = false;
1155         }
1156         else if (success)
1157                 if (readDocument(lex))
1158                         success = false;
1159         return success;
1160 }
1161
1162
1163 Buffer::ReadStatus Buffer::readFile(FileName const & fn)
1164 {
1165         FileName fname(fn);
1166         Lexer lex;
1167         if (!lex.setFile(fname)) {
1168                 Alert::error(_("File Not Found"),
1169                         bformat(_("Unable to open file `%1$s'."),
1170                                 from_utf8(fn.absFileName())));
1171                 return ReadFileNotFound;
1172         }
1173
1174         int file_format;
1175         ReadStatus const ret_plf = parseLyXFormat(lex, fn, file_format);
1176         if (ret_plf != ReadSuccess)
1177                 return ret_plf;
1178
1179         if (file_format != LYX_FORMAT) {
1180                 FileName tmpFile;
1181                 ReadStatus ret_clf = convertLyXFormat(fn, tmpFile, file_format);
1182                 if (ret_clf != ReadSuccess)
1183                         return ret_clf;
1184                 ret_clf = readFile(tmpFile);
1185                 if (ret_clf == ReadSuccess) {
1186                         d->file_format = file_format;
1187                         d->need_format_backup = true;
1188                 }
1189                 return ret_clf;
1190         }
1191
1192         // FIXME: InsetInfo needs to know whether the file is under VCS
1193         // during the parse process, so this has to be done before.
1194         lyxvc().file_found_hook(d->filename);
1195
1196         if (readDocument(lex)) {
1197                 Alert::error(_("Document format failure"),
1198                         bformat(_("%1$s ended unexpectedly, which means"
1199                                 " that it is probably corrupted."),
1200                                         from_utf8(fn.absFileName())));
1201                 return ReadDocumentFailure;
1202         }
1203
1204         d->file_fully_loaded = true;
1205         d->read_only = !d->filename.isWritable();
1206         params().compressed = theFormats().isZippedFile(d->filename);
1207         saveCheckSum();
1208         return ReadSuccess;
1209 }
1210
1211
1212 bool Buffer::isFullyLoaded() const
1213 {
1214         return d->file_fully_loaded;
1215 }
1216
1217
1218 void Buffer::setFullyLoaded(bool value)
1219 {
1220         d->file_fully_loaded = value;
1221 }
1222
1223
1224 bool Buffer::lastPreviewError() const
1225 {
1226         return d->preview_error_;
1227 }
1228
1229
1230 PreviewLoader * Buffer::loader() const
1231 {
1232         if (!isExporting() && lyxrc.preview == LyXRC::PREVIEW_OFF)
1233                 return 0;
1234         if (!d->preview_loader_)
1235                 d->preview_loader_ = new PreviewLoader(*this);
1236         return d->preview_loader_;
1237 }
1238
1239
1240 void Buffer::removePreviews() const
1241 {
1242         delete d->preview_loader_;
1243         d->preview_loader_ = 0;
1244 }
1245
1246
1247 void Buffer::updatePreviews() const
1248 {
1249         PreviewLoader * ploader = loader();
1250         if (!ploader)
1251                 return;
1252
1253         InsetIterator it = inset_iterator_begin(*d->inset);
1254         InsetIterator const end = inset_iterator_end(*d->inset);
1255         for (; it != end; ++it)
1256                 it->addPreview(it, *ploader);
1257
1258         ploader->startLoading();
1259 }
1260
1261
1262 Buffer::ReadStatus Buffer::parseLyXFormat(Lexer & lex,
1263         FileName const & fn, int & file_format) const
1264 {
1265         if(!lex.checkFor("\\lyxformat")) {
1266                 Alert::error(_("Document format failure"),
1267                         bformat(_("%1$s is not a readable LyX document."),
1268                                 from_utf8(fn.absFileName())));
1269                 return ReadNoLyXFormat;
1270         }
1271
1272         string tmp_format;
1273         lex >> tmp_format;
1274
1275         // LyX formats 217 and earlier were written as 2.17. This corresponds
1276         // to files from LyX versions < 1.1.6.3. We just remove the dot in
1277         // these cases. See also: www.lyx.org/trac/changeset/1313.
1278         size_t dot = tmp_format.find_first_of(".,");
1279         if (dot != string::npos)
1280                 tmp_format.erase(dot, 1);
1281
1282         file_format = convert<int>(tmp_format);
1283         return ReadSuccess;
1284 }
1285
1286
1287 Buffer::ReadStatus Buffer::convertLyXFormat(FileName const & fn,
1288         FileName & tmpfile, int from_format)
1289 {
1290         TempFile tempfile("Buffer_convertLyXFormatXXXXXX.lyx");
1291         tempfile.setAutoRemove(false);
1292         tmpfile = tempfile.name();
1293         if(tmpfile.empty()) {
1294                 Alert::error(_("Conversion failed"),
1295                         bformat(_("%1$s is from a different"
1296                                 " version of LyX, but a temporary"
1297                                 " file for converting it could"
1298                                 " not be created."),
1299                                 from_utf8(fn.absFileName())));
1300                 return LyX2LyXNoTempFile;
1301         }
1302
1303         FileName const lyx2lyx = libFileSearch("lyx2lyx", "lyx2lyx");
1304         if (lyx2lyx.empty()) {
1305                 Alert::error(_("Conversion script not found"),
1306                      bformat(_("%1$s is from a different"
1307                                " version of LyX, but the"
1308                                " conversion script lyx2lyx"
1309                                " could not be found."),
1310                                from_utf8(fn.absFileName())));
1311                 return LyX2LyXNotFound;
1312         }
1313
1314         // Run lyx2lyx:
1315         //   $python$ "$lyx2lyx$" -t $LYX_FORMAT$ -o "$tempfile$" "$filetoread$"
1316         ostringstream command;
1317         command << os::python()
1318                 << ' ' << quoteName(lyx2lyx.toFilesystemEncoding())
1319                 << " -t " << convert<string>(LYX_FORMAT)
1320                 << " -o " << quoteName(tmpfile.toSafeFilesystemEncoding())
1321                 << ' ' << quoteName(fn.toSafeFilesystemEncoding());
1322         string const command_str = command.str();
1323
1324         LYXERR(Debug::INFO, "Running '" << command_str << '\'');
1325
1326         cmd_ret const ret = runCommand(command_str);
1327         if (ret.first != 0) {
1328                 if (from_format < LYX_FORMAT) {
1329                         Alert::error(_("Conversion script failed"),
1330                                 bformat(_("%1$s is from an older version"
1331                                         " of LyX and the lyx2lyx script"
1332                                         " failed to convert it."),
1333                                         from_utf8(fn.absFileName())));
1334                         return LyX2LyXOlderFormat;
1335                 } else {
1336                         Alert::error(_("Conversion script failed"),
1337                                 bformat(_("%1$s is from a newer version"
1338                                         " of LyX and the lyx2lyx script"
1339                                         " failed to convert it."),
1340                                         from_utf8(fn.absFileName())));
1341                         return LyX2LyXNewerFormat;
1342                 }
1343         }
1344         return ReadSuccess;
1345 }
1346
1347
1348 FileName Buffer::getBackupName() const {
1349         FileName const & fn = fileName();
1350         string const fname = fn.onlyFileNameWithoutExt();
1351         string const fext  = fn.extension() + "~";
1352         string const fpath = lyxrc.backupdir_path.empty() ?
1353                 fn.onlyPath().absFileName() :
1354                 lyxrc.backupdir_path;
1355         string const fform = convert<string>(d->file_format);
1356         string const backname = fname + "-lyxformat-" + fform;
1357         FileName backup(addName(fpath, addExtension(backname, fext)));
1358
1359         // limit recursion, just in case
1360         int v = 1;
1361         unsigned long orig_checksum = 0;
1362         while (backup.exists() && v < 100) {
1363                 if (orig_checksum == 0)
1364                         orig_checksum = fn.checksum();
1365                 unsigned long new_checksum = backup.checksum();
1366                 if (orig_checksum == new_checksum) {
1367                         LYXERR(Debug::FILES, "Not backing up " << fn <<
1368                                "since " << backup << "has the same checksum.");
1369                         // a bit of a hack, but we have to check this anyway
1370                         // below, and setting this is simpler than introducing
1371                         // a special boolean for this purpose.
1372                         v = 1000;
1373                         break;
1374                 }
1375                 string const newbackname = backname + "-" + convert<string>(v);
1376                 backup.set(addName(fpath, addExtension(newbackname, fext)));
1377                 v++;
1378         }
1379         return v < 100 ? backup : FileName();
1380 }
1381
1382
1383 // Should probably be moved to somewhere else: BufferView? GuiView?
1384 bool Buffer::save() const
1385 {
1386         FileMonitorBlocker block = d->blockFileMonitor();
1387         docstring const file = makeDisplayPath(absFileName(), 20);
1388         d->filename.refresh();
1389
1390         // check the read-only status before moving the file as a backup
1391         if (d->filename.exists()) {
1392                 bool const read_only = !d->filename.isWritable();
1393                 if (read_only) {
1394                         Alert::warning(_("File is read-only"),
1395                                 bformat(_("The file %1$s cannot be written because it "
1396                                 "is marked as read-only."), file));
1397                         return false;
1398                 }
1399         }
1400
1401         // ask if the disk file has been externally modified (use checksum method)
1402         if (fileName().exists() && isChecksumModified()) {
1403                 docstring text =
1404                         bformat(_("Document %1$s has been externally modified. "
1405                                 "Are you sure you want to overwrite this file?"), file);
1406                 int const ret = Alert::prompt(_("Overwrite modified file?"),
1407                         text, 1, 1, _("&Overwrite"), _("&Cancel"));
1408                 if (ret == 1)
1409                         return false;
1410         }
1411
1412         // We don't need autosaves in the immediate future. (Asger)
1413         resetAutosaveTimers();
1414
1415         // if the file does not yet exist, none of the backup activity
1416         // that follows is necessary
1417         if (!fileName().exists()) {
1418                 if (!writeFile(fileName()))
1419                         return false;
1420                 markClean();
1421                 return true;
1422         }
1423
1424         // we first write the file to a new name, then move it to its
1425         // proper location once that has been done successfully. that
1426         // way we preserve the original file if something goes wrong.
1427         string const justname = fileName().onlyFileNameWithoutExt();
1428         auto tempfile = make_unique<TempFile>(fileName().onlyPath(),
1429                                               justname + "-XXXXXX.lyx");
1430         bool const symlink = fileName().isSymLink();
1431         if (!symlink)
1432                 tempfile->setAutoRemove(false);
1433
1434         FileName savefile(tempfile->name());
1435         LYXERR(Debug::FILES, "Saving to " << savefile.absFileName());
1436         if (!writeFile(savefile))
1437                 return false;
1438
1439         // we will set this to false if we fail
1440         bool made_backup = true;
1441
1442         FileName backupName;
1443         bool const needBackup = lyxrc.make_backup || d->need_format_backup;
1444         if (needBackup) {
1445                 if (d->need_format_backup)
1446                         backupName = getBackupName();
1447
1448                 // If we for some reason failed to find a backup name in case of
1449                 // a format change, this will still set one. It's the best we can
1450                 // do in this case.
1451                 if (backupName.empty()) {
1452                         backupName.set(fileName().absFileName() + "~");
1453                         if (!lyxrc.backupdir_path.empty()) {
1454                                 string const mangledName =
1455                                         subst(subst(backupName.absFileName(), '/', '!'), ':', '!');
1456                                 backupName.set(addName(lyxrc.backupdir_path, mangledName));
1457                         }
1458                 }
1459
1460                 LYXERR(Debug::FILES, "Backing up original file to " <<
1461                                 backupName.absFileName());
1462                 // Except file is symlink do not copy because of #6587.
1463                 // Hard links have bad luck.
1464                 made_backup = symlink ?
1465                         fileName().copyTo(backupName):
1466                         fileName().moveTo(backupName);
1467
1468                 if (!made_backup) {
1469                         Alert::error(_("Backup failure"),
1470                                      bformat(_("Cannot create backup file %1$s.\n"
1471                                                "Please check whether the directory exists and is writable."),
1472                                              from_utf8(backupName.absFileName())));
1473                         //LYXERR(Debug::DEBUG, "Fs error: " << fe.what());
1474                 } else if (d->need_format_backup) {
1475                         // the original file has been backed up successfully, so we
1476                         // will not need to do that again
1477                         d->need_format_backup = false;
1478                 }
1479         }
1480
1481         // Destroy tempfile since it keeps the file locked on windows (bug 9234)
1482         // Only do this if tempfile is not in autoremove mode
1483         if (!symlink)
1484                 tempfile.reset();
1485         // If we have no symlink, we can simply rename the temp file.
1486         // Otherwise, we need to copy it so the symlink stays intact.
1487         if (made_backup && symlink ? savefile.copyTo(fileName(), true) :
1488                                            savefile.moveTo(fileName()))
1489         {
1490                 // saveCheckSum() was already called by writeFile(), but the
1491                 // time stamp is invalidated by copying/moving
1492                 saveCheckSum();
1493                 markClean();
1494                 if (d->file_format != LYX_FORMAT)
1495                         // the file associated with this buffer is now in the current format
1496                         d->file_format = LYX_FORMAT;
1497                 return true;
1498         }
1499         // else we saved the file, but failed to move it to the right location.
1500
1501         if (needBackup && made_backup && !symlink) {
1502                 // the original file was moved to some new location, so it will look
1503                 // to the user as if it was deleted. (see bug #9234.) we could try
1504                 // to restore it, but that would basically mean trying to do again
1505                 // what we just failed to do. better to leave things as they are.
1506                 Alert::error(_("Write failure"),
1507                              bformat(_("The file has successfully been saved as:\n  %1$s.\n"
1508                                        "But LyX could not move it to:\n  %2$s.\n"
1509                                        "Your original file has been backed up to:\n  %3$s"),
1510                                      from_utf8(savefile.absFileName()),
1511                                      from_utf8(fileName().absFileName()),
1512                                      from_utf8(backupName.absFileName())));
1513         } else {
1514                 // either we did not try to make a backup, or else we tried and failed,
1515                 // or else the original file was a symlink, in which case it was copied,
1516                 // not moved. so the original file is intact.
1517                 Alert::error(_("Write failure"),
1518                              bformat(_("Cannot move saved file to:\n  %1$s.\n"
1519                                        "But the file has successfully been saved as:\n  %2$s."),
1520                                      from_utf8(fileName().absFileName()),
1521                          from_utf8(savefile.absFileName())));
1522         }
1523         return false;
1524 }
1525
1526
1527 bool Buffer::writeFile(FileName const & fname) const
1528 {
1529         if (d->read_only && fname == d->filename)
1530                 return false;
1531
1532         bool retval = false;
1533
1534         docstring const str = bformat(_("Saving document %1$s..."),
1535                 makeDisplayPath(fname.absFileName()));
1536         message(str);
1537
1538         string const encoded_fname = fname.toSafeFilesystemEncoding(os::CREATE);
1539
1540         if (params().compressed) {
1541                 gz::ogzstream ofs(encoded_fname.c_str(), ios::out|ios::trunc);
1542                 retval = ofs && write(ofs);
1543         } else {
1544                 ofstream ofs(encoded_fname.c_str(), ios::out|ios::trunc);
1545                 retval = ofs && write(ofs);
1546         }
1547
1548         if (!retval) {
1549                 message(str + _(" could not write file!"));
1550                 return false;
1551         }
1552
1553         // see bug 6587
1554         // removeAutosaveFile();
1555
1556         saveCheckSum();
1557         message(str + _(" done."));
1558
1559         return true;
1560 }
1561
1562
1563 docstring Buffer::emergencyWrite()
1564 {
1565         // No need to save if the buffer has not changed.
1566         if (isClean())
1567                 return docstring();
1568
1569         string const doc = isUnnamed() ? onlyFileName(absFileName()) : absFileName();
1570
1571         docstring user_message = bformat(
1572                 _("LyX: Attempting to save document %1$s\n"), from_utf8(doc));
1573
1574         // We try to save three places:
1575         // 1) Same place as document. Unless it is an unnamed doc.
1576         if (!isUnnamed()) {
1577                 string s = absFileName();
1578                 s += ".emergency";
1579                 LYXERR0("  " << s);
1580                 if (writeFile(FileName(s))) {
1581                         markClean();
1582                         user_message += "  " + bformat(_("Saved to %1$s. Phew.\n"), from_utf8(s));
1583                         return user_message;
1584                 } else {
1585                         user_message += "  " + _("Save failed! Trying again...\n");
1586                 }
1587         }
1588
1589         // 2) In HOME directory.
1590         string s = addName(Package::get_home_dir().absFileName(), absFileName());
1591         s += ".emergency";
1592         lyxerr << ' ' << s << endl;
1593         if (writeFile(FileName(s))) {
1594                 markClean();
1595                 user_message += "  " + bformat(_("Saved to %1$s. Phew.\n"), from_utf8(s));
1596                 return user_message;
1597         }
1598
1599         user_message += "  " + _("Save failed! Trying yet again...\n");
1600
1601         // 3) In "/tmp" directory.
1602         // MakeAbsPath to prepend the current
1603         // drive letter on OS/2
1604         s = addName(package().temp_dir().absFileName(), absFileName());
1605         s += ".emergency";
1606         lyxerr << ' ' << s << endl;
1607         if (writeFile(FileName(s))) {
1608                 markClean();
1609                 user_message += "  " + bformat(_("Saved to %1$s. Phew.\n"), from_utf8(s));
1610                 return user_message;
1611         }
1612
1613         user_message += "  " + _("Save failed! Document is lost.");
1614         // Don't try again.
1615         markClean();
1616         return user_message;
1617 }
1618
1619
1620 bool Buffer::write(ostream & ofs) const
1621 {
1622 #ifdef HAVE_LOCALE
1623         // Use the standard "C" locale for file output.
1624         ofs.imbue(locale::classic());
1625 #endif
1626
1627         // The top of the file should not be written by params().
1628
1629         // write out a comment in the top of the file
1630         // Important: Keep the version formatting in sync with lyx2lyx and
1631         //            tex2lyx (bug 7951)
1632         ofs << "#LyX " << lyx_version_major << "." << lyx_version_minor
1633             << " created this file. For more info see http://www.lyx.org/\n"
1634             << "\\lyxformat " << LYX_FORMAT << "\n"
1635             << "\\begin_document\n";
1636
1637         /// For each author, set 'used' to true if there is a change
1638         /// by this author in the document; otherwise set it to 'false'.
1639         AuthorList::Authors::const_iterator a_it = params().authors().begin();
1640         AuthorList::Authors::const_iterator a_end = params().authors().end();
1641         for (; a_it != a_end; ++a_it)
1642                 a_it->setUsed(false);
1643
1644         ParIterator const end = const_cast<Buffer *>(this)->par_iterator_end();
1645         ParIterator it = const_cast<Buffer *>(this)->par_iterator_begin();
1646         for ( ; it != end; ++it)
1647                 it->checkAuthors(params().authors());
1648
1649         // now write out the buffer parameters.
1650         ofs << "\\begin_header\n";
1651         params().writeFile(ofs, this);
1652         ofs << "\\end_header\n";
1653
1654         // write the text
1655         ofs << "\n\\begin_body\n";
1656         text().write(ofs);
1657         ofs << "\n\\end_body\n";
1658
1659         // Write marker that shows file is complete
1660         ofs << "\\end_document" << endl;
1661
1662         // Shouldn't really be needed....
1663         //ofs.close();
1664
1665         // how to check if close went ok?
1666         // Following is an attempt... (BE 20001011)
1667
1668         // good() returns false if any error occurred, including some
1669         //        formatting error.
1670         // bad()  returns true if something bad happened in the buffer,
1671         //        which should include file system full errors.
1672
1673         bool status = true;
1674         if (!ofs) {
1675                 status = false;
1676                 lyxerr << "File was not closed properly." << endl;
1677         }
1678
1679         return status;
1680 }
1681
1682
1683 bool Buffer::makeLaTeXFile(FileName const & fname,
1684                            string const & original_path,
1685                            OutputParams const & runparams_in,
1686                            OutputWhat output) const
1687 {
1688         OutputParams runparams = runparams_in;
1689
1690         // XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740),
1691         // Check here, because the "flavor" is not known in BufferParams::encoding()
1692         // (power users can override this safety measure selecting "utf8-plain").
1693         if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
1694             && (runparams.encoding->name() != "utf8-plain"))
1695                 runparams.encoding = encodings.fromLyXName("ascii");
1696
1697         string const encoding = runparams.encoding->iconvName();
1698         LYXERR(Debug::LATEX, "makeLaTeXFile encoding: " << encoding << ", fname=" << fname.realPath());
1699
1700         ofdocstream ofs;
1701         try { ofs.reset(encoding); }
1702         catch (iconv_codecvt_facet_exception const & e) {
1703                 lyxerr << "Caught iconv exception: " << e.what() << endl;
1704                 Alert::error(_("Iconv software exception Detected"), bformat(_("Please "
1705                         "verify that the support software for your encoding (%1$s) is "
1706                         "properly installed"), from_ascii(encoding)));
1707                 return false;
1708         }
1709         if (!openFileWrite(ofs, fname))
1710                 return false;
1711
1712         ErrorList & errorList = d->errorLists["Export"];
1713         errorList.clear();
1714         bool failed_export = false;
1715         otexstream os(ofs);
1716
1717         // make sure we are ready to export
1718         // this needs to be done before we validate
1719         // FIXME Do we need to do this all the time? I.e., in children
1720         // of a master we are exporting?
1721         updateBuffer();
1722         updateMacroInstances(OutputUpdate);
1723
1724         try {
1725                 writeLaTeXSource(os, original_path, runparams, output);
1726         }
1727         catch (EncodingException const & e) {
1728                 docstring const failed(1, e.failed_char);
1729                 ostringstream oss;
1730                 oss << "0x" << hex << e.failed_char << dec;
1731                 docstring msg = bformat(_("Could not find LaTeX command for character '%1$s'"
1732                                           " (code point %2$s)"),
1733                                           failed, from_utf8(oss.str()));
1734                 errorList.push_back(ErrorItem(msg, _("Some characters of your document are probably not "
1735                                 "representable in the chosen encoding.\n"
1736                                 "Changing the document encoding to utf8 could help."),
1737                                               {e.par_id, e.pos}, {e.par_id, e.pos + 1}));
1738                 failed_export = true;
1739         }
1740         catch (iconv_codecvt_facet_exception const & e) {
1741                 errorList.push_back(ErrorItem(_("iconv conversion failed"),
1742                                               _(e.what())));
1743                 failed_export = true;
1744         }
1745         catch (exception const & e) {
1746                 errorList.push_back(ErrorItem(_("conversion failed"),
1747                                               _(e.what())));
1748                 lyxerr << e.what() << endl;
1749                 failed_export = true;
1750         }
1751         catch (...) {
1752                 lyxerr << "Caught some really weird exception..." << endl;
1753                 lyx_exit(1);
1754         }
1755
1756         d->texrow = move(os.texrow());
1757
1758         ofs.close();
1759         if (ofs.fail()) {
1760                 failed_export = true;
1761                 lyxerr << "File '" << fname << "' was not closed properly." << endl;
1762         }
1763
1764         if (runparams_in.silent)
1765                 errorList.clear();
1766         else
1767                 errors("Export");
1768         return !failed_export;
1769 }
1770
1771
1772 void Buffer::writeLaTeXSource(otexstream & os,
1773                            string const & original_path,
1774                            OutputParams const & runparams_in,
1775                            OutputWhat output) const
1776 {
1777         // The child documents, if any, shall be already loaded at this point.
1778
1779         OutputParams runparams = runparams_in;
1780
1781         // XeTeX with TeX fonts is only safe with ASCII encoding,
1782         // Check here, because the "flavor" is not known in BufferParams::encoding()
1783         // (power users can override this safety measure selecting "utf8-plain").
1784         if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
1785             && (runparams.encoding->name() != "utf8-plain"))
1786                 runparams.encoding = encodings.fromLyXName("ascii");
1787         // FIXME: when only the current paragraph is shown, this is ignored
1788         //        (or not reached) and characters encodable in the current
1789         //        encoding are not converted to ASCII-representation.
1790
1791         // If we are compiling a file standalone, even if this is the
1792         // child of some other buffer, let's cut the link here, so the
1793         // file is really independent and no concurring settings from
1794         // the master (e.g. branch state) interfere (see #8100).
1795         if (!runparams.is_child)
1796                 d->ignore_parent = true;
1797
1798         // Classify the unicode characters appearing in math insets
1799         BufferEncodings::initUnicodeMath(*this);
1800
1801         // validate the buffer.
1802         LYXERR(Debug::LATEX, "  Validating buffer...");
1803         LaTeXFeatures features(*this, params(), runparams);
1804         validate(features);
1805         // This is only set once per document (in master)
1806         if (!runparams.is_child)
1807                 runparams.use_polyglossia = features.usePolyglossia();
1808         LYXERR(Debug::LATEX, "  Buffer validation done.");
1809
1810         bool const output_preamble =
1811                 output == FullSource || output == OnlyPreamble;
1812         bool const output_body =
1813                 output == FullSource || output == OnlyBody;
1814
1815         // The starting paragraph of the coming rows is the
1816         // first paragraph of the document. (Asger)
1817         if (output_preamble && runparams.nice) {
1818                 os << "%% LyX " << lyx_version << " created this file.  "
1819                         "For more info, see http://www.lyx.org/.\n"
1820                         "%% Do not edit unless you really know what "
1821                         "you are doing.\n";
1822         }
1823         LYXERR(Debug::INFO, "lyx document header finished");
1824
1825         // There are a few differences between nice LaTeX and usual files:
1826         // usual files have \batchmode and special input@path to allow
1827         // inclusion of figures specified by an explicitly relative path
1828         // (i.e., a path starting with './' or '../') with either \input or
1829         // \includegraphics, as the TEXINPUTS method doesn't work in this case.
1830         // input@path is set when the actual parameter original_path is set.
1831         // This is done for usual tex-file, but not for nice-latex-file.
1832         // (Matthias 250696)
1833         // Note that input@path is only needed for something the user does
1834         // in the preamble, included .tex files or ERT, files included by
1835         // LyX work without it.
1836         if (output_preamble) {
1837                 if (!runparams.nice) {
1838                         // code for usual, NOT nice-latex-file
1839                         os << "\\batchmode\n"; // changed from \nonstopmode
1840                 }
1841                 if (!original_path.empty()) {
1842                         // FIXME UNICODE
1843                         // We don't know the encoding of inputpath
1844                         docstring const inputpath = from_utf8(original_path);
1845                         docstring uncodable_glyphs;
1846                         Encoding const * const enc = runparams.encoding;
1847                         if (enc) {
1848                                 for (size_t n = 0; n < inputpath.size(); ++n) {
1849                                         if (!enc->encodable(inputpath[n])) {
1850                                                 docstring const glyph(1, inputpath[n]);
1851                                                 LYXERR0("Uncodable character '"
1852                                                         << glyph
1853                                                         << "' in input path!");
1854                                                 uncodable_glyphs += glyph;
1855                                         }
1856                                 }
1857                         }
1858
1859                         // warn user if we found uncodable glyphs.
1860                         if (!uncodable_glyphs.empty()) {
1861                                 frontend::Alert::warning(
1862                                         _("Uncodable character in file path"),
1863                                         support::bformat(
1864                                           _("The path of your document\n"
1865                                             "(%1$s)\n"
1866                                             "contains glyphs that are unknown "
1867                                             "in the current document encoding "
1868                                             "(namely %2$s). This may result in "
1869                                             "incomplete output, unless "
1870                                             "TEXINPUTS contains the document "
1871                                             "directory and you don't use "
1872                                             "explicitly relative paths (i.e., "
1873                                             "paths starting with './' or "
1874                                             "'../') in the preamble or in ERT."
1875                                             "\n\nIn case of problems, choose "
1876                                             "an appropriate document encoding\n"
1877                                             "(such as utf8) or change the "
1878                                             "file path name."),
1879                                           inputpath, uncodable_glyphs));
1880                         } else {
1881                                 string docdir =
1882                                         support::latex_path(original_path);
1883                                 if (contains(docdir, '#')) {
1884                                         docdir = subst(docdir, "#", "\\#");
1885                                         os << "\\catcode`\\#=11"
1886                                               "\\def\\#{#}\\catcode`\\#=6\n";
1887                                 }
1888                                 if (contains(docdir, '%')) {
1889                                         docdir = subst(docdir, "%", "\\%");
1890                                         os << "\\catcode`\\%=11"
1891                                               "\\def\\%{%}\\catcode`\\%=14\n";
1892                                 }
1893                                 os << "\\makeatletter\n"
1894                                    << "\\def\\input@path{{"
1895                                    << docdir << "}}\n"
1896                                    << "\\makeatother\n";
1897                         }
1898                 }
1899
1900                 // get parent macros (if this buffer has a parent) which will be
1901                 // written at the document begin further down.
1902                 MacroSet parentMacros;
1903                 listParentMacros(parentMacros, features);
1904
1905                 // Write the preamble
1906                 runparams.use_babel = params().writeLaTeX(os, features,
1907                                                           d->filename.onlyPath());
1908
1909                 // Biblatex bibliographies are loaded here
1910                 if (params().useBiblatex()) {
1911                         vector<docstring> const bibfiles =
1912                                 prepareBibFilePaths(runparams, getBibfilesCache(), true);
1913                         for (docstring const & file: bibfiles)
1914                                 os << "\\addbibresource{" << file << "}\n";
1915                 }
1916
1917                 if (!runparams.dryrun && features.hasPolyglossiaExclusiveLanguages()
1918                     && !features.hasOnlyPolyglossiaLanguages()) {
1919                         docstring blangs;
1920                         docstring plangs;
1921                         vector<string> bll = features.getBabelExclusiveLanguages();
1922                         vector<string> pll = features.getPolyglossiaExclusiveLanguages();
1923                         if (!bll.empty()) {
1924                                 docstring langs;
1925                                 for (vector<string>::const_iterator it = bll.begin(); it != bll.end(); ++it) {
1926                                         if (!langs.empty())
1927                                                 langs += ", ";
1928                                         langs += _(*it);
1929                                 }
1930                                 blangs = bll.size() > 1 ?
1931                                             support::bformat(_("The languages %1$s are only supported by Babel."), langs)
1932                                           : support::bformat(_("The language %1$s is only supported by Babel."), langs);
1933                         }
1934                         if (!pll.empty()) {
1935                                 docstring langs;
1936                                 for (vector<string>::const_iterator it = pll.begin(); it != pll.end(); ++it) {
1937                                         if (!langs.empty())
1938                                                 langs += ", ";
1939                                         langs += _(*it);
1940                                 }
1941                                 plangs = pll.size() > 1 ?
1942                                             support::bformat(_("The languages %1$s are only supported by Polyglossia."), langs)
1943                                           : support::bformat(_("The language %1$s is only supported by Polyglossia."), langs);
1944                                 if (!blangs.empty())
1945                                         plangs += "\n";
1946                         }
1947
1948                         frontend::Alert::warning(
1949                                 _("Incompatible Languages!"),
1950                                 support::bformat(
1951                                   _("You cannot use the following languages "
1952                                     "together in one LaTeX document because "
1953                                     "they require conflicting language packages:\n"
1954                                     "%1$s%2$s"),
1955                                   plangs, blangs));
1956                 }
1957
1958                 // Japanese might be required only in some children of a document,
1959                 // but once required, we must keep use_japanese true.
1960                 runparams.use_japanese |= features.isRequired("japanese");
1961
1962                 if (!output_body) {
1963                         // Restore the parenthood if needed
1964                         if (!runparams.is_child)
1965                                 d->ignore_parent = false;
1966                         return;
1967                 }
1968
1969                 // make the body.
1970                 // mark the beginning of the body to separate it from InPreamble insets
1971                 os.texrow().start(TexRow::beginDocument());
1972                 os << "\\begin{document}\n";
1973
1974                 // mark the start of a new paragraph by simulating a newline,
1975                 // so that os.afterParbreak() returns true at document start
1976                 os.lastChar('\n');
1977
1978                 // output the parent macros
1979                 MacroSet::iterator it = parentMacros.begin();
1980                 MacroSet::iterator end = parentMacros.end();
1981                 for (; it != end; ++it) {
1982                         int num_lines = (*it)->write(os.os(), true);
1983                         os.texrow().newlines(num_lines);
1984                 }
1985
1986         } // output_preamble
1987
1988         LYXERR(Debug::INFO, "preamble finished, now the body.");
1989
1990         // the real stuff
1991         latexParagraphs(*this, text(), os, runparams);
1992
1993         // Restore the parenthood if needed
1994         if (!runparams.is_child)
1995                 d->ignore_parent = false;
1996
1997         // add this just in case after all the paragraphs
1998         os << endl;
1999
2000         if (output_preamble) {
2001                 os << "\\end{document}\n";
2002                 LYXERR(Debug::LATEX, "makeLaTeXFile...done");
2003         } else {
2004                 LYXERR(Debug::LATEX, "LaTeXFile for inclusion made.");
2005         }
2006         runparams_in.encoding = runparams.encoding;
2007
2008         LYXERR(Debug::INFO, "Finished making LaTeX file.");
2009         LYXERR(Debug::INFO, "Row count was " << os.texrow().rows() - 1 << '.');
2010 }
2011
2012
2013 void Buffer::makeDocBookFile(FileName const & fname,
2014                               OutputParams const & runparams,
2015                               OutputWhat output) const
2016 {
2017         LYXERR(Debug::LATEX, "makeDocBookFile...");
2018
2019         ofdocstream ofs;
2020         if (!openFileWrite(ofs, fname))
2021                 return;
2022
2023         // make sure we are ready to export
2024         // this needs to be done before we validate
2025         updateBuffer();
2026         updateMacroInstances(OutputUpdate);
2027
2028         writeDocBookSource(ofs, fname.absFileName(), runparams, output);
2029
2030         ofs.close();
2031         if (ofs.fail())
2032                 lyxerr << "File '" << fname << "' was not closed properly." << endl;
2033 }
2034
2035
2036 void Buffer::writeDocBookSource(odocstream & os, string const & fname,
2037                              OutputParams const & runparams,
2038                              OutputWhat output) const
2039 {
2040         LaTeXFeatures features(*this, params(), runparams);
2041         validate(features);
2042
2043         d->texrow.reset();
2044
2045         DocumentClass const & tclass = params().documentClass();
2046         string const & top_element = tclass.latexname();
2047
2048         bool const output_preamble =
2049                 output == FullSource || output == OnlyPreamble;
2050         bool const output_body =
2051           output == FullSource || output == OnlyBody;
2052
2053         if (output_preamble) {
2054                 if (runparams.flavor == OutputParams::XML)
2055                         os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
2056
2057                 // FIXME UNICODE
2058                 os << "<!DOCTYPE " << from_ascii(top_element) << ' ';
2059
2060                 // FIXME UNICODE
2061                 if (! tclass.class_header().empty())
2062                         os << from_ascii(tclass.class_header());
2063                 else if (runparams.flavor == OutputParams::XML)
2064                         os << "PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\" "
2065                             << "\"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\"";
2066                 else
2067                         os << " PUBLIC \"-//OASIS//DTD DocBook V4.2//EN\"";
2068
2069                 docstring preamble = params().preamble;
2070                 if (runparams.flavor != OutputParams::XML ) {
2071                         preamble += "<!ENTITY % output.print.png \"IGNORE\">\n";
2072                         preamble += "<!ENTITY % output.print.pdf \"IGNORE\">\n";
2073                         preamble += "<!ENTITY % output.print.eps \"IGNORE\">\n";
2074                         preamble += "<!ENTITY % output.print.bmp \"IGNORE\">\n";
2075                 }
2076
2077                 string const name = runparams.nice
2078                         ? changeExtension(absFileName(), ".sgml") : fname;
2079                 preamble += features.getIncludedFiles(name);
2080                 preamble += features.getLyXSGMLEntities();
2081
2082                 if (!preamble.empty()) {
2083                         os << "\n [ " << preamble << " ]";
2084                 }
2085                 os << ">\n\n";
2086         }
2087
2088         if (output_body) {
2089                 string top = top_element;
2090                 top += " lang=\"";
2091                 if (runparams.flavor == OutputParams::XML)
2092                         top += params().language->code();
2093                 else
2094                         top += params().language->code().substr(0, 2);
2095                 top += '"';
2096
2097                 if (!params().options.empty()) {
2098                         top += ' ';
2099                         top += params().options;
2100                 }
2101
2102                 os << "<!-- " << ((runparams.flavor == OutputParams::XML)? "XML" : "SGML")
2103                                 << " file was created by LyX " << lyx_version
2104                                 << "\n  See http://www.lyx.org/ for more information -->\n";
2105
2106                 params().documentClass().counters().reset();
2107
2108                 sgml::openTag(os, top);
2109                 os << '\n';
2110                 docbookParagraphs(text(), *this, os, runparams);
2111                 sgml::closeTag(os, top_element);
2112         }
2113 }
2114
2115
2116 void Buffer::makeLyXHTMLFile(FileName const & fname,
2117                               OutputParams const & runparams) const
2118 {
2119         LYXERR(Debug::LATEX, "makeLyXHTMLFile...");
2120
2121         ofdocstream ofs;
2122         if (!openFileWrite(ofs, fname))
2123                 return;
2124
2125         // make sure we are ready to export
2126         // this has to be done before we validate
2127         updateBuffer(UpdateMaster, OutputUpdate);
2128         updateMacroInstances(OutputUpdate);
2129
2130         writeLyXHTMLSource(ofs, runparams, FullSource);
2131
2132         ofs.close();
2133         if (ofs.fail())
2134                 lyxerr << "File '" << fname << "' was not closed properly." << endl;
2135 }
2136
2137
2138 void Buffer::writeLyXHTMLSource(odocstream & os,
2139                              OutputParams const & runparams,
2140                              OutputWhat output) const
2141 {
2142         LaTeXFeatures features(*this, params(), runparams);
2143         validate(features);
2144         d->bibinfo_.makeCitationLabels(*this);
2145
2146         bool const output_preamble =
2147                 output == FullSource || output == OnlyPreamble;
2148         bool const output_body =
2149           output == FullSource || output == OnlyBody || output == IncludedFile;
2150
2151         if (output_preamble) {
2152                 os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
2153                    << "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN\" \"http://www.w3.org/Math/DTD/mathml2/xhtml-math11-f.dtd\">\n"
2154                    // FIXME Language should be set properly.
2155                    << "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n"
2156                    << "<head>\n"
2157                    << "<meta name=\"GENERATOR\" content=\"" << PACKAGE_STRING << "\" />\n"
2158                    // FIXME Presumably need to set this right
2159                    << "<meta http-equiv=\"Content-type\" content=\"text/html;charset=UTF-8\" />\n";
2160
2161                 docstring const & doctitle = features.htmlTitle();
2162                 os << "<title>"
2163                    << (doctitle.empty() ?
2164                          from_ascii("LyX Document") :
2165                          html::htmlize(doctitle, XHTMLStream::ESCAPE_ALL))
2166                    << "</title>\n";
2167
2168                 docstring styles = features.getTClassHTMLPreamble();
2169                 if (!styles.empty())
2170                         os << "\n<!-- Text Class Preamble -->\n" << styles << '\n';
2171
2172                 styles = features.getPreambleSnippets().str;
2173                 if (!styles.empty())
2174                         os << "\n<!-- Preamble Snippets -->\n" << styles << '\n';
2175
2176                 // we will collect CSS information in a stream, and then output it
2177                 // either here, as part of the header, or else in a separate file.
2178                 odocstringstream css;
2179                 styles = features.getCSSSnippets();
2180                 if (!styles.empty())
2181                         css << "/* LyX Provided Styles */\n" << styles << '\n';
2182
2183                 styles = features.getTClassHTMLStyles();
2184                 if (!styles.empty())
2185                         css << "/* Layout-provided Styles */\n" << styles << '\n';
2186
2187                 bool const needfg = params().fontcolor != RGBColor(0, 0, 0);
2188                 bool const needbg = params().backgroundcolor != RGBColor(0xFF, 0xFF, 0xFF);
2189                 if (needfg || needbg) {
2190                                 css << "\nbody {\n";
2191                                 if (needfg)
2192                                    css << "  color: "
2193                                             << from_ascii(X11hexname(params().fontcolor))
2194                                             << ";\n";
2195                                 if (needbg)
2196                                    css << "  background-color: "
2197                                             << from_ascii(X11hexname(params().backgroundcolor))
2198                                             << ";\n";
2199                                 css << "}\n";
2200                 }
2201
2202                 docstring const dstyles = css.str();
2203                 if (!dstyles.empty()) {
2204                         bool written = false;
2205                         if (params().html_css_as_file) {
2206                                 // open a file for CSS info
2207                                 ofdocstream ocss;
2208                                 string const fcssname = addName(temppath(), "docstyle.css");
2209                                 FileName const fcssfile = FileName(fcssname);
2210                                 if (openFileWrite(ocss, fcssfile)) {
2211                                         ocss << dstyles;
2212                                         ocss.close();
2213                                         written = true;
2214                                         // write link to header
2215                                         os << "<link rel='stylesheet' href='docstyle.css' type='text/css' />\n";
2216                                         // register file
2217                                         runparams.exportdata->addExternalFile("xhtml", fcssfile);
2218                                 }
2219                         }
2220                         // we are here if the CSS is supposed to be written to the header
2221                         // or if we failed to write it to an external file.
2222                         if (!written) {
2223                                 os << "<style type='text/css'>\n"
2224                                          << dstyles
2225                                          << "\n</style>\n";
2226                         }
2227                 }
2228                 os << "</head>\n";
2229         }
2230
2231         if (output_body) {
2232                 bool const output_body_tag = (output != IncludedFile);
2233                 if (output_body_tag)
2234                         os << "<body dir=\"auto\">\n";
2235                 XHTMLStream xs(os);
2236                 if (output != IncludedFile)
2237                         // if we're an included file, the counters are in the master.
2238                         params().documentClass().counters().reset();
2239                 xhtmlParagraphs(text(), *this, xs, runparams);
2240                 if (output_body_tag)
2241                         os << "</body>\n";
2242         }
2243
2244         if (output_preamble)
2245                 os << "</html>\n";
2246 }
2247
2248
2249 // chktex should be run with these flags disabled: 3, 22, 25, 30, 38(?)
2250 // Other flags: -wall -v0 -x
2251 int Buffer::runChktex()
2252 {
2253         setBusy(true);
2254
2255         // get LaTeX-Filename
2256         FileName const path(temppath());
2257         string const name = addName(path.absFileName(), latexName());
2258         string const org_path = filePath();
2259
2260         PathChanger p(path); // path to LaTeX file
2261         message(_("Running chktex..."));
2262
2263         // Generate the LaTeX file if neccessary
2264         OutputParams runparams(&params().encoding());
2265         runparams.flavor = OutputParams::LATEX;
2266         runparams.nice = false;
2267         runparams.linelen = lyxrc.plaintext_linelen;
2268         makeLaTeXFile(FileName(name), org_path, runparams);
2269
2270         TeXErrors terr;
2271         Chktex chktex(lyxrc.chktex_command, onlyFileName(name), filePath());
2272         int const res = chktex.run(terr); // run chktex
2273
2274         if (res == -1) {
2275                 Alert::error(_("chktex failure"),
2276                              _("Could not run chktex successfully."));
2277         } else {
2278                 ErrorList & errlist = d->errorLists["ChkTeX"];
2279                 errlist.clear();
2280                 bufferErrors(terr, errlist);
2281         }
2282
2283         setBusy(false);
2284
2285         if (runparams.silent)
2286                 d->errorLists["ChkTeX"].clear();
2287         else
2288                 errors("ChkTeX");
2289
2290         return res;
2291 }
2292
2293
2294 void Buffer::validate(LaTeXFeatures & features) const
2295 {
2296         // Validate the buffer params, but not for included
2297         // files, since they also use the parent buffer's
2298         // params (# 5941)
2299         if (!features.runparams().is_child)
2300                 params().validate(features);
2301
2302         for (Paragraph const & p : paragraphs())
2303                 p.validate(features);
2304
2305         if (lyxerr.debugging(Debug::LATEX)) {
2306                 features.showStruct();
2307         }
2308 }
2309
2310
2311 void Buffer::getLabelList(vector<docstring> & list) const
2312 {
2313         // If this is a child document, use the master's list instead.
2314         if (parent()) {
2315                 masterBuffer()->getLabelList(list);
2316                 return;
2317         }
2318
2319         list.clear();
2320         shared_ptr<Toc> toc = d->toc_backend.toc("label");
2321         Toc::const_iterator toc_it = toc->begin();
2322         Toc::const_iterator end = toc->end();
2323         for (; toc_it != end; ++toc_it) {
2324                 if (toc_it->depth() == 0)
2325                         list.push_back(toc_it->str());
2326         }
2327 }
2328
2329
2330 void Buffer::updateBibfilesCache(UpdateScope scope) const
2331 {
2332         // FIXME This is probably unnecssary, given where we call this.
2333         // If this is a child document, use the parent's cache instead.
2334         if (parent() && scope != UpdateChildOnly) {
2335                 masterBuffer()->updateBibfilesCache();
2336                 return;
2337         }
2338
2339         d->bibfiles_cache_.clear();
2340         for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
2341                 if (it->lyxCode() == BIBTEX_CODE) {
2342                         InsetBibtex const & inset = static_cast<InsetBibtex const &>(*it);
2343                         support::FileNamePairList const bibfiles = inset.getBibFiles();
2344                         d->bibfiles_cache_.insert(d->bibfiles_cache_.end(),
2345                                 bibfiles.begin(),
2346                                 bibfiles.end());
2347                 } else if (it->lyxCode() == INCLUDE_CODE) {
2348                         InsetInclude & inset = static_cast<InsetInclude &>(*it);
2349                         Buffer const * const incbuf = inset.getChildBuffer();
2350                         if (!incbuf)
2351                                 continue;
2352                         support::FileNamePairList const & bibfiles =
2353                                         incbuf->getBibfilesCache(UpdateChildOnly);
2354                         if (!bibfiles.empty()) {
2355                                 d->bibfiles_cache_.insert(d->bibfiles_cache_.end(),
2356                                         bibfiles.begin(),
2357                                         bibfiles.end());
2358                         }
2359                 }
2360         }
2361         d->bibfile_cache_valid_ = true;
2362         d->bibinfo_cache_valid_ = false;
2363         d->cite_labels_valid_ = false;
2364 }
2365
2366
2367 void Buffer::invalidateBibinfoCache() const
2368 {
2369         d->bibinfo_cache_valid_ = false;
2370         d->cite_labels_valid_ = false;
2371         // also invalidate the cache for the parent buffer
2372         Buffer const * const pbuf = d->parent();
2373         if (pbuf)
2374                 pbuf->invalidateBibinfoCache();
2375 }
2376
2377
2378 void Buffer::invalidateBibfileCache() const
2379 {
2380         d->bibfile_cache_valid_ = false;
2381         d->bibinfo_cache_valid_ = false;
2382         d->cite_labels_valid_ = false;
2383         // also invalidate the cache for the parent buffer
2384         Buffer const * const pbuf = d->parent();
2385         if (pbuf)
2386                 pbuf->invalidateBibfileCache();
2387 }
2388
2389
2390 support::FileNamePairList const & Buffer::getBibfilesCache(UpdateScope scope) const
2391 {
2392         // FIXME This is probably unnecessary, given where we call this.
2393         // If this is a child document, use the master's cache instead.
2394         Buffer const * const pbuf = masterBuffer();
2395         if (pbuf != this && scope != UpdateChildOnly)
2396                 return pbuf->getBibfilesCache();
2397
2398         if (!d->bibfile_cache_valid_)
2399                 this->updateBibfilesCache(scope);
2400
2401         return d->bibfiles_cache_;
2402 }
2403
2404
2405 BiblioInfo const & Buffer::masterBibInfo() const
2406 {
2407         Buffer const * const tmp = masterBuffer();
2408         if (tmp != this)
2409                 return tmp->masterBibInfo();
2410         return d->bibinfo_;
2411 }
2412
2413
2414 void Buffer::checkIfBibInfoCacheIsValid() const
2415 {
2416         // use the master's cache
2417         Buffer const * const tmp = masterBuffer();
2418         if (tmp != this) {
2419                 tmp->checkIfBibInfoCacheIsValid();
2420                 return;
2421         }
2422
2423         // compare the cached timestamps with the actual ones.
2424         FileNamePairList const & bibfiles_cache = getBibfilesCache();
2425         FileNamePairList::const_iterator ei = bibfiles_cache.begin();
2426         FileNamePairList::const_iterator en = bibfiles_cache.end();
2427         for (; ei != en; ++ ei) {
2428                 FileName const fn = ei->second;
2429                 time_t lastw = fn.lastModified();
2430                 time_t prevw = d->bibfile_status_[fn];
2431                 if (lastw != prevw) {
2432                         d->bibinfo_cache_valid_ = false;
2433                         d->cite_labels_valid_ = false;
2434                         d->bibfile_status_[fn] = lastw;
2435                 }
2436         }
2437 }
2438
2439
2440 void Buffer::reloadBibInfoCache() const
2441 {
2442         // use the master's cache
2443         Buffer const * const tmp = masterBuffer();
2444         if (tmp != this) {
2445                 tmp->reloadBibInfoCache();
2446                 return;
2447         }
2448
2449         checkIfBibInfoCacheIsValid();
2450         if (d->bibinfo_cache_valid_)
2451                 return;
2452
2453         d->bibinfo_.clear();
2454         collectBibKeys();
2455         d->bibinfo_cache_valid_ = true;
2456 }
2457
2458
2459 void Buffer::collectBibKeys() const
2460 {
2461         for (InsetIterator it = inset_iterator_begin(inset()); it; ++it)
2462                 it->collectBibKeys(it);
2463 }
2464
2465
2466 void Buffer::addBiblioInfo(BiblioInfo const & bi) const
2467 {
2468         Buffer const * tmp = masterBuffer();
2469         BiblioInfo & masterbi = (tmp == this) ?
2470                 d->bibinfo_ : tmp->d->bibinfo_;
2471         masterbi.mergeBiblioInfo(bi);
2472 }
2473
2474
2475 void Buffer::addBibTeXInfo(docstring const & key, BibTeXInfo const & bi) const
2476 {
2477         Buffer const * tmp = masterBuffer();
2478         BiblioInfo & masterbi = (tmp == this) ?
2479                 d->bibinfo_ : tmp->d->bibinfo_;
2480         masterbi[key] = bi;
2481 }
2482
2483
2484 void Buffer::makeCitationLabels() const
2485 {
2486         Buffer const * const master = masterBuffer();
2487         return d->bibinfo_.makeCitationLabels(*master);
2488 }
2489
2490
2491 bool Buffer::citeLabelsValid() const
2492 {
2493         return masterBuffer()->d->cite_labels_valid_;
2494 }
2495
2496
2497 void Buffer::removeBiblioTempFiles() const
2498 {
2499         // We remove files that contain LaTeX commands specific to the
2500         // particular bibliographic style being used, in order to avoid
2501         // LaTeX errors when we switch style.
2502         FileName const aux_file(addName(temppath(), changeExtension(latexName(),".aux")));
2503         FileName const bbl_file(addName(temppath(), changeExtension(latexName(),".bbl")));
2504         LYXERR(Debug::FILES, "Removing the .aux file " << aux_file);
2505         aux_file.removeFile();
2506         LYXERR(Debug::FILES, "Removing the .bbl file " << bbl_file);
2507         bbl_file.removeFile();
2508         // Also for the parent buffer
2509         Buffer const * const pbuf = parent();
2510         if (pbuf)
2511                 pbuf->removeBiblioTempFiles();
2512 }
2513
2514
2515 bool Buffer::isDepClean(string const & name) const
2516 {
2517         DepClean::const_iterator const it = d->dep_clean.find(name);
2518         if (it == d->dep_clean.end())
2519                 return true;
2520         return it->second;
2521 }
2522
2523
2524 void Buffer::markDepClean(string const & name)
2525 {
2526         d->dep_clean[name] = true;
2527 }
2528
2529
2530 bool Buffer::getStatus(FuncRequest const & cmd, FuncStatus & flag)
2531 {
2532         if (isInternal()) {
2533                 // FIXME? if there is an Buffer LFUN that can be dispatched even
2534                 // if internal, put a switch '(cmd.action)' here.
2535                 return false;
2536         }
2537
2538         bool enable = true;
2539
2540         switch (cmd.action()) {
2541
2542         case LFUN_BUFFER_TOGGLE_READ_ONLY:
2543                 flag.setOnOff(hasReadonlyFlag());
2544                 break;
2545
2546                 // FIXME: There is need for a command-line import.
2547                 //case LFUN_BUFFER_IMPORT:
2548
2549         case LFUN_BUFFER_AUTO_SAVE:
2550                 break;
2551
2552         case LFUN_BUFFER_EXPORT_CUSTOM:
2553                 // FIXME: Nothing to check here?
2554                 break;
2555
2556         case LFUN_BUFFER_EXPORT: {
2557                 docstring const arg = cmd.argument();
2558                 if (arg == "custom") {
2559                         enable = true;
2560                         break;
2561                 }
2562                 string format = (arg.empty() || arg == "default") ?
2563                         params().getDefaultOutputFormat() : to_utf8(arg);
2564                 size_t pos = format.find(' ');
2565                 if (pos != string::npos)
2566                         format = format.substr(0, pos);
2567                 enable = params().isExportable(format, false);
2568                 if (!enable)
2569                         flag.message(bformat(
2570                                              _("Don't know how to export to format: %1$s"), arg));
2571                 break;
2572         }
2573
2574         case LFUN_BUFFER_CHKTEX:
2575                 enable = params().isLatex() && !lyxrc.chktex_command.empty();
2576                 break;
2577
2578         case LFUN_BUILD_PROGRAM:
2579                 enable = params().isExportable("program", false);
2580                 break;
2581
2582         case LFUN_BRANCH_ACTIVATE:
2583         case LFUN_BRANCH_DEACTIVATE:
2584         case LFUN_BRANCH_MASTER_ACTIVATE:
2585         case LFUN_BRANCH_MASTER_DEACTIVATE: {
2586                 bool const master = (cmd.action() == LFUN_BRANCH_MASTER_ACTIVATE
2587                                      || cmd.action() == LFUN_BRANCH_MASTER_DEACTIVATE);
2588                 BranchList const & branchList = master ? masterBuffer()->params().branchlist()
2589                         : params().branchlist();
2590                 docstring const branchName = cmd.argument();
2591                 flag.setEnabled(!branchName.empty() && branchList.find(branchName));
2592                 break;
2593         }
2594
2595         case LFUN_BRANCH_ADD:
2596         case LFUN_BRANCHES_RENAME:
2597                 // if no Buffer is present, then of course we won't be called!
2598                 break;
2599
2600         case LFUN_BUFFER_LANGUAGE:
2601                 enable = !isReadonly();
2602                 break;
2603
2604         case LFUN_BUFFER_VIEW_CACHE:
2605                 (d->preview_file_).refresh();
2606                 enable = (d->preview_file_).exists() && !(d->preview_file_).isFileEmpty();
2607                 break;
2608
2609         case LFUN_CHANGES_TRACK:
2610                 flag.setEnabled(true);
2611                 flag.setOnOff(params().track_changes);
2612                 break;
2613
2614         case LFUN_CHANGES_OUTPUT:
2615                 flag.setEnabled(true);
2616                 flag.setOnOff(params().output_changes);
2617                 break;
2618
2619         case LFUN_BUFFER_TOGGLE_COMPRESSION: {
2620                 flag.setOnOff(params().compressed);
2621                 break;
2622         }
2623
2624         case LFUN_BUFFER_TOGGLE_OUTPUT_SYNC: {
2625                 flag.setOnOff(params().output_sync);
2626                 break;
2627         }
2628
2629         default:
2630                 return false;
2631         }
2632         flag.setEnabled(enable);
2633         return true;
2634 }
2635
2636
2637 void Buffer::dispatch(string const & command, DispatchResult & result)
2638 {
2639         return dispatch(lyxaction.lookupFunc(command), result);
2640 }
2641
2642
2643 // NOTE We can end up here even if we have no GUI, because we are called
2644 // by LyX::exec to handled command-line requests. So we may need to check
2645 // whether we have a GUI or not. The boolean use_gui holds this information.
2646 void Buffer::dispatch(FuncRequest const & func, DispatchResult & dr)
2647 {
2648         if (isInternal()) {
2649                 // FIXME? if there is an Buffer LFUN that can be dispatched even
2650                 // if internal, put a switch '(cmd.action())' here.
2651                 dr.dispatched(false);
2652                 return;
2653         }
2654         string const argument = to_utf8(func.argument());
2655         // We'll set this back to false if need be.
2656         bool dispatched = true;
2657         undo().beginUndoGroup();
2658
2659         switch (func.action()) {
2660         case LFUN_BUFFER_TOGGLE_READ_ONLY:
2661                 if (lyxvc().inUse()) {
2662                         string log = lyxvc().toggleReadOnly();
2663                         if (!log.empty())
2664                                 dr.setMessage(log);
2665                 }
2666                 else
2667                         setReadonly(!hasReadonlyFlag());
2668                 break;
2669
2670         case LFUN_BUFFER_EXPORT: {
2671                 string const format = (argument.empty() || argument == "default") ?
2672                         params().getDefaultOutputFormat() : argument;
2673                 ExportStatus const status = doExport(format, false);
2674                 dr.setError(status != ExportSuccess);
2675                 if (status != ExportSuccess)
2676                         dr.setMessage(bformat(_("Error exporting to format: %1$s."),
2677                                               from_utf8(format)));
2678                 break;
2679         }
2680
2681         case LFUN_BUILD_PROGRAM: {
2682                 ExportStatus const status = doExport("program", true);
2683                 dr.setError(status != ExportSuccess);
2684                 if (status != ExportSuccess)
2685                         dr.setMessage(_("Error generating literate programming code."));
2686                 break;
2687         }
2688
2689         case LFUN_BUFFER_CHKTEX:
2690                 runChktex();
2691                 break;
2692
2693         case LFUN_BUFFER_EXPORT_CUSTOM: {
2694                 string format_name;
2695                 string command = split(argument, format_name, ' ');
2696                 Format const * format = theFormats().getFormat(format_name);
2697                 if (!format) {
2698                         lyxerr << "Format \"" << format_name
2699                                 << "\" not recognized!"
2700                                 << endl;
2701                         break;
2702                 }
2703
2704                 // The name of the file created by the conversion process
2705                 string filename;
2706
2707                 // Output to filename
2708                 if (format->name() == "lyx") {
2709                         string const latexname = latexName(false);
2710                         filename = changeExtension(latexname,
2711                                 format->extension());
2712                         filename = addName(temppath(), filename);
2713
2714                         if (!writeFile(FileName(filename)))
2715                                 break;
2716
2717                 } else {
2718                         doExport(format_name, true, filename);
2719                 }
2720
2721                 // Substitute $$FName for filename
2722                 if (!contains(command, "$$FName"))
2723                         command = "( " + command + " ) < $$FName";
2724                 command = subst(command, "$$FName", filename);
2725
2726                 // Execute the command in the background
2727                 Systemcall call;
2728                 call.startscript(Systemcall::DontWait, command,
2729                                  filePath(), layoutPos());
2730                 break;
2731         }
2732
2733         // FIXME: There is need for a command-line import.
2734         /*
2735         case LFUN_BUFFER_IMPORT:
2736                 doImport(argument);
2737                 break;
2738         */
2739
2740         case LFUN_BUFFER_AUTO_SAVE:
2741                 autoSave();
2742                 resetAutosaveTimers();
2743                 break;
2744
2745         case LFUN_BRANCH_ACTIVATE:
2746         case LFUN_BRANCH_DEACTIVATE:
2747         case LFUN_BRANCH_MASTER_ACTIVATE:
2748         case LFUN_BRANCH_MASTER_DEACTIVATE: {
2749                 bool const master = (func.action() == LFUN_BRANCH_MASTER_ACTIVATE
2750                                      || func.action() == LFUN_BRANCH_MASTER_DEACTIVATE);
2751                 Buffer * buf = master ? const_cast<Buffer *>(masterBuffer())
2752                                       : this;
2753
2754                 docstring const branch_name = func.argument();
2755                 // the case without a branch name is handled elsewhere
2756                 if (branch_name.empty()) {
2757                         dispatched = false;
2758                         break;
2759                 }
2760                 Branch * branch = buf->params().branchlist().find(branch_name);
2761                 if (!branch) {
2762                         LYXERR0("Branch " << branch_name << " does not exist.");
2763                         dr.setError(true);
2764                         docstring const msg =
2765                                 bformat(_("Branch \"%1$s\" does not exist."), branch_name);
2766                         dr.setMessage(msg);
2767                         break;
2768                 }
2769                 bool const activate = (func.action() == LFUN_BRANCH_ACTIVATE
2770                                        || func.action() == LFUN_BRANCH_MASTER_ACTIVATE);
2771                 if (branch->isSelected() != activate) {
2772                         buf->undo().recordUndoBufferParams(CursorData());
2773                         branch->setSelected(activate);
2774                         dr.setError(false);
2775                         dr.screenUpdate(Update::Force);
2776                         dr.forceBufferUpdate();
2777                 }
2778                 break;
2779         }
2780
2781         case LFUN_BRANCH_ADD: {
2782                 docstring branch_name = func.argument();
2783                 if (branch_name.empty()) {
2784                         dispatched = false;
2785                         break;
2786                 }
2787                 BranchList & branch_list = params().branchlist();
2788                 vector<docstring> const branches =
2789                         getVectorFromString(branch_name, branch_list.separator());
2790                 docstring msg;
2791                 for (vector<docstring>::const_iterator it = branches.begin();
2792                      it != branches.end(); ++it) {
2793                         branch_name = *it;
2794                         Branch * branch = branch_list.find(branch_name);
2795                         if (branch) {
2796                                 LYXERR0("Branch " << branch_name << " already exists.");
2797                                 dr.setError(true);
2798                                 if (!msg.empty())
2799                                         msg += ("\n");
2800                                 msg += bformat(_("Branch \"%1$s\" already exists."), branch_name);
2801                         } else {
2802                                 undo().recordUndoBufferParams(CursorData());
2803                                 branch_list.add(branch_name);
2804                                 branch = branch_list.find(branch_name);
2805                                 string const x11hexname = X11hexname(branch->color());
2806                                 docstring const str = branch_name + ' ' + from_ascii(x11hexname);
2807                                 lyx::dispatch(FuncRequest(LFUN_SET_COLOR, str));
2808                                 dr.setError(false);
2809                                 dr.screenUpdate(Update::Force);
2810                         }
2811                 }
2812                 if (!msg.empty())
2813                         dr.setMessage(msg);
2814                 break;
2815         }
2816
2817         case LFUN_BRANCHES_RENAME: {
2818                 if (func.argument().empty())
2819                         break;
2820
2821                 docstring const oldname = from_utf8(func.getArg(0));
2822                 docstring const newname = from_utf8(func.getArg(1));
2823                 InsetIterator it  = inset_iterator_begin(inset());
2824                 InsetIterator const end = inset_iterator_end(inset());
2825                 bool success = false;
2826                 for (; it != end; ++it) {
2827                         if (it->lyxCode() == BRANCH_CODE) {
2828                                 InsetBranch & ins = static_cast<InsetBranch &>(*it);
2829                                 if (ins.branch() == oldname) {
2830                                         undo().recordUndo(CursorData(it));
2831                                         ins.rename(newname);
2832                                         success = true;
2833                                         continue;
2834                                 }
2835                         }
2836                         if (it->lyxCode() == INCLUDE_CODE) {
2837                                 // get buffer of external file
2838                                 InsetInclude const & ins =
2839                                         static_cast<InsetInclude const &>(*it);
2840                                 Buffer * child = ins.getChildBuffer();
2841                                 if (!child)
2842                                         continue;
2843                                 child->dispatch(func, dr);
2844                         }
2845                 }
2846
2847                 if (success) {
2848                         dr.screenUpdate(Update::Force);
2849                         dr.forceBufferUpdate();
2850                 }
2851                 break;
2852         }
2853
2854         case LFUN_BUFFER_VIEW_CACHE:
2855                 if (!theFormats().view(*this, d->preview_file_,
2856                                   d->preview_format_))
2857                         dr.setMessage(_("Error viewing the output file."));
2858                 break;
2859
2860         case LFUN_CHANGES_TRACK:
2861                 if (params().save_transient_properties)
2862                         undo().recordUndoBufferParams(CursorData());
2863                 params().track_changes = !params().track_changes;
2864                 if (!params().track_changes)
2865                         dr.forceChangesUpdate();
2866                 break;
2867
2868         case LFUN_CHANGES_OUTPUT:
2869                 if (params().save_transient_properties)
2870                         undo().recordUndoBufferParams(CursorData());
2871                 params().output_changes = !params().output_changes;
2872                 if (params().output_changes) {
2873                         bool dvipost    = LaTeXFeatures::isAvailable("dvipost");
2874                         bool xcolorulem = LaTeXFeatures::isAvailable("ulem") &&
2875                                           LaTeXFeatures::isAvailable("xcolor");
2876
2877                         if (!dvipost && !xcolorulem) {
2878                                 Alert::warning(_("Changes not shown in LaTeX output"),
2879                                                _("Changes will not be highlighted in LaTeX output, "
2880                                                  "because neither dvipost nor xcolor/ulem are installed.\n"
2881                                                  "Please install these packages or redefine "
2882                                                  "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
2883                         } else if (!xcolorulem) {
2884                                 Alert::warning(_("Changes not shown in LaTeX output"),
2885                                                _("Changes will not be highlighted in LaTeX output "
2886                                                  "when using pdflatex, because xcolor and ulem are not installed.\n"
2887                                                  "Please install both packages or redefine "
2888                                                  "\\lyxadded and \\lyxdeleted in the LaTeX preamble."));
2889                         }
2890                 }
2891                 break;
2892
2893         case LFUN_BUFFER_TOGGLE_COMPRESSION:
2894                 // turn compression on/off
2895                 undo().recordUndoBufferParams(CursorData());
2896                 params().compressed = !params().compressed;
2897                 break;
2898
2899         case LFUN_BUFFER_TOGGLE_OUTPUT_SYNC:
2900                 undo().recordUndoBufferParams(CursorData());
2901                 params().output_sync = !params().output_sync;
2902                 break;
2903
2904         default:
2905                 dispatched = false;
2906                 break;
2907         }
2908         dr.dispatched(dispatched);
2909         undo().endUndoGroup();
2910 }
2911
2912
2913 void Buffer::changeLanguage(Language const * from, Language const * to)
2914 {
2915         LASSERT(from, return);
2916         LASSERT(to, return);
2917
2918         for_each(par_iterator_begin(),
2919                  par_iterator_end(),
2920                  bind(&Paragraph::changeLanguage, _1, params(), from, to));
2921 }
2922
2923
2924 bool Buffer::isMultiLingual() const
2925 {
2926         ParConstIterator end = par_iterator_end();
2927         for (ParConstIterator it = par_iterator_begin(); it != end; ++it)
2928                 if (it->isMultiLingual(params()))
2929                         return true;
2930
2931         return false;
2932 }
2933
2934
2935 std::set<Language const *> Buffer::getLanguages() const
2936 {
2937         std::set<Language const *> languages;
2938         getLanguages(languages);
2939         return languages;
2940 }
2941
2942
2943 void Buffer::getLanguages(std::set<Language const *> & languages) const
2944 {
2945         ParConstIterator end = par_iterator_end();
2946         // add the buffer language, even if it's not actively used
2947         languages.insert(language());
2948         // iterate over the paragraphs
2949         for (ParConstIterator it = par_iterator_begin(); it != end; ++it)
2950                 it->getLanguages(languages);
2951         // also children
2952         ListOfBuffers clist = getDescendents();
2953         ListOfBuffers::const_iterator cit = clist.begin();
2954         ListOfBuffers::const_iterator const cen = clist.end();
2955         for (; cit != cen; ++cit)
2956                 (*cit)->getLanguages(languages);
2957 }
2958
2959
2960 DocIterator Buffer::getParFromID(int const id) const
2961 {
2962         Buffer * buf = const_cast<Buffer *>(this);
2963         if (id < 0)
2964                 // This means non-existent
2965                 return doc_iterator_end(buf);
2966
2967         for (DocIterator it = doc_iterator_begin(buf); !it.atEnd(); it.forwardPar())
2968                 if (it.paragraph().id() == id)
2969                         return it;
2970
2971         return doc_iterator_end(buf);
2972 }
2973
2974
2975 bool Buffer::hasParWithID(int const id) const
2976 {
2977         return !getParFromID(id).atEnd();
2978 }
2979
2980
2981 ParIterator Buffer::par_iterator_begin()
2982 {
2983         return ParIterator(doc_iterator_begin(this));
2984 }
2985
2986
2987 ParIterator Buffer::par_iterator_end()
2988 {
2989         return ParIterator(doc_iterator_end(this));
2990 }
2991
2992
2993 ParConstIterator Buffer::par_iterator_begin() const
2994 {
2995         return ParConstIterator(doc_iterator_begin(this));
2996 }
2997
2998
2999 ParConstIterator Buffer::par_iterator_end() const
3000 {
3001         return ParConstIterator(doc_iterator_end(this));
3002 }
3003
3004
3005 Language const * Buffer::language() const
3006 {
3007         return params().language;
3008 }
3009
3010
3011 docstring const Buffer::B_(string const & l10n) const
3012 {
3013         return params().B_(l10n);
3014 }
3015
3016
3017 bool Buffer::isClean() const
3018 {
3019         return d->lyx_clean;
3020 }
3021
3022
3023 bool Buffer::isChecksumModified() const
3024 {
3025         LASSERT(d->filename.exists(), return false);
3026         return d->checksum_ != d->filename.checksum();
3027 }
3028
3029
3030 void Buffer::saveCheckSum() const
3031 {
3032         FileName const & file = d->filename;
3033         file.refresh();
3034         d->checksum_ = file.exists() ? file.checksum()
3035                 : 0; // in the case of save to a new file.
3036 }
3037
3038
3039 void Buffer::markClean() const
3040 {
3041         if (!d->lyx_clean) {
3042                 d->lyx_clean = true;
3043                 updateTitles();
3044         }
3045         // if the .lyx file has been saved, we don't need an
3046         // autosave
3047         d->bak_clean = true;
3048         d->undo_.markDirty();
3049         clearExternalModification();
3050 }
3051
3052
3053 void Buffer::setUnnamed(bool flag)
3054 {
3055         d->unnamed = flag;
3056 }
3057
3058
3059 bool Buffer::isUnnamed() const
3060 {
3061         return d->unnamed;
3062 }
3063
3064
3065 /// \note
3066 /// Don't check unnamed, here: isInternal() is used in
3067 /// newBuffer(), where the unnamed flag has not been set by anyone
3068 /// yet. Also, for an internal buffer, there should be no need for
3069 /// retrieving fileName() nor for checking if it is unnamed or not.
3070 bool Buffer::isInternal() const
3071 {
3072         return d->internal_buffer;
3073 }
3074
3075
3076 void Buffer::setInternal(bool flag)
3077 {
3078         d->internal_buffer = flag;
3079 }
3080
3081
3082 void Buffer::markDirty()
3083 {
3084         if (d->lyx_clean) {
3085                 d->lyx_clean = false;
3086                 updateTitles();
3087         }
3088         d->bak_clean = false;
3089
3090         DepClean::iterator it = d->dep_clean.begin();
3091         DepClean::const_iterator const end = d->dep_clean.end();
3092
3093         for (; it != end; ++it)
3094                 it->second = false;
3095 }
3096
3097
3098 FileName Buffer::fileName() const
3099 {
3100         return d->filename;
3101 }
3102
3103
3104 string Buffer::absFileName() const
3105 {
3106         return d->filename.absFileName();
3107 }
3108
3109
3110 string Buffer::filePath() const
3111 {
3112         string const abs = d->filename.onlyPath().absFileName();
3113         if (abs.empty())
3114                 return abs;
3115         int last = abs.length() - 1;
3116
3117         return abs[last] == '/' ? abs : abs + '/';
3118 }
3119
3120
3121 DocFileName Buffer::getReferencedFileName(string const & fn) const
3122 {
3123         DocFileName result;
3124         if (FileName::isAbsolute(fn) || !FileName::isAbsolute(params().origin))
3125                 result.set(fn, filePath());
3126         else {
3127                 // filePath() ends with a path separator
3128                 FileName const test(filePath() + fn);
3129                 if (test.exists())
3130                         result.set(fn, filePath());
3131                 else
3132                         result.set(fn, params().origin);
3133         }
3134
3135         return result;
3136 }
3137
3138
3139 string const Buffer::prepareFileNameForLaTeX(string const & name,
3140                                              string const & ext, bool nice) const
3141 {
3142         string const fname = makeAbsPath(name, filePath()).absFileName();
3143         if (FileName::isAbsolute(name) || !FileName(fname + ext).isReadableFile())
3144                 return name;
3145         if (!nice)
3146                 return fname;
3147
3148         // FIXME UNICODE
3149         return to_utf8(makeRelPath(from_utf8(fname),
3150                 from_utf8(masterBuffer()->filePath())));
3151 }
3152
3153
3154 vector<docstring> const Buffer::prepareBibFilePaths(OutputParams const & runparams,
3155                                                 FileNamePairList const bibfilelist,
3156                                                 bool const add_extension) const
3157 {
3158         // If we are processing the LaTeX file in a temp directory then
3159         // copy the .bib databases to this temp directory, mangling their
3160         // names in the process. Store this mangled name in the list of
3161         // all databases.
3162         // (We need to do all this because BibTeX *really*, *really*
3163         // can't handle "files with spaces" and Windows users tend to
3164         // use such filenames.)
3165         // Otherwise, store the (maybe absolute) path to the original,
3166         // unmangled database name.
3167
3168         vector<docstring> res;
3169
3170         // determine the export format
3171         string const tex_format = flavor2format(runparams.flavor);
3172
3173         // check for spaces in paths
3174         bool found_space = false;
3175
3176         FileNamePairList::const_iterator it = bibfilelist.begin();
3177         FileNamePairList::const_iterator en = bibfilelist.end();
3178         for (; it != en; ++it) {
3179                 string utf8input = to_utf8(it->first);
3180                 string database =
3181                         prepareFileNameForLaTeX(utf8input, ".bib", runparams.nice);
3182                 FileName const try_in_file =
3183                         makeAbsPath(database + ".bib", filePath());
3184                 bool const not_from_texmf = try_in_file.isReadableFile();
3185
3186                 if (!runparams.inComment && !runparams.dryrun && !runparams.nice &&
3187                     not_from_texmf) {
3188                         // mangledFileName() needs the extension
3189                         DocFileName const in_file = DocFileName(try_in_file);
3190                         database = removeExtension(in_file.mangledFileName());
3191                         FileName const out_file = makeAbsPath(database + ".bib",
3192                                         masterBuffer()->temppath());
3193                         bool const success = in_file.copyTo(out_file);
3194                         if (!success) {
3195                                 LYXERR0("Failed to copy '" << in_file
3196                                        << "' to '" << out_file << "'");
3197                         }
3198                 } else if (!runparams.inComment && runparams.nice && not_from_texmf) {
3199                         runparams.exportdata->addExternalFile(tex_format, try_in_file, database + ".bib");
3200                         if (!isValidLaTeXFileName(database)) {
3201                                 frontend::Alert::warning(_("Invalid filename"),
3202                                          _("The following filename will cause troubles "
3203                                                "when running the exported file through LaTeX: ") +
3204                                              from_utf8(database));
3205                         }
3206                         if (!isValidDVIFileName(database)) {
3207                                 frontend::Alert::warning(_("Problematic filename for DVI"),
3208                                          _("The following filename can cause troubles "
3209                                                "when running the exported file through LaTeX "
3210                                                    "and opening the resulting DVI: ") +
3211                                              from_utf8(database), true);
3212                         }
3213                 }
3214
3215                 if (add_extension)
3216                         database += ".bib";
3217
3218                 // FIXME UNICODE
3219                 docstring const path = from_utf8(latex_path(database));
3220
3221                 if (contains(path, ' '))
3222                         found_space = true;
3223
3224                 if (find(res.begin(), res.end(), path) == res.end())
3225                         res.push_back(path);
3226         }
3227
3228         // Check if there are spaces in the path and warn BibTeX users, if so.
3229         // (biber can cope with such paths)
3230         if (!prefixIs(runparams.bibtex_command, "biber")) {
3231                 // Post this warning only once.
3232                 static bool warned_about_spaces = false;
3233                 if (!warned_about_spaces &&
3234                     runparams.nice && found_space) {
3235                         warned_about_spaces = true;
3236                         Alert::warning(_("Export Warning!"),
3237                                        _("There are spaces in the paths to your BibTeX databases.\n"
3238                                                       "BibTeX will be unable to find them."));
3239                 }
3240         }
3241
3242         return res;
3243 }
3244
3245
3246
3247 string Buffer::layoutPos() const
3248 {
3249         return d->layout_position;
3250 }
3251
3252
3253 void Buffer::setLayoutPos(string const & path)
3254 {
3255         if (path.empty()) {
3256                 d->layout_position.clear();
3257                 return;
3258         }
3259
3260         LATTEST(FileName::isAbsolute(path));
3261
3262         d->layout_position =
3263                 to_utf8(makeRelPath(from_utf8(path), from_utf8(filePath())));
3264
3265         if (d->layout_position.empty())
3266                 d->layout_position = ".";
3267 }
3268
3269
3270 bool Buffer::hasReadonlyFlag() const
3271 {
3272         return d->read_only;
3273 }
3274
3275
3276 bool Buffer::isReadonly() const
3277 {
3278         return hasReadonlyFlag() || notifiesExternalModification();
3279 }
3280
3281
3282 void Buffer::setParent(Buffer const * buffer)
3283 {
3284         // Avoids recursive include.
3285         d->setParent(buffer == this ? 0 : buffer);
3286         updateMacros();
3287 }
3288
3289
3290 Buffer const * Buffer::parent() const
3291 {
3292         return d->parent();
3293 }
3294
3295
3296 ListOfBuffers Buffer::allRelatives() const
3297 {
3298         ListOfBuffers lb = masterBuffer()->getDescendents();
3299         lb.push_front(const_cast<Buffer *>(masterBuffer()));
3300         return lb;
3301 }
3302
3303
3304 Buffer const * Buffer::masterBuffer() const
3305 {
3306         // FIXME Should be make sure we are not in some kind
3307         // of recursive include? A -> B -> A will crash this.
3308         Buffer const * const pbuf = d->parent();
3309         if (!pbuf)
3310                 return this;
3311
3312         return pbuf->masterBuffer();
3313 }
3314
3315
3316 bool Buffer::isChild(Buffer * child) const
3317 {
3318         return d->children_positions.find(child) != d->children_positions.end();
3319 }
3320
3321
3322 DocIterator Buffer::firstChildPosition(Buffer const * child)
3323 {
3324         Impl::BufferPositionMap::iterator it;
3325         it = d->children_positions.find(child);
3326         if (it == d->children_positions.end())
3327                 return DocIterator(this);
3328         return it->second;
3329 }
3330
3331
3332 bool Buffer::hasChildren() const
3333 {
3334         return !d->children_positions.empty();
3335 }
3336
3337
3338 void Buffer::collectChildren(ListOfBuffers & clist, bool grand_children) const
3339 {
3340         // loop over children
3341         Impl::BufferPositionMap::iterator it = d->children_positions.begin();
3342         Impl::BufferPositionMap::iterator end = d->children_positions.end();
3343         for (; it != end; ++it) {
3344                 Buffer * child = const_cast<Buffer *>(it->first);
3345                 // No duplicates
3346                 ListOfBuffers::const_iterator bit = find(clist.begin(), clist.end(), child);
3347                 if (bit != clist.end())
3348                         continue;
3349                 clist.push_back(child);
3350                 if (grand_children)
3351                         // there might be grandchildren
3352                         child->collectChildren(clist, true);
3353         }
3354 }
3355
3356
3357 ListOfBuffers Buffer::getChildren() const
3358 {
3359         ListOfBuffers v;
3360         collectChildren(v, false);
3361         // Make sure we have not included ourselves.
3362         ListOfBuffers::iterator bit = find(v.begin(), v.end(), this);
3363         if (bit != v.end()) {
3364                 LYXERR0("Recursive include detected in `" << fileName() << "'.");
3365                 v.erase(bit);
3366         }
3367         return v;
3368 }
3369
3370
3371 ListOfBuffers Buffer::getDescendents() const
3372 {
3373         ListOfBuffers v;
3374         collectChildren(v, true);
3375         // Make sure we have not included ourselves.
3376         ListOfBuffers::iterator bit = find(v.begin(), v.end(), this);
3377         if (bit != v.end()) {
3378                 LYXERR0("Recursive include detected in `" << fileName() << "'.");
3379                 v.erase(bit);
3380         }
3381         return v;
3382 }
3383
3384
3385 template<typename M>
3386 typename M::const_iterator greatest_below(M & m, typename M::key_type const & x)
3387 {
3388         if (m.empty())
3389                 return m.end();
3390
3391         typename M::const_iterator it = m.lower_bound(x);
3392         if (it == m.begin())
3393                 return m.end();
3394
3395         it--;
3396         return it;
3397 }
3398
3399
3400 MacroData const * Buffer::Impl::getBufferMacro(docstring const & name,
3401                                          DocIterator const & pos) const
3402 {
3403         LYXERR(Debug::MACROS, "Searching for " << to_ascii(name) << " at " << pos);
3404
3405         // if paragraphs have no macro context set, pos will be empty
3406         if (pos.empty())
3407                 return 0;
3408
3409         // we haven't found anything yet
3410         DocIterator bestPos = owner_->par_iterator_begin();
3411         MacroData const * bestData = 0;
3412
3413         // find macro definitions for name
3414         NamePositionScopeMacroMap::const_iterator nameIt = macros.find(name);
3415         if (nameIt != macros.end()) {
3416                 // find last definition in front of pos or at pos itself
3417                 PositionScopeMacroMap::const_iterator it
3418                         = greatest_below(nameIt->second, pos);
3419                 if (it != nameIt->second.end()) {
3420                         while (true) {
3421                                 // scope ends behind pos?
3422                                 if (pos < it->second.scope) {
3423                                         // Looks good, remember this. If there
3424                                         // is no external macro behind this,
3425                                         // we found the right one already.
3426                                         bestPos = it->first;
3427                                         bestData = &it->second.macro;
3428                                         break;
3429                                 }
3430
3431                                 // try previous macro if there is one
3432                                 if (it == nameIt->second.begin())
3433                                         break;
3434                                 --it;
3435                         }
3436                 }
3437         }
3438
3439         // find macros in included files
3440         PositionScopeBufferMap::const_iterator it
3441                 = greatest_below(position_to_children, pos);
3442         if (it == position_to_children.end())
3443                 // no children before
3444                 return bestData;
3445
3446         while (true) {
3447                 // do we know something better (i.e. later) already?
3448                 if (it->first < bestPos )
3449                         break;
3450
3451                 // scope ends behind pos?
3452                 if (pos < it->second.scope
3453                         && (cloned_buffer_ ||
3454                             theBufferList().isLoaded(it->second.buffer))) {
3455                         // look for macro in external file
3456                         macro_lock = true;
3457                         MacroData const * data
3458                                 = it->second.buffer->getMacro(name, false);
3459                         macro_lock = false;
3460                         if (data) {
3461                                 bestPos = it->first;
3462                                 bestData = data;
3463                                 break;
3464                         }
3465                 }
3466
3467                 // try previous file if there is one
3468                 if (it == position_to_children.begin())
3469                         break;
3470                 --it;
3471         }
3472
3473         // return the best macro we have found
3474         return bestData;
3475 }
3476
3477
3478 MacroData const * Buffer::getMacro(docstring const & name,
3479         DocIterator const & pos, bool global) const
3480 {
3481         if (d->macro_lock)
3482                 return 0;
3483
3484         // query buffer macros
3485         MacroData const * data = d->getBufferMacro(name, pos);
3486         if (data != 0)
3487                 return data;
3488
3489         // If there is a master buffer, query that
3490         Buffer const * const pbuf = d->parent();
3491         if (pbuf) {
3492                 d->macro_lock = true;
3493                 MacroData const * macro = pbuf->getMacro(
3494                         name, *this, false);
3495                 d->macro_lock = false;
3496                 if (macro)
3497                         return macro;
3498         }
3499
3500         if (global) {
3501                 data = MacroTable::globalMacros().get(name);
3502                 if (data != 0)
3503                         return data;
3504         }
3505
3506         return 0;
3507 }
3508
3509
3510 MacroData const * Buffer::getMacro(docstring const & name, bool global) const
3511 {
3512         // set scope end behind the last paragraph
3513         DocIterator scope = par_iterator_begin();
3514         scope.pit() = scope.lastpit() + 1;
3515
3516         return getMacro(name, scope, global);
3517 }
3518
3519
3520 MacroData const * Buffer::getMacro(docstring const & name,
3521         Buffer const & child, bool global) const
3522 {
3523         // look where the child buffer is included first
3524         Impl::BufferPositionMap::iterator it = d->children_positions.find(&child);
3525         if (it == d->children_positions.end())
3526                 return 0;
3527
3528         // check for macros at the inclusion position
3529         return getMacro(name, it->second, global);
3530 }
3531
3532
3533 void Buffer::Impl::updateMacros(DocIterator & it, DocIterator & scope)
3534 {
3535         pit_type const lastpit = it.lastpit();
3536
3537         // look for macros in each paragraph
3538         while (it.pit() <= lastpit) {
3539                 Paragraph & par = it.paragraph();
3540
3541                 // iterate over the insets of the current paragraph
3542                 InsetList const & insets = par.insetList();
3543                 InsetList::const_iterator iit = insets.begin();
3544                 InsetList::const_iterator end = insets.end();
3545                 for (; iit != end; ++iit) {
3546                         it.pos() = iit->pos;
3547
3548                         // is it a nested text inset?
3549                         if (iit->inset->asInsetText()) {
3550                                 // Inset needs its own scope?
3551                                 InsetText const * itext = iit->inset->asInsetText();
3552                                 bool newScope = itext->isMacroScope();
3553
3554                                 // scope which ends just behind the inset
3555                                 DocIterator insetScope = it;
3556                                 ++insetScope.pos();
3557
3558                                 // collect macros in inset
3559                                 it.push_back(CursorSlice(*iit->inset));
3560                                 updateMacros(it, newScope ? insetScope : scope);
3561                                 it.pop_back();
3562                                 continue;
3563                         }
3564
3565                         if (iit->inset->asInsetTabular()) {
3566                                 CursorSlice slice(*iit->inset);
3567                                 size_t const numcells = slice.nargs();
3568                                 for (; slice.idx() < numcells; slice.forwardIdx()) {
3569                                         it.push_back(slice);
3570                                         updateMacros(it, scope);
3571                                         it.pop_back();
3572                                 }
3573                                 continue;
3574                         }
3575
3576                         // is it an external file?
3577                         if (iit->inset->lyxCode() == INCLUDE_CODE) {
3578                                 // get buffer of external file
3579                                 InsetInclude const & inset =
3580                                         static_cast<InsetInclude const &>(*iit->inset);
3581                                 macro_lock = true;
3582                                 Buffer * child = inset.getChildBuffer();
3583                                 macro_lock = false;
3584                                 if (!child)
3585                                         continue;
3586
3587                                 // register its position, but only when it is
3588                                 // included first in the buffer
3589                                 if (children_positions.find(child) ==
3590                                         children_positions.end())
3591                                                 children_positions[child] = it;
3592
3593                                 // register child with its scope
3594                                 position_to_children[it] = Impl::ScopeBuffer(scope, child);
3595                                 continue;
3596                         }
3597
3598                         InsetMath * im = iit->inset->asInsetMath();
3599                         if (doing_export && im)  {
3600                                 InsetMathHull * hull = im->asHullInset();
3601                                 if (hull)
3602                                         hull->recordLocation(it);
3603                         }
3604
3605                         if (iit->inset->lyxCode() != MATHMACRO_CODE)
3606                                 continue;
3607
3608                         // get macro data
3609                         MathMacroTemplate & macroTemplate =
3610                                 *iit->inset->asInsetMath()->asMacroTemplate();
3611                         MacroContext mc(owner_, it);
3612                         macroTemplate.updateToContext(mc);
3613
3614                         // valid?
3615                         bool valid = macroTemplate.validMacro();
3616                         // FIXME: Should be fixNameAndCheckIfValid() in fact,
3617                         // then the BufferView's cursor will be invalid in
3618                         // some cases which leads to crashes.
3619                         if (!valid)
3620                                 continue;
3621
3622                         // register macro
3623                         // FIXME (Abdel), I don't understand why we pass 'it' here
3624                         // instead of 'macroTemplate' defined above... is this correct?
3625                         macros[macroTemplate.name()][it] =
3626                                 Impl::ScopeMacro(scope, MacroData(const_cast<Buffer *>(owner_), it));
3627                 }
3628
3629                 // next paragraph
3630                 it.pit()++;
3631                 it.pos() = 0;
3632         }
3633 }
3634
3635
3636 void Buffer::updateMacros() const
3637 {
3638         if (d->macro_lock)
3639                 return;
3640
3641         LYXERR(Debug::MACROS, "updateMacro of " << d->filename.onlyFileName());
3642
3643         // start with empty table
3644         d->macros.clear();
3645         d->children_positions.clear();
3646         d->position_to_children.clear();
3647
3648         // Iterate over buffer, starting with first paragraph
3649         // The scope must be bigger than any lookup DocIterator
3650         // later. For the global lookup, lastpit+1 is used, hence
3651         // we use lastpit+2 here.
3652         DocIterator it = par_iterator_begin();
3653         DocIterator outerScope = it;
3654         outerScope.pit() = outerScope.lastpit() + 2;
3655         d->updateMacros(it, outerScope);
3656 }
3657
3658
3659 void Buffer::getUsedBranches(std::list<docstring> & result, bool const from_master) const
3660 {
3661         InsetIterator it  = inset_iterator_begin(inset());
3662         InsetIterator const end = inset_iterator_end(inset());
3663         for (; it != end; ++it) {
3664                 if (it->lyxCode() == BRANCH_CODE) {
3665                         InsetBranch & br = static_cast<InsetBranch &>(*it);
3666                         docstring const name = br.branch();
3667                         if (!from_master && !params().branchlist().find(name))
3668                                 result.push_back(name);
3669                         else if (from_master && !masterBuffer()->params().branchlist().find(name))
3670                                 result.push_back(name);
3671                         continue;
3672                 }
3673                 if (it->lyxCode() == INCLUDE_CODE) {
3674                         // get buffer of external file
3675                         InsetInclude const & ins =
3676                                 static_cast<InsetInclude const &>(*it);
3677                         Buffer * child = ins.getChildBuffer();
3678                         if (!child)
3679                                 continue;
3680                         child->getUsedBranches(result, true);
3681                 }
3682         }
3683         // remove duplicates
3684         result.unique();
3685 }
3686
3687
3688 void Buffer::updateMacroInstances(UpdateType utype) const
3689 {
3690         LYXERR(Debug::MACROS, "updateMacroInstances for "
3691                 << d->filename.onlyFileName());
3692         DocIterator it = doc_iterator_begin(this);
3693         it.forwardInset();
3694         DocIterator const end = doc_iterator_end(this);
3695         for (; it != end; it.forwardInset()) {
3696                 // look for MathData cells in InsetMathNest insets
3697                 InsetMath * minset = it.nextInset()->asInsetMath();
3698                 if (!minset)
3699                         continue;
3700
3701                 // update macro in all cells of the InsetMathNest
3702                 DocIterator::idx_type n = minset->nargs();
3703                 MacroContext mc = MacroContext(this, it);
3704                 for (DocIterator::idx_type i = 0; i < n; ++i) {
3705                         MathData & data = minset->cell(i);
3706                         data.updateMacros(0, mc, utype, 0);
3707                 }
3708         }
3709 }
3710
3711
3712 void Buffer::listMacroNames(MacroNameSet & macros) const
3713 {
3714         if (d->macro_lock)
3715                 return;
3716
3717         d->macro_lock = true;
3718
3719         // loop over macro names
3720         Impl::NamePositionScopeMacroMap::iterator nameIt = d->macros.begin();
3721         Impl::NamePositionScopeMacroMap::iterator nameEnd = d->macros.end();
3722         for (; nameIt != nameEnd; ++nameIt)
3723                 macros.insert(nameIt->first);
3724
3725         // loop over children
3726         Impl::BufferPositionMap::iterator it = d->children_positions.begin();
3727         Impl::BufferPositionMap::iterator end = d->children_positions.end();
3728         for (; it != end; ++it)
3729                 it->first->listMacroNames(macros);
3730
3731         // call parent
3732         Buffer const * const pbuf = d->parent();
3733         if (pbuf)
3734                 pbuf->listMacroNames(macros);
3735
3736         d->macro_lock = false;
3737 }
3738
3739
3740 void Buffer::listParentMacros(MacroSet & macros, LaTeXFeatures & features) const
3741 {
3742         Buffer const * const pbuf = d->parent();
3743         if (!pbuf)
3744                 return;
3745
3746         MacroNameSet names;
3747         pbuf->listMacroNames(names);
3748
3749         // resolve macros
3750         MacroNameSet::iterator it = names.begin();
3751         MacroNameSet::iterator end = names.end();
3752         for (; it != end; ++it) {
3753                 // defined?
3754                 MacroData const * data =
3755                 pbuf->getMacro(*it, *this, false);
3756                 if (data) {
3757                         macros.insert(data);
3758
3759                         // we cannot access the original MathMacroTemplate anymore
3760                         // here to calls validate method. So we do its work here manually.
3761                         // FIXME: somehow make the template accessible here.
3762                         if (data->optionals() > 0)
3763                                 features.require("xargs");
3764                 }
3765         }
3766 }
3767
3768
3769 Buffer::References & Buffer::getReferenceCache(docstring const & label)
3770 {
3771         if (d->parent())
3772                 return const_cast<Buffer *>(masterBuffer())->getReferenceCache(label);
3773
3774         RefCache::iterator it = d->ref_cache_.find(label);
3775         if (it != d->ref_cache_.end())
3776                 return it->second.second;
3777
3778         static InsetLabel const * dummy_il = 0;
3779         static References const dummy_refs = References();
3780         it = d->ref_cache_.insert(
3781                 make_pair(label, make_pair(dummy_il, dummy_refs))).first;
3782         return it->second.second;
3783 }
3784
3785
3786 Buffer::References const & Buffer::references(docstring const & label) const
3787 {
3788         return const_cast<Buffer *>(this)->getReferenceCache(label);
3789 }
3790
3791
3792 void Buffer::addReference(docstring const & label, Inset * inset, ParIterator it)
3793 {
3794         References & refs = getReferenceCache(label);
3795         refs.push_back(make_pair(inset, it));
3796 }
3797
3798
3799 void Buffer::setInsetLabel(docstring const & label, InsetLabel const * il)
3800 {
3801         masterBuffer()->d->ref_cache_[label].first = il;
3802 }
3803
3804
3805 InsetLabel const * Buffer::insetLabel(docstring const & label) const
3806 {
3807         return masterBuffer()->d->ref_cache_[label].first;
3808 }
3809
3810
3811 void Buffer::clearReferenceCache() const
3812 {
3813         if (!d->parent())
3814                 d->ref_cache_.clear();
3815 }
3816
3817
3818 void Buffer::changeRefsIfUnique(docstring const & from, docstring const & to)
3819 {
3820         //FIXME: This does not work for child documents yet.
3821         reloadBibInfoCache();
3822
3823         // Check if the label 'from' appears more than once
3824         BiblioInfo const & keys = masterBibInfo();
3825         BiblioInfo::const_iterator bit  = keys.begin();
3826         BiblioInfo::const_iterator bend = keys.end();
3827         vector<docstring> labels;
3828
3829         for (; bit != bend; ++bit)
3830                 // FIXME UNICODE
3831                 labels.push_back(bit->first);
3832
3833         if (count(labels.begin(), labels.end(), from) > 1)
3834                 return;
3835
3836         string const paramName = "key";
3837         for (InsetIterator it = inset_iterator_begin(inset()); it; ++it) {
3838                 if (it->lyxCode() != CITE_CODE)
3839                         continue;
3840                 InsetCommand * inset = it->asInsetCommand();
3841                 docstring const oldValue = inset->getParam(paramName);
3842                 if (oldValue == from)
3843                         inset->setParam(paramName, to);
3844         }
3845 }
3846
3847 // returns NULL if id-to-row conversion is unsupported
3848 unique_ptr<TexRow> Buffer::getSourceCode(odocstream & os, string const & format,
3849                                          pit_type par_begin, pit_type par_end,
3850                                          OutputWhat output, bool master) const
3851 {
3852         unique_ptr<TexRow> texrow;
3853         OutputParams runparams(&params().encoding());
3854         runparams.nice = true;
3855         runparams.flavor = params().getOutputFlavor(format);
3856         runparams.linelen = lyxrc.plaintext_linelen;
3857         // No side effect of file copying and image conversion
3858         runparams.dryrun = true;
3859
3860         if (output == CurrentParagraph) {
3861                 runparams.par_begin = par_begin;
3862                 runparams.par_end = par_end;
3863                 if (par_begin + 1 == par_end) {
3864                         os << "% "
3865                            << bformat(_("Preview source code for paragraph %1$d"), par_begin)
3866                            << "\n\n";
3867                 } else {
3868                         os << "% "
3869                            << bformat(_("Preview source code from paragraph %1$s to %2$s"),
3870                                         convert<docstring>(par_begin),
3871                                         convert<docstring>(par_end - 1))
3872                            << "\n\n";
3873                 }
3874                 // output paragraphs
3875                 if (runparams.flavor == OutputParams::LYX) {
3876                         Paragraph const & par = text().paragraphs()[par_begin];
3877                         ostringstream ods;
3878                         depth_type dt = par.getDepth();
3879                         par.write(ods, params(), dt);
3880                         os << from_utf8(ods.str());
3881                 } else if (runparams.flavor == OutputParams::HTML) {
3882                         XHTMLStream xs(os);
3883                         setMathFlavor(runparams);
3884                         xhtmlParagraphs(text(), *this, xs, runparams);
3885                 } else if (runparams.flavor == OutputParams::TEXT) {
3886                         bool dummy = false;
3887                         // FIXME Handles only one paragraph, unlike the others.
3888                         // Probably should have some routine with a signature like them.
3889                         writePlaintextParagraph(*this,
3890                                 text().paragraphs()[par_begin], os, runparams, dummy);
3891                 } else if (params().isDocBook()) {
3892                         docbookParagraphs(text(), *this, os, runparams);
3893                 } else {
3894                         // If we are previewing a paragraph, even if this is the
3895                         // child of some other buffer, let's cut the link here,
3896                         // so that no concurring settings from the master
3897                         // (e.g. branch state) interfere (see #8101).
3898                         if (!master)
3899                                 d->ignore_parent = true;
3900                         // We need to validate the Buffer params' features here
3901                         // in order to know if we should output polyglossia
3902                         // macros (instead of babel macros)
3903                         LaTeXFeatures features(*this, params(), runparams);
3904                         validate(features);
3905                         runparams.use_polyglossia = features.usePolyglossia();
3906                         // latex or literate
3907                         otexstream ots(os);
3908                         // output above
3909                         ots.texrow().newlines(2);
3910                         // the real stuff
3911                         latexParagraphs(*this, text(), ots, runparams);
3912                         texrow = ots.releaseTexRow();
3913
3914                         // Restore the parenthood
3915                         if (!master)
3916                                 d->ignore_parent = false;
3917                 }
3918         } else {
3919                 os << "% ";
3920                 if (output == FullSource)
3921                         os << _("Preview source code");
3922                 else if (output == OnlyPreamble)
3923                         os << _("Preview preamble");
3924                 else if (output == OnlyBody)
3925                         os << _("Preview body");
3926                 os << "\n\n";
3927                 if (runparams.flavor == OutputParams::LYX) {
3928                         ostringstream ods;
3929                         if (output == FullSource)
3930                                 write(ods);
3931                         else if (output == OnlyPreamble)
3932                                 params().writeFile(ods, this);
3933                         else if (output == OnlyBody)
3934                                 text().write(ods);
3935                         os << from_utf8(ods.str());
3936                 } else if (runparams.flavor == OutputParams::HTML) {
3937                         writeLyXHTMLSource(os, runparams, output);
3938                 } else if (runparams.flavor == OutputParams::TEXT) {
3939                         if (output == OnlyPreamble) {
3940                                 os << "% "<< _("Plain text does not have a preamble.");
3941                         } else
3942                                 writePlaintextFile(*this, os, runparams);
3943                 } else if (params().isDocBook()) {
3944                                 writeDocBookSource(os, absFileName(), runparams, output);
3945                 } else {
3946                         // latex or literate
3947                         otexstream ots(os);
3948                         // output above
3949                         ots.texrow().newlines(2);
3950                         if (master)
3951                                 runparams.is_child = true;
3952                         updateBuffer();
3953                         writeLaTeXSource(ots, string(), runparams, output);
3954                         texrow = ots.releaseTexRow();
3955                 }
3956         }
3957         return texrow;
3958 }
3959
3960
3961 ErrorList & Buffer::errorList(string const & type) const
3962 {
3963         return d->errorLists[type];
3964 }
3965
3966
3967 void Buffer::updateTocItem(std::string const & type,
3968         DocIterator const & dit) const
3969 {
3970         if (d->gui_)
3971                 d->gui_->updateTocItem(type, dit);
3972 }
3973
3974
3975 void Buffer::structureChanged() const
3976 {
3977         if (d->gui_)
3978                 d->gui_->structureChanged();
3979 }
3980
3981
3982 void Buffer::errors(string const & err, bool from_master) const
3983 {
3984         if (d->gui_)
3985                 d->gui_->errors(err, from_master);
3986 }
3987
3988
3989 void Buffer::message(docstring const & msg) const
3990 {
3991         if (d->gui_)
3992                 d->gui_->message(msg);
3993 }
3994
3995
3996 void Buffer::setBusy(bool on) const
3997 {
3998         if (d->gui_)
3999                 d->gui_->setBusy(on);
4000 }
4001
4002
4003 void Buffer::updateTitles() const
4004 {
4005         if (d->wa_)
4006                 d->wa_->updateTitles();
4007 }
4008
4009
4010 void Buffer::resetAutosaveTimers() const
4011 {
4012         if (d->gui_)
4013                 d->gui_->resetAutosaveTimers();
4014 }
4015
4016
4017 bool Buffer::hasGuiDelegate() const
4018 {
4019         return d->gui_;
4020 }
4021
4022
4023 void Buffer::setGuiDelegate(frontend::GuiBufferDelegate * gui)
4024 {
4025         d->gui_ = gui;
4026 }
4027
4028
4029
4030 namespace {
4031
4032 class AutoSaveBuffer : public ForkedProcess {
4033 public:
4034         ///
4035         AutoSaveBuffer(Buffer const & buffer, FileName const & fname)
4036                 : buffer_(buffer), fname_(fname) {}
4037         ///
4038         virtual shared_ptr<ForkedProcess> clone() const
4039         {
4040                 return make_shared<AutoSaveBuffer>(*this);
4041         }
4042         ///
4043         int start()
4044         {
4045                 command_ = to_utf8(bformat(_("Auto-saving %1$s"),
4046                                                  from_utf8(fname_.absFileName())));
4047                 return run(DontWait);
4048         }
4049 private:
4050         ///
4051         virtual int generateChild();
4052         ///
4053         Buffer const & buffer_;
4054         FileName fname_;
4055 };
4056
4057
4058 int AutoSaveBuffer::generateChild()
4059 {
4060 #if defined(__APPLE__)
4061         /* FIXME fork() is not usable for autosave on Mac OS X 10.6 (snow leopard)
4062          *   We should use something else like threads.
4063          *
4064          * Since I do not know how to determine at run time what is the OS X
4065          * version, I just disable forking altogether for now (JMarc)
4066          */
4067         pid_t const pid = -1;
4068 #else
4069         // tmp_ret will be located (usually) in /tmp
4070         // will that be a problem?
4071         // Note that this calls ForkedCalls::fork(), so it's
4072         // ok cross-platform.
4073         pid_t const pid = fork();
4074         // If you want to debug the autosave
4075         // you should set pid to -1, and comment out the fork.
4076         if (pid != 0 && pid != -1)
4077                 return pid;
4078 #endif
4079
4080         // pid = -1 signifies that lyx was unable
4081         // to fork. But we will do the save
4082         // anyway.
4083         bool failed = false;
4084         TempFile tempfile("lyxautoXXXXXX.lyx");
4085         tempfile.setAutoRemove(false);
4086         FileName const tmp_ret = tempfile.name();
4087         if (!tmp_ret.empty()) {
4088                 if (!buffer_.writeFile(tmp_ret))
4089                         failed = true;
4090                 else if (!tmp_ret.moveTo(fname_))
4091                         failed = true;
4092         } else
4093                 failed = true;
4094
4095         if (failed) {
4096                 // failed to write/rename tmp_ret so try writing direct
4097                 if (!buffer_.writeFile(fname_)) {
4098                         // It is dangerous to do this in the child,
4099                         // but safe in the parent, so...
4100                         if (pid == -1) // emit message signal.
4101                                 buffer_.message(_("Autosave failed!"));
4102                 }
4103         }
4104
4105         if (pid == 0) // we are the child so...
4106                 _exit(0);
4107
4108         return pid;
4109 }
4110
4111 } // namespace anon
4112
4113
4114 FileName Buffer::getEmergencyFileName() const
4115 {
4116         return FileName(d->filename.absFileName() + ".emergency");
4117 }
4118
4119
4120 FileName Buffer::getAutosaveFileName() const
4121 {
4122         // if the document is unnamed try to save in the backup dir, else
4123         // in the default document path, and as a last try in the filePath,
4124         // which will most often be the temporary directory
4125         string fpath;
4126         if (isUnnamed())
4127                 fpath = lyxrc.backupdir_path.empty() ? lyxrc.document_path
4128                         : lyxrc.backupdir_path;
4129         if (!isUnnamed() || fpath.empty() || !FileName(fpath).exists())
4130                 fpath = filePath();
4131
4132         string const fname = "#" + d->filename.onlyFileName() + "#";
4133
4134         return makeAbsPath(fname, fpath);
4135 }
4136
4137
4138 void Buffer::removeAutosaveFile() const
4139 {
4140         FileName const f = getAutosaveFileName();
4141         if (f.exists())
4142                 f.removeFile();
4143 }
4144
4145
4146 void Buffer::moveAutosaveFile(support::FileName const & oldauto) const
4147 {
4148         FileName const newauto = getAutosaveFileName();
4149         oldauto.refresh();
4150         if (newauto != oldauto && oldauto.exists())
4151                 if (!oldauto.moveTo(newauto))
4152                         LYXERR0("Unable to move autosave file `" << oldauto << "'!");
4153 }
4154
4155
4156 bool Buffer::autoSave() const
4157 {
4158         Buffer const * buf = d->cloned_buffer_ ? d->cloned_buffer_ : this;
4159         if (buf->d->bak_clean || hasReadonlyFlag())
4160                 return true;
4161
4162         message(_("Autosaving current document..."));
4163         buf->d->bak_clean = true;
4164
4165         FileName const fname = getAutosaveFileName();
4166         LASSERT(d->cloned_buffer_, return false);
4167
4168         // If this buffer is cloned, we assume that
4169         // we are running in a separate thread already.
4170         TempFile tempfile("lyxautoXXXXXX.lyx");
4171         tempfile.setAutoRemove(false);
4172         FileName const tmp_ret = tempfile.name();
4173         if (!tmp_ret.empty()) {
4174                 writeFile(tmp_ret);
4175                 // assume successful write of tmp_ret
4176                 if (tmp_ret.moveTo(fname))
4177                         return true;
4178         }
4179         // failed to write/rename tmp_ret so try writing direct
4180         return writeFile(fname);
4181 }
4182
4183
4184 void Buffer::setExportStatus(bool e) const
4185 {
4186         d->doing_export = e;
4187         ListOfBuffers clist = getDescendents();
4188         ListOfBuffers::const_iterator cit = clist.begin();
4189         ListOfBuffers::const_iterator const cen = clist.end();
4190         for (; cit != cen; ++cit)
4191                 (*cit)->d->doing_export = e;
4192 }
4193
4194
4195 bool Buffer::isExporting() const
4196 {
4197         return d->doing_export;
4198 }
4199
4200
4201 Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir)
4202         const
4203 {
4204         string result_file;
4205         return doExport(target, put_in_tempdir, result_file);
4206 }
4207
4208 Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir,
4209         string & result_file) const
4210 {
4211         bool const update_unincluded =
4212                         params().maintain_unincluded_children
4213                         && !params().getIncludedChildren().empty();
4214
4215         // (1) export with all included children (omit \includeonly)
4216         if (update_unincluded) {
4217                 ExportStatus const status =
4218                         doExport(target, put_in_tempdir, true, result_file);
4219                 if (status != ExportSuccess)
4220                         return status;
4221         }
4222         // (2) export with included children only
4223         return doExport(target, put_in_tempdir, false, result_file);
4224 }
4225
4226
4227 void Buffer::setMathFlavor(OutputParams & op) const
4228 {
4229         switch (params().html_math_output) {
4230         case BufferParams::MathML:
4231                 op.math_flavor = OutputParams::MathAsMathML;
4232                 break;
4233         case BufferParams::HTML:
4234                 op.math_flavor = OutputParams::MathAsHTML;
4235                 break;
4236         case BufferParams::Images:
4237                 op.math_flavor = OutputParams::MathAsImages;
4238                 break;
4239         case BufferParams::LaTeX:
4240                 op.math_flavor = OutputParams::MathAsLaTeX;
4241                 break;
4242         }
4243 }
4244
4245
4246 Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir,
4247         bool includeall, string & result_file) const
4248 {
4249         LYXERR(Debug::FILES, "target=" << target);
4250         OutputParams runparams(&params().encoding());
4251         string format = target;
4252         string dest_filename;
4253         size_t pos = target.find(' ');
4254         if (pos != string::npos) {
4255                 dest_filename = target.substr(pos + 1, target.length() - pos - 1);
4256                 format = target.substr(0, pos);
4257                 if (format == "default")
4258                         format = params().getDefaultOutputFormat();
4259                 runparams.export_folder = FileName(dest_filename).onlyPath().realPath();
4260                 FileName(dest_filename).onlyPath().createPath();
4261                 LYXERR(Debug::FILES, "format=" << format << ", dest_filename=" << dest_filename << ", export_folder=" << runparams.export_folder);
4262         }
4263         MarkAsExporting exporting(this);
4264         string backend_format;
4265         runparams.flavor = OutputParams::LATEX;
4266         runparams.linelen = lyxrc.plaintext_linelen;
4267         runparams.includeall = includeall;
4268         vector<string> backs = params().backends();
4269         Converters converters = theConverters();
4270         bool need_nice_file = false;
4271         if (find(backs.begin(), backs.end(), format) == backs.end()) {
4272                 // Get shortest path to format
4273                 converters.buildGraph();
4274                 Graph::EdgePath path;
4275                 for (vector<string>::const_iterator it = backs.begin();
4276                      it != backs.end(); ++it) {
4277                         Graph::EdgePath p = converters.getPath(*it, format);
4278                         if (!p.empty() && (path.empty() || p.size() < path.size())) {
4279                                 backend_format = *it;
4280                                 path = p;
4281                         }
4282                 }
4283                 if (path.empty()) {
4284                         if (!put_in_tempdir) {
4285                                 // Only show this alert if this is an export to a non-temporary
4286                                 // file (not for previewing).
4287                                 Alert::error(_("Couldn't export file"), bformat(
4288                                         _("No information for exporting the format %1$s."),
4289                                         theFormats().prettyName(format)));
4290                         }
4291                         return ExportNoPathToFormat;
4292                 }
4293                 runparams.flavor = converters.getFlavor(path, this);
4294                 Graph::EdgePath::const_iterator it = path.begin();
4295                 Graph::EdgePath::const_iterator en = path.end();
4296                 for (; it != en; ++it)
4297                         if (theConverters().get(*it).nice()) {
4298                                 need_nice_file = true;
4299                                 break;
4300                         }
4301
4302         } else {
4303                 backend_format = format;
4304                 LYXERR(Debug::FILES, "backend_format=" << backend_format);
4305                 // FIXME: Don't hardcode format names here, but use a flag
4306                 if (backend_format == "pdflatex")
4307                         runparams.flavor = OutputParams::PDFLATEX;
4308                 else if (backend_format == "luatex")
4309                         runparams.flavor = OutputParams::LUATEX;
4310                 else if (backend_format == "dviluatex")
4311                         runparams.flavor = OutputParams::DVILUATEX;
4312                 else if (backend_format == "xetex")
4313                         runparams.flavor = OutputParams::XETEX;
4314         }
4315
4316         string filename = latexName(false);
4317         filename = addName(temppath(), filename);
4318         filename = changeExtension(filename,
4319                                    theFormats().extension(backend_format));
4320         LYXERR(Debug::FILES, "filename=" << filename);
4321
4322         // Plain text backend
4323         if (backend_format == "text") {
4324                 runparams.flavor = OutputParams::TEXT;
4325                 writePlaintextFile(*this, FileName(filename), runparams);
4326         }
4327         // HTML backend
4328         else if (backend_format == "xhtml") {
4329                 runparams.flavor = OutputParams::HTML;
4330                 setMathFlavor(runparams);
4331                 makeLyXHTMLFile(FileName(filename), runparams);
4332         } else if (backend_format == "lyx")
4333                 writeFile(FileName(filename));
4334         // Docbook backend
4335         else if (params().isDocBook()) {
4336                 runparams.nice = !put_in_tempdir;
4337                 makeDocBookFile(FileName(filename), runparams);
4338         }
4339         // LaTeX backend
4340         else if (backend_format == format || need_nice_file) {
4341                 runparams.nice = true;
4342                 bool const success = makeLaTeXFile(FileName(filename), string(), runparams);
4343                 if (d->cloned_buffer_)
4344                         d->cloned_buffer_->d->errorLists["Export"] = d->errorLists["Export"];
4345                 if (!success)
4346                         return ExportError;
4347         } else if (!lyxrc.tex_allows_spaces
4348                    && contains(filePath(), ' ')) {
4349                 Alert::error(_("File name error"),
4350                            _("The directory path to the document cannot contain spaces."));
4351                 return ExportTexPathHasSpaces;
4352         } else {
4353                 runparams.nice = false;
4354                 bool const success = makeLaTeXFile(
4355                         FileName(filename), filePath(), runparams);
4356                 if (d->cloned_buffer_)
4357                         d->cloned_buffer_->d->errorLists["Export"] = d->errorLists["Export"];
4358                 if (!success)
4359                         return ExportError;
4360         }
4361
4362         string const error_type = (format == "program")
4363                 ? "Build" : params().bufferFormat();
4364         ErrorList & error_list = d->errorLists[error_type];
4365         string const ext = theFormats().extension(format);
4366         FileName const tmp_result_file(changeExtension(filename, ext));
4367         bool const success = converters.convert(this, FileName(filename),
4368                 tmp_result_file, FileName(absFileName()), backend_format, format,
4369                 error_list);
4370
4371         // Emit the signal to show the error list or copy it back to the
4372         // cloned Buffer so that it can be emitted afterwards.
4373         if (format != backend_format) {
4374                 if (runparams.silent)
4375                         error_list.clear();
4376                 else if (d->cloned_buffer_)
4377                         d->cloned_buffer_->d->errorLists[error_type] =
4378                                 d->errorLists[error_type];
4379                 else
4380                         errors(error_type);
4381                 // also to the children, in case of master-buffer-view
4382                 ListOfBuffers clist = getDescendents();
4383                 ListOfBuffers::const_iterator cit = clist.begin();
4384                 ListOfBuffers::const_iterator const cen = clist.end();
4385                 for (; cit != cen; ++cit) {
4386                         if (runparams.silent)
4387                                 (*cit)->d->errorLists[error_type].clear();
4388                         else if (d->cloned_buffer_) {
4389                                 // Enable reverse search by copying back the
4390                                 // texrow object to the cloned buffer.
4391                                 // FIXME: this is not thread safe.
4392                                 (*cit)->d->cloned_buffer_->d->texrow = (*cit)->d->texrow;
4393                                 (*cit)->d->cloned_buffer_->d->errorLists[error_type] =
4394                                         (*cit)->d->errorLists[error_type];
4395                         } else
4396                                 (*cit)->errors(error_type, true);
4397                 }
4398         }
4399
4400         if (d->cloned_buffer_) {
4401                 // Enable reverse dvi or pdf to work by copying back the texrow
4402                 // object to the cloned buffer.
4403                 // FIXME: There is a possibility of concurrent access to texrow
4404                 // here from the main GUI thread that should be securized.
4405                 d->cloned_buffer_->d->texrow = d->texrow;
4406                 string const error_type = params().bufferFormat();
4407                 d->cloned_buffer_->d->errorLists[error_type] = d->errorLists[error_type];
4408         }
4409
4410
4411         if (put_in_tempdir) {
4412                 result_file = tmp_result_file.absFileName();
4413                 return success ? ExportSuccess : ExportConverterError;
4414         }
4415
4416         if (dest_filename.empty())
4417                 result_file = changeExtension(d->exportFileName().absFileName(), ext);
4418         else
4419                 result_file = dest_filename;
4420         // We need to copy referenced files (e. g. included graphics
4421         // if format == "dvi") to the result dir.
4422         vector<ExportedFile> const files =
4423                 runparams.exportdata->externalFiles(format);
4424         string const dest = runparams.export_folder.empty() ?
4425                 onlyPath(result_file) : runparams.export_folder;
4426         bool use_force = use_gui ? lyxrc.export_overwrite == ALL_FILES
4427                                  : force_overwrite == ALL_FILES;
4428         CopyStatus status = use_force ? FORCE : SUCCESS;
4429
4430         vector<ExportedFile>::const_iterator it = files.begin();
4431         vector<ExportedFile>::const_iterator const en = files.end();
4432         for (; it != en && status != CANCEL; ++it) {
4433                 string const fmt = theFormats().getFormatFromFile(it->sourceName);
4434                 string fixedName = it->exportName;
4435                 if (!runparams.export_folder.empty()) {
4436                         // Relative pathnames starting with ../ will be sanitized
4437                         // if exporting to a different folder
4438                         while (fixedName.substr(0, 3) == "../")
4439                                 fixedName = fixedName.substr(3, fixedName.length() - 3);
4440                 }
4441                 FileName fixedFileName = makeAbsPath(fixedName, dest);
4442                 fixedFileName.onlyPath().createPath();
4443                 status = copyFile(fmt, it->sourceName,
4444                         fixedFileName,
4445                         it->exportName, status == FORCE,
4446                         runparams.export_folder.empty());
4447         }
4448
4449         if (status == CANCEL) {
4450                 message(_("Document export cancelled."));
4451                 return ExportCancel;
4452         }
4453
4454         if (tmp_result_file.exists()) {
4455                 // Finally copy the main file
4456                 use_force = use_gui ? lyxrc.export_overwrite != NO_FILES
4457                                     : force_overwrite != NO_FILES;
4458                 if (status == SUCCESS && use_force)
4459                         status = FORCE;
4460                 status = copyFile(format, tmp_result_file,
4461                         FileName(result_file), result_file,
4462                         status == FORCE);
4463                 if (status == CANCEL) {
4464                         message(_("Document export cancelled."));
4465                         return ExportCancel;
4466                 } else {
4467                         message(bformat(_("Document exported as %1$s "
4468                                 "to file `%2$s'"),
4469                                 theFormats().prettyName(format),
4470                                 makeDisplayPath(result_file)));
4471                 }
4472         } else {
4473                 // This must be a dummy converter like fax (bug 1888)
4474                 message(bformat(_("Document exported as %1$s"),
4475                         theFormats().prettyName(format)));
4476         }
4477
4478         return success ? ExportSuccess : ExportConverterError;
4479 }
4480
4481
4482 Buffer::ExportStatus Buffer::preview(string const & format) const
4483 {
4484         bool const update_unincluded =
4485                         params().maintain_unincluded_children
4486                         && !params().getIncludedChildren().empty();
4487         return preview(format, update_unincluded);
4488 }
4489
4490
4491 Buffer::ExportStatus Buffer::preview(string const & format, bool includeall) const
4492 {
4493         MarkAsExporting exporting(this);
4494         string result_file;
4495         // (1) export with all included children (omit \includeonly)
4496         if (includeall) {
4497                 ExportStatus const status = doExport(format, true, true, result_file);
4498                 if (status != ExportSuccess)
4499                         return status;
4500         }
4501         // (2) export with included children only
4502         ExportStatus const status = doExport(format, true, false, result_file);
4503         FileName const previewFile(result_file);
4504
4505         Impl * theimpl = isClone() ? d->cloned_buffer_->d : d;
4506         theimpl->preview_file_ = previewFile;
4507         theimpl->preview_format_ = format;
4508         theimpl->preview_error_ = (status != ExportSuccess);
4509
4510         if (status != ExportSuccess)
4511                 return status;
4512
4513         if (previewFile.exists())
4514                 return theFormats().view(*this, previewFile, format) ?
4515                         PreviewSuccess : PreviewError;
4516
4517         // Successful export but no output file?
4518         // Probably a bug in error detection.
4519         LATTEST(status != ExportSuccess);
4520         return status;
4521 }
4522
4523
4524 Buffer::ReadStatus Buffer::extractFromVC()
4525 {
4526         bool const found = LyXVC::file_not_found_hook(d->filename);
4527         if (!found)
4528                 return ReadFileNotFound;
4529         if (!d->filename.isReadableFile())
4530                 return ReadVCError;
4531         return ReadSuccess;
4532 }
4533
4534
4535 Buffer::ReadStatus Buffer::loadEmergency()
4536 {
4537         FileName const emergencyFile = getEmergencyFileName();
4538         if (!emergencyFile.exists()
4539                   || emergencyFile.lastModified() <= d->filename.lastModified())
4540                 return ReadFileNotFound;
4541
4542         docstring const file = makeDisplayPath(d->filename.absFileName(), 20);
4543         docstring const text = bformat(_("An emergency save of the document "
4544                 "%1$s exists.\n\nRecover emergency save?"), file);
4545
4546         int const load_emerg = Alert::prompt(_("Load emergency save?"), text,
4547                 0, 2, _("&Recover"), _("&Load Original"), _("&Cancel"));
4548
4549         switch (load_emerg)
4550         {
4551         case 0: {
4552                 docstring str;
4553                 ReadStatus const ret_llf = loadThisLyXFile(emergencyFile);
4554                 bool const success = (ret_llf == ReadSuccess);
4555                 if (success) {
4556                         if (hasReadonlyFlag()) {
4557                                 Alert::warning(_("File is read-only"),
4558                                         bformat(_("An emergency file is successfully loaded, "
4559                                         "but the original file %1$s is marked read-only. "
4560                                         "Please make sure to save the document as a different "
4561                                         "file."), from_utf8(d->filename.absFileName())));
4562                         }
4563                         markDirty();
4564                         lyxvc().file_found_hook(d->filename);
4565                         str = _("Document was successfully recovered.");
4566                 } else
4567                         str = _("Document was NOT successfully recovered.");
4568                 str += "\n\n" + bformat(_("Remove emergency file now?\n(%1$s)"),
4569                         makeDisplayPath(emergencyFile.absFileName()));
4570
4571                 int const del_emerg =
4572                         Alert::prompt(_("Delete emergency file?"), str, 1, 1,
4573                                 _("&Remove"), _("&Keep"));
4574                 if (del_emerg == 0) {
4575                         emergencyFile.removeFile();
4576                         if (success)
4577                                 Alert::warning(_("Emergency file deleted"),
4578                                         _("Do not forget to save your file now!"), true);
4579                         }
4580                 return success ? ReadSuccess : ReadEmergencyFailure;
4581         }
4582         case 1: {
4583                 int const del_emerg =
4584                         Alert::prompt(_("Delete emergency file?"),
4585                                 _("Remove emergency file now?"), 1, 1,
4586                                 _("&Remove"), _("&Keep"));
4587                 if (del_emerg == 0)
4588                         emergencyFile.removeFile();
4589                 return ReadOriginal;
4590         }
4591
4592         default:
4593                 break;
4594         }
4595         return ReadCancel;
4596 }
4597
4598
4599 Buffer::ReadStatus Buffer::loadAutosave()
4600 {
4601         // Now check if autosave file is newer.
4602         FileName const autosaveFile = getAutosaveFileName();
4603         if (!autosaveFile.exists()
4604                   || autosaveFile.lastModified() <= d->filename.lastModified())
4605                 return ReadFileNotFound;
4606
4607         docstring const file = makeDisplayPath(d->filename.absFileName(), 20);
4608         docstring const text = bformat(_("The backup of the document %1$s "
4609                 "is newer.\n\nLoad the backup instead?"), file);
4610         int const ret = Alert::prompt(_("Load backup?"), text, 0, 2,
4611                 _("&Load backup"), _("Load &original"), _("&Cancel"));
4612
4613         switch (ret)
4614         {
4615         case 0: {
4616                 ReadStatus const ret_llf = loadThisLyXFile(autosaveFile);
4617                 // the file is not saved if we load the autosave file.
4618                 if (ret_llf == ReadSuccess) {
4619                         if (hasReadonlyFlag()) {
4620                                 Alert::warning(_("File is read-only"),
4621                                         bformat(_("A backup file is successfully loaded, "
4622                                         "but the original file %1$s is marked read-only. "
4623                                         "Please make sure to save the document as a "
4624                                         "different file."),
4625                                         from_utf8(d->filename.absFileName())));
4626                         }
4627                         markDirty();
4628                         lyxvc().file_found_hook(d->filename);
4629                         return ReadSuccess;
4630                 }
4631                 return ReadAutosaveFailure;
4632         }
4633         case 1:
4634                 // Here we delete the autosave
4635                 autosaveFile.removeFile();
4636                 return ReadOriginal;
4637         default:
4638                 break;
4639         }
4640         return ReadCancel;
4641 }
4642
4643
4644 Buffer::ReadStatus Buffer::loadLyXFile()
4645 {
4646         if (!d->filename.isReadableFile()) {
4647                 ReadStatus const ret_rvc = extractFromVC();
4648                 if (ret_rvc != ReadSuccess)
4649                         return ret_rvc;
4650         }
4651
4652         ReadStatus const ret_re = loadEmergency();
4653         if (ret_re == ReadSuccess || ret_re == ReadCancel)
4654                 return ret_re;
4655
4656         ReadStatus const ret_ra = loadAutosave();
4657         if (ret_ra == ReadSuccess || ret_ra == ReadCancel)
4658                 return ret_ra;
4659
4660         return loadThisLyXFile(d->filename);
4661 }
4662
4663
4664 Buffer::ReadStatus Buffer::loadThisLyXFile(FileName const & fn)
4665 {
4666         return readFile(fn);
4667 }
4668
4669
4670 void Buffer::bufferErrors(TeXErrors const & terr, ErrorList & errorList) const
4671 {
4672         for (auto const & err : terr) {
4673                 TexRow::TextEntry start = TexRow::text_none, end = TexRow::text_none;
4674                 int errorRow = err.error_in_line;
4675                 Buffer const * buf = 0;
4676                 Impl const * p = d;
4677                 if (err.child_name.empty())
4678                         tie(start, end) = p->texrow.getEntriesFromRow(errorRow);
4679                 else {
4680                         // The error occurred in a child
4681                         for (Buffer const * child : getDescendents()) {
4682                                 string const child_name =
4683                                         DocFileName(changeExtension(child->absFileName(), "tex")).
4684                                         mangledFileName();
4685                                 if (err.child_name != child_name)
4686                                         continue;
4687                                 tie(start, end) = child->d->texrow.getEntriesFromRow(errorRow);
4688                                 if (!TexRow::isNone(start)) {
4689                                         buf = d->cloned_buffer_
4690                                                 ? child->d->cloned_buffer_->d->owner_
4691                                                 : child->d->owner_;
4692                                         p = child->d;
4693                                         break;
4694                                 }
4695                         }
4696                 }
4697                 errorList.push_back(ErrorItem(err.error_desc, err.error_text,
4698                                               start, end, buf));
4699         }
4700 }
4701
4702
4703 void Buffer::setBuffersForInsets() const
4704 {
4705         inset().setBuffer(const_cast<Buffer &>(*this));
4706 }
4707
4708
4709 void Buffer::updateBuffer(UpdateScope scope, UpdateType utype) const
4710 {
4711         LBUFERR(!text().paragraphs().empty());
4712
4713         // Use the master text class also for child documents
4714         Buffer const * const master = masterBuffer();
4715         DocumentClass const & textclass = master->params().documentClass();
4716
4717         // do this only if we are the top-level Buffer
4718         if (master == this) {
4719                 textclass.counters().reset(from_ascii("bibitem"));
4720                 reloadBibInfoCache();
4721         }
4722
4723         // keep the buffers to be children in this set. If the call from the
4724         // master comes back we can see which of them were actually seen (i.e.
4725         // via an InsetInclude). The remaining ones in the set need still be updated.
4726         static std::set<Buffer const *> bufToUpdate;
4727         if (scope == UpdateMaster) {
4728                 // If this is a child document start with the master
4729                 if (master != this) {
4730                         bufToUpdate.insert(this);
4731                         master->updateBuffer(UpdateMaster, utype);
4732                         // If the master buffer has no gui associated with it, then the TocModel is
4733                         // not updated during the updateBuffer call and TocModel::toc_ is invalid
4734                         // (bug 5699). The same happens if the master buffer is open in a different
4735                         // window. This test catches both possibilities.
4736                         // See: http://marc.info/?l=lyx-devel&m=138590578911716&w=2
4737                         // There remains a problem here: If there is another child open in yet a third
4738                         // window, that TOC is not updated. So some more general solution is needed at
4739                         // some point.
4740                         if (master->d->gui_ != d->gui_)
4741                                 structureChanged();
4742
4743                         // was buf referenced from the master (i.e. not in bufToUpdate anymore)?
4744                         if (bufToUpdate.find(this) == bufToUpdate.end())
4745                                 return;
4746                 }
4747
4748                 // start over the counters in the master
4749                 textclass.counters().reset();
4750         }
4751
4752         // update will be done below for this buffer
4753         bufToUpdate.erase(this);
4754
4755         // update all caches
4756         clearReferenceCache();
4757         updateMacros();
4758         setChangesPresent(false);
4759
4760         Buffer & cbuf = const_cast<Buffer &>(*this);
4761
4762         // do the real work
4763         ParIterator parit = cbuf.par_iterator_begin();
4764         updateBuffer(parit, utype);
4765
4766         if (master != this)
4767                 // If this document has siblings, then update the TocBackend later. The
4768                 // reason is to ensure that later siblings are up to date when e.g. the
4769                 // broken or not status of references is computed. The update is called
4770                 // in InsetInclude::addToToc.
4771                 return;
4772
4773         d->bibinfo_cache_valid_ = true;
4774         d->cite_labels_valid_ = true;
4775         /// FIXME: Perf
4776         cbuf.tocBackend().update(true, utype);
4777         if (scope == UpdateMaster)
4778                 cbuf.structureChanged();
4779 }
4780
4781
4782 static depth_type getDepth(DocIterator const & it)
4783 {
4784         depth_type depth = 0;
4785         for (size_t i = 0 ; i < it.depth() ; ++i)
4786                 if (!it[i].inset().inMathed())
4787                         depth += it[i].paragraph().getDepth() + 1;
4788         // remove 1 since the outer inset does not count
4789         // we should have at least one non-math inset, so
4790         // depth should nevery be 0. but maybe it is worth
4791         // marking this, just in case.
4792         LATTEST(depth > 0);
4793         // coverity[INTEGER_OVERFLOW]
4794         return depth - 1;
4795 }
4796
4797 static depth_type getItemDepth(ParIterator const & it)
4798 {
4799         Paragraph const & par = *it;
4800         LabelType const labeltype = par.layout().labeltype;
4801
4802         if (labeltype != LABEL_ENUMERATE && labeltype != LABEL_ITEMIZE)
4803                 return 0;
4804
4805         // this will hold the lowest depth encountered up to now.
4806         depth_type min_depth = getDepth(it);
4807         ParIterator prev_it = it;
4808         while (true) {
4809                 if (prev_it.pit())
4810                         --prev_it.top().pit();
4811                 else {
4812                         // start of nested inset: go to outer par
4813                         prev_it.pop_back();
4814                         if (prev_it.empty()) {
4815                                 // start of document: nothing to do
4816                                 return 0;
4817                         }
4818                 }
4819
4820                 // We search for the first paragraph with same label
4821                 // that is not more deeply nested.
4822                 Paragraph & prev_par = *prev_it;
4823                 depth_type const prev_depth = getDepth(prev_it);
4824                 if (labeltype == prev_par.layout().labeltype) {
4825                         if (prev_depth < min_depth)
4826                                 return prev_par.itemdepth + 1;
4827                         if (prev_depth == min_depth)
4828                                 return prev_par.itemdepth;
4829                 }
4830                 min_depth = min(min_depth, prev_depth);
4831                 // small optimization: if we are at depth 0, we won't
4832                 // find anything else
4833                 if (prev_depth == 0)
4834                         return 0;
4835         }
4836 }
4837
4838
4839 static bool needEnumCounterReset(ParIterator const & it)
4840 {
4841         Paragraph const & par = *it;
4842         LASSERT(par.layout().labeltype == LABEL_ENUMERATE, return false);
4843         depth_type const cur_depth = par.getDepth();
4844         ParIterator prev_it = it;
4845         while (prev_it.pit()) {
4846                 --prev_it.top().pit();
4847                 Paragraph const & prev_par = *prev_it;
4848                 if (prev_par.getDepth() <= cur_depth)
4849                         return prev_par.layout().name() != par.layout().name();
4850         }
4851         // start of nested inset: reset
4852         return true;
4853 }
4854
4855
4856 // set the label of a paragraph. This includes the counters.
4857 void Buffer::Impl::setLabel(ParIterator & it, UpdateType utype) const
4858 {
4859         BufferParams const & bp = owner_->masterBuffer()->params();
4860         DocumentClass const & textclass = bp.documentClass();
4861         Paragraph & par = it.paragraph();
4862         Layout const & layout = par.layout();
4863         Counters & counters = textclass.counters();
4864
4865         if (par.params().startOfAppendix()) {
4866                 // We want to reset the counter corresponding to toplevel sectioning
4867                 Layout const & lay = textclass.getTOCLayout();
4868                 docstring const cnt = lay.counter;
4869                 if (!cnt.empty())
4870                         counters.reset(cnt);
4871                 counters.appendix(true);
4872         }
4873         par.params().appendix(counters.appendix());
4874
4875         // Compute the item depth of the paragraph
4876         par.itemdepth = getItemDepth(it);
4877
4878         if (layout.margintype == MARGIN_MANUAL) {
4879                 if (par.params().labelWidthString().empty())
4880                         par.params().labelWidthString(par.expandLabel(layout, bp));
4881         } else if (layout.latextype == LATEX_BIB_ENVIRONMENT) {
4882                 // we do not need to do anything here, since the empty case is
4883                 // handled during export.
4884         } else {
4885                 par.params().labelWidthString(docstring());
4886         }
4887
4888         switch(layout.labeltype) {
4889         case LABEL_ITEMIZE: {
4890                 // At some point of time we should do something more
4891                 // clever here, like:
4892                 //   par.params().labelString(
4893                 //    bp.user_defined_bullet(par.itemdepth).getText());
4894                 // for now, use a simple hardcoded label
4895                 docstring itemlabel;
4896                 switch (par.itemdepth) {
4897                 case 0:
4898                         itemlabel = char_type(0x2022);
4899                         break;
4900                 case 1:
4901                         itemlabel = char_type(0x2013);
4902                         break;
4903                 case 2:
4904                         itemlabel = char_type(0x2217);
4905                         break;
4906                 case 3:
4907                         itemlabel = char_type(0x2219); // or 0x00b7
4908                         break;
4909                 }
4910                 par.params().labelString(itemlabel);
4911                 break;
4912         }
4913
4914         case LABEL_ENUMERATE: {
4915                 docstring enumcounter = layout.counter.empty() ? from_ascii("enum") : layout.counter;
4916
4917                 switch (par.itemdepth) {
4918                 case 2:
4919                         enumcounter += 'i';
4920                 case 1:
4921                         enumcounter += 'i';
4922                 case 0:
4923                         enumcounter += 'i';
4924                         break;
4925                 case 3:
4926                         enumcounter += "iv";
4927                         break;
4928                 default:
4929                         // not a valid enumdepth...
4930                         break;
4931                 }
4932
4933                 // Increase the master counter?
4934                 if (layout.stepmastercounter && needEnumCounterReset(it))
4935                         counters.stepMaster(enumcounter, utype);
4936
4937                 // Maybe we have to reset the enumeration counter.
4938                 if (!layout.resumecounter && needEnumCounterReset(it))
4939                         counters.reset(enumcounter);
4940                 counters.step(enumcounter, utype);
4941
4942                 string const & lang = par.getParLanguage(bp)->code();
4943                 par.params().labelString(counters.theCounter(enumcounter, lang));
4944
4945                 break;
4946         }
4947
4948         case LABEL_SENSITIVE: {
4949                 string const & type = counters.current_float();
4950                 docstring full_label;
4951                 if (type.empty())
4952                         full_label = owner_->B_("Senseless!!! ");
4953                 else {
4954                         docstring name = owner_->B_(textclass.floats().getType(type).name());
4955                         if (counters.hasCounter(from_utf8(type))) {
4956                                 string const & lang = par.getParLanguage(bp)->code();
4957                                 counters.step(from_utf8(type), utype);
4958                                 full_label = bformat(from_ascii("%1$s %2$s:"),
4959                                                      name,
4960                                                      counters.theCounter(from_utf8(type), lang));
4961                         } else
4962                                 full_label = bformat(from_ascii("%1$s #:"), name);
4963                 }
4964                 par.params().labelString(full_label);
4965                 break;
4966         }
4967
4968         case LABEL_NO_LABEL:
4969                 par.params().labelString(docstring());
4970                 break;
4971
4972         case LABEL_ABOVE:
4973         case LABEL_CENTERED:
4974         case LABEL_STATIC: {
4975                 docstring const & lcounter = layout.counter;
4976                 if (!lcounter.empty()) {
4977                         if (layout.toclevel <= bp.secnumdepth
4978                                                 && (layout.latextype != LATEX_ENVIRONMENT
4979                                         || it.text()->isFirstInSequence(it.pit()))) {
4980                                 if (counters.hasCounter(lcounter))
4981                                         counters.step(lcounter, utype);
4982                                 par.params().labelString(par.expandLabel(layout, bp));
4983                         } else
4984                                 par.params().labelString(docstring());
4985                 } else
4986                         par.params().labelString(par.expandLabel(layout, bp));
4987                 break;
4988         }
4989
4990         case LABEL_MANUAL:
4991         case LABEL_BIBLIO:
4992                 par.params().labelString(par.expandLabel(layout, bp));
4993         }
4994 }
4995
4996
4997 void Buffer::updateBuffer(ParIterator & parit, UpdateType utype) const
4998 {
4999         // LASSERT: Is it safe to continue here, or should we just return?
5000         LASSERT(parit.pit() == 0, /**/);
5001
5002         // Set the position of the text in the buffer to be able
5003         // to resolve macros in it.
5004         parit.text()->setMacrocontextPosition(parit);
5005
5006         depth_type maxdepth = 0;
5007         pit_type const lastpit = parit.lastpit();
5008         for ( ; parit.pit() <= lastpit ; ++parit.pit()) {
5009                 // reduce depth if necessary
5010                 if (parit->params().depth() > maxdepth) {
5011                         /** FIXME: this function is const, but
5012                          * nevertheless it modifies the buffer. To be
5013                          * cleaner, one should modify the buffer in
5014                          * another function, which is actually
5015                          * non-const. This would however be costly in
5016                          * terms of code duplication.
5017                          */
5018                         const_cast<Buffer *>(this)->undo().recordUndo(CursorData(parit));
5019                         parit->params().depth(maxdepth);
5020                 }
5021                 maxdepth = parit->getMaxDepthAfter();
5022
5023                 if (utype == OutputUpdate) {
5024                         // track the active counters
5025                         // we have to do this for the master buffer, since the local
5026                         // buffer isn't tracking anything.
5027                         masterBuffer()->params().documentClass().counters().
5028                                         setActiveLayout(parit->layout());
5029                 }
5030
5031                 // set the counter for this paragraph
5032                 d->setLabel(parit, utype);
5033
5034                 // update change-tracking flag
5035                 parit->addChangesToBuffer(*this);
5036
5037                 // now the insets
5038                 InsetList::const_iterator iit = parit->insetList().begin();
5039                 InsetList::const_iterator end = parit->insetList().end();
5040                 for (; iit != end; ++iit) {
5041                         parit.pos() = iit->pos;
5042                         iit->inset->updateBuffer(parit, utype);
5043                 }
5044         }
5045 }
5046
5047
5048 int Buffer::spellCheck(DocIterator & from, DocIterator & to,
5049         WordLangTuple & word_lang, docstring_list & suggestions) const
5050 {
5051         int progress = 0;
5052         WordLangTuple wl;
5053         suggestions.clear();
5054         word_lang = WordLangTuple();
5055         bool const to_end = to.empty();
5056         DocIterator const end = to_end ? doc_iterator_end(this) : to;
5057         // OK, we start from here.
5058         for (; from != end; from.forwardPos()) {
5059                 // This skips all insets with spell check disabled.
5060                 while (!from.allowSpellCheck()) {
5061                         from.pop_back();
5062                         from.pos()++;
5063                 }
5064                 // If from is at the end of the document (which is possible
5065                 // when "from" was changed above) LyX will crash later otherwise.
5066                 if (from.atEnd() || (!to_end && from >= end))
5067                         break;
5068                 to = from;
5069                 from.paragraph().spellCheck();
5070                 SpellChecker::Result res = from.paragraph().spellCheck(from.pos(), to.pos(), wl, suggestions);
5071                 if (SpellChecker::misspelled(res)) {
5072                         word_lang = wl;
5073                         break;
5074                 }
5075                 // Do not increase progress when from == to, otherwise the word
5076                 // count will be wrong.
5077                 if (from != to) {
5078                         from = to;
5079                         ++progress;
5080                 }
5081         }
5082         return progress;
5083 }
5084
5085
5086 void Buffer::Impl::updateStatistics(DocIterator & from, DocIterator & to, bool skipNoOutput)
5087 {
5088         bool inword = false;
5089         word_count_ = 0;
5090         char_count_ = 0;
5091         blank_count_ = 0;
5092
5093         for (DocIterator dit = from ; dit != to && !dit.atEnd(); ) {
5094                 if (!dit.inTexted()) {
5095                         dit.forwardPos();
5096                         continue;
5097                 }
5098
5099                 Paragraph const & par = dit.paragraph();
5100                 pos_type const pos = dit.pos();
5101
5102                 // Copied and adapted from isWordSeparator() in Paragraph
5103                 if (pos == dit.lastpos()) {
5104                         inword = false;
5105                 } else {
5106                         Inset const * ins = par.getInset(pos);
5107                         if (ins && skipNoOutput && !ins->producesOutput()) {
5108                                 // skip this inset
5109                                 ++dit.top().pos();
5110                                 // stop if end of range was skipped
5111                                 if (!to.atEnd() && dit >= to)
5112                                         break;
5113                                 continue;
5114                         } else if (!par.isDeleted(pos)) {
5115                                 if (par.isWordSeparator(pos))
5116                                         inword = false;
5117                                 else if (!inword) {
5118                                         ++word_count_;
5119                                         inword = true;
5120                                 }
5121                                 if (ins && ins->isLetter())
5122                                         ++char_count_;
5123                                 else if (ins && ins->isSpace())
5124                                         ++blank_count_;
5125                                 else {
5126                                         char_type const c = par.getChar(pos);
5127                                         if (isPrintableNonspace(c))
5128                                                 ++char_count_;
5129                                         else if (isSpace(c))
5130                                                 ++blank_count_;
5131                                 }
5132                         }
5133                 }
5134                 dit.forwardPos();
5135         }
5136 }
5137
5138
5139 void Buffer::updateStatistics(DocIterator & from, DocIterator & to, bool skipNoOutput) const
5140 {
5141         d->updateStatistics(from, to, skipNoOutput);
5142 }
5143
5144
5145 int Buffer::wordCount() const
5146 {
5147         return d->wordCount();
5148 }
5149
5150
5151 int Buffer::charCount(bool with_blanks) const
5152 {
5153         return d->charCount(with_blanks);
5154 }
5155
5156
5157 Buffer::ReadStatus Buffer::reload()
5158 {
5159         setBusy(true);
5160         // c.f. bug http://www.lyx.org/trac/ticket/6587
5161         removeAutosaveFile();
5162         // e.g., read-only status could have changed due to version control
5163         d->filename.refresh();
5164         docstring const disp_fn = makeDisplayPath(d->filename.absFileName());
5165
5166         // clear parent. this will get reset if need be.
5167         d->setParent(0);
5168         ReadStatus const status = loadLyXFile();
5169         if (status == ReadSuccess) {
5170                 updateBuffer();
5171                 changed(true);
5172                 updateTitles();
5173                 markClean();
5174                 message(bformat(_("Document %1$s reloaded."), disp_fn));
5175                 d->undo_.clear();
5176         } else {
5177                 message(bformat(_("Could not reload document %1$s."), disp_fn));
5178         }
5179         setBusy(false);
5180         removePreviews();
5181         updatePreviews();
5182         errors("Parse");
5183         return status;
5184 }
5185
5186
5187 bool Buffer::saveAs(FileName const & fn)
5188 {
5189         FileName const old_name = fileName();
5190         FileName const old_auto = getAutosaveFileName();
5191         bool const old_unnamed = isUnnamed();
5192         bool success = true;
5193         d->old_position = filePath();
5194
5195         setFileName(fn);
5196         markDirty();
5197         setUnnamed(false);
5198
5199         if (save()) {
5200                 // bring the autosave file with us, just in case.
5201                 moveAutosaveFile(old_auto);
5202                 // validate version control data and
5203                 // correct buffer title
5204                 lyxvc().file_found_hook(fileName());
5205                 updateTitles();
5206                 // the file has now been saved to the new location.
5207                 // we need to check that the locations of child buffers
5208                 // are still valid.
5209                 checkChildBuffers();
5210                 checkMasterBuffer();
5211         } else {
5212                 // save failed
5213                 // reset the old filename and unnamed state
5214                 setFileName(old_name);
5215                 setUnnamed(old_unnamed);
5216                 success = false;
5217         }
5218
5219         d->old_position.clear();
5220         return success;
5221 }
5222
5223
5224 void Buffer::checkChildBuffers()
5225 {
5226         Impl::BufferPositionMap::iterator it = d->children_positions.begin();
5227         Impl::BufferPositionMap::iterator const en = d->children_positions.end();
5228         for (; it != en; ++it) {
5229                 DocIterator dit = it->second;
5230                 Buffer * cbuf = const_cast<Buffer *>(it->first);
5231                 if (!cbuf || !theBufferList().isLoaded(cbuf))
5232                         continue;
5233                 Inset * inset = dit.nextInset();
5234                 LASSERT(inset && inset->lyxCode() == INCLUDE_CODE, continue);
5235                 InsetInclude * inset_inc = static_cast<InsetInclude *>(inset);
5236                 docstring const & incfile = inset_inc->getParam("filename");
5237                 string oldloc = cbuf->absFileName();
5238                 string newloc = makeAbsPath(to_utf8(incfile),
5239                                 onlyPath(absFileName())).absFileName();
5240                 if (oldloc == newloc)
5241                         continue;
5242                 // the location of the child file is incorrect.
5243                 cbuf->setParent(0);
5244                 inset_inc->setChildBuffer(0);
5245         }
5246         // invalidate cache of children
5247         d->children_positions.clear();
5248         d->position_to_children.clear();
5249 }
5250
5251
5252 // If a child has been saved under a different name/path, it might have been
5253 // orphaned. Therefore the master needs to be reset (bug 8161).
5254 void Buffer::checkMasterBuffer()
5255 {
5256         Buffer const * const master = masterBuffer();
5257         if (master == this)
5258                 return;
5259
5260         // necessary to re-register the child (bug 5873)
5261         // FIXME: clean up updateMacros (here, only
5262         // child registering is needed).
5263         master->updateMacros();
5264         // (re)set master as master buffer, but only
5265         // if we are a real child
5266         if (master->isChild(this))
5267                 setParent(master);
5268         else
5269                 setParent(0);
5270 }
5271
5272
5273 string Buffer::includedFilePath(string const & name, string const & ext) const
5274 {
5275         if (d->old_position.empty() ||
5276             equivalent(FileName(d->old_position), FileName(filePath())))
5277                 return name;
5278
5279         bool isabsolute = FileName::isAbsolute(name);
5280         // both old_position and filePath() end with a path separator
5281         string absname = isabsolute ? name : d->old_position + name;
5282
5283         // if old_position is set to origin, we need to do the equivalent of
5284         // getReferencedFileName() (see readDocument())
5285         if (!isabsolute && d->old_position == params().origin) {
5286                 FileName const test(addExtension(filePath() + name, ext));
5287                 if (test.exists())
5288                         absname = filePath() + name;
5289         }
5290
5291         if (!FileName(addExtension(absname, ext)).exists())
5292                 return name;
5293
5294         if (isabsolute)
5295                 return to_utf8(makeRelPath(from_utf8(name), from_utf8(filePath())));
5296
5297         return to_utf8(makeRelPath(from_utf8(FileName(absname).realPath()),
5298                                    from_utf8(filePath())));
5299 }
5300
5301
5302 void Buffer::setChangesPresent(bool b) const
5303 {
5304         d->tracked_changes_present_ = b;
5305 }
5306
5307
5308 bool Buffer::areChangesPresent() const
5309 {
5310         return d->tracked_changes_present_;
5311 }
5312
5313
5314 void Buffer::updateChangesPresent() const
5315 {
5316         LYXERR(Debug::CHANGES, "Buffer::updateChangesPresent");
5317         setChangesPresent(false);
5318         ParConstIterator it = par_iterator_begin();
5319         ParConstIterator const end = par_iterator_end();
5320         for (; !areChangesPresent() && it != end; ++it)
5321                 it->addChangesToBuffer(*this);
5322 }
5323
5324
5325 void Buffer::Impl::refreshFileMonitor()
5326 {
5327         if (file_monitor_ && file_monitor_->filename() == filename.absFileName()) {
5328                 file_monitor_->refresh();
5329                 return;
5330         }
5331
5332         // The previous file monitor is invalid
5333         // This also destroys the previous file monitor and all its connections
5334         file_monitor_ = FileSystemWatcher::monitor(filename);
5335         fileExternallyModified(false);
5336         // file_monitor_ will be destroyed with *this, so it is not going to call a
5337         // destroyed object method.
5338         file_monitor_->connect([this](){ fileExternallyModified(true); });
5339 }
5340
5341
5342 void Buffer::Impl::fileExternallyModified(bool modified) const
5343 {
5344         if (modified) {
5345                 // prevent false positives, because FileMonitorBlocker is not enough on
5346                 // OSX.
5347                 if (filename.exists() && checksum_ == filename.checksum()) {
5348                         LYXERR(Debug::FILES, "External modification but "
5349                                "checksum unchanged: " << filename);
5350                         return;
5351                 }
5352                 lyx_clean = bak_clean = false;
5353         }
5354         externally_modified_ = modified;
5355         if (wa_)
5356                 wa_->updateTitles();
5357 }
5358
5359
5360 bool Buffer::notifiesExternalModification() const
5361 {
5362         return d->isExternallyModified();
5363 }
5364
5365
5366 void Buffer::clearExternalModification() const
5367 {
5368         d->fileExternallyModified(false);
5369 }
5370
5371
5372 } // namespace lyx