2 * \file ConverterCache.cpp
3 * This file is part of LyX, the document processor.
4 * Licence details can be found in the file COPYING.
7 * \author Angus Leeming
10 * Full author contact details are available in file CREDITS.
15 #include "ConverterCache.h"
23 #include "support/convert.h"
24 #include "support/filetools.h"
25 #include "support/lyxlib.h"
26 #include "support/lyxtime.h"
27 #include "support/Package.h"
29 #include <boost/crc.hpp>
30 #include <boost/filesystem/operations.hpp>
31 #include <boost/current_function.hpp>
38 using lyx::support::addName;
42 namespace fs = boost::filesystem;
46 using support::FileName;
50 unsigned long do_crc(string const & s)
52 boost::crc_32_type crc;
53 crc = std::for_each(s.begin(), s.end(), crc);
54 return crc.checksum();
58 static FileName cache_dir;
64 CacheItem(FileName const & orig_from, string const & to_format,
65 time_t t, unsigned long c)
66 : timestamp(t), checksum(c)
68 std::ostringstream os;
69 os << std::setw(10) << std::setfill('0') << do_crc(orig_from.absFilename())
71 cache_name = FileName(addName(cache_dir.absFilename(), os.str()));
72 LYXERR(Debug::FILES) << "Add file cache item " << orig_from
73 << ' ' << to_format << ' ' << cache_name
74 << ' ' << timestamp << ' ' << checksum
80 unsigned long checksum;
86 /** The cache contains one item per orig file and target format, so use a
87 * nested map to find the cache item quickly by filename and format.
89 typedef std::map<string, CacheItem> FormatCacheType;
92 /// Format of the source file
94 /// Cache target format -> item to quickly find the item by format
95 FormatCacheType cache;
97 typedef std::map<FileName, FormatCache> CacheType;
100 class ConverterCache::Impl {
107 CacheItem * find(FileName const & from, string const & format);
112 void ConverterCache::Impl::readIndex()
114 time_t const now = current_time();
115 FileName const index(addName(cache_dir.absFilename(), "index"));
116 std::ifstream is(index.toFilesystemEncoding().c_str());
122 string const orig_from = lex.getString();
125 string const to_format = lex.getString();
128 time_t const timestamp =
129 convert<unsigned long>(lex.getString());
132 unsigned long const checksum =
133 convert<unsigned long>(lex.getString());
134 FileName const orig_from_name(orig_from);
135 CacheItem item(orig_from_name, to_format, timestamp, checksum);
137 // Don't cache files that do not exist anymore
138 if (!fs::exists(orig_from_name.toFilesystemEncoding())) {
139 LYXERR(Debug::FILES) << "Not caching file `"
140 << orig_from << "' (does not exist anymore)."
142 support::unlink(item.cache_name);
146 // Don't add items that are not in the cache anymore
147 // This can happen if two instances of LyX are running
148 // at the same time and update the index file independantly.
149 if (!fs::exists(item.cache_name.toFilesystemEncoding())) {
150 LYXERR(Debug::FILES) << "Not caching file `"
152 << "' (cached copy does not exist anymore)."
157 // Delete the cached file if it is too old
158 if (difftime(now, fs::last_write_time(item.cache_name.toFilesystemEncoding())) >
159 lyxrc.converter_cache_maxage) {
160 LYXERR(Debug::FILES) << "Not caching file `"
161 << orig_from << "' (too old)." << std::endl;
162 support::unlink(item.cache_name);
166 FormatCache & format_cache = cache[orig_from_name];
167 if (format_cache.from_format.empty())
168 format_cache.from_format =
169 formats.getFormatFromFile(orig_from_name);
170 format_cache.cache[to_format] = item;
176 void ConverterCache::Impl::writeIndex()
178 FileName const index(addName(cache_dir.absFilename(), "index"));
179 std::ofstream os(index.toFilesystemEncoding().c_str());
181 if (!lyx::support::chmod(index, 0600))
183 os.open(index.toFilesystemEncoding().c_str());
184 CacheType::iterator it1 = cache.begin();
185 CacheType::iterator const end1 = cache.end();
186 for (; it1 != end1; ++it1) {
187 FormatCacheType const & format_cache = it1->second.cache;
188 FormatCacheType::const_iterator it2 = format_cache.begin();
189 FormatCacheType::const_iterator const end2 = format_cache.end();
190 for (; it2 != end2; ++it2)
191 os << Lexer::quoteString(it1->first.absFilename())
192 << ' ' << it2->first << ' '
193 << it2->second.timestamp << ' '
194 << it2->second.checksum << '\n';
200 CacheItem * ConverterCache::Impl::find(FileName const & from,
201 string const & format)
203 if (!lyxrc.use_converter_cache)
205 CacheType::iterator const it1 = cache.find(from);
206 if (it1 == cache.end())
208 FormatCacheType & format_cache = it1->second.cache;
209 FormatCacheType::iterator const it2 = format_cache.find(format);
210 if (it2 == format_cache.end())
212 return &(it2->second);
216 ConverterCache & ConverterCache::get()
218 // Now return the cache
219 static ConverterCache singleton;
224 void ConverterCache::init()
226 if (!lyxrc.use_converter_cache)
228 // We do this here and not in the constructor because package() gets
229 // initialized after all static variables.
230 cache_dir = FileName(addName(support::package().user_support().absFilename(), "cache"));
231 if (!fs::exists(cache_dir.toFilesystemEncoding()))
232 if (support::mkdir(cache_dir, 0700) != 0) {
233 lyxerr << "Could not create cache directory `"
234 << cache_dir << "'." << std::endl;
237 get().pimpl_->readIndex();
241 ConverterCache::ConverterCache()
246 ConverterCache::~ConverterCache()
248 if (!lyxrc.use_converter_cache)
250 pimpl_->writeIndex();
254 void ConverterCache::add(FileName const & orig_from, string const & to_format,
255 FileName const & converted_file) const
257 if (!lyxrc.use_converter_cache || orig_from.empty() ||
258 converted_file.empty())
260 LYXERR(Debug::FILES) << BOOST_CURRENT_FUNCTION << ' ' << orig_from
261 << ' ' << to_format << ' ' << converted_file
264 // FIXME: Should not hardcode this (see bug 3819 for details)
265 if (to_format == "pstex") {
266 FileName const converted_eps(support::changeExtension(converted_file.absFilename(), "eps"));
267 add(orig_from, "eps", converted_eps);
268 } else if (to_format == "pdftex") {
269 FileName const converted_pdf(support::changeExtension(converted_file.absFilename(), "pdf"));
270 add(orig_from, "pdf", converted_pdf);
273 // Is the file in the cache already?
274 CacheItem * item = pimpl_->find(orig_from, to_format);
276 time_t const timestamp = fs::last_write_time(orig_from.toFilesystemEncoding());
277 Mover const & mover = getMover(to_format);
279 LYXERR(Debug::FILES) << "ConverterCache::add(" << orig_from << "):\n"
280 "The file is already in the cache."
282 // First test for timestamp
283 if (timestamp == item->timestamp) {
284 LYXERR(Debug::FILES) << "Same timestamp."
288 // Maybe the contents is still the same?
289 item->timestamp = timestamp;
290 unsigned long const checksum = support::sum(orig_from);
291 if (checksum == item->checksum) {
292 LYXERR(Debug::FILES) << "Same checksum."
296 item->checksum = checksum;
298 if (!mover.copy(converted_file, item->cache_name,
299 support::onlyFilename(item->cache_name.absFilename()), 0600)) {
300 LYXERR(Debug::FILES) << "ConverterCache::add("
301 << orig_from << "):\n"
302 "Could not copy file."
306 CacheItem new_item(orig_from, to_format, timestamp,
307 support::sum(orig_from));
308 if (mover.copy(converted_file, new_item.cache_name,
309 support::onlyFilename(new_item.cache_name.absFilename()), 0600)) {
310 FormatCache & format_cache = pimpl_->cache[orig_from];
311 if (format_cache.from_format.empty())
312 format_cache.from_format =
313 formats.getFormatFromFile(orig_from);
314 format_cache.cache[to_format] = new_item;
316 LYXERR(Debug::FILES) << "ConverterCache::add("
317 << orig_from << "):\n"
318 "Could not copy file."
324 void ConverterCache::remove(FileName const & orig_from,
325 string const & to_format) const
327 if (!lyxrc.use_converter_cache || orig_from.empty())
329 LYXERR(Debug::FILES) << BOOST_CURRENT_FUNCTION << ' ' << orig_from
330 << ' ' << to_format << std::endl;
332 CacheType::iterator const it1 = pimpl_->cache.find(orig_from);
333 if (it1 == pimpl_->cache.end())
335 FormatCacheType & format_cache = it1->second.cache;
336 FormatCacheType::iterator const it2 = format_cache.find(to_format);
337 if (it2 == format_cache.end())
340 format_cache.erase(it2);
341 if (format_cache.empty())
342 pimpl_->cache.erase(it1);
346 void ConverterCache::remove_all(string const & from_format,
347 string const & to_format) const
349 if (!lyxrc.use_converter_cache)
351 CacheType::iterator it1 = pimpl_->cache.begin();
352 while (it1 != pimpl_->cache.end()) {
353 if (it1->second.from_format != from_format) {
357 FormatCacheType & format_cache = it1->second.cache;
358 FormatCacheType::iterator it2 = format_cache.begin();
359 while (it2 != format_cache.end()) {
360 if (it2->first == to_format) {
362 << "Removing file cache item "
364 << ' ' << to_format << std::endl;
365 support::unlink(it2->second.cache_name);
366 format_cache.erase(it2);
367 // Have to start over again since items in a
368 // map are not ordered
369 it2 = format_cache.begin();
373 if (format_cache.empty()) {
374 pimpl_->cache.erase(it1);
375 // Have to start over again since items in a map are
377 it1 = pimpl_->cache.begin();
381 pimpl_->writeIndex();
385 bool ConverterCache::inCache(FileName const & orig_from,
386 string const & to_format) const
388 if (!lyxrc.use_converter_cache || orig_from.empty())
390 LYXERR(Debug::FILES) << BOOST_CURRENT_FUNCTION << ' ' << orig_from
391 << ' ' << to_format << std::endl;
393 CacheItem * const item = pimpl_->find(orig_from, to_format);
395 LYXERR(Debug::FILES) << "not in cache." << std::endl;
398 time_t const timestamp = fs::last_write_time(orig_from.toFilesystemEncoding());
399 if (item->timestamp == timestamp) {
400 LYXERR(Debug::FILES) << "identical timestamp." << std::endl;
403 if (item->checksum == support::sum(orig_from)) {
404 item->timestamp = timestamp;
405 LYXERR(Debug::FILES) << "identical checksum." << std::endl;
408 LYXERR(Debug::FILES) << "in cache, but too old." << std::endl;
413 FileName const & ConverterCache::cacheName(FileName const & orig_from,
414 string const & to_format) const
416 LYXERR(Debug::FILES) << BOOST_CURRENT_FUNCTION << ' ' << orig_from
417 << ' ' << to_format << std::endl;
419 CacheItem * const item = pimpl_->find(orig_from, to_format);
421 return item->cache_name;
425 bool ConverterCache::copy(FileName const & orig_from, string const & to_format,
426 FileName const & dest) const
428 if (!lyxrc.use_converter_cache || orig_from.empty() || dest.empty())
430 LYXERR(Debug::FILES) << BOOST_CURRENT_FUNCTION << ' ' << orig_from
431 << ' ' << to_format << ' ' << dest << std::endl;
433 // FIXME: Should not hardcode this (see bug 3819 for details)
434 if (to_format == "pstex") {
435 FileName const dest_eps(support::changeExtension(dest.absFilename(), "eps"));
436 if (!copy(orig_from, "eps", dest_eps))
438 } else if (to_format == "pdftex") {
439 FileName const dest_pdf(support::changeExtension(dest.absFilename(), "pdf"));
440 if (!copy(orig_from, "pdf", dest_pdf))
444 CacheItem * const item = pimpl_->find(orig_from, to_format);
446 Mover const & mover = getMover(to_format);
447 return mover.copy(item->cache_name, dest,
448 support::onlyFilename(dest.absFilename()));