src/converter.C

   1 /**
   2  * \file converter.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Dekel Tsur
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 #include <config.h>
  12
  13 #include "converter.h"
  14
  15 #include "ConverterCache.h"
  16 #include "buffer.h"
  17 #include "buffer_funcs.h"
  18 #include "bufferparams.h"
  19 #include "debug.h"
  20 #include "format.h"
  21 #include "gettext.h"
  22 #include "language.h"
  23 #include "LaTeX.h"
  24 #include "mover.h"
  25
  26 #include "frontends/Alert.h"
  27
  28 #include "support/filetools.h"
  29 #include "support/lyxlib.h"
  30 #include "support/os.h"
  31 #include "support/path.h"
  32 #include "support/systemcall.h"
  33
  34
  35 namespace lyx {
  36
  37 using support::addName;
  38 using support::bformat;
  39 using support::changeExtension;
  40 using support::compare_ascii_no_case;
  41 using support::contains;
  42 using support::dirList;
  43 using support::FileName;
  44 using support::getExtension;
  45 using support::isFileReadable;
  46 using support::libFileSearch;
  47 using support::libScriptSearch;
  48 using support::makeRelPath;
  49 using support::onlyFilename;
  50 using support::onlyPath;
  51 using support::Path;
  52 using support::prefixIs;
  53 using support::quoteName;
  54 using support::split;
  55 using support::subst;
  56 using support::Systemcall;
  57
  58 using std::endl;
  59 using std::find_if;
  60 using std::string;
  61 using std::vector;
  62 using std::distance;
  63
  64 namespace Alert = lyx::frontend::Alert;
  65
  66
  67 namespace {
  68
  69 string const token_from("$$i");
  70 string const token_base("$$b");
  71 string const token_to("$$o");
  72 string const token_path("$$p");
  73
  74
  75
  76 string const add_options(string const & command, string const & options)
  77 {
  78         string head;
  79         string const tail = split(command, head, ' ');
  80         return head + ' ' + options + ' ' + tail;
  81 }
  82
  83
  84 string const dvipdfm_options(BufferParams const & bp)
  85 {
  86         string result;
  87
  88         if (bp.papersize != PAPER_CUSTOM) {
  89                 string const paper_size = bp.paperSizeName();
  90                 if (paper_size != "b5" && paper_size != "foolscap")
  91                         result = "-p "+ paper_size;
  92
  93                 if (bp.orientation == ORIENTATION_LANDSCAPE)
  94                         result += " -l";
  95         }
  96
  97         return result;
  98 }
  99
 100
 101 class ConverterEqual : public std::binary_function<string, string, bool> {
 102 public:
 103         ConverterEqual(string const & from, string const & to)
 104                 : from_(from), to_(to) {}
 105         bool operator()(Converter const & c) const {
 106                 return c.from == from_ && c.to == to_;
 107         }
 108 private:
 109         string const from_;
 110         string const to_;
 111 };
 112
 113 } // namespace anon
 114
 115
 116 Converter::Converter(string const & f, string const & t,
 117                      string const & c, string const & l)
 118         : from(f), to(t), command(c), flags(l),
 119           From(0), To(0), latex(false), xml(false),
 120           original_dir(false), need_aux(false)
 121 {}
 122
 123
 124 void Converter::readFlags()
 125 {
 126         string flag_list(flags);
 127         while (!flag_list.empty()) {
 128                 string flag_name, flag_value;
 129                 flag_list = split(flag_list, flag_value, ',');
 130                 flag_value = split(flag_value, flag_name, '=');
 131                 if (flag_name == "latex")
 132                         latex = true;
 133                 else if (flag_name == "xml")
 134                         xml = true;
 135                 else if (flag_name == "originaldir")
 136                         original_dir = true;
 137                 else if (flag_name == "needaux")
 138                         need_aux = true;
 139                 else if (flag_name == "resultdir")
 140                         result_dir = (flag_value.empty())
 141                                 ? token_base : flag_value;
 142                 else if (flag_name == "resultfile")
 143                         result_file = flag_value;
 144                 else if (flag_name == "parselog")
 145                         parselog = flag_value;
 146         }
 147         if (!result_dir.empty() && result_file.empty())
 148                 result_file = "index." + formats.extension(to);
 149         //if (!contains(command, token_from))
 150         //      latex = true;
 151 }
 152
 153
 154 bool operator<(Converter const & a, Converter const & b)
 155 {
 156         // use the compare_ascii_no_case instead of compare_no_case,
 157         // because in turkish, 'i' is not the lowercase version of 'I',
 158         // and thus turkish locale breaks parsing of tags.
 159         int const i = compare_ascii_no_case(a.From->prettyname(),
 160                                             b.From->prettyname());
 161         if (i == 0)
 162                 return compare_ascii_no_case(a.To->prettyname(),
 163                                              b.To->prettyname()) < 0;
 164         else
 165                 return i < 0;
 166 }
 167
 168
 169 Converter const * Converters::getConverter(string const & from,
 170                                             string const & to) const
 171 {
 172         ConverterList::const_iterator const cit =
 173                 find_if(converterlist_.begin(), converterlist_.end(),
 174                         ConverterEqual(from, to));
 175         if (cit != converterlist_.end())
 176                 return &(*cit);
 177         else
 178                 return 0;
 179 }
 180
 181
 182 int Converters::getNumber(string const & from, string const & to) const
 183 {
 184         ConverterList::const_iterator const cit =
 185                 find_if(converterlist_.begin(), converterlist_.end(),
 186                         ConverterEqual(from, to));
 187         if (cit != converterlist_.end())
 188                 return distance(converterlist_.begin(), cit);
 189         else
 190                 return -1;
 191 }
 192
 193
 194 void Converters::add(string const & from, string const & to,
 195                      string const & command, string const & flags)
 196 {
 197         formats.add(from);
 198         formats.add(to);
 199         ConverterList::iterator it = find_if(converterlist_.begin(),
 200                                              converterlist_.end(),
 201                                              ConverterEqual(from , to));
 202
 203         Converter converter(from, to, command, flags);
 204         if (it != converterlist_.end() && !flags.empty() && flags[0] == '*') {
 205                 converter = *it;
 206                 converter.command = command;
 207                 converter.flags = flags;
 208         }
 209         converter.readFlags();
 210
 211         if (converter.latex && (latex_command_.empty() || to == "dvi"))
 212                 latex_command_ = subst(command, token_from, "");
 213         // If we have both latex & pdflatex, we set latex_command to latex.
 214         // The latex_command is used to update the .aux file when running
 215         // a converter that uses it.
 216
 217         if (it == converterlist_.end()) {
 218                 converterlist_.push_back(converter);
 219         } else {
 220                 converter.From = it->From;
 221                 converter.To = it->To;
 222                 *it = converter;
 223         }
 224 }
 225
 226
 227 void Converters::erase(string const & from, string const & to)
 228 {
 229         ConverterList::iterator const it =
 230                 find_if(converterlist_.begin(),
 231                         converterlist_.end(),
 232                         ConverterEqual(from, to));
 233         if (it != converterlist_.end())
 234                 converterlist_.erase(it);
 235 }
 236
 237
 238 // This method updates the pointers From and To in all the converters.
 239 // The code is not very efficient, but it doesn't matter as the number
 240 // of formats and converters is small.
 241 // Furthermore, this method is called only on startup, or after
 242 // adding/deleting a format in FormPreferences (the latter calls can be
 243 // eliminated if the formats in the Formats class are stored using a map or
 244 // a list (instead of a vector), but this will cause other problems).
 245 void Converters::update(Formats const & formats)
 246 {
 247         ConverterList::iterator it = converterlist_.begin();
 248         ConverterList::iterator end = converterlist_.end();
 249         for (; it != end; ++it) {
 250                 it->From = formats.getFormat(it->from);
 251                 it->To = formats.getFormat(it->to);
 252         }
 253 }
 254
 255
 256 // This method updates the pointers From and To in the last converter.
 257 // It is called when adding a new converter in FormPreferences
 258 void Converters::updateLast(Formats const & formats)
 259 {
 260         if (converterlist_.begin() != converterlist_.end()) {
 261                 ConverterList::iterator it = converterlist_.end() - 1;
 262                 it->From = formats.getFormat(it->from);
 263                 it->To = formats.getFormat(it->to);
 264         }
 265 }
 266
 267
 268 void Converters::sort()
 269 {
 270         std::sort(converterlist_.begin(), converterlist_.end());
 271 }
 272
 273
 274 OutputParams::FLAVOR Converters::getFlavor(Graph::EdgePath const & path)
 275 {
 276         for (Graph::EdgePath::const_iterator cit = path.begin();
 277              cit != path.end(); ++cit) {
 278                 Converter const & conv = converterlist_[*cit];
 279                 if (conv.latex)
 280                         if (contains(conv.to, "pdf"))
 281                                 return OutputParams::PDFLATEX;
 282                 if (conv.xml)
 283                         return OutputParams::XML;
 284         }
 285         return OutputParams::LATEX;
 286 }
 287
 288
 289 bool Converters::convert(Buffer const * buffer,
 290                          FileName const & from_file, FileName const & to_file,
 291                          FileName const & orig_from,
 292                          string const & from_format, string const & to_format,
 293                          ErrorList & errorList, int conversionflags)
 294 {
 295         if (from_format == to_format)
 296                 return move(from_format, from_file, to_file, false);
 297
 298         if ((conversionflags & try_cache) &&
 299             ConverterCache::get().inCache(orig_from, to_format))
 300                 return ConverterCache::get().copy(orig_from, to_format, to_file);
 301
 302         Graph::EdgePath edgepath = getPath(from_format, to_format);
 303         if (edgepath.empty()) {
 304                 if (conversionflags & try_default) {
 305                         // if no special converter defined, then we take the
 306                         // default one from ImageMagic.
 307                         string const from_ext = from_format.empty() ?
 308                                 getExtension(from_file.absFilename()) :
 309                                 formats.extension(from_format);
 310                         string const to_ext = formats.extension(to_format);
 311                         string const command =
 312                                 support::os::python() + ' ' +
 313                                 quoteName(libFileSearch("scripts", "convertDefault.py").toFilesystemEncoding()) +
 314                                 ' ' +
 315                                 quoteName(from_ext + ':' + from_file.toFilesystemEncoding()) +
 316                                 ' ' +
 317                                 quoteName(to_ext + ':' + to_file.toFilesystemEncoding());
 318                         lyxerr[Debug::FILES]
 319                                 << "No converter defined! "
 320                                    "I use convertDefault.py:\n\t"
 321                                 << command << endl;
 322                         Systemcall one;
 323                         one.startscript(Systemcall::Wait, command);
 324                         if (isFileReadable(to_file)) {
 325                                 if (conversionflags & try_cache)
 326                                         ConverterCache::get().add(orig_from,
 327                                                         to_format, to_file);
 328                                 return true;
 329                         }
 330                 }
 331                 Alert::error(_("Cannot convert file"),
 332                              bformat(_("No information for converting %1$s "
 333                                                     "format files to %2$s.\n"
 334                                                     "Define a converter in the preferences."),
 335                                                         from_ascii(from_format), from_ascii(to_format)));
 336                 return false;
 337         }
 338         OutputParams runparams;
 339         runparams.flavor = getFlavor(edgepath);
 340
 341         // Some converters (e.g. lilypond) can only output files to the
 342         // current directory, so we need to change the current directory.
 343         // This has the added benefit that all other files that may be
 344         // generated by the converter are deleted when LyX closes and do not
 345         // clutter the real working directory.
 346         string path = onlyPath(from_file.absFilename());
 347         Path p(path);
 348
 349         // empty the error list before any new conversion takes place.
 350         errorList.clear();
 351
 352         bool run_latex = false;
 353         string from_base = changeExtension(from_file.absFilename(), "");
 354         string to_base = changeExtension(to_file.absFilename(), "");
 355         FileName infile;
 356         FileName outfile = from_file;
 357         for (Graph::EdgePath::const_iterator cit = edgepath.begin();
 358              cit != edgepath.end(); ++cit) {
 359                 Converter const & conv = converterlist_[*cit];
 360                 bool dummy = conv.To->dummy() && conv.to != "program";
 361                 if (!dummy)
 362                         lyxerr[Debug::FILES] << "Converting from  "
 363                                << conv.from << " to " << conv.to << endl;
 364                 infile = outfile;
 365                 outfile = FileName(conv.result_dir.empty()
 366                         ? changeExtension(from_file.absFilename(), conv.To->extension())
 367                         : addName(subst(conv.result_dir,
 368                                         token_base, from_base),
 369                                   subst(conv.result_file,
 370                                         token_base, onlyFilename(from_base))));
 371
 372                 // if input and output files are equal, we use a
 373                 // temporary file as intermediary (JMarc)
 374                 FileName real_outfile;
 375                 if (outfile == infile) {
 376                         real_outfile = infile;
 377                         outfile = FileName(addName(buffer->temppath(), "tmpfile.out"));
 378                 }
 379
 380                 if (conv.latex) {
 381                         run_latex = true;
 382                         string const command = subst(conv.command, token_from, "");
 383                         lyxerr[Debug::FILES] << "Running " << command << endl;
 384                         if (!runLaTeX(*buffer, command, runparams, errorList))
 385                                 return false;
 386                 } else {
 387                         if (conv.need_aux && !run_latex
 388                             && !latex_command_.empty()) {
 389                                 lyxerr[Debug::FILES]
 390                                         << "Running " << latex_command_
 391                                         << " to update aux file"<<  endl;
 392                                 runLaTeX(*buffer, latex_command_, runparams, errorList);
 393                         }
 394
 395                         string const infile2 = (conv.original_dir)
 396                                 ? infile.absFilename() : makeRelPath(infile.absFilename(), path);
 397                         string const outfile2 = (conv.original_dir)
 398                                 ? outfile.absFilename() : makeRelPath(outfile.absFilename(), path);
 399
 400                         string command = conv.command;
 401                         command = subst(command, token_from, quoteName(infile2));
 402                         command = subst(command, token_base, quoteName(from_base));
 403                         command = subst(command, token_to, quoteName(outfile2));
 404                         command = libScriptSearch(command);
 405
 406                         if (!conv.parselog.empty())
 407                                 command += " 2> " + quoteName(infile2 + ".out");
 408
 409                         if (conv.from == "dvi" && conv.to == "ps")
 410                                 command = add_options(command,
 411                                                       buffer->params().dvips_options());
 412                         else if (conv.from == "dvi" && prefixIs(conv.to, "pdf"))
 413                                 command = add_options(command,
 414                                                       dvipdfm_options(buffer->params()));
 415
 416                         lyxerr[Debug::FILES] << "Calling " << command << endl;
 417                         if (buffer)
 418                                 buffer->message(_("Executing command: ")
 419                                 + from_utf8(command));
 420
 421                         Systemcall::Starttype const type = (dummy)
 422                                 ? Systemcall::DontWait : Systemcall::Wait;
 423                         Systemcall one;
 424                         int res;
 425                         if (conv.original_dir) {
 426                                 Path p(buffer->filePath());
 427                                 res = one.startscript(type, command);
 428                         } else
 429                                 res = one.startscript(type, command);
 430
 431                         if (!real_outfile.empty()) {
 432                                 Mover const & mover = movers(conv.to);
 433                                 if (!mover.rename(outfile, real_outfile))
 434                                         res = -1;
 435                                 else
 436                                         lyxerr[Debug::FILES]
 437                                                 << "renaming file " << outfile
 438                                                 << " to " << real_outfile
 439                                                 << endl;
 440                                 // Finally, don't forget to tell any future
 441                                 // converters to use the renamed file...
 442                                 outfile = real_outfile;
 443                         }
 444
 445                         if (!conv.parselog.empty()) {
 446                                 string const logfile =  infile2 + ".log";
 447                                 string const script = libScriptSearch(conv.parselog);
 448                                 string const command2 = script +
 449                                         " < " + quoteName(infile2 + ".out") +
 450                                         " > " + quoteName(logfile);
 451                                 one.startscript(Systemcall::Wait, command2);
 452                                 if (!scanLog(*buffer, command, logfile, errorList))
 453                                         return false;
 454                         }
 455
 456                         if (res) {
 457                                 if (conv.to == "program") {
 458                                         Alert::error(_("Build errors"),
 459                                                 _("There were errors during the build process."));
 460                                 } else {
 461 // FIXME: this should go out of here. For example, here we cannot say if
 462 // it is a document (.lyx) or something else. Same goes for elsewhere.
 463                                         Alert::error(_("Cannot convert file"),
 464                                                 bformat(_("An error occurred whilst running %1$s"),
 465                                                 from_ascii(command.substr(0, 50))));
 466                                 }
 467                                 return false;
 468                         }
 469                 }
 470         }
 471
 472         Converter const & conv = converterlist_[edgepath.back()];
 473         if (conv.To->dummy())
 474                 return true;
 475
 476         if (!conv.result_dir.empty()) {
 477                 // The converter has put the file(s) in a directory.
 478                 // In this case we ignore the given to_file.
 479                 if (from_base != to_base) {
 480                         string const from = subst(conv.result_dir,
 481                                             token_base, from_base);
 482                         string const to = subst(conv.result_dir,
 483                                           token_base, to_base);
 484                         Mover const & mover = movers(conv.from);
 485                         if (!mover.rename(FileName(from), FileName(to))) {
 486                                 Alert::error(_("Cannot convert file"),
 487                                         bformat(_("Could not move a temporary directory from %1$s to %2$s."),
 488                                                 from_ascii(from), from_ascii(to)));
 489                                 return false;
 490                         }
 491                 }
 492                 return true;
 493         } else {
 494                 if (conversionflags & try_cache)
 495                         ConverterCache::get().add(orig_from, to_format, outfile);
 496                 return move(conv.to, outfile, to_file, conv.latex);
 497         }
 498 }
 499
 500
 501 bool Converters::move(string const & fmt,
 502                       FileName const & from, FileName const & to, bool copy)
 503 {
 504         if (from == to)
 505                 return true;
 506
 507         bool no_errors = true;
 508         string const path = onlyPath(from.absFilename());
 509         string const base = onlyFilename(changeExtension(from.absFilename(), string()));
 510         string const to_base = changeExtension(to.absFilename(), string());
 511         string const to_extension = getExtension(to.absFilename());
 512
 513         vector<string> files = dirList(onlyPath(from.absFilename()), getExtension(from.absFilename()));
 514         for (vector<string>::const_iterator it = files.begin();
 515              it != files.end(); ++it)
 516                 if (prefixIs(*it, base)) {
 517                         string const from2 = path + *it;
 518                         string to2 = to_base + it->substr(base.length());
 519                         to2 = changeExtension(to2, to_extension);
 520                         lyxerr[Debug::FILES] << "moving " << from2
 521                                              << " to " << to2 << endl;
 522
 523                         Mover const & mover = movers(fmt);
 524                         bool const moved = copy
 525                                 ? mover.copy(FileName(from2), FileName(to2))
 526                                 : mover.rename(FileName(from2), FileName(to2));
 527                         if (!moved && no_errors) {
 528                                 Alert::error(_("Cannot convert file"),
 529                                         bformat(copy ?
 530                                                 _("Could not copy a temporary file from %1$s to %2$s.") :
 531                                                 _("Could not move a temporary file from %1$s to %2$s."),
 532                                                 from_ascii(from2), from_ascii(to2)));
 533                                 no_errors = false;
 534                         }
 535                 }
 536         return no_errors;
 537 }
 538
 539
 540 bool Converters::formatIsUsed(string const & format)
 541 {
 542         ConverterList::const_iterator cit = converterlist_.begin();
 543         ConverterList::const_iterator end = converterlist_.end();
 544         for (; cit != end; ++cit) {
 545                 if (cit->from == format || cit->to == format)
 546                         return true;
 547         }
 548         return false;
 549 }
 550
 551
 552 bool Converters::scanLog(Buffer const & buffer, string const & /*command*/,
 553                          string const & filename, ErrorList & errorList)
 554 {
 555         OutputParams runparams;
 556         runparams.flavor = OutputParams::LATEX;
 557         LaTeX latex("", runparams, filename);
 558         TeXErrors terr;
 559         int const result = latex.scanLogFile(terr);
 560
 561         if (result & LaTeX::ERRORS)
 562                 bufferErrors(buffer, terr, errorList);
 563
 564         return true;
 565 }
 566
 567
 568 namespace {
 569
 570 class showMessage : public std::unary_function<docstring, void>, public boost::signals::trackable {
 571 public:
 572         showMessage(Buffer const & b) : buffer_(b) {};
 573         void operator()(docstring const & m) const
 574         {
 575                 buffer_.message(m);
 576         }
 577 private:
 578         Buffer const & buffer_;
 579 };
 580
 581 }
 582
 583
 584 bool Converters::runLaTeX(Buffer const & buffer, string const & command,
 585                           OutputParams const & runparams, ErrorList & errorList)
 586 {
 587         buffer.busy(true);
 588         buffer.message(_("Running LaTeX..."));
 589
 590         runparams.document_language = buffer.params().language->babel();
 591
 592         // do the LaTeX run(s)
 593         string const name = buffer.getLatexName();
 594         LaTeX latex(command, runparams, name);
 595         TeXErrors terr;
 596         showMessage show(buffer);
 597         latex.message.connect(show);
 598         int const result = latex.run(terr);
 599
 600         if (result & LaTeX::ERRORS)
 601                 bufferErrors(buffer, terr, errorList);
 602
 603         // check return value from latex.run().
 604         if ((result & LaTeX::NO_LOGFILE)) {
 605                 docstring const str =
 606                         bformat(_("LaTeX did not run successfully. "
 607                                                "Additionally, LyX could not locate "
 608                                                "the LaTeX log %1$s."), from_utf8(name));
 609                 Alert::error(_("LaTeX failed"), str);
 610         } else if (result & LaTeX::NO_OUTPUT) {
 611                 Alert::warning(_("Output is empty"),
 612                                _("An empty output file was generated."));
 613         }
 614
 615
 616         buffer.busy(false);
 617
 618         int const ERROR_MASK =
 619                         LaTeX::NO_LOGFILE |
 620                         LaTeX::ERRORS |
 621                         LaTeX::NO_OUTPUT;
 622
 623         return (result & ERROR_MASK) == 0;
 624
 625 }
 626
 627
 628
 629 void Converters::buildGraph()
 630 {
 631         G_.init(formats.size());
 632         ConverterList::iterator beg = converterlist_.begin();
 633         ConverterList::iterator const end = converterlist_.end();
 634         for (ConverterList::iterator it = beg; it != end ; ++it) {
 635                 int const s = formats.getNumber(it->from);
 636                 int const t = formats.getNumber(it->to);
 637                 G_.addEdge(s,t);
 638         }
 639 }
 640
 641
 642 std::vector<Format const *> const
 643 Converters::intToFormat(std::vector<int> const & input)
 644 {
 645         vector<Format const *> result(input.size());
 646
 647         vector<int>::const_iterator it = input.begin();
 648         vector<int>::const_iterator const end = input.end();
 649         vector<Format const *>::iterator rit = result.begin();
 650         for ( ; it != end; ++it, ++rit) {
 651                 *rit = &formats.get(*it);
 652         }
 653         return result;
 654 }
 655
 656
 657 vector<Format const *> const
 658 Converters::getReachableTo(string const & target, bool const clear_visited)
 659 {
 660         vector<int> const & reachablesto =
 661                 G_.getReachableTo(formats.getNumber(target), clear_visited);
 662
 663         return intToFormat(reachablesto);
 664 }
 665
 666
 667 vector<Format const *> const
 668 Converters::getReachable(string const & from, bool const only_viewable,
 669                          bool const clear_visited)
 670 {
 671         vector<int> const & reachables =
 672                 G_.getReachable(formats.getNumber(from),
 673                                 only_viewable,
 674                                 clear_visited);
 675
 676         return intToFormat(reachables);
 677 }
 678
 679
 680 bool Converters::isReachable(string const & from, string const & to)
 681 {
 682         return G_.isReachable(formats.getNumber(from),
 683                               formats.getNumber(to));
 684 }
 685
 686
 687 Graph::EdgePath const
 688 Converters::getPath(string const & from, string const & to)
 689 {
 690         return G_.getPath(formats.getNumber(from),
 691                           formats.getNumber(to));
 692 }
 693
 694
 695 /// The global instance
 696 Converters converters;
 697
 698 // The global copy after reading lyxrc.defaults
 699 Converters system_converters;
 700
 701
 702 } // namespace lyx