src/insets/InsetBibtex.cpp

   1 /**
   2  * \file InsetBibtex.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Alejandro Aguilar Sierra
   7  * \author Richard Kimberly Heck (BibTeX parser improvements)
   8  * \author Jürgen Spitzmüller
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "InsetBibtex.h"
  16
  17 #include "BiblioInfo.h"
  18 #include "Buffer.h"
  19 #include "BufferParams.h"
  20 #include "CiteEnginesList.h"
  21 #include "Cursor.h"
  22 #include "DispatchResult.h"
  23 #include "Encoding.h"
  24 #include "Exporter.h"
  25 #include "Format.h"
  26 #include "FuncRequest.h"
  27 #include "FuncStatus.h"
  28 #include "LaTeXFeatures.h"
  29 #include "output_latex.h"
  30 #include "xml.h"
  31 #include "PDFOptions.h"
  32 #include "texstream.h"
  33 #include "TextClass.h"
  34 #include "TocBackend.h"
  35
  36 #include "frontends/alert.h"
  37
  38 #include "support/convert.h"
  39 #include "support/debug.h"
  40 #include "support/docstream.h"
  41 #include "support/docstring_list.h"
  42 #include "support/ExceptionMessage.h"
  43 #include "support/FileNameList.h"
  44 #include "support/filetools.h"
  45 #include "support/gettext.h"
  46 #include "support/lstrings.h"
  47 #include "support/os.h"
  48 #include "support/PathChanger.h"
  49 #include "support/textutils.h"
  50
  51 #include <limits>
  52 #include <map>
  53 #include <regex>
  54 #include <utility>
  55
  56 #include <iostream>
  57
  58 using namespace std;
  59 using namespace lyx::support;
  60
  61 namespace lyx {
  62
  63 namespace Alert = frontend::Alert;
  64 namespace os = support::os;
  65
  66
  67 InsetBibtex::InsetBibtex(Buffer * buf, InsetCommandParams const & p)
  68         : InsetCommand(buf, p)
  69 {}
  70
  71
  72 ParamInfo const & InsetBibtex::findInfo(string const & /* cmdName */)
  73 {
  74         static ParamInfo param_info_;
  75         if (param_info_.empty()) {
  76                 param_info_.add("btprint", ParamInfo::LATEX_OPTIONAL);
  77                 param_info_.add("bibfiles", ParamInfo::LATEX_REQUIRED);
  78                 param_info_.add("options", ParamInfo::LYX_INTERNAL);
  79                 param_info_.add("encoding", ParamInfo::LYX_INTERNAL);
  80                 param_info_.add("file_encodings", ParamInfo::LYX_INTERNAL);
  81                 param_info_.add("biblatexopts", ParamInfo::LATEX_OPTIONAL);
  82         }
  83         return param_info_;
  84 }
  85
  86
  87 void InsetBibtex::doDispatch(Cursor & cur, FuncRequest & cmd)
  88 {
  89         switch (cmd.action()) {
  90
  91         case LFUN_INSET_EDIT:
  92                 editDatabases(cmd.argument());
  93                 break;
  94
  95         case LFUN_INSET_MODIFY: {
  96                 InsetCommandParams p(BIBTEX_CODE);
  97                 try {
  98                         if (!InsetCommand::string2params(to_utf8(cmd.argument()), p)) {
  99                                 cur.noScreenUpdate();
 100                                 break;
 101                         }
 102                 } catch (ExceptionMessage const & message) {
 103                         if (message.type_ == WarningException) {
 104                                 Alert::warning(message.title_, message.details_);
 105                                 cur.noScreenUpdate();
 106                         } else
 107                                 throw;
 108                         break;
 109                 }
 110
 111                 cur.recordUndo();
 112                 setParams(p);
 113                 cur.buffer()->clearBibFileCache();
 114                 cur.forceBufferUpdate();
 115                 break;
 116         }
 117
 118         default:
 119                 InsetCommand::doDispatch(cur, cmd);
 120                 break;
 121         }
 122 }
 123
 124
 125 bool InsetBibtex::getStatus(Cursor & cur, FuncRequest const & cmd,
 126                 FuncStatus & flag) const
 127 {
 128         switch (cmd.action()) {
 129         case LFUN_INSET_EDIT:
 130                 flag.setEnabled(true);
 131                 return true;
 132
 133         default:
 134                 return InsetCommand::getStatus(cur, cmd, flag);
 135         }
 136 }
 137
 138
 139 void InsetBibtex::editDatabases(docstring const & db) const
 140 {
 141         vector<docstring> bibfilelist = getVectorFromString(getParam("bibfiles"));
 142
 143         if (bibfilelist.empty())
 144                 return;
 145
 146         size_t nr_databases = bibfilelist.size();
 147         if (nr_databases > 1 && db.empty()) {
 148                         docstring const engine = usingBiblatex() ? _("Biblatex") : _("BibTeX");
 149                         docstring message = bformat(_("The %1$s[[BibTeX/Biblatex]] inset includes %2$s databases.\n"
 150                                                        "If you proceed, all of them will be opened."),
 151                                                         engine, convert<docstring>(nr_databases));
 152                         int const ret = Alert::prompt(_("Open Databases?"),
 153                                 message, 0, 1, _("&Cancel"), _("&Proceed"));
 154
 155                         if (ret == 0)
 156                                 return;
 157         }
 158
 159         vector<docstring>::const_iterator it = bibfilelist.begin();
 160         vector<docstring>::const_iterator en = bibfilelist.end();
 161         for (; it != en; ++it) {
 162                 if (!db.empty() && db != *it)
 163                         continue;
 164                 FileName const bibfile = buffer().getBibfilePath(*it);
 165                 theFormats().edit(buffer(), bibfile,
 166                      theFormats().getFormatFromFile(bibfile));
 167         }
 168 }
 169
 170
 171 bool InsetBibtex::usingBiblatex() const
 172 {
 173         return buffer().masterParams().useBiblatex();
 174 }
 175
 176
 177 docstring InsetBibtex::screenLabel() const
 178 {
 179         docstring res;
 180         if (getParam("bibfiles").empty())
 181                 res = _("EMPTY: ");
 182         res += usingBiblatex() ? _("Biblatex Generated Bibliography")
 183                                : _("BibTeX Generated Bibliography");
 184         return res;
 185 }
 186
 187
 188 docstring InsetBibtex::toolTip(BufferView const & /*bv*/, int /*x*/, int /*y*/) const
 189 {
 190         docstring tip = _("Databases:");
 191         vector<docstring> bibfilelist = getVectorFromString(getParam("bibfiles"));
 192
 193         tip += "<ul>";
 194         if (bibfilelist.empty())
 195                 tip += "<li>" + _("None[[bib databases]], please fill in!") + "</li>";
 196         else
 197                 for (docstring const & bibfile : bibfilelist)
 198                         tip += "<li>" + bibfile + "</li>";
 199         tip += "</ul>";
 200
 201         // Style-Options
 202         bool toc = false;
 203         docstring style = getParam("options"); // maybe empty! and with bibtotoc
 204         docstring bibtotoc = from_ascii("bibtotoc");
 205         if (prefixIs(style, bibtotoc)) {
 206                 toc = true;
 207                 if (contains(style, char_type(',')))
 208                         style = split(style, bibtotoc, char_type(','));
 209         }
 210
 211         docstring const btprint = getParam("btprint");
 212         if (!usingBiblatex()) {
 213                 tip += _("Style File:");
 214                 tip += "<ul><li>" + (style.empty() ? _("none") : style) + "</li></ul>";
 215
 216                 tip += _("Lists:") + " ";
 217                 if (btprint == "btPrintAll")
 218                         tip += _("all references");
 219                 else if (btprint == "btPrintNotCited")
 220                         tip += _("all uncited references");
 221                 else
 222                         tip += _("all cited references");
 223                 if (toc) {
 224                         tip += ", ";
 225                         tip += _("included in TOC");
 226                 }
 227                 if (!buffer().parent()
 228                     && buffer().params().multibib == "child") {
 229                         tip += "<br />";
 230                         tip += _("Note: This bibliography is not output, since bibliographies in the master file "
 231                                  "are not allowed with the setting 'Multiple bibliographies per child document'");
 232                 }
 233         } else {
 234                 tip += _("Lists:") + " ";
 235                 if (btprint == "bibbysection")
 236                         tip += _("all reference units");
 237                 else if (btprint == "btPrintAll")
 238                         tip += _("all references");
 239                 else
 240                         tip += _("all cited references");
 241                 if (toc) {
 242                         tip += ", ";
 243                         tip += _("included in TOC");
 244                 }
 245                 if (!getParam("biblatexopts").empty()) {
 246                         tip += "<br />";
 247                         tip += _("Options: ") + getParam("biblatexopts");
 248                 }
 249         }
 250
 251         return tip;
 252 }
 253
 254
 255 void InsetBibtex::latex(otexstream & os, OutputParams const & runparams) const
 256 {
 257         // The sequence of the commands:
 258         // With normal BibTeX:
 259         // 1. \bibliographystyle{style}
 260         // 2. \addcontentsline{...} - if option bibtotoc set
 261         // 3. \bibliography{database}
 262         // With bibtopic:
 263         // 1. \bibliographystyle{style}
 264         // 2. \begin{btSect}{database}
 265         // 3. \btPrint{Cited|NotCited|All}
 266         // 4. \end{btSect}
 267         // With Biblatex:
 268         // \printbibliography[biblatexopts]
 269         // or
 270         // \bibbysection[biblatexopts] - if btprint is "bibbysection"
 271
 272         // chapterbib does not allow bibliographies in the master
 273         if (!usingBiblatex() && !runparams.is_child
 274             && buffer().params().multibib == "child")
 275                 return;
 276
 277         if (runparams.inDeletedInset) {
 278                 // We cannot strike-out bibligraphies,
 279                 // so we just output a note.
 280                 os << "\\textbf{"
 281                    << buffer().B_("[BIBLIOGRAPHY DELETED!]")
 282                    << "}";
 283                 return;
 284         }
 285
 286         string style = to_utf8(getParam("options")); // maybe empty! and with bibtotoc
 287         string bibtotoc;
 288         if (prefixIs(style, "bibtotoc")) {
 289                 bibtotoc = "bibtotoc";
 290                 if (contains(style, ','))
 291                         style = split(style, bibtotoc, ',');
 292         }
 293
 294         if (usingBiblatex()) {
 295                 // Options
 296                 string opts = to_utf8(getParam("biblatexopts"));
 297                 // bibtotoc-Option
 298                 if (!bibtotoc.empty())
 299                         opts = opts.empty() ? "heading=bibintoc" : "heading=bibintoc," + opts;
 300                 // The bibliography command
 301                 docstring btprint = getParam("btprint");
 302                 if (btprint == "btPrintAll")
 303                         os << "\\nocite{*}\n";
 304                 if (btprint == "bibbysection" && !buffer().masterParams().multibib.empty())
 305                         os << "\\bibbysection";
 306                 else
 307                         os << "\\printbibliography";
 308                 if (!opts.empty())
 309                         os << "[" << opts << "]";
 310                 os << "\n";
 311         } else {// using BibTeX
 312                 // Database(s)
 313                 vector<pair<docstring, string>> const dbs =
 314                         buffer().prepareBibFilePaths(runparams, getBibFiles(), false);
 315                 vector<docstring> db_out;
 316                 db_out.reserve(dbs.size());
 317                 for (pair<docstring, string> const & db : dbs)
 318                         db_out.push_back(db.first);
 319                 // Style options
 320                 if (style == "default")
 321                         style = buffer().masterParams().defaultBiblioStyle();
 322                 if (!style.empty() && !buffer().masterParams().useBibtopic()) {
 323                         string base = buffer().masterBuffer()->prepareFileNameForLaTeX(style, ".bst", runparams.nice);
 324                         FileName const try_in_file =
 325                                 makeAbsPath(base + ".bst", buffer().filePath());
 326                         bool const not_from_texmf = try_in_file.isReadableFile();
 327                         // If this style does not come from texmf and we are not
 328                         // exporting to .tex copy it to the tmp directory.
 329                         // This prevents problems with spaces and 8bit characters
 330                         // in the file name.
 331                         if (!runparams.inComment && !runparams.dryrun && !runparams.nice &&
 332                             not_from_texmf) {
 333                                 // use new style name
 334                                 DocFileName const in_file = DocFileName(try_in_file);
 335                                 base = removeExtension(in_file.mangledFileName());
 336                                 FileName const out_file = makeAbsPath(base + ".bst",
 337                                                 buffer().masterBuffer()->temppath());
 338                                 bool const success = in_file.copyTo(out_file);
 339                                 if (!success) {
 340                                         LYXERR0("Failed to copy '" << in_file
 341                                                << "' to '" << out_file << "'");
 342                                 }
 343                         }
 344                         // FIXME UNICODE
 345                         os << "\\bibliographystyle{"
 346                            << from_utf8(latex_path(buffer().prepareFileNameForLaTeX(base, ".bst", runparams.nice)))
 347                            << "}\n";
 348                 }
 349                 // Warn about spaces in bst path. Warn only once.
 350                 static bool warned_about_bst_spaces = false;
 351                 if (!warned_about_bst_spaces && runparams.nice && contains(style, ' ')) {
 352                         warned_about_bst_spaces = true;
 353                         Alert::warning(_("Export Warning!"),
 354                                        _("There are spaces in the path to your BibTeX style file.\n"
 355                                                       "BibTeX will be unable to find it."));
 356                 }
 357                 // Encoding
 358                 bool encoding_switched = false;
 359                 Encoding const * const save_enc = runparams.encoding;
 360                 docstring const encoding = getParam("encoding");
 361                 if (!encoding.empty() && encoding != from_ascii("default")) {
 362                         Encoding const * const enc = encodings.fromLyXName(to_ascii(encoding));
 363                         if (enc != runparams.encoding) {
 364                                 os << "\\bgroup";
 365                                 switchEncoding(os.os(), buffer().params(), runparams, *enc, true);
 366                                 runparams.encoding = enc;
 367                                 encoding_switched = true;
 368                         }
 369                 }
 370                 // Handle the bibtopic case
 371                 if (!db_out.empty() && buffer().masterParams().useBibtopic()) {
 372                         os << "\\begin{btSect}";
 373                         if (!style.empty())
 374                                 os << "[" << style << "]";
 375                         os << "{" << getStringFromVector(db_out) << "}\n";
 376                         docstring btprint = getParam("btprint");
 377                         if (btprint.empty())
 378                                 // default
 379                                 btprint = from_ascii("btPrintCited");
 380                         os << "\\" << btprint << "\n"
 381                            << "\\end{btSect}\n";
 382                 }
 383                 // bibtotoc option
 384                 if (!bibtotoc.empty() && !buffer().masterParams().useBibtopic()
 385                     && !buffer().masterParams().documentClass().bibInToc()) {
 386                         // set label for hyperref, see http://www.lyx.org/trac/ticket/6470
 387                         if (buffer().masterParams().pdfoptions().use_hyperref)
 388                                         os << "\\phantomsection";
 389                         if (buffer().masterParams().documentClass().hasLaTeXLayout("chapter"))
 390                                 os << "\\addcontentsline{toc}{chapter}{\\bibname}";
 391                         else if (buffer().masterParams().documentClass().hasLaTeXLayout("section"))
 392                                 os << "\\addcontentsline{toc}{section}{\\refname}";
 393                 }
 394                 // The bibliography command
 395                 if (!db_out.empty() && !buffer().masterParams().useBibtopic()) {
 396                         docstring btprint = getParam("btprint");
 397                         if (btprint == "btPrintAll") {
 398                                 os << "\\nocite{*}\n";
 399                         }
 400                         os << "\\bibliography{" << getStringFromVector(db_out) << "}\n";
 401                 }
 402                 if (encoding_switched){
 403                         // Switch back
 404                         switchEncoding(os.os(), buffer().params(),
 405                                        runparams, *save_enc, true, true);
 406                         os << "\\egroup" << breakln;
 407                         runparams.encoding = save_enc;
 408                 }
 409         }
 410 }
 411
 412
 413 docstring_list InsetBibtex::getBibFiles() const
 414 {
 415         return getVectorFromString(getParam("bibfiles"));
 416 }
 417
 418 namespace {
 419
 420         // methods for parsing bibtex files
 421
 422         typedef map<docstring, docstring> VarMap;
 423
 424         /// remove whitespace characters, optionally a single comma,
 425         /// and further whitespace characters from the stream.
 426         /// @return true if a comma was found, false otherwise
 427         ///
 428         bool removeWSAndComma(ifdocstream & ifs) {
 429                 char_type ch;
 430
 431                 if (!ifs)
 432                         return false;
 433
 434                 // skip whitespace
 435                 do {
 436                         ifs.get(ch);
 437                 } while (ifs && isSpace(ch));
 438
 439                 if (!ifs)
 440                         return false;
 441
 442                 if (ch != ',') {
 443                         ifs.putback(ch);
 444                         return false;
 445                 }
 446
 447                 // skip whitespace
 448                 do {
 449                         ifs.get(ch);
 450                 } while (ifs && isSpace(ch));
 451
 452                 if (ifs) {
 453                         ifs.putback(ch);
 454                 }
 455
 456                 return true;
 457         }
 458
 459
 460         enum charCase {
 461                 makeLowerCase,
 462                 keepCase
 463         };
 464
 465         /// remove whitespace characters, read character sequence
 466         /// not containing whitespace characters or characters in
 467         /// delimChars, and remove further whitespace characters.
 468         ///
 469         /// @return true if a string of length > 0 could be read.
 470         ///
 471         bool readTypeOrKey(docstring & val, ifdocstream & ifs,
 472                 docstring const & delimChars, docstring const & illegalChars,
 473                 charCase chCase) {
 474
 475                 char_type ch;
 476
 477                 val.clear();
 478
 479                 if (!ifs)
 480                         return false;
 481
 482                 // skip whitespace
 483                 do {
 484                         ifs.get(ch);
 485                 } while (ifs && isSpace(ch));
 486
 487                 if (!ifs)
 488                         return false;
 489
 490                 // read value
 491                 while (ifs && !isSpace(ch) &&
 492                        delimChars.find(ch) == docstring::npos &&
 493                        illegalChars.find(ch) == docstring::npos)
 494                 {
 495                         if (chCase == makeLowerCase)
 496                                 val += lowercase(ch);
 497                         else
 498                                 val += ch;
 499                         ifs.get(ch);
 500                 }
 501
 502                 if (illegalChars.find(ch) != docstring::npos) {
 503                         ifs.putback(ch);
 504                         return false;
 505                 }
 506
 507                 // skip whitespace
 508                 while (ifs && isSpace(ch)) {
 509                         ifs.get(ch);
 510                 }
 511
 512                 if (ifs) {
 513                         ifs.putback(ch);
 514                 }
 515
 516                 return val.length() > 0;
 517         }
 518
 519         /// read subsequent bibtex values that are delimited with a #-character.
 520         /// Concatenate all parts and replace names with the associated string in
 521         /// the variable strings.
 522         /// @return true if reading was successful (all single parts were delimited
 523         /// correctly)
 524         bool readValue(docstring & val, ifdocstream & ifs, const VarMap & strings) {
 525
 526                 char_type ch;
 527
 528                 val.clear();
 529
 530                 if (!ifs)
 531                         return false;
 532
 533                 do {
 534                         // skip whitespace
 535                         do {
 536                                 ifs.get(ch);
 537                         } while (ifs && isSpace(ch));
 538
 539                         if (!ifs)
 540                                 return false;
 541
 542                         // check for field type
 543                         if (isDigitASCII(ch)) {
 544
 545                                 // read integer value
 546                                 do {
 547                                         val += ch;
 548                                         ifs.get(ch);
 549                                 } while (ifs && isDigitASCII(ch));
 550
 551                                 if (!ifs)
 552                                         return false;
 553
 554                         } else if (ch == '"' || ch == '{') {
 555                                 // set end delimiter
 556                                 char_type delim = ch == '"' ? '"': '}';
 557
 558                                 // Skip whitespace
 559                                 do {
 560                                         ifs.get(ch);
 561                                 } while (ifs && isSpace(ch));
 562
 563                                 if (!ifs)
 564                                         return false;
 565
 566                                 // We now have the first non-whitespace character
 567                                 // We'll collapse adjacent whitespace.
 568                                 bool lastWasWhiteSpace = false;
 569
 570                                 // inside this delimited text braces must match.
 571                                 // Thus we can have a closing delimiter only
 572                                 // when nestLevel == 0
 573                                 int nestLevel = 0;
 574
 575                                 while (ifs && (nestLevel > 0 || ch != delim)) {
 576                                         if (isSpace(ch)) {
 577                                                 lastWasWhiteSpace = true;
 578                                                 ifs.get(ch);
 579                                                 continue;
 580                                         }
 581                                         // We output the space only after we stop getting
 582                                         // whitespace so as not to output any whitespace
 583                                         // at the end of the value.
 584                                         if (lastWasWhiteSpace) {
 585                                                 lastWasWhiteSpace = false;
 586                                                 val += ' ';
 587                                         }
 588
 589                                         val += ch;
 590
 591                                         // update nesting level
 592                                         switch (ch) {
 593                                                 case '{':
 594                                                         ++nestLevel;
 595                                                         break;
 596                                                 case '}':
 597                                                         --nestLevel;
 598                                                         if (nestLevel < 0)
 599                                                                 return false;
 600                                                         break;
 601                                         }
 602
 603                                         if (ifs)
 604                                                 ifs.get(ch);
 605                                 }
 606
 607                                 if (!ifs)
 608                                         return false;
 609
 610                                 // FIXME Why is this here?
 611                                 ifs.get(ch);
 612
 613                                 if (!ifs)
 614                                         return false;
 615
 616                         } else {
 617
 618                                 // reading a string name
 619                                 docstring strName;
 620
 621                                 while (ifs && !isSpace(ch) && ch != '#' && ch != ',' && ch != '}' && ch != ')') {
 622                                         strName += lowercase(ch);
 623                                         ifs.get(ch);
 624                                 }
 625
 626                                 if (!ifs)
 627                                         return false;
 628
 629                                 // replace the string with its assigned value or
 630                                 // discard it if it's not assigned
 631                                 if (strName.length()) {
 632                                         VarMap::const_iterator pos = strings.find(strName);
 633                                         if (pos != strings.end()) {
 634                                                 val += pos->second;
 635                                         }
 636                                 }
 637                         }
 638
 639                         // skip WS
 640                         while (ifs && isSpace(ch)) {
 641                                 ifs.get(ch);
 642                         }
 643
 644                         if (!ifs)
 645                                 return false;
 646
 647                         // continue reading next value on concatenate with '#'
 648                 } while (ch == '#');
 649
 650                 ifs.putback(ch);
 651
 652                 return true;
 653         }
 654 } // namespace
 655
 656
 657 void InsetBibtex::collectBibKeys(InsetIterator const & /*di*/, FileNameList & checkedFiles) const
 658 {
 659         parseBibTeXFiles(checkedFiles);
 660 }
 661
 662
 663 void InsetBibtex::parseBibTeXFiles(FileNameList & checkedFiles) const
 664 {
 665         // This bibtex parser is a first step to parse bibtex files
 666         // more precisely.
 667         //
 668         // - it reads the whole bibtex entry and does a syntax check
 669         //   (matching delimiters, missing commas,...
 670         // - it recovers from errors starting with the next @-character
 671         // - it reads @string definitions and replaces them in the
 672         //   field values.
 673         // - it accepts more characters in keys or value names than
 674         //   bibtex does.
 675         //
 676         // Officially bibtex does only support ASCII, but in practice
 677         // you can use any encoding as long as some elements like keys
 678         // and names are pure ASCII. We support specifying an encoding,
 679         // and we convert the file from that (default is buffer encoding).
 680         // We don't restrict keys to ASCII in LyX, since our own
 681         // InsetBibitem can generate non-ASCII keys, and nonstandard
 682         // 8bit clean bibtex forks exist.
 683
 684         BiblioInfo keylist;
 685
 686         docstring_list const files = getBibFiles();
 687         for (auto const & bf : files) {
 688                 FileName const bibfile = buffer().getBibfilePath(bf);
 689                 if (bibfile.empty()) {
 690                         LYXERR0("Unable to find path for " << bf << "!");
 691                         continue;
 692                 }
 693                 if (find(checkedFiles.begin(), checkedFiles.end(), bibfile) != checkedFiles.end())
 694                         // already checked this one. Skip.
 695                         continue;
 696                 else
 697                         // record that we check this.
 698                         checkedFiles.push_back(bibfile);
 699                 string encoding = buffer().masterParams().encoding().iconvName();
 700                 string ienc = buffer().masterParams().bibFileEncoding(to_utf8(bf));
 701                 if (ienc.empty() || ienc == "general")
 702                         ienc = to_ascii(params()["encoding"]);
 703
 704                 if (!ienc.empty() && ienc != "auto-legacy-plain" && ienc != "auto-legacy" && encodings.fromLyXName(ienc))
 705                         encoding = encodings.fromLyXName(ienc)->iconvName();
 706                 ifdocstream ifs(bibfile.toFilesystemEncoding().c_str(),
 707                         ios_base::in, encoding);
 708
 709                 char_type ch;
 710                 VarMap strings;
 711
 712                 while (ifs) {
 713                         ifs.get(ch);
 714                         if (!ifs)
 715                                 break;
 716
 717                         if (ch != '@')
 718                                 continue;
 719
 720                         docstring entryType;
 721
 722                         if (!readTypeOrKey(entryType, ifs, from_ascii("{("), docstring(), makeLowerCase)) {
 723                                 lyxerr << "BibTeX Parser: Error reading entry type." << std::endl;
 724                                 continue;
 725                         }
 726
 727                         if (!ifs) {
 728                                 lyxerr << "BibTeX Parser: Unexpected end of file." << std::endl;
 729                                 continue;
 730                         }
 731
 732                         if (entryType == from_ascii("comment")) {
 733                                 ifs.ignore(numeric_limits<int>::max(), '\n');
 734                                 continue;
 735                         }
 736
 737                         ifs.get(ch);
 738                         if (!ifs) {
 739                                 lyxerr << "BibTeX Parser: Unexpected end of file." << std::endl;
 740                                 break;
 741                         }
 742
 743                         if ((ch != '(') && (ch != '{')) {
 744                                 lyxerr << "BibTeX Parser: Invalid entry delimiter." << std::endl;
 745                                 ifs.putback(ch);
 746                                 continue;
 747                         }
 748
 749                         // process the entry
 750                         if (entryType == from_ascii("string")) {
 751
 752                                 // read string and add it to the strings map
 753                                 // (or replace it's old value)
 754                                 docstring name;
 755                                 docstring value;
 756
 757                                 if (!readTypeOrKey(name, ifs, from_ascii("="), from_ascii("#{}(),"), makeLowerCase)) {
 758                                         lyxerr << "BibTeX Parser: Error reading string name." << std::endl;
 759                                         continue;
 760                                 }
 761
 762                                 if (!ifs) {
 763                                         lyxerr << "BibTeX Parser: Unexpected end of file." << std::endl;
 764                                         continue;
 765                                 }
 766
 767                                 // next char must be an equal sign
 768                                 ifs.get(ch);
 769                                 if (!ifs || ch != '=') {
 770                                         lyxerr << "BibTeX Parser: No `=' after string name: " <<
 771                                                         name << "." << std::endl;
 772                                         continue;
 773                                 }
 774
 775                                 if (!readValue(value, ifs, strings)) {
 776                                         lyxerr << "BibTeX Parser: Unable to read value for string: " <<
 777                                                         name << "." << std::endl;
 778                                         continue;
 779                                 }
 780
 781                                 strings[name] = value;
 782
 783                         } else if (entryType == from_ascii("preamble")) {
 784
 785                                 // preamble definitions are discarded.
 786                                 // can they be of any use in lyx?
 787                                 docstring value;
 788
 789                                 if (!readValue(value, ifs, strings)) {
 790                                         lyxerr << "BibTeX Parser: Unable to read preamble value." << std::endl;
 791                                         continue;
 792                                 }
 793
 794                         } else {
 795
 796                                 // Citation entry. Try to read the key.
 797                                 docstring key;
 798
 799                                 if (!readTypeOrKey(key, ifs, from_ascii(","), from_ascii("}"), keepCase)) {
 800                                         lyxerr << "BibTeX Parser: Unable to read key for entry type:" <<
 801                                                         entryType << "." << std::endl;
 802                                         continue;
 803                                 }
 804
 805                                 if (!ifs) {
 806                                         lyxerr << "BibTeX Parser: Unexpected end of file." << std::endl;
 807                                         continue;
 808                                 }
 809
 810                                 /////////////////////////////////////////////
 811                                 // now we have a key, so we will add an entry
 812                                 // (even if it's empty, as bibtex does)
 813                                 //
 814                                 // we now read the field = value pairs.
 815                                 // all items must be separated by a comma. If
 816                                 // it is missing the scanning of this entry is
 817                                 // stopped and the next is searched.
 818                                 docstring name;
 819                                 docstring value;
 820                                 docstring data;
 821                                 BibTeXInfo keyvalmap(key, entryType);
 822
 823                                 bool readNext = removeWSAndComma(ifs);
 824
 825                                 while (ifs && readNext) {
 826
 827                                         // read field name
 828                                         if (!readTypeOrKey(name, ifs, from_ascii("="),
 829                                                            from_ascii("{}(),"), makeLowerCase) || !ifs)
 830                                                 break;
 831
 832                                         // next char must be an equal sign
 833                                         // FIXME Whitespace??
 834                                         ifs.get(ch);
 835                                         if (!ifs) {
 836                                                 lyxerr << "BibTeX Parser: Unexpected end of file." << std::endl;
 837                                                 break;
 838                                         }
 839                                         if (ch != '=') {
 840                                                 lyxerr << "BibTeX Parser: Missing `=' after field name: " <<
 841                                                                 name << ", for key: " << key << "." << std::endl;
 842                                                 ifs.putback(ch);
 843                                                 break;
 844                                         }
 845
 846                                         // read field value
 847                                         if (!readValue(value, ifs, strings)) {
 848                                                 lyxerr << "BibTeX Parser: Unable to read value for field: " <<
 849                                                                 name << ", for key: " << key << "." << std::endl;
 850                                                 break;
 851                                         }
 852
 853                                         keyvalmap[name] = value;
 854                                         data += "\n\n" + value;
 855                                         keylist.addFieldName(name);
 856                                         readNext = removeWSAndComma(ifs);
 857                                 }
 858
 859                                 // add the new entry
 860                                 keylist.addEntryType(entryType);
 861                                 keyvalmap.setAllData(data);
 862                                 keylist[key] = keyvalmap;
 863                         } //< else (citation entry)
 864                 } //< searching '@'
 865         } //< for loop over files
 866
 867         buffer().addBiblioInfo(keylist);
 868 }
 869
 870
 871 bool InsetBibtex::addDatabase(docstring const & db)
 872 {
 873         docstring bibfiles = getParam("bibfiles");
 874         if (tokenPos(bibfiles, ',', db) != -1)
 875                 return false;
 876         if (!bibfiles.empty())
 877                 bibfiles += ',';
 878         setParam("bibfiles", bibfiles + db);
 879         return true;
 880 }
 881
 882
 883 bool InsetBibtex::delDatabase(docstring const & db)
 884 {
 885         docstring bibfiles = getParam("bibfiles");
 886         if (contains(bibfiles, db)) {
 887                 int const n = tokenPos(bibfiles, ',', db);
 888                 docstring bd = db;
 889                 if (n > 0) {
 890                         // this is not the first database
 891                         docstring tmp = ',' + bd;
 892                         setParam("bibfiles", subst(bibfiles, tmp, docstring()));
 893                 } else if (n == 0)
 894                         // this is the first (or only) database
 895                         setParam("bibfiles", split(bibfiles, bd, ','));
 896                 else
 897                         return false;
 898         }
 899         return true;
 900 }
 901
 902
 903 void InsetBibtex::validate(LaTeXFeatures & features) const
 904 {
 905         BufferParams const & mparams = features.buffer().masterParams();
 906         if (mparams.useBibtopic())
 907                 features.require("bibtopic");
 908         else if (!mparams.useBiblatex() && mparams.multibib == "child")
 909                 features.require("chapterbib");
 910         // FIXME XHTML
 911         // It'd be better to be able to get this from an InsetLayout, but at present
 912         // InsetLayouts do not seem really to work for things that aren't InsetTexts.
 913         if (features.runparams().flavor == Flavor::Html)
 914                 features.addCSSSnippet("div.bibtexentry { margin-left: 2em; text-indent: -2em; }\n"
 915                         "span.bibtexlabel:before{ content: \"[\"; }\n"
 916                         "span.bibtexlabel:after{ content: \"] \"; }");
 917 }
 918
 919
 920 void InsetBibtex::updateBuffer(ParIterator const &, UpdateType, bool const /*deleted*/)
 921 {
 922         buffer().registerBibfiles(getBibFiles());
 923         // record encoding of bib files for biblatex
 924         string const enc = (params()["encoding"] == from_ascii("default")) ?
 925                                 string() : to_ascii(params()["encoding"]);
 926         bool invalidate = false;
 927         if (buffer().params().bibEncoding() != enc) {
 928                 buffer().params().setBibEncoding(enc);
 929                 invalidate = true;
 930         }
 931         map<string, string> encs = getFileEncodings();
 932         map<string, string>::const_iterator it = encs.begin();
 933         for (; it != encs.end(); ++it) {
 934                 if (buffer().params().bibFileEncoding(it->first) != it->second) {
 935                         buffer().params().setBibFileEncoding(it->first, it->second);
 936                         invalidate = true;
 937                 }
 938         }
 939         if (invalidate)
 940                 buffer().invalidateBibinfoCache();
 941
 942         setBroken(getParam("bibfiles").empty());
 943 }
 944
 945
 946 map<string, string> InsetBibtex::getFileEncodings() const
 947 {
 948         vector<string> ps =
 949                 getVectorFromString(to_utf8(getParam("file_encodings")), "\t");
 950         std::map<string, string> res;
 951         for (string const & s: ps) {
 952                 string key;
 953                 string val = split(s, key, ' ');
 954                 res[key] = val;
 955         }
 956         return res;
 957 }
 958
 959
 960 docstring InsetBibtex::getRefLabel() const
 961 {
 962         if (buffer().masterParams().documentClass().hasLaTeXLayout("chapter"))
 963                 return buffer().B_("Bibliography");
 964         return buffer().B_("References");
 965 }
 966
 967
 968 void InsetBibtex::addToToc(DocIterator const & cpit, bool output_active,
 969                            UpdateType, TocBackend & backend) const
 970 {
 971         if (!prefixIs(to_utf8(getParam("options")), "bibtotoc"))
 972                 return;
 973
 974         docstring const str = getRefLabel();
 975         shared_ptr<Toc> toc = backend.toc("tableofcontents");
 976         // Assign to appropriate level
 977         int const item_depth =
 978                 (buffer().masterParams().documentClass().hasLaTeXLayout("chapter"))
 979                         ? 1 : 2;
 980         toc->push_back(TocItem(cpit, item_depth, str, output_active));
 981 }
 982
 983
 984 int InsetBibtex::plaintext(odocstringstream & os,
 985        OutputParams const & op, size_t max_length) const
 986 {
 987         docstring const reflabel = getRefLabel();
 988
 989         // We could output more information here, e.g., what databases are included
 990         // and information about options. But I don't necessarily see any reason to
 991         // do this right now.
 992         if (op.for_tooltip || op.for_toc || op.find_effective()) {
 993                 os << '[' << reflabel << ']' << '\n';
 994                 return PLAINTEXT_NEWLINE;
 995         }
 996
 997         BiblioInfo bibinfo = buffer().masterBibInfo();
 998         bibinfo.makeCitationLabels(buffer());
 999         vector<docstring> const & cites = bibinfo.citedEntries();
1000
1001         size_t start_size = os.str().size();
1002         docstring refoutput;
1003         refoutput += reflabel + "\n\n";
1004
1005         // Tell BiblioInfo our purpose
1006         CiteItem ci;
1007         ci.context = CiteItem::Export;
1008
1009         // Now we loop over the entries
1010         vector<docstring>::const_iterator vit = cites.begin();
1011         vector<docstring>::const_iterator const ven = cites.end();
1012         for (; vit != ven; ++vit) {
1013                 if (start_size + refoutput.size() >= max_length)
1014                         break;
1015                 BiblioInfo::const_iterator const biit = bibinfo.find(*vit);
1016                 if (biit == bibinfo.end())
1017                         continue;
1018                 BibTeXInfo const & entry = biit->second;
1019                 refoutput += "[" + entry.label() + "] ";
1020                 // FIXME Right now, we are calling BibInfo::getInfo on the key,
1021                 // which will give us all the cross-referenced info. But for every
1022                 // entry, so there's a lot of repetition. This should be fixed.
1023                 refoutput += bibinfo.getInfo(entry.key(), buffer(), ci) + "\n\n";
1024         }
1025         os << refoutput;
1026         return int(refoutput.size());
1027 }
1028
1029
1030 // FIXME
1031 // docstring InsetBibtex::entriesAsXHTML(vector<docstring> const & entries)
1032 // And then here just: entriesAsXHTML(buffer().masterBibInfo().citedEntries())
1033 docstring InsetBibtex::xhtml(XMLStream & xs, OutputParams const &) const
1034 {
1035         BiblioInfo const & bibinfo = buffer().masterBibInfo();
1036         bool const all_entries = getParam("btprint") == "btPrintAll";
1037         vector<docstring> const & cites =
1038             all_entries ? bibinfo.getKeys() : bibinfo.citedEntries();
1039
1040         docstring const reflabel = buffer().B_("References");
1041
1042         // tell BiblioInfo our purpose
1043         CiteItem ci;
1044         ci.context = CiteItem::Export;
1045         ci.richtext = true;
1046         ci.max_key_size = UINT_MAX;
1047
1048         xs << xml::StartTag("h2", "class='bibtex'")
1049                 << reflabel
1050                 << xml::EndTag("h2")
1051                 << xml::StartTag("div", "class='bibtex'");
1052
1053         // Now we loop over the entries
1054         vector<docstring>::const_iterator vit = cites.begin();
1055         vector<docstring>::const_iterator const ven = cites.end();
1056         for (; vit != ven; ++vit) {
1057                 BiblioInfo::const_iterator const biit = bibinfo.find(*vit);
1058                 if (biit == bibinfo.end())
1059                         continue;
1060
1061                 BibTeXInfo const & entry = biit->second;
1062                 string const attr = "class='bibtexentry' id='LyXCite-"
1063                     + to_utf8(xml::cleanAttr(entry.key())) + "'";
1064                 xs << xml::StartTag("div", attr);
1065
1066                 // don't print labels if we're outputting all entries
1067                 if (!all_entries) {
1068                         xs << xml::StartTag("span", "class='bibtexlabel'")
1069                                 << entry.label()
1070                                 << xml::EndTag("span");
1071                 }
1072
1073                 // FIXME Right now, we are calling BibInfo::getInfo on the key,
1074                 // which will give us all the cross-referenced info. But for every
1075                 // entry, so there's a lot of repetition. This should be fixed.
1076                 xs << xml::StartTag("span", "class='bibtexinfo'")
1077                    << XMLStream::ESCAPE_AND
1078                    << bibinfo.getInfo(entry.key(), buffer(), ci)
1079                    << xml::EndTag("span")
1080                    << xml::EndTag("div")
1081                    << xml::CR();
1082         }
1083         xs << xml::EndTag("div");
1084         return docstring();
1085 }
1086
1087
1088 void InsetBibtex::docbook(XMLStream & xs, OutputParams const &) const
1089 {
1090         BiblioInfo const & bibinfo = buffer().masterBibInfo();
1091         bool const all_entries = getParam("btprint") == "btPrintAll";
1092         vector<docstring> const & cites =
1093                         all_entries ? bibinfo.getKeys() : bibinfo.citedEntries();
1094
1095         docstring const reflabel = buffer().B_("References");
1096
1097         // Check that the bibliography is not empty, to ensure that the document is valid.
1098         if (cites.empty()) {
1099                 xs << XMLStream::ESCAPE_NONE << "<!-- The bibliography is empty! -->";
1100                 xs << xml::CR();
1101                 return;
1102         }
1103
1104         // Tell BiblioInfo our purpose (i.e. generate HTML rich text).
1105         CiteItem ci;
1106         ci.context = CiteItem::Export;
1107         ci.richtext = true;
1108         ci.max_key_size = UINT_MAX;
1109
1110         // Header for bibliography (title required).
1111         xs << xml::StartTag("bibliography");
1112         xs << xml::CR();
1113         xs << xml::StartTag("title");
1114         xs << reflabel;
1115         xs << xml::EndTag("title");
1116         xs << xml::CR();
1117
1118         // Translation between keys in each entry and DocBook tags.
1119         // IDs for publications; list: http://tdg.docbook.org/tdg/5.2/biblioid.html.
1120         vector<pair<string, string>> biblioId = { // <bibtex, docbook>
1121                 make_pair("doi", "doi"),
1122                 make_pair("isbn", "isbn"),
1123                 make_pair("issn", "issn"),
1124                 make_pair("isrn", "isrn"),
1125                 make_pair("istc", "istc"),
1126                 make_pair("lccn", "libraryofcongress"),
1127                 make_pair("number", "pubsnumber"),
1128                 make_pair("url", "uri")
1129         };
1130         // Relations between documents.
1131         vector<pair<string, string>> relations = { // <bibtex, docbook biblioset relation>
1132                 make_pair("journal", "journal"),
1133                 make_pair("journaltitle", "journal"),
1134                 make_pair("fulljournaltitle", "journal"),
1135                 make_pair("booktitle", "book"),
1136                 make_pair("fullbooktitle", "book"),
1137                 make_pair("series", "series")
1138         };
1139         // Various things that do not fit DocBook.
1140         vector<string> misc = { "language", "school", "note" };
1141
1142         // Store the mapping between BibTeX and DocBook.
1143         map<string, string> toDocBookTag;
1144         toDocBookTag["fullnames:author"] = "SPECIFIC"; // No direct translation to DocBook: <authorgroup>.
1145         toDocBookTag["fullbynames:bookauthor"] = "SPECIFIC"; // No direct translation to DocBook: <authorgroup>.
1146         toDocBookTag["publisher"] = "SPECIFIC"; // No direct translation to DocBook: <publisher>.
1147         toDocBookTag["address"] = "SPECIFIC"; // No direct translation to DocBook: <publisher>.
1148         toDocBookTag["editor"] = "SPECIFIC";  // No direct translation to DocBook: <editor><personname/orgname>.
1149         toDocBookTag["fullbynames:editor"] = "SPECIFIC";  // No direct translation to DocBook: <editor><personname/orgname>.
1150         toDocBookTag["institution"] = "SPECIFIC"; // No direct translation to DocBook: <org>.
1151
1152         toDocBookTag["title"] = "title";
1153         toDocBookTag["fulltitle"] = "title";
1154         toDocBookTag["quotetitle"] = "title";
1155         toDocBookTag["volume"] = "volumenum";
1156         toDocBookTag["edition"] = "edition";
1157         toDocBookTag["pages"] = "artpagenums";
1158
1159         toDocBookTag["abstract"] = "SPECIFIC"; // No direct translation to DocBook: <abstract>.
1160         toDocBookTag["keywords"] = "SPECIFIC"; // No direct translation to DocBook: <keywordset>.
1161         toDocBookTag["year"] = "SPECIFIC"; // No direct translation to DocBook: <pubdate>.
1162         toDocBookTag["month"] = "SPECIFIC"; // No direct translation to DocBook: <pubdate>.
1163
1164         toDocBookTag["journal"] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1165         toDocBookTag["journaltitle"] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1166         toDocBookTag["fulljournaltitle"] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1167         toDocBookTag["booktitle"] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1168         toDocBookTag["fullbooktitle"] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1169         toDocBookTag["series"] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1170
1171         for (auto const & id: biblioId)
1172             toDocBookTag[id.first] = "SPECIFIC"; // No direct translation to DocBook: <biblioid>.
1173         for (auto const & id: relations)
1174             toDocBookTag[id.first] = "SPECIFIC"; // No direct translation to DocBook: <biblioset>.
1175         for (auto const & id: misc)
1176             toDocBookTag[id] = "SPECIFIC"; // No direct translation to DocBook: <bibliomisc>.
1177
1178         // Loop over the entries. If there are no entries, add a comment to say so.
1179         auto vit = cites.begin();
1180         auto ven = cites.end();
1181
1182         for (; vit != ven; ++vit) {
1183                 auto const biit = bibinfo.find(*vit);
1184                 if (biit == bibinfo.end())
1185                         continue;
1186
1187                 BibTeXInfo const & entry = biit->second;
1188                 string const attr = "xml:id=\"" + to_utf8(xml::cleanID(entry.key())) + "\"";
1189                 xs << xml::StartTag("biblioentry", attr);
1190                 xs << xml::CR();
1191
1192                 // FIXME Right now, we are calling BibInfo::getInfo on the key,
1193                 // which will give us all the cross-referenced info. But for every
1194                 // entry, so there's a lot of repetition. This should be fixed.
1195
1196                 // Parse the results of getInfo and emit the corresponding DocBook tags. Interesting pieces have the form
1197                 // "<span class="bib-STH">STH</span>", the rest of the text may be discarded.
1198                 // Could have written a DocBook version of expandFormat (that parses a citation into HTML), but it implements
1199                 // some kind of recursion. Still, a (static) conversion step between the citation format and DocBook would have
1200                 // been required. All in all, both codes approaches would have been similar, but this parsing allows relying
1201                 // on existing building blocks.
1202
1203                 string html = to_utf8(bibinfo.getInfo(entry.key(), buffer(), ci));
1204                 regex tagRegex("<span class=\"bib-([^\"]*)\">([^<]*)</span>");
1205                 smatch match;
1206                 auto tagIt = sregex_iterator(html.cbegin(), html.cend(), tagRegex, regex_constants::match_default);
1207                 auto tagEnd = sregex_iterator();
1208                 map<string, string> delayedTags;
1209
1210                 // Read all tags from HTML and convert those that have a 1:1 matching.
1211                 // Avoid outputting the same tag twice in DocBook: several bibliography tags might map to the same DocBook
1212                 // element, avoid outputting the same DocBook tag twice to keep a valid output. "SPECIFIC" tags are handled in
1213                 // a more specific way later on (among the delayed tags).
1214                 set<string> alreadyOutputDocBookTags;
1215                 while (tagIt != tagEnd) {
1216                         string tag = tagIt->str(); // regex_match cannot work with temporary strings.
1217                         ++tagIt;
1218
1219                         if (regex_match(tag, match, tagRegex)) {
1220                                 const string docbookTag = toDocBookTag[match[1]];
1221                                 if (docbookTag == "SPECIFIC") {
1222                                         delayedTags[match[1]] = match[2];
1223                                 } else {
1224                                         if (alreadyOutputDocBookTags.find(docbookTag) != alreadyOutputDocBookTags.end()) {
1225                                                 xs << XMLStream::ESCAPE_NONE <<
1226                                                    from_utf8("<!-- Several similar tags in the reference for ") + from_utf8(docbookTag) +
1227                                                    from_utf8(". New tag: ") + from_utf8(match[1]) + from_utf8(". Corresponding value: ") +
1228                                                    from_utf8(match[2].str()) + from_utf8(" -->\n");
1229                                         } else {
1230                                                 xs << xml::StartTag(docbookTag);
1231                                                 xs << from_utf8(match[2].str());
1232                                                 xs << xml::EndTag(docbookTag);
1233                                                 xs << xml::CR();
1234                                         }
1235                                 }
1236                         } else {
1237                                 LYXERR0("The BibTeX field " << match[1].str() << " is unknown.");
1238                                 xs << XMLStream::ESCAPE_NONE <<
1239                                         from_utf8("<!-- Output Error: The BibTeX field " + match[1].str() + " is unknown -->\n");
1240                         }
1241                 }
1242
1243                 // Type of document (book, journal paper, etc.).
1244                 xs << xml::StartTag("bibliomisc", "role=\"type\"");
1245                 xs << entry.entryType();
1246                 xs << xml::EndTag("bibliomisc");
1247                 xs << xml::CR();
1248
1249                 // Handle tags that have complex transformations.
1250                 if (! delayedTags.empty()) {
1251                         unsigned long remainingTags = delayedTags.size(); // Used as a workaround. With GCC 7, when erasing all
1252                         // elements one by one, some elements may still pop in later on (even though they were deleted previously).
1253                         auto hasTag = [&delayedTags](const string & key) { return delayedTags.find(key) != delayedTags.end(); };
1254                         auto getTag = [&delayedTags](const string & key) { return from_utf8(delayedTags[key]); };
1255                         auto eraseTag = [&delayedTags, &remainingTags](const string & key) {
1256                                 remainingTags -= 1;
1257                                 delayedTags.erase(key);
1258                         };
1259
1260                         // Notes on order of checks.
1261                         // - address goes with publisher if there is one, so check this first. Otherwise, the address goes with
1262                         //   the entry without other details.
1263
1264                         // <publisher>
1265                         if (hasTag("publisher")) {
1266                                 xs << xml::StartTag("publisher");
1267                                 xs << xml::CR();
1268                                 xs << xml::StartTag("publishername");
1269                                 xs << getTag("publisher");
1270                                 xs << xml::EndTag("publishername");
1271                                 xs << xml::CR();
1272
1273                                 if (hasTag("address")) {
1274                                         xs << xml::StartTag("address");
1275                                         xs << getTag("address");
1276                                         xs << xml::EndTag("address");
1277                                         eraseTag("address");
1278                                 }
1279
1280                                 xs << xml::EndTag("publisher");
1281                                 xs << xml::CR();
1282                                 eraseTag("publisher");
1283                         }
1284
1285                         if (hasTag("address")) {
1286                                 xs << xml::StartTag("address");
1287                                 xs << getTag("address");
1288                                 xs << xml::EndTag("address");
1289                                 eraseTag("address");
1290                         }
1291
1292                         // <keywordset>
1293                         if (hasTag("keywords")) {
1294                                 // Split the keywords on comma.
1295                                 docstring keywordSet = getTag("keywords");
1296                                 vector<docstring> keywords;
1297                                 if (keywordSet.find(from_utf8(",")) == string::npos) {
1298                                         keywords = { keywordSet };
1299                                 } else {
1300                                         size_t pos = 0;
1301                                         while ((pos = keywordSet.find(from_utf8(","))) != string::npos) {
1302                                                 keywords.push_back(keywordSet.substr(0, pos));
1303                                                 keywordSet.erase(0, pos + 1);
1304                                         }
1305                                         keywords.push_back(keywordSet);
1306                                 }
1307
1308                                 xs << xml::StartTag("keywordset") << xml::CR();
1309                                 for (auto & kw: keywords) {
1310                                         kw.erase(kw.begin(), std::find_if(kw.begin(), kw.end(),
1311                                                                           [](char_type c) {return !lyx::isSpace(c);}));
1312                                         xs << xml::StartTag("keyword");
1313                                         xs << kw;
1314                                         xs << xml::EndTag("keyword");
1315                                         xs << xml::CR();
1316                                 }
1317                                 xs << xml::EndTag("keywordset") << xml::CR();
1318                                 eraseTag("keywords");
1319                         }
1320
1321                         // <copyright>
1322                         // Example: http://tdg.docbook.org/tdg/5.1/biblioset.html
1323                         if (hasTag("year")) {
1324                                 docstring value = getTag("year");
1325                                 eraseTag("year");
1326
1327                                 // Follow xsd:gYearMonth format (http://books.xmlschemata.org/relaxng/ch19-77135.html).
1328                                 if (hasTag("month")) {
1329                                         value += "-" + getTag("month");
1330                                         eraseTag("month");
1331                                 }
1332
1333                                 xs << xml::StartTag("pubdate");
1334                                 xs << value;
1335                                 xs << xml::EndTag("pubdate");
1336                                 xs << xml::CR();
1337                         }
1338
1339                         // <institution>
1340                         if (hasTag("institution")) {
1341                                 xs << xml::StartTag("org");
1342                                 xs << xml::CR();
1343                                 xs << xml::StartTag("orgname");
1344                                 xs << getTag("institution");
1345                                 xs << xml::EndTag("orgname");
1346                                 xs << xml::CR();
1347                                 xs << xml::EndTag("org");
1348                                 xs << xml::CR();
1349                                 eraseTag("institution");
1350                         }
1351
1352                         // <biblioset>
1353                         // Example: http://tdg.docbook.org/tdg/5.1/biblioset.html
1354                         for (auto const & id: relations) {
1355                                 std::string keptJournal;
1356                                 std::string keptBook;
1357
1358                                 if (hasTag(id.first)) {
1359                                         bool outputThisTag = true;
1360
1361                                         // Deal with duplicate entries for the same semantics.
1362                                         if (id.first == "journal" || id.first == "journaltitle" || id.first == "fulljournaltitle") {
1363                                                 if (!keptJournal.empty()) {
1364                                                         xs << XMLStream::ESCAPE_NONE <<
1365                                                                 from_utf8("<!-- Several journal tags in the reference. Kept journal entry: ") +
1366                                                                         from_utf8(keptJournal) + from_utf8(". Other journal tag: ") +
1367                                                                     from_utf8(id.first) + from_utf8(". Corresponding value: ") +
1368                                                                     getTag(id.first) + from_utf8(" -->\n");
1369                                                         outputThisTag = false;
1370                                                 } else {
1371                                                         keptJournal = id.first;
1372                                                 }
1373                                         } else if (id.first == "booktitle" || id.first == "fullbooktitle") {
1374                                                 if (!keptBook.empty()) {
1375                                                         xs << XMLStream::ESCAPE_NONE <<
1376                                                            from_utf8("<!-- Several book tags in the reference. Kept book entry: ") +
1377                                                            from_utf8(keptBook) + from_utf8(". Other book tag: ") +
1378                                                            from_utf8(id.first) + from_utf8(". Corresponding value: ") +
1379                                                            getTag(id.first) + from_utf8(" -->\n");
1380                                                         outputThisTag = false;
1381                                                 } else {
1382                                                         keptBook = id.first;
1383                                                 }
1384                                         }
1385
1386                                         // Output this tag only if it is not a duplicate of a previously output tag.
1387                                         if (outputThisTag) {
1388                                                 xs << xml::StartTag("biblioset", "relation=\"" + id.second + "\"");
1389                                                 xs << xml::CR();
1390                                                 xs << xml::StartTag("title");
1391                                                 xs << getTag(id.first);
1392                                                 xs << xml::EndTag("title");
1393                                                 xs << xml::CR();
1394                                                 xs << xml::EndTag("biblioset");
1395                                                 xs << xml::CR();
1396                                         }
1397
1398                                         // In all cases, erase this tag: it has been dealt with.
1399                                         eraseTag(id.first);
1400                                 }
1401                         }
1402
1403                         // <authorgroup>
1404                         // Example: http://tdg.docbook.org/tdg/5.1/authorgroup.html
1405                         // Perform full parsing of the BibTeX string, dealing with the many corner cases that might
1406                         // be encountered.
1407                         if (hasTag("fullnames:author")) {
1408                                 authorsToDocBookAuthorGroup(getTag("fullnames:author"), xs, buffer(), "author");
1409                                 eraseTag("fullnames:author");
1410                         }
1411                         if (hasTag("fullbynames:bookauthor")) {
1412                                 authorsToDocBookAuthorGroup(getTag("fullbynames:bookauthor"), xs, buffer(), "book");
1413                                 eraseTag("fullbynames:bookauthor");
1414                         }
1415
1416                         // <editor>
1417                         // Example: http://tdg.docbook.org/tdg/5.1/editor.html
1418                         if (hasTag("editor") || hasTag("fullbynames:editor")) {
1419                                 // If several editor tags are present, only output one.
1420                                 const docstring editorName = getTag(hasTag("editor") ? "editor" : "fullbynames:editor");
1421
1422                                 // Arbitrarily decide that the editor is always a person. There is no reliable information in the input
1423                                 // to make the distinction between a person (<personname>) and an organisation (<orgname>).
1424                                 xs << xml::StartTag("editor");
1425                                 xs << xml::CR();
1426                                 xs << xml::StartTag("personname");
1427                                 xs << editorName;
1428                                 xs << xml::EndTag("personname");
1429                                 xs << xml::CR();
1430                                 xs << xml::EndTag("editor");
1431                                 xs << xml::CR();
1432
1433                                 if (hasTag("editor") && hasTag("fullbynames:editor")) {
1434                                         xs << XMLStream::ESCAPE_NONE <<
1435                                                         from_utf8("<!-- Several editor tags in the reference. Other editor tag: ") +
1436                                                         from_utf8("fullbynames:editor. Corresponding value: ") +
1437                                                         getTag("fullbynames:editor") + from_utf8(" -->\n");
1438                                 }
1439
1440                                 // Erase all editor tags that might be present, even if only one is output.
1441                                 if (hasTag("editor")) {
1442                                         eraseTag("editor");
1443                                 }
1444                                 if (hasTag("fullbynames:editor")) {
1445                                         eraseTag("fullbynames:editor");
1446                                 }
1447                         }
1448
1449                         // <abstract>
1450                         if (hasTag("abstract")) {
1451                                 // Split the paragraphs on new line.
1452                                 docstring abstract = getTag("abstract");
1453                                 vector<docstring> paragraphs;
1454                                 if (abstract.find(from_utf8("\n")) == string::npos) {
1455                                         paragraphs = { abstract };
1456                                 } else {
1457                                         size_t pos = 0;
1458                                         while ((pos = abstract.find(from_utf8(","))) != string::npos) {
1459                                                 paragraphs.push_back(abstract.substr(0, pos));
1460                                                 abstract.erase(0, pos + 1);
1461                                         }
1462                                         paragraphs.push_back(abstract);
1463                                 }
1464
1465                                 xs << xml::StartTag("abstract");
1466                                 xs << xml::CR();
1467                                 for (auto const & para: paragraphs) {
1468                                         if (para.empty())
1469                                                 continue;
1470                                         xs << xml::StartTag("para");
1471                                         xs << para;
1472                                         xs << xml::EndTag("para");
1473                                 }
1474                                 xs << xml::CR();
1475                                 xs << xml::EndTag("abstract");
1476                                 xs << xml::CR();
1477                                 eraseTag("abstract");
1478                         }
1479
1480                         // <biblioid>
1481                         for (auto const & id: biblioId) {
1482                                 if (hasTag(id.first)) {
1483                                         xs << xml::StartTag("biblioid", "class=\"" + id.second + "\"");
1484                                         xs << getTag(id.first);
1485                                         xs << xml::EndTag("biblioid");
1486                                         xs << xml::CR();
1487                                         eraseTag(id.first);
1488                                 }
1489                         }
1490
1491                         // <bibliomisc>
1492                         for (auto const & id: misc) {
1493                                 if (hasTag(id)) {
1494                                         xs << xml::StartTag("bibliomisc", "role=\"" + id + "\"");
1495                                         xs << getTag(id);
1496                                         xs << xml::EndTag("bibliomisc");
1497                                         xs << xml::CR();
1498                                         eraseTag(id);
1499                                 }
1500                         }
1501
1502                         // After all tags are processed, check for errors.
1503                         if (remainingTags > 0) {
1504                                 LYXERR0("Still delayed tags not yet handled.");
1505                                 xs << XMLStream::ESCAPE_NONE << from_utf8("<!-- Output Error: still delayed tags not yet handled.\n");
1506                                 for (auto const & item: delayedTags) {
1507                                         xs << from_utf8(" " + item.first + ": " + item.second + "\n");
1508                                 }
1509                                 xs << XMLStream::ESCAPE_NONE << from_utf8(" -->\n");
1510                         }
1511                 }
1512
1513                 xs << xml::EndTag("biblioentry");
1514                 xs << xml::CR();
1515         }
1516
1517         // Footer for bibliography.
1518         xs << xml::EndTag("bibliography");
1519         xs << xml::CR();
1520 }
1521
1522
1523 void InsetBibtex::write(ostream & os) const
1524 {
1525         params().Write(os, &buffer());
1526 }
1527
1528
1529 string InsetBibtex::contextMenuName() const
1530 {
1531         return "context-bibtex";
1532 }
1533
1534
1535 } // namespace lyx