src/Paragraph.cpp

   1 /**
   2  * \file Paragraph.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Asger Alstrup
   7  * \author Lars Gullik Bjønnes
   8  * \author Jean-Marc Lasgouttes
   9  * \author Angus Leeming
  10  * \author John Levon
  11  * \author André Pönitz
  12  * \author Dekel Tsur
  13  * \author Jürgen Vigna
  14  *
  15  * Full author contact details are available in file CREDITS.
  16  */
  17
  18 #include <config.h>
  19
  20 #include "Paragraph.h"
  21
  22 #include "Buffer.h"
  23 #include "BufferParams.h"
  24 #include "Counters.h"
  25 #include "Encoding.h"
  26 #include "debug.h"
  27 #include "gettext.h"
  28 #include "Language.h"
  29 #include "LaTeXFeatures.h"
  30 #include "Color.h"
  31 #include "Layout.h"
  32 #include "Length.h"
  33 #include "Font.h"
  34 #include "LyXRC.h"
  35 #include "Messages.h"
  36 #include "OutputParams.h"
  37 #include "output_latex.h"
  38 #include "paragraph_funcs.h"
  39 #include "ParagraphParameters.h"
  40 #include "sgml.h"
  41 #include "TexRow.h"
  42 #include "VSpace.h"
  43
  44 #include "frontends/alert.h"
  45 #include "frontends/FontMetrics.h"
  46
  47 #include "insets/InsetBibitem.h"
  48 #include "insets/InsetOptArg.h"
  49
  50 #include "support/lstrings.h"
  51 #include "support/textutils.h"
  52 #include "support/convert.h"
  53 #include "support/unicode.h"
  54
  55 #include <boost/bind.hpp>
  56 #include <boost/next_prior.hpp>
  57
  58 #include <algorithm>
  59 #include <sstream>
  60
  61 using std::distance;
  62 using std::endl;
  63 using std::string;
  64 using std::ostream;
  65
  66 namespace lyx {
  67
  68 using support::contains;
  69 using support::prefixIs;
  70 using support::suffixIs;
  71 using support::rsplit;
  72 using support::rtrim;
  73
  74
  75 /////////////////////////////////////////////////////////////////////
  76 //
  77 // Paragraph::Pimpl
  78 //
  79 /////////////////////////////////////////////////////////////////////
  80
  81 class Encoding;
  82 class Layout;
  83
  84
  85 class Paragraph::Pimpl {
  86 public:
  87         ///
  88         Pimpl(Paragraph * owner);
  89         /// "Copy constructor"
  90         Pimpl(Pimpl const &, Paragraph * owner);
  91
  92         //
  93         // Change tracking
  94         //
  95         /// look up change at given pos
  96         Change const & lookupChange(pos_type pos) const;
  97         /// is there a change within the given range ?
  98         bool isChanged(pos_type start, pos_type end) const;
  99         /// will the paragraph be physically merged with the next
 100         /// one if the imaginary end-of-par character is logically deleted?
 101         bool isMergedOnEndOfParDeletion(bool trackChanges) const;
 102         /// set change for the entire par
 103         void setChange(Change const & change);
 104         /// set change at given pos
 105         void setChange(pos_type pos, Change const & change);
 106         /// accept changes within the given range
 107         void acceptChanges(BufferParams const & bparams, pos_type start, pos_type end);
 108         /// reject changes within the given range
 109         void rejectChanges(BufferParams const & bparams, pos_type start, pos_type end);
 110
 111         ///
 112         value_type getChar(pos_type pos) const;
 113         ///
 114         void insertChar(pos_type pos, value_type c, Change const & change);
 115         ///
 116         void insertInset(pos_type pos, Inset * inset, Change const & change);
 117         /// (logically) erase the char at pos; return true if it was actually erased
 118         bool eraseChar(pos_type pos, bool trackChanges);
 119         /// (logically) erase the given range; return the number of chars actually erased
 120         int eraseChars(pos_type start, pos_type end, bool trackChanges);
 121         ///
 122         Inset * inset_owner;
 123
 124         /** A font entry covers a range of positions. Notice that the
 125             entries in the list are inserted in random order.
 126             I don't think it's worth the effort to implement a more effective
 127             datastructure, because the number of different fonts in a paragraph
 128             is limited. (Asger)
 129             Nevertheless, I decided to store fontlist using a sorted vector:
 130             fontlist = { {pos_1,font_1} , {pos_2,font_2} , ... } where
 131             pos_1 < pos_2 < ..., font_{i-1} != font_i for all i,
 132             and font_i covers the chars in positions pos_{i-1}+1,...,pos_i
 133             (font_1 covers the chars 0,...,pos_1) (Dekel)
 134         */
 135         class FontTable  {
 136         public:
 137                 ///
 138                 FontTable(pos_type p, Font const & f)
 139                         : pos_(p), font_(f)
 140                 {}
 141                 ///
 142                 pos_type pos() const { return pos_; }
 143                 ///
 144                 void pos(pos_type p) { pos_ = p; }
 145                 ///
 146                 Font const & font() const { return font_; }
 147                 ///
 148                 void font(Font const & f) { font_ = f;}
 149         private:
 150                 /// End position of paragraph this font attribute covers
 151                 pos_type pos_;
 152                 /** Font. Interpretation of the font values:
 153                     If a value is Font::INHERIT_*, it means that the font
 154                     attribute is inherited from either the layout of this
 155                     paragraph or, in the case of nested paragraphs, from the
 156                     layout in the environment one level up until completely
 157                     resolved.
 158                     The values Font::IGNORE_* and Font::TOGGLE are NOT
 159                     allowed in these font tables.
 160                 */
 161                 Font font_;
 162         };
 163         ///
 164         friend class matchFT;
 165         ///
 166         class matchFT {
 167         public:
 168                 /// used by lower_bound and upper_bound
 169                 int operator()(FontTable const & a, FontTable const & b) const {
 170                         return a.pos() < b.pos();
 171                 }
 172         };
 173
 174         ///
 175         typedef std::vector<FontTable> FontList;
 176         ///
 177         FontList fontlist;
 178
 179         /// Output the surrogate pair formed by \p c and \p next to \p os.
 180         /// \return the number of characters written.
 181         int latexSurrogatePair(odocstream & os, value_type c, value_type next,
 182                                Encoding const &);
 183         /// Output a space in appropriate formatting (or a surrogate pair
 184         /// if the next character is a combining character).
 185         /// \return whether a surrogate pair was output.
 186         bool simpleTeXBlanks(Encoding const &,
 187                              odocstream &, TexRow & texrow,
 188                              pos_type & i,
 189                              unsigned int & column,
 190                              Font const & font,
 191                              Layout const & style);
 192         /// Output consecutive known unicode chars, belonging to the same
 193         /// language as specified by \p preamble, to \p os starting from \p c.
 194         /// \return the number of characters written.
 195         int knownLangChars(odocstream & os, value_type c, string & preamble,
 196                            Change &, Encoding const &, pos_type &);
 197         ///
 198         void simpleTeXSpecialChars(Buffer const &, BufferParams const &,
 199                                    odocstream &,
 200                                    TexRow & texrow, OutputParams &,
 201                                    Font & running_font,
 202                                    Font & basefont,
 203                                    Font const & outerfont,
 204                                    bool & open_font,
 205                                    Change & running_change,
 206                                    Layout const & style,
 207                                    pos_type & i,
 208                                    unsigned int & column, value_type const c);
 209
 210         ///
 211         void validate(LaTeXFeatures & features,
 212                       Layout const & layout) const;
 213
 214         ///
 215         unsigned int id_;
 216         ///
 217         static unsigned int paragraph_id;
 218         ///
 219         ParagraphParameters params;
 220
 221 //private:
 222         ///
 223         pos_type size() const { return owner_->size(); }
 224         /// match a string against a particular point in the paragraph
 225         bool isTextAt(std::string const & str, pos_type pos) const;
 226
 227         /// for recording and looking up changes
 228         Changes changes_;
 229
 230         /// Who owns us?
 231         Paragraph * owner_;
 232 };
 233
 234
 235
 236
 237 using std::endl;
 238 using std::upper_bound;
 239 using std::lower_bound;
 240 using std::string;
 241
 242
 243 // Initialization of the counter for the paragraph id's,
 244 unsigned int Paragraph::Pimpl::paragraph_id = 0;
 245
 246 namespace {
 247
 248 struct special_phrase {
 249         string phrase;
 250         docstring macro;
 251         bool builtin;
 252 };
 253
 254 special_phrase const special_phrases[] = {
 255         { "LyX", from_ascii("\\LyX{}"), false },
 256         { "TeX", from_ascii("\\TeX{}"), true },
 257         { "LaTeX2e", from_ascii("\\LaTeXe{}"), true },
 258         { "LaTeX", from_ascii("\\LaTeX{}"), true },
 259 };
 260
 261 size_t const phrases_nr = sizeof(special_phrases)/sizeof(special_phrase);
 262
 263 } // namespace anon
 264
 265
 266 Paragraph::Pimpl::Pimpl(Paragraph * owner)
 267         : owner_(owner)
 268 {
 269         inset_owner = 0;
 270         id_ = paragraph_id++;
 271 }
 272
 273
 274 Paragraph::Pimpl::Pimpl(Pimpl const & p, Paragraph * owner)
 275         : params(p.params), changes_(p.changes_), owner_(owner)
 276 {
 277         inset_owner = p.inset_owner;
 278         fontlist = p.fontlist;
 279         id_ = paragraph_id++;
 280 }
 281
 282
 283 bool Paragraph::Pimpl::isChanged(pos_type start, pos_type end) const
 284 {
 285         BOOST_ASSERT(start >= 0 && start <= size());
 286         BOOST_ASSERT(end > start && end <= size() + 1);
 287
 288         return changes_.isChanged(start, end);
 289 }
 290
 291
 292 bool Paragraph::Pimpl::isMergedOnEndOfParDeletion(bool trackChanges) const {
 293         // keep the logic here in sync with the logic of eraseChars()
 294
 295         if (!trackChanges) {
 296                 return true;
 297         }
 298
 299         Change change = changes_.lookup(size());
 300
 301         return change.type == Change::INSERTED && change.author == 0;
 302 }
 303
 304
 305 void Paragraph::Pimpl::setChange(Change const & change)
 306 {
 307         // beware of the imaginary end-of-par character!
 308         changes_.set(change, 0, size() + 1);
 309
 310         /*
 311          * Propagate the change recursively - but not in case of DELETED!
 312          *
 313          * Imagine that your co-author makes changes in an existing inset. He
 314          * sends your document to you and you come to the conclusion that the
 315          * inset should go completely. If you erase it, LyX must not delete all
 316          * text within the inset. Otherwise, the change tracked insertions of
 317          * your co-author get lost and there is no way to restore them later.
 318          *
 319          * Conclusion: An inset's content should remain untouched if you delete it
 320          */
 321
 322         if (change.type != Change::DELETED) {
 323                 for (pos_type pos = 0; pos < size(); ++pos) {
 324                         if (owner_->isInset(pos)) {
 325                                 owner_->getInset(pos)->setChange(change);
 326                         }
 327                 }
 328         }
 329 }
 330
 331
 332 void Paragraph::Pimpl::setChange(pos_type pos, Change const & change)
 333 {
 334         BOOST_ASSERT(pos >= 0 && pos <= size());
 335
 336         changes_.set(change, pos);
 337
 338         // see comment in setChange(Change const &) above
 339
 340         if (change.type != Change::DELETED &&
 341             pos < size() && owner_->isInset(pos)) {
 342                 owner_->getInset(pos)->setChange(change);
 343         }
 344 }
 345
 346
 347 Change const & Paragraph::Pimpl::lookupChange(pos_type pos) const
 348 {
 349         BOOST_ASSERT(pos >= 0 && pos <= size());
 350
 351         return changes_.lookup(pos);
 352 }
 353
 354
 355 void Paragraph::Pimpl::acceptChanges(BufferParams const & bparams, pos_type start, pos_type end)
 356 {
 357         BOOST_ASSERT(start >= 0 && start <= size());
 358         BOOST_ASSERT(end > start && end <= size() + 1);
 359
 360         for (pos_type pos = start; pos < end; ++pos) {
 361                 switch (lookupChange(pos).type) {
 362                         case Change::UNCHANGED:
 363                                 // accept changes in nested inset
 364                                 if (pos < size() && owner_->isInset(pos)) {
 365                                         owner_->getInset(pos)->acceptChanges(bparams);
 366                                 }
 367
 368                                 break;
 369
 370                         case Change::INSERTED:
 371                                 changes_.set(Change(Change::UNCHANGED), pos);
 372                                 // also accept changes in nested inset
 373                                 if (pos < size() && owner_->isInset(pos)) {
 374                                         owner_->getInset(pos)->acceptChanges(bparams);
 375                                 }
 376                                 break;
 377
 378                         case Change::DELETED:
 379                                 // Suppress access to non-existent
 380                                 // "end-of-paragraph char"
 381                                 if (pos < size()) {
 382                                         eraseChar(pos, false);
 383                                         --end;
 384                                         --pos;
 385                                 }
 386                                 break;
 387                 }
 388
 389         }
 390 }
 391
 392
 393 void Paragraph::Pimpl::rejectChanges(BufferParams const & bparams, pos_type start, pos_type end)
 394 {
 395         BOOST_ASSERT(start >= 0 && start <= size());
 396         BOOST_ASSERT(end > start && end <= size() + 1);
 397
 398         for (pos_type pos = start; pos < end; ++pos) {
 399                 switch (lookupChange(pos).type) {
 400                         case Change::UNCHANGED:
 401                                 // reject changes in nested inset
 402                                 if (pos < size() && owner_->isInset(pos)) {
 403                                         owner_->getInset(pos)->rejectChanges(bparams);
 404                                 }
 405                                 break;
 406
 407                         case Change::INSERTED:
 408                                 // Suppress access to non-existent
 409                                 // "end-of-paragraph char"
 410                                 if (pos < size()) {
 411                                         eraseChar(pos, false);
 412                                         --end;
 413                                         --pos;
 414                                 }
 415                                 break;
 416
 417                         case Change::DELETED:
 418                                 changes_.set(Change(Change::UNCHANGED), pos);
 419
 420                                 // Do NOT reject changes within a deleted inset!
 421                                 // There may be insertions of a co-author inside of it!
 422
 423                                 break;
 424                 }
 425         }
 426 }
 427
 428
 429 Paragraph::value_type Paragraph::Pimpl::getChar(pos_type pos) const
 430 {
 431         BOOST_ASSERT(pos >= 0 && pos <= size());
 432
 433         return owner_->getChar(pos);
 434 }
 435
 436
 437 void Paragraph::Pimpl::insertChar(pos_type pos, value_type c, Change const & change)
 438 {
 439         BOOST_ASSERT(pos >= 0 && pos <= size());
 440
 441         // track change
 442         changes_.insert(change, pos);
 443
 444         // This is actually very common when parsing buffers (and
 445         // maybe inserting ascii text)
 446         if (pos == size()) {
 447                 // when appending characters, no need to update tables
 448                 owner_->text_.push_back(c);
 449                 return;
 450         }
 451
 452         owner_->text_.insert(owner_->text_.begin() + pos, c);
 453
 454         // Update the font table.
 455         FontTable search_font(pos, Font());
 456         for (FontList::iterator it
 457               = lower_bound(fontlist.begin(), fontlist.end(), search_font, matchFT());
 458              it != fontlist.end(); ++it)
 459         {
 460                 it->pos(it->pos() + 1);
 461         }
 462
 463         // Update the insets
 464         owner_->insetlist.increasePosAfterPos(pos);
 465 }
 466
 467
 468 void Paragraph::Pimpl::insertInset(pos_type pos, Inset * inset,
 469                                    Change const & change)
 470 {
 471         BOOST_ASSERT(inset);
 472         BOOST_ASSERT(pos >= 0 && pos <= size());
 473
 474         insertChar(pos, META_INSET, change);
 475         BOOST_ASSERT(owner_->text_[pos] == META_INSET);
 476
 477         // Add a new entry in the insetlist.
 478         owner_->insetlist.insert(inset, pos);
 479 }
 480
 481
 482 bool Paragraph::Pimpl::eraseChar(pos_type pos, bool trackChanges)
 483 {
 484         BOOST_ASSERT(pos >= 0 && pos <= size());
 485
 486         // keep the logic here in sync with the logic of isMergedOnEndOfParDeletion()
 487
 488         if (trackChanges) {
 489                 Change change = changes_.lookup(pos);
 490
 491                 // set the character to DELETED if
 492                 //  a) it was previously unchanged or
 493                 //  b) it was inserted by a co-author
 494
 495                 if (change.type == Change::UNCHANGED ||
 496                     (change.type == Change::INSERTED && change.author != 0)) {
 497                         setChange(pos, Change(Change::DELETED));
 498                         return false;
 499                 }
 500
 501                 if (change.type == Change::DELETED)
 502                         return false;
 503         }
 504
 505         // Don't physically access the imaginary end-of-paragraph character.
 506         // eraseChar() can only mark it as DELETED. A physical deletion of
 507         // end-of-par must be handled externally.
 508         if (pos == size()) {
 509                 return false;
 510         }
 511
 512         // track change
 513         changes_.erase(pos);
 514
 515         // if it is an inset, delete the inset entry
 516         if (owner_->text_[pos] == Paragraph::META_INSET) {
 517                 owner_->insetlist.erase(pos);
 518         }
 519
 520         owner_->text_.erase(owner_->text_.begin() + pos);
 521
 522         // Erase entries in the tables.
 523         FontTable search_font(pos, Font());
 524
 525         FontList::iterator it =
 526                 lower_bound(fontlist.begin(),
 527                             fontlist.end(),
 528                             search_font, matchFT());
 529         if (it != fontlist.end() && it->pos() == pos &&
 530             (pos == 0 ||
 531              (it != fontlist.begin()
 532               && boost::prior(it)->pos() == pos - 1))) {
 533                 // If it is a multi-character font
 534                 // entry, we just make it smaller
 535                 // (see update below), otherwise we
 536                 // should delete it.
 537                 unsigned int const i = it - fontlist.begin();
 538                 fontlist.erase(fontlist.begin() + i);
 539                 it = fontlist.begin() + i;
 540                 if (i > 0 && i < fontlist.size() &&
 541                     fontlist[i - 1].font() == fontlist[i].font()) {
 542                         fontlist.erase(fontlist.begin() + i - 1);
 543                         it = fontlist.begin() + i - 1;
 544                 }
 545         }
 546
 547         // Update all other entries
 548         FontList::iterator fend = fontlist.end();
 549         for (; it != fend; ++it)
 550                 it->pos(it->pos() - 1);
 551
 552         // Update the insetlist
 553         owner_->insetlist.decreasePosAfterPos(pos);
 554
 555         return true;
 556 }
 557
 558
 559 int Paragraph::Pimpl::eraseChars(pos_type start, pos_type end, bool trackChanges)
 560 {
 561         BOOST_ASSERT(start >= 0 && start <= size());
 562         BOOST_ASSERT(end >= start && end <= size() + 1);
 563
 564         pos_type i = start;
 565         for (pos_type count = end - start; count; --count) {
 566                 if (!eraseChar(i, trackChanges))
 567                         ++i;
 568         }
 569         return end - i;
 570 }
 571
 572
 573 int Paragraph::Pimpl::latexSurrogatePair(odocstream & os, value_type c,
 574                 value_type next, Encoding const & encoding)
 575 {
 576         // Writing next here may circumvent a possible font change between
 577         // c and next. Since next is only output if it forms a surrogate pair
 578         // with c we can ignore this:
 579         // A font change inside a surrogate pair does not make sense and is
 580         // hopefully impossible to input.
 581         // FIXME: change tracking
 582         // Is this correct WRT change tracking?
 583         docstring const latex1 = encoding.latexChar(next);
 584         docstring const latex2 = encoding.latexChar(c);
 585         os << latex1 << '{' << latex2 << '}';
 586         return latex1.length() + latex2.length() + 2;
 587 }
 588
 589
 590 bool Paragraph::Pimpl::simpleTeXBlanks(Encoding const & encoding,
 591                                        odocstream & os, TexRow & texrow,
 592                                        pos_type & i,
 593                                        unsigned int & column,
 594                                        Font const & font,
 595                                        Layout const & style)
 596 {
 597         if (style.pass_thru)
 598                 return false;
 599
 600         if (i < size() - 1) {
 601                 char_type next = getChar(i + 1);
 602                 if (Encodings::isCombiningChar(next)) {
 603                         // This space has an accent, so we must always output it.
 604                         column += latexSurrogatePair(os, ' ', next, encoding) - 1;
 605                         ++i;
 606                         return true;
 607                 }
 608         }
 609
 610         if (lyxrc.plaintext_linelen > 0
 611             && column > lyxrc.plaintext_linelen
 612             && i
 613             && getChar(i - 1) != ' '
 614             && (i < size() - 1)
 615             // same in FreeSpacing mode
 616             && !owner_->isFreeSpacing()
 617             // In typewriter mode, we want to avoid
 618             // ! . ? : at the end of a line
 619             && !(font.family() == Font::TYPEWRITER_FAMILY
 620                  && (getChar(i - 1) == '.'
 621                      || getChar(i - 1) == '?'
 622                      || getChar(i - 1) == ':'
 623                      || getChar(i - 1) == '!'))) {
 624                 os << '\n';
 625                 texrow.newline();
 626                 texrow.start(owner_->id(), i + 1);
 627                 column = 0;
 628         } else if (style.free_spacing) {
 629                 os << '~';
 630         } else {
 631                 os << ' ';
 632         }
 633         return false;
 634 }
 635
 636
 637 int Paragraph::Pimpl::knownLangChars(odocstream & os,
 638                                      value_type c,
 639                                      string & preamble,
 640                                      Change & runningChange,
 641                                      Encoding const & encoding,
 642                                      pos_type & i)
 643 {
 644         // The latex command is "\textLANG{<spec>}" and we have to retain
 645         // "\textLANG{<spec>" for the first char but only "<spec>" for all
 646         // subsequent chars (this also works when we are passed untranslated
 647         // unicode).
 648         docstring const latex1 = rtrim(encoding.latexChar(c), "}");
 649         int length = latex1.length();
 650         os << latex1;
 651         while (i < size() - 1) {
 652                 char_type next = getChar(i + 1);
 653                 if (!Encodings::isKnownLangChar(next, preamble) ||
 654                     runningChange != lookupChange(i + 1))
 655                         break;
 656                 Font prev_font;
 657                 bool found = false;
 658                 FontList::const_iterator cit = fontlist.begin();
 659                 FontList::const_iterator end = fontlist.end();
 660                 for (; cit != end; ++cit) {
 661                         if (cit->pos() >= i && !found) {
 662                                 prev_font = cit->font();
 663                                 found = true;
 664                         }
 665                         if (cit->pos() >= i + 1)
 666                                 break;
 667                 }
 668                 if (found && cit != end && prev_font != cit->font())
 669                         break;
 670                 docstring const latex = rtrim(encoding.latexChar(next), "}");
 671                 docstring::size_type const j =
 672                                         latex.find_first_of(from_ascii("{"));
 673                 if (j == docstring::npos) {
 674                         os << latex;
 675                         length += latex.length();
 676                 } else {
 677                         os << latex.substr(j + 1);
 678                         length += latex.substr(j + 1).length();
 679                 }
 680                 ++i;
 681         }
 682         // When the proper language is set, we are passed the straight unicode,
 683         // so we should not try to close the \textLANG command.
 684         if (prefixIs(latex1, from_ascii("\\" + preamble))) {
 685                 os << '}';
 686                 ++length;
 687         }
 688         return length;
 689 }
 690
 691
 692 bool Paragraph::Pimpl::isTextAt(string const & str, pos_type pos) const
 693 {
 694         pos_type const len = str.length();
 695
 696         // is the paragraph large enough?
 697         if (pos + len > size())
 698                 return false;
 699
 700         // does the wanted text start at point?
 701         for (string::size_type i = 0; i < str.length(); ++i) {
 702                 // Caution: direct comparison of characters works only
 703                 // because str is pure ASCII.
 704                 if (str[i] != owner_->text_[pos + i])
 705                         return false;
 706         }
 707
 708         // is there a font change in middle of the word?
 709         FontList::const_iterator cit = fontlist.begin();
 710         FontList::const_iterator end = fontlist.end();
 711         for (; cit != end; ++cit) {
 712                 if (cit->pos() >= pos)
 713                         break;
 714         }
 715         if (cit != end && pos + len - 1 > cit->pos())
 716                 return false;
 717
 718         return true;
 719 }
 720
 721
 722 void Paragraph::Pimpl::simpleTeXSpecialChars(Buffer const & buf,
 723                                              BufferParams const & bparams,
 724                                              odocstream & os,
 725                                              TexRow & texrow,
 726                                              OutputParams & runparams,
 727                                              Font & running_font,
 728                                              Font & basefont,
 729                                              Font const & outerfont,
 730                                              bool & open_font,
 731                                              Change & running_change,
 732                                              Layout const & style,
 733                                              pos_type & i,
 734                                              unsigned int & column,
 735                                              value_type const c)
 736 {
 737         if (style.pass_thru) {
 738                 if (c != Paragraph::META_INSET) {
 739                         if (c != '\0')
 740                                 // FIXME UNICODE: This can fail if c cannot
 741                                 // be encoded in the current encoding.
 742                                 os.put(c);
 743                 } else
 744                         owner_->getInset(i)->plaintext(buf, os, runparams);
 745                 return;
 746         }
 747
 748         // Two major modes:  LaTeX or plain
 749         // Handle here those cases common to both modes
 750         // and then split to handle the two modes separately.
 751         switch (c) {
 752         case Paragraph::META_INSET: {
 753                 Inset * inset = owner_->getInset(i);
 754
 755                 // FIXME: remove this check
 756                 if (!inset)
 757                         break;
 758
 759                 // FIXME: move this to InsetNewline::latex
 760                 if (inset->lyxCode() == Inset::NEWLINE_CODE) {
 761                         // newlines are handled differently here than
 762                         // the default in simpleTeXSpecialChars().
 763                         if (!style.newline_allowed) {
 764                                 os << '\n';
 765                         } else {
 766                                 if (open_font) {
 767                                         column += running_font.latexWriteEndChanges(
 768                                                 os, bparams, runparams,
 769                                                 basefont, basefont);
 770                                         open_font = false;
 771                                 }
 772
 773                                 if (running_font.family() == Font::TYPEWRITER_FAMILY)
 774                                         os << '~';
 775
 776                                 basefont = owner_->getLayoutFont(bparams, outerfont);
 777                                 running_font = basefont;
 778
 779                                 if (runparams.moving_arg)
 780                                         os << "\\protect ";
 781
 782                                 os << "\\\\\n";
 783                         }
 784                         texrow.newline();
 785                         texrow.start(owner_->id(), i + 1);
 786                         column = 0;
 787                         break;
 788                 }
 789
 790                 if (lookupChange(i).type == Change::DELETED) {
 791                         if( ++runparams.inDeletedInset == 1)
 792                                 runparams.changeOfDeletedInset = lookupChange(i);
 793                 }
 794
 795                 if (inset->canTrackChanges()) {
 796                         column += Changes::latexMarkChange(os, bparams, running_change,
 797                                 Change(Change::UNCHANGED));
 798                         running_change = Change(Change::UNCHANGED);
 799                 }
 800
 801                 bool close = false;
 802                 odocstream::pos_type const len = os.tellp();
 803
 804                 if ((inset->lyxCode() == Inset::GRAPHICS_CODE
 805                      || inset->lyxCode() == Inset::MATH_CODE
 806                      || inset->lyxCode() == Inset::URL_CODE)
 807                     && running_font.isRightToLeft()) {
 808                         if (running_font.language()->lang() == "farsi")
 809                                 os << "\\beginL{}";
 810                         else
 811                                 os << "\\L{";
 812                         close = true;
 813                 }
 814
 815 // FIXME: Bug: we can have an empty font change here!
 816 // if there has just been a font change, we are going to close it
 817 // right now, which means stupid latex code like \textsf{}. AFAIK,
 818 // this does not harm dvi output. A minor bug, thus (JMarc)
 819                 // Some insets cannot be inside a font change command.
 820                 // However, even such insets *can* be placed in \L or \R
 821                 // or their equivalents (for RTL language switches), so we don't
 822                 // close the language in those cases.
 823                 // ArabTeX, though, cannot handle this special behavior, it seems.
 824                 bool arabtex = basefont.language()->lang() == "arabic_arabtex" ||
 825                                            running_font.language()->lang() == "arabic_arabtex";
 826                 if (open_font && inset->noFontChange()) {
 827                         bool closeLanguage = arabtex ||
 828                                 basefont.isRightToLeft() == running_font.isRightToLeft();
 829                         unsigned int count = running_font.latexWriteEndChanges(
 830                                         os, bparams, runparams,
 831                                                 basefont, basefont, closeLanguage);
 832                         column += count;
 833                         // if any font properties were closed, update the running_font,
 834                         // making sure, however, to leave the language as it was
 835                         if (count > 0) {
 836                                 // FIXME: probably a better way to keep track of the old
 837                                 // language, than copying the entire font?
 838                                 Font const copy_font(running_font);
 839                                 basefont = owner_->getLayoutFont(bparams, outerfont);
 840                                 running_font = basefont;
 841                                 if (!closeLanguage)
 842                                         running_font.setLanguage(copy_font.language());
 843                                 // leave font open if language is still open
 844                                 open_font = (running_font.language() == basefont.language());
 845                                 if (closeLanguage)
 846                                         runparams.local_font = &basefont;
 847                         }
 848                 }
 849
 850                 int tmp = inset->latex(buf, os, runparams);
 851
 852                 if (close) {
 853                         if (running_font.language()->lang() == "farsi")
 854                                 os << "\\endL{}";
 855                         else
 856                                 os << '}';
 857                 }
 858
 859                 if (tmp) {
 860                         for (int j = 0; j < tmp; ++j) {
 861                                 texrow.newline();
 862                         }
 863                         texrow.start(owner_->id(), i + 1);
 864                         column = 0;
 865                 } else {
 866                         column += os.tellp() - len;
 867                 }
 868
 869                 if (lookupChange(i).type == Change::DELETED) {
 870                         --runparams.inDeletedInset;
 871                 }
 872         }
 873         break;
 874
 875         default:
 876                 // And now for the special cases within each mode
 877
 878                 switch (c) {
 879                 case '\\':
 880                         os << "\\textbackslash{}";
 881                         column += 15;
 882                         break;
 883
 884                 case '|': case '<': case '>':
 885                         // In T1 encoding, these characters exist
 886                         if (lyxrc.fontenc == "T1") {
 887                                 os.put(c);
 888                                 //... but we should avoid ligatures
 889                                 if ((c == '>' || c == '<')
 890                                     && i <= size() - 2
 891                                     && getChar(i + 1) == c) {
 892                                         //os << "\\textcompwordmark{}";
 893                                         //column += 19;
 894                                         // Jean-Marc, have a look at
 895                                         // this. I think this works
 896                                         // equally well:
 897                                         os << "\\,{}";
 898                                         // Lgb
 899                                         column += 3;
 900                                 }
 901                                 break;
 902                         }
 903                         // Typewriter font also has them
 904                         if (running_font.family() == Font::TYPEWRITER_FAMILY) {
 905                                 os.put(c);
 906                                 break;
 907                         }
 908                         // Otherwise, we use what LaTeX
 909                         // provides us.
 910                         switch (c) {
 911                         case '<':
 912                                 os << "\\textless{}";
 913                                 column += 10;
 914                                 break;
 915                         case '>':
 916                                 os << "\\textgreater{}";
 917                                 column += 13;
 918                                 break;
 919                         case '|':
 920                                 os << "\\textbar{}";
 921                                 column += 9;
 922                                 break;
 923                         }
 924                         break;
 925
 926                 case '-': // "--" in Typewriter mode -> "-{}-"
 927                         if (i <= size() - 2 &&
 928                             getChar(i + 1) == '-' &&
 929                             running_font.family() == Font::TYPEWRITER_FAMILY) {
 930                                 os << "-{}";
 931                                 column += 2;
 932                         } else {
 933                                 os << '-';
 934                         }
 935                         break;
 936
 937                 case '\"':
 938                         os << "\\char`\\\"{}";
 939                         column += 9;
 940                         break;
 941
 942                 case '$': case '&':
 943                 case '%': case '#': case '{':
 944                 case '}': case '_':
 945                         os << '\\';
 946                         os.put(c);
 947                         column += 1;
 948                         break;
 949
 950                 case '~':
 951                         os << "\\textasciitilde{}";
 952                         column += 16;
 953                         break;
 954
 955                 case '^':
 956                         os << "\\textasciicircum{}";
 957                         column += 17;
 958                         break;
 959
 960                 case '*': case '[':
 961                         // avoid being mistaken for optional arguments
 962                         os << '{';
 963                         os.put(c);
 964                         os << '}';
 965                         column += 2;
 966                         break;
 967
 968                 case ' ':
 969                         // Blanks are printed before font switching.
 970                         // Sure? I am not! (try nice-latex)
 971                         // I am sure it's correct. LyX might be smarter
 972                         // in the future, but for now, nothing wrong is
 973                         // written. (Asger)
 974                         break;
 975
 976                 default:
 977
 978                         // I assume this is hack treating typewriter as verbatim
 979                         // FIXME UNICODE: This can fail if c cannot be encoded
 980                         // in the current encoding.
 981                         if (running_font.family() == Font::TYPEWRITER_FAMILY) {
 982                                 if (c != '\0') {
 983                                         os.put(c);
 984                                 }
 985                                 break;
 986                         }
 987
 988                         // LyX, LaTeX etc.
 989
 990                         // FIXME: if we have "LaTeX" with a font
 991                         // change in the middle (before the 'T', then
 992                         // the "TeX" part is still special cased.
 993                         // Really we should only operate this on
 994                         // "words" for some definition of word
 995
 996                         size_t pnr = 0;
 997
 998                         for (; pnr < phrases_nr; ++pnr) {
 999                                 if (isTextAt(special_phrases[pnr].phrase, i)) {
1000                                         os << special_phrases[pnr].macro;
1001                                         i += special_phrases[pnr].phrase.length() - 1;
1002                                         column += special_phrases[pnr].macro.length() - 1;
1003                                         break;
1004                                 }
1005                         }
1006
1007                         if (pnr == phrases_nr && c != '\0') {
1008                                 Encoding const & encoding = *(runparams.encoding);
1009                                 if (i < size() - 1) {
1010                                         char_type next = getChar(i + 1);
1011                                         if (Encodings::isCombiningChar(next)) {
1012                                                 column += latexSurrogatePair(os, c, next, encoding) - 1;
1013                                                 ++i;
1014                                                 break;
1015                                         }
1016                                 }
1017                                 string preamble;
1018                                 if (Encodings::isKnownLangChar(c, preamble)) {
1019                                         column +=
1020                                                 knownLangChars(os, c, preamble,
1021                                                         running_change,
1022                                                         encoding, i) - 1;
1023                                         break;
1024                                 }
1025                                 docstring const latex = encoding.latexChar(c);
1026                                 if (latex.length() > 1 &&
1027                                     latex[latex.length() - 1] != '}') {
1028                                         // Prevent eating of a following
1029                                         // space or command corruption by
1030                                         // following characters
1031                                         column += latex.length() + 1;
1032                                         os << latex << "{}";
1033                                 } else {
1034                                         column += latex.length() - 1;
1035                                         os << latex;
1036                                 }
1037                         }
1038                         break;
1039                 }
1040         }
1041 }
1042
1043
1044 void Paragraph::Pimpl::validate(LaTeXFeatures & features,
1045                                 Layout const & layout) const
1046 {
1047         BufferParams const & bparams = features.bufferParams();
1048
1049         // check the params.
1050         if (!params.spacing().isDefault())
1051                 features.require("setspace");
1052
1053         // then the layouts
1054         features.useLayout(layout.name());
1055
1056         // then the fonts
1057         Language const * doc_language = bparams.language;
1058
1059         FontList::const_iterator fcit = fontlist.begin();
1060         FontList::const_iterator fend = fontlist.end();
1061         for (; fcit != fend; ++fcit) {
1062                 if (fcit->font().noun() == Font::ON) {
1063                         LYXERR(Debug::LATEX) << "font.noun: "
1064                                              << fcit->font().noun()
1065                                              << endl;
1066                         features.require("noun");
1067                         LYXERR(Debug::LATEX) << "Noun enabled. Font: "
1068                                              << to_utf8(fcit->font().stateText(0))
1069                                              << endl;
1070                 }
1071                 switch (fcit->font().color()) {
1072                 case Color::none:
1073                 case Color::inherit:
1074                 case Color::ignore:
1075                         // probably we should put here all interface colors used for
1076                         // font displaying! For now I just add this ones I know of (Jug)
1077                 case Color::latex:
1078                 case Color::note:
1079                         break;
1080                 default:
1081                         features.require("color");
1082                         LYXERR(Debug::LATEX) << "Color enabled. Font: "
1083                                              << to_utf8(fcit->font().stateText(0))
1084                                              << endl;
1085                 }
1086
1087                 Language const * language = fcit->font().language();
1088                 if (language->babel() != doc_language->babel() &&
1089                     language != ignore_language &&
1090                     language != latex_language)
1091                 {
1092                         features.useLanguage(language);
1093                         LYXERR(Debug::LATEX) << "Found language "
1094                                              << language->lang() << endl;
1095                 }
1096         }
1097
1098         if (!params.leftIndent().zero())
1099                 features.require("ParagraphLeftIndent");
1100
1101         // then the insets
1102         InsetList::const_iterator icit = owner_->insetlist.begin();
1103         InsetList::const_iterator iend = owner_->insetlist.end();
1104         for (; icit != iend; ++icit) {
1105                 if (icit->inset) {
1106                         icit->inset->validate(features);
1107                         if (layout.needprotect &&
1108                             icit->inset->lyxCode() == Inset::FOOT_CODE)
1109                                 features.require("NeedLyXFootnoteCode");
1110                 }
1111         }
1112
1113         // then the contents
1114         for (pos_type i = 0; i < size() ; ++i) {
1115                 for (size_t pnr = 0; pnr < phrases_nr; ++pnr) {
1116                         if (!special_phrases[pnr].builtin
1117                             && isTextAt(special_phrases[pnr].phrase, i)) {
1118                                 features.require(special_phrases[pnr].phrase);
1119                                 break;
1120                         }
1121                 }
1122                 Encodings::validate(getChar(i), features);
1123         }
1124 }
1125
1126
1127 } // namespace lyx
1128
1129
1130 /////////////////////////////////////////////////////////////////////
1131 //
1132 // Paragraph
1133 //
1134 /////////////////////////////////////////////////////////////////////
1135
1136 namespace lyx {
1137
1138 Paragraph::Paragraph()
1139         : begin_of_body_(0), pimpl_(new Paragraph::Pimpl(this))
1140 {
1141         itemdepth = 0;
1142         params().clear();
1143 }
1144
1145
1146 Paragraph::Paragraph(Paragraph const & par)
1147         : itemdepth(par.itemdepth), insetlist(par.insetlist),
1148         layout_(par.layout_),
1149         text_(par.text_), begin_of_body_(par.begin_of_body_),
1150         pimpl_(new Paragraph::Pimpl(*par.pimpl_, this))
1151 {
1152         //lyxerr << "Paragraph::Paragraph(Paragraph const&)" << endl;
1153         InsetList::iterator it = insetlist.begin();
1154         InsetList::iterator end = insetlist.end();
1155         for (; it != end; ++it)
1156                 it->inset = it->inset->clone();
1157 }
1158
1159
1160 Paragraph & Paragraph::operator=(Paragraph const & par)
1161 {
1162         // needed as we will destroy the pimpl_ before copying it
1163         if (&par != this) {
1164                 itemdepth = par.itemdepth;
1165
1166                 insetlist = par.insetlist;
1167                 InsetList::iterator it = insetlist.begin();
1168                 InsetList::iterator end = insetlist.end();
1169                 for (; it != end; ++it)
1170                         it->inset = it->inset->clone();
1171
1172                 layout_ = par.layout();
1173                 text_ = par.text_;
1174                 begin_of_body_ = par.begin_of_body_;
1175
1176                 delete pimpl_;
1177                 pimpl_ = new Pimpl(*par.pimpl_, this);
1178         }
1179         return *this;
1180 }
1181
1182
1183 Paragraph::~Paragraph()
1184 {
1185         delete pimpl_;
1186         //
1187         //lyxerr << "Paragraph::paragraph_id = "
1188         //       << Paragraph::paragraph_id << endl;
1189 }
1190
1191
1192 void Paragraph::write(Buffer const & buf, ostream & os,
1193                           BufferParams const & bparams,
1194                           depth_type & dth) const
1195 {
1196         // The beginning or end of a deeper (i.e. nested) area?
1197         if (dth != params().depth()) {
1198                 if (params().depth() > dth) {
1199                         while (params().depth() > dth) {
1200                                 os << "\n\\begin_deeper";
1201                                 ++dth;
1202                         }
1203                 } else {
1204                         while (params().depth() < dth) {
1205                                 os << "\n\\end_deeper";
1206                                 --dth;
1207                         }
1208                 }
1209         }
1210
1211         // First write the layout
1212         os << "\n\\begin_layout " << to_utf8(layout()->name()) << '\n';
1213
1214         params().write(os);
1215
1216         Font font1(Font::ALL_INHERIT, bparams.language);
1217
1218         Change running_change = Change(Change::UNCHANGED);
1219
1220         int column = 0;
1221         for (pos_type i = 0; i <= size(); ++i) {
1222
1223                 Change change = pimpl_->lookupChange(i);
1224                 Changes::lyxMarkChange(os, column, running_change, change);
1225                 running_change = change;
1226
1227                 if (i == size())
1228                         break;
1229
1230                 // Write font changes
1231                 Font font2 = getFontSettings(bparams, i);
1232                 if (font2 != font1) {
1233                         font2.lyxWriteChanges(font1, os);
1234                         column = 0;
1235                         font1 = font2;
1236                 }
1237
1238                 value_type const c = getChar(i);
1239                 switch (c) {
1240                 case META_INSET:
1241                 {
1242                         Inset const * inset = getInset(i);
1243                         if (inset)
1244                                 if (inset->directWrite()) {
1245                                         // international char, let it write
1246                                         // code directly so it's shorter in
1247                                         // the file
1248                                         inset->write(buf, os);
1249                                 } else {
1250                                         if (i)
1251                                                 os << '\n';
1252                                         os << "\\begin_inset ";
1253                                         inset->write(buf, os);
1254                                         os << "\n\\end_inset\n\n";
1255                                         column = 0;
1256                                 }
1257                 }
1258                 break;
1259                 case '\\':
1260                         os << "\n\\backslash\n";
1261                         column = 0;
1262                         break;
1263                 case '.':
1264                         if (i + 1 < size() && getChar(i + 1) == ' ') {
1265                                 os << ".\n";
1266                                 column = 0;
1267                         } else
1268                                 os << '.';
1269                         break;
1270                 default:
1271                         if ((column > 70 && c == ' ')
1272                             || column > 79) {
1273                                 os << '\n';
1274                                 column = 0;
1275                         }
1276                         // this check is to amend a bug. LyX sometimes
1277                         // inserts '\0' this could cause problems.
1278                         if (c != '\0') {
1279                                 std::vector<char> tmp = ucs4_to_utf8(c);
1280                                 tmp.push_back('\0');
1281                                 os << &tmp[0];
1282                         } else
1283                                 lyxerr << "ERROR (Paragraph::writeFile):"
1284                                         " NULL char in structure." << endl;
1285                         ++column;
1286                         break;
1287                 }
1288         }
1289
1290         os << "\n\\end_layout\n";
1291 }
1292
1293
1294 void Paragraph::validate(LaTeXFeatures & features) const
1295 {
1296         pimpl_->validate(features, *layout());
1297 }
1298
1299
1300 bool Paragraph::eraseChar(pos_type pos, bool trackChanges)
1301 {
1302         return pimpl_->eraseChar(pos, trackChanges);
1303 }
1304
1305
1306 int Paragraph::eraseChars(pos_type start, pos_type end, bool trackChanges)
1307 {
1308         return pimpl_->eraseChars(start, end, trackChanges);
1309 }
1310
1311
1312 void Paragraph::insert(pos_type start, docstring const & str,
1313                        Font const & font, Change const & change)
1314 {
1315         for (size_t i = 0, n = str.size(); i != n ; ++i)
1316                 insertChar(start + i, str[i], font, change);
1317 }
1318
1319
1320 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
1321                            bool trackChanges)
1322 {
1323         pimpl_->insertChar(pos, c, Change(trackChanges ?
1324                            Change::INSERTED : Change::UNCHANGED));
1325 }
1326
1327
1328 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
1329                            Font const & font, bool trackChanges)
1330 {
1331         pimpl_->insertChar(pos, c, Change(trackChanges ?
1332                            Change::INSERTED : Change::UNCHANGED));
1333         setFont(pos, font);
1334 }
1335
1336
1337 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
1338                            Font const & font, Change const & change)
1339 {
1340         pimpl_->insertChar(pos, c, change);
1341         setFont(pos, font);
1342 }
1343
1344
1345 void Paragraph::insertInset(pos_type pos, Inset * inset,
1346                             Change const & change)
1347 {
1348         pimpl_->insertInset(pos, inset, change);
1349 }
1350
1351
1352 void Paragraph::insertInset(pos_type pos, Inset * inset,
1353                             Font const & font, Change const & change)
1354 {
1355         pimpl_->insertInset(pos, inset, change);
1356         // Set the font/language of the inset...
1357         setFont(pos, font);
1358 }
1359
1360
1361 bool Paragraph::insetAllowed(Inset_code code)
1362 {
1363         return !pimpl_->inset_owner || pimpl_->inset_owner->insetAllowed(code);
1364 }
1365
1366
1367 // Gets uninstantiated font setting at position.
1368 Font const Paragraph::getFontSettings(BufferParams const & bparams,
1369                                          pos_type pos) const
1370 {
1371         if (pos > size()) {
1372                 lyxerr << " pos: " << pos << " size: " << size() << endl;
1373                 BOOST_ASSERT(pos <= size());
1374         }
1375
1376         Pimpl::FontList::const_iterator cit = pimpl_->fontlist.begin();
1377         Pimpl::FontList::const_iterator end = pimpl_->fontlist.end();
1378         for (; cit != end; ++cit)
1379                 if (cit->pos() >= pos)
1380                         break;
1381
1382         if (cit != end)
1383                 return cit->font();
1384
1385         if (pos == size() && !empty())
1386                 return getFontSettings(bparams, pos - 1);
1387
1388         return Font(Font::ALL_INHERIT, getParLanguage(bparams));
1389 }
1390
1391
1392 FontSpan Paragraph::fontSpan(pos_type pos) const
1393 {
1394         BOOST_ASSERT(pos <= size());
1395         pos_type start = 0;
1396
1397         Pimpl::FontList::const_iterator cit = pimpl_->fontlist.begin();
1398         Pimpl::FontList::const_iterator end = pimpl_->fontlist.end();
1399         for (; cit != end; ++cit) {
1400                 if (cit->pos() >= pos) {
1401                         if (pos >= beginOfBody())
1402                                 return FontSpan(std::max(start, beginOfBody()),
1403                                                 cit->pos());
1404                         else
1405                                 return FontSpan(start,
1406                                                 std::min(beginOfBody() - 1,
1407                                                          cit->pos()));
1408                 }
1409                 start = cit->pos() + 1;
1410         }
1411
1412         // This should not happen, but if so, we take no chances.
1413         //lyxerr << "Paragraph::getEndPosOfFontSpan: This should not happen!"
1414         //      << endl;
1415         return FontSpan(pos, pos);
1416 }
1417
1418
1419 // Gets uninstantiated font setting at position 0
1420 Font const Paragraph::getFirstFontSettings(BufferParams const & bparams) const
1421 {
1422         if (!empty() && !pimpl_->fontlist.empty())
1423                 return pimpl_->fontlist[0].font();
1424
1425         return Font(Font::ALL_INHERIT, bparams.language);
1426 }
1427
1428
1429 // Gets the fully instantiated font at a given position in a paragraph
1430 // This is basically the same function as Text::GetFont() in text2.cpp.
1431 // The difference is that this one is used for generating the LaTeX file,
1432 // and thus cosmetic "improvements" are disallowed: This has to deliver
1433 // the true picture of the buffer. (Asger)
1434 Font const Paragraph::getFont(BufferParams const & bparams, pos_type pos,
1435                                  Font const & outerfont) const
1436 {
1437         BOOST_ASSERT(pos >= 0);
1438
1439         Font font = getFontSettings(bparams, pos);
1440
1441         pos_type const body_pos = beginOfBody();
1442         if (pos < body_pos)
1443                 font.realize(layout_->labelfont);
1444         else
1445                 font.realize(layout_->font);
1446
1447         font.realize(outerfont);
1448         font.realize(bparams.getFont());
1449
1450         return font;
1451 }
1452
1453
1454 Font const Paragraph::getLabelFont
1455         (BufferParams const & bparams, Font const & outerfont) const
1456 {
1457         Font tmpfont = layout()->labelfont;
1458         tmpfont.setLanguage(getParLanguage(bparams));
1459         tmpfont.realize(outerfont);
1460         tmpfont.realize(bparams.getFont());
1461         return tmpfont;
1462 }
1463
1464
1465 Font const Paragraph::getLayoutFont
1466         (BufferParams const & bparams, Font const & outerfont) const
1467 {
1468         Font tmpfont = layout()->font;
1469         tmpfont.setLanguage(getParLanguage(bparams));
1470         tmpfont.realize(outerfont);
1471         tmpfont.realize(bparams.getFont());
1472         return tmpfont;
1473 }
1474
1475
1476 /// Returns the height of the highest font in range
1477 Font_size Paragraph::highestFontInRange
1478         (pos_type startpos, pos_type endpos, Font_size def_size) const
1479 {
1480         if (pimpl_->fontlist.empty())
1481                 return def_size;
1482
1483         Pimpl::FontList::const_iterator end_it = pimpl_->fontlist.begin();
1484         Pimpl::FontList::const_iterator const end = pimpl_->fontlist.end();
1485         for (; end_it != end; ++end_it) {
1486                 if (end_it->pos() >= endpos)
1487                         break;
1488         }
1489
1490         if (end_it != end)
1491                 ++end_it;
1492
1493         Pimpl::FontList::const_iterator cit = pimpl_->fontlist.begin();
1494         for (; cit != end; ++cit) {
1495                 if (cit->pos() >= startpos)
1496                         break;
1497         }
1498
1499         Font::FONT_SIZE maxsize = Font::SIZE_TINY;
1500         for (; cit != end_it; ++cit) {
1501                 Font::FONT_SIZE size = cit->font().size();
1502                 if (size == Font::INHERIT_SIZE)
1503                         size = def_size;
1504                 if (size > maxsize && size <= Font::SIZE_HUGER)
1505                         maxsize = size;
1506         }
1507         return maxsize;
1508 }
1509
1510
1511 Paragraph::value_type
1512 Paragraph::getUChar(BufferParams const & bparams, pos_type pos) const
1513 {
1514         value_type c = getChar(pos);
1515         if (!lyxrc.rtl_support)
1516                 return c;
1517
1518         value_type uc = c;
1519         switch (c) {
1520         case '(':
1521                 uc = ')';
1522                 break;
1523         case ')':
1524                 uc = '(';
1525                 break;
1526         case '[':
1527                 uc = ']';
1528                 break;
1529         case ']':
1530                 uc = '[';
1531                 break;
1532         case '{':
1533                 uc = '}';
1534                 break;
1535         case '}':
1536                 uc = '{';
1537                 break;
1538         case '<':
1539                 uc = '>';
1540                 break;
1541         case '>':
1542                 uc = '<';
1543                 break;
1544         }
1545         if (uc != c && getFontSettings(bparams, pos).isRightToLeft())
1546                 return uc;
1547         else
1548                 return c;
1549 }
1550
1551
1552 void Paragraph::setFont(pos_type pos, Font const & font)
1553 {
1554         BOOST_ASSERT(pos <= size());
1555
1556         // First, reduce font against layout/label font
1557         // Update: The setCharFont() routine in text2.cpp already
1558         // reduces font, so we don't need to do that here. (Asger)
1559         // No need to simplify this because it will disappear
1560         // in a new kernel. (Asger)
1561         // Next search font table
1562
1563         Pimpl::FontList::iterator beg = pimpl_->fontlist.begin();
1564         Pimpl::FontList::iterator it = beg;
1565         Pimpl::FontList::iterator endit = pimpl_->fontlist.end();
1566         for (; it != endit; ++it) {
1567                 if (it->pos() >= pos)
1568                         break;
1569         }
1570         size_t const i = distance(beg, it);
1571         bool notfound = (it == endit);
1572
1573         if (!notfound && pimpl_->fontlist[i].font() == font)
1574                 return;
1575
1576         bool begin = pos == 0 || notfound ||
1577                 (i > 0 && pimpl_->fontlist[i - 1].pos() == pos - 1);
1578         // Is position pos is a beginning of a font block?
1579         bool end = !notfound && pimpl_->fontlist[i].pos() == pos;
1580         // Is position pos is the end of a font block?
1581         if (begin && end) { // A single char block
1582                 if (i + 1 < pimpl_->fontlist.size() &&
1583                     pimpl_->fontlist[i + 1].font() == font) {
1584                         // Merge the singleton block with the next block
1585                         pimpl_->fontlist.erase(pimpl_->fontlist.begin() + i);
1586                         if (i > 0 && pimpl_->fontlist[i - 1].font() == font)
1587                                 pimpl_->fontlist.erase(pimpl_->fontlist.begin() + i - 1);
1588                 } else if (i > 0 && pimpl_->fontlist[i - 1].font() == font) {
1589                         // Merge the singleton block with the previous block
1590                         pimpl_->fontlist[i - 1].pos(pos);
1591                         pimpl_->fontlist.erase(pimpl_->fontlist.begin() + i);
1592                 } else
1593                         pimpl_->fontlist[i].font(font);
1594         } else if (begin) {
1595                 if (i > 0 && pimpl_->fontlist[i - 1].font() == font)
1596                         pimpl_->fontlist[i - 1].pos(pos);
1597                 else
1598                         pimpl_->fontlist.insert(pimpl_->fontlist.begin() + i,
1599                                         Pimpl::FontTable(pos, font));
1600         } else if (end) {
1601                 pimpl_->fontlist[i].pos(pos - 1);
1602                 if (!(i + 1 < pimpl_->fontlist.size() &&
1603                       pimpl_->fontlist[i + 1].font() == font))
1604                         pimpl_->fontlist.insert(pimpl_->fontlist.begin() + i + 1,
1605                                         Pimpl::FontTable(pos, font));
1606         } else { // The general case. The block is splitted into 3 blocks
1607                 pimpl_->fontlist.insert(pimpl_->fontlist.begin() + i,
1608                                 Pimpl::FontTable(pos - 1, pimpl_->fontlist[i].font()));
1609                 pimpl_->fontlist.insert(pimpl_->fontlist.begin() + i + 1,
1610                                 Pimpl::FontTable(pos, font));
1611         }
1612 }
1613
1614
1615 void Paragraph::makeSameLayout(Paragraph const & par)
1616 {
1617         layout(par.layout());
1618         // move to pimpl?
1619         params() = par.params();
1620 }
1621
1622
1623 bool Paragraph::stripLeadingSpaces(bool trackChanges)
1624 {
1625         if (isFreeSpacing())
1626                 return false;
1627
1628         int pos = 0;
1629         int count = 0;
1630
1631         while (pos < size() && (isNewline(pos) || isLineSeparator(pos))) {
1632                 if (eraseChar(pos, trackChanges))
1633                         ++count;
1634                 else
1635                         ++pos;
1636         }
1637
1638         return count > 0 || pos > 0;
1639 }
1640
1641
1642 bool Paragraph::hasSameLayout(Paragraph const & par) const
1643 {
1644         return par.layout() == layout() && params().sameLayout(par.params());
1645 }
1646
1647
1648 depth_type Paragraph::getDepth() const
1649 {
1650         return params().depth();
1651 }
1652
1653
1654 depth_type Paragraph::getMaxDepthAfter() const
1655 {
1656         if (layout()->isEnvironment())
1657                 return params().depth() + 1;
1658         else
1659                 return params().depth();
1660 }
1661
1662
1663 char Paragraph::getAlign() const
1664 {
1665         if (params().align() == LYX_ALIGN_LAYOUT)
1666                 return layout()->align;
1667         else
1668                 return params().align();
1669 }
1670
1671
1672 docstring const & Paragraph::getLabelstring() const
1673 {
1674         return params().labelString();
1675 }
1676
1677
1678 // the next two functions are for the manual labels
1679 docstring const Paragraph::getLabelWidthString() const
1680 {
1681         if (layout()->margintype == MARGIN_MANUAL)
1682                 return params().labelWidthString();
1683         else
1684                 return _("Senseless with this layout!");
1685 }
1686
1687
1688 void Paragraph::setLabelWidthString(docstring const & s)
1689 {
1690         params().labelWidthString(s);
1691 }
1692
1693
1694 docstring const Paragraph::translateIfPossible(docstring const & s,
1695                 BufferParams const & bparams) const
1696 {
1697         if (!support::isAscii(s) || s.empty()) {
1698                 // This must be a user defined layout. We cannot translate
1699                 // this, since gettext accepts only ascii keys.
1700                 return s;
1701         }
1702         // Probably standard layout, try to translate
1703         Messages & m = getMessages(getParLanguage(bparams)->code());
1704         return m.get(to_ascii(s));
1705 }
1706
1707
1708 docstring Paragraph::expandLabel(LayoutPtr const & layout,
1709                 BufferParams const & bparams, bool process_appendix) const
1710 {
1711         TextClass const & tclass = bparams.getTextClass();
1712
1713         docstring fmt;
1714         if (process_appendix && params().appendix())
1715                 fmt = translateIfPossible(layout->labelstring_appendix(),
1716                         bparams);
1717         else
1718                 fmt = translateIfPossible(layout->labelstring(), bparams);
1719
1720         if (fmt.empty() && layout->labeltype == LABEL_COUNTER
1721             && !layout->counter.empty())
1722                 fmt = "\\the" + layout->counter;
1723
1724         // handle 'inherited level parts' in 'fmt',
1725         // i.e. the stuff between '@' in   '@Section@.\arabic{subsection}'
1726         size_t const i = fmt.find('@', 0);
1727         if (i != docstring::npos) {
1728                 size_t const j = fmt.find('@', i + 1);
1729                 if (j != docstring::npos) {
1730                         docstring parent(fmt, i + 1, j - i - 1);
1731                         docstring label = from_ascii("??");
1732                         if (tclass.hasLayout(parent))
1733                                 docstring label = expandLabel(tclass[parent], bparams,
1734                                                       process_appendix);
1735                         fmt = docstring(fmt, 0, i) + label
1736                                 + docstring(fmt, j + 1, docstring::npos);
1737                 }
1738         }
1739
1740         return tclass.counters().counterLabel(fmt);
1741 }
1742
1743
1744 void Paragraph::applyLayout(LayoutPtr const & new_layout)
1745 {
1746         layout(new_layout);
1747         LyXAlignment const oldAlign = params().align();
1748
1749         if (!(oldAlign & layout()->alignpossible)) {
1750                 frontend::Alert::warning(_("Alignment not permitted"),
1751                         _("The new layout does not permit the alignment previously used.\nSetting to default."));
1752                 params().align(LYX_ALIGN_LAYOUT);
1753         }
1754 }
1755
1756
1757 pos_type Paragraph::beginOfBody() const
1758 {
1759         return begin_of_body_;
1760 }
1761
1762
1763 void Paragraph::setBeginOfBody()
1764 {
1765         if (layout()->labeltype != LABEL_MANUAL) {
1766                 begin_of_body_ = 0;
1767                 return;
1768         }
1769
1770         // Unroll the first two cycles of the loop
1771         // and remember the previous character to
1772         // remove unnecessary getChar() calls
1773         pos_type i = 0;
1774         pos_type end = size();
1775         if (i < end && !isNewline(i)) {
1776                 ++i;
1777                 char_type previous_char = 0;
1778                 char_type temp = 0;
1779                 if (i < end) {
1780                         previous_char = text_[i];
1781                         if (!isNewline(i)) {
1782                                 ++i;
1783                                 while (i < end && previous_char != ' ') {
1784                                         temp = text_[i];
1785                                         if (isNewline(i))
1786                                                 break;
1787                                         ++i;
1788                                         previous_char = temp;
1789                                 }
1790                         }
1791                 }
1792         }
1793
1794         begin_of_body_ = i;
1795 }
1796
1797
1798 // returns -1 if inset not found
1799 int Paragraph::getPositionOfInset(Inset const * inset) const
1800 {
1801         // Find the entry.
1802         InsetList::const_iterator it = insetlist.begin();
1803         InsetList::const_iterator end = insetlist.end();
1804         for (; it != end; ++it)
1805                 if (it->inset == inset)
1806                         return it->pos;
1807         return -1;
1808 }
1809
1810
1811 InsetBibitem * Paragraph::bibitem() const
1812 {
1813         if (!insetlist.empty()) {
1814                 Inset * inset = insetlist.begin()->inset;
1815                 if (inset->lyxCode() == Inset::BIBITEM_CODE)
1816                         return static_cast<InsetBibitem *>(inset);
1817         }
1818         return 0;
1819 }
1820
1821
1822 bool Paragraph::forceDefaultParagraphs() const
1823 {
1824         return inInset() && inInset()->forceDefaultParagraphs(0);
1825 }
1826
1827
1828 namespace {
1829
1830 // paragraphs inside floats need different alignment tags to avoid
1831 // unwanted space
1832
1833 bool noTrivlistCentering(Inset::Code code)
1834 {
1835         return code == Inset::FLOAT_CODE || code == Inset::WRAP_CODE;
1836 }
1837
1838
1839 string correction(string const & orig)
1840 {
1841         if (orig == "flushleft")
1842                 return "raggedright";
1843         if (orig == "flushright")
1844                 return "raggedleft";
1845         if (orig == "center")
1846                 return "centering";
1847         return orig;
1848 }
1849
1850
1851 string const corrected_env(string const & suffix, string const & env,
1852         Inset::Code code)
1853 {
1854         string output = suffix + "{";
1855         if (noTrivlistCentering(code))
1856                 output += correction(env);
1857         else
1858                 output += env;
1859         output += "}";
1860         if (suffix == "\\begin")
1861                 output += "\n";
1862         return output;
1863 }
1864
1865
1866 void adjust_row_column(string const & str, TexRow & texrow, int & column)
1867 {
1868         if (!contains(str, "\n"))
1869                 column += str.size();
1870         else {
1871                 string tmp;
1872                 texrow.newline();
1873                 column = rsplit(str, tmp, '\n').size();
1874         }
1875 }
1876
1877 } // namespace anon
1878
1879
1880 // This could go to ParagraphParameters if we want to
1881 int Paragraph::startTeXParParams(BufferParams const & bparams,
1882                                  odocstream & os, TexRow & texrow,
1883                                  bool moving_arg) const
1884 {
1885         int column = 0;
1886
1887         if (params().noindent()) {
1888                 os << "\\noindent ";
1889                 column += 10;
1890         }
1891
1892         LyXAlignment const curAlign = params().align();
1893
1894         if (curAlign == layout()->align)
1895                 return column;
1896
1897         switch (curAlign) {
1898         case LYX_ALIGN_NONE:
1899         case LYX_ALIGN_BLOCK:
1900         case LYX_ALIGN_LAYOUT:
1901         case LYX_ALIGN_SPECIAL:
1902                 break;
1903         case LYX_ALIGN_LEFT:
1904         case LYX_ALIGN_RIGHT:
1905         case LYX_ALIGN_CENTER:
1906                 if (moving_arg) {
1907                         os << "\\protect";
1908                         column += 8;
1909                 }
1910                 break;
1911         }
1912
1913         switch (curAlign) {
1914         case LYX_ALIGN_NONE:
1915         case LYX_ALIGN_BLOCK:
1916         case LYX_ALIGN_LAYOUT:
1917         case LYX_ALIGN_SPECIAL:
1918                 break;
1919         case LYX_ALIGN_LEFT: {
1920                 string output;
1921                 if (getParLanguage(bparams)->babel() != "hebrew")
1922                         output = corrected_env("\\begin", "flushleft", ownerCode());
1923                 else
1924                         output = corrected_env("\\begin", "flushright", ownerCode());
1925                 os << from_ascii(output);
1926                 adjust_row_column(output, texrow, column);
1927                 break;
1928         } case LYX_ALIGN_RIGHT: {
1929                 string output;
1930                 if (getParLanguage(bparams)->babel() != "hebrew")
1931                         output = corrected_env("\\begin", "flushright", ownerCode());
1932                 else
1933                         output = corrected_env("\\begin", "flushleft", ownerCode());
1934                 os << from_ascii(output);
1935                 adjust_row_column(output, texrow, column);
1936                 break;
1937         } case LYX_ALIGN_CENTER: {
1938                 string output;
1939                 output = corrected_env("\\begin", "center", ownerCode());
1940                 os << from_ascii(output);
1941                 adjust_row_column(output, texrow, column);
1942                 break;
1943         }
1944         }
1945
1946         return column;
1947 }
1948
1949
1950 // This could go to ParagraphParameters if we want to
1951 int Paragraph::endTeXParParams(BufferParams const & bparams,
1952                                odocstream & os, TexRow & texrow,
1953                                bool moving_arg) const
1954 {
1955         int column = 0;
1956
1957         switch (params().align()) {
1958         case LYX_ALIGN_NONE:
1959         case LYX_ALIGN_BLOCK:
1960         case LYX_ALIGN_LAYOUT:
1961         case LYX_ALIGN_SPECIAL:
1962                 break;
1963         case LYX_ALIGN_LEFT:
1964         case LYX_ALIGN_RIGHT:
1965         case LYX_ALIGN_CENTER:
1966                 if (moving_arg) {
1967                         os << "\\protect";
1968                         column = 8;
1969                 }
1970                 break;
1971         }
1972
1973         switch (params().align()) {
1974         case LYX_ALIGN_NONE:
1975         case LYX_ALIGN_BLOCK:
1976         case LYX_ALIGN_LAYOUT:
1977         case LYX_ALIGN_SPECIAL:
1978                 break;
1979         case LYX_ALIGN_LEFT: {
1980                 string output;
1981                 if (getParLanguage(bparams)->babel() != "hebrew")
1982                         output = corrected_env("\n\\par\\end", "flushleft", ownerCode());
1983                 else
1984                         output = corrected_env("\n\\par\\end", "flushright", ownerCode());
1985                 os << from_ascii(output);
1986                 adjust_row_column(output, texrow, column);
1987                 break;
1988         } case LYX_ALIGN_RIGHT: {
1989                 string output;
1990                 if (getParLanguage(bparams)->babel() != "hebrew")
1991                         output = corrected_env("\n\\par\\end", "flushright", ownerCode());
1992                 else
1993                         output = corrected_env("\n\\par\\end", "flushleft", ownerCode());
1994                 os << from_ascii(output);
1995                 adjust_row_column(output, texrow, column);
1996                 break;
1997         } case LYX_ALIGN_CENTER: {
1998                 string output;
1999                 output = corrected_env("\n\\par\\end", "center", ownerCode());
2000                 os << from_ascii(output);
2001                 adjust_row_column(output, texrow, column);
2002                 break;
2003         }
2004         }
2005
2006         return column;
2007 }
2008
2009
2010 // This one spits out the text of the paragraph
2011 bool Paragraph::simpleTeXOnePar(Buffer const & buf,
2012                                 BufferParams const & bparams,
2013                                 Font const & outerfont,
2014                                 odocstream & os, TexRow & texrow,
2015                                 OutputParams const & runparams) const
2016 {
2017         LYXERR(Debug::LATEX) << "SimpleTeXOnePar...     " << this << endl;
2018
2019         bool return_value = false;
2020
2021         LayoutPtr style;
2022
2023         // well we have to check if we are in an inset with unlimited
2024         // length (all in one row) if that is true then we don't allow
2025         // any special options in the paragraph and also we don't allow
2026         // any environment other than the default layout of the text class
2027         // to be valid!
2028         bool asdefault = forceDefaultParagraphs();
2029
2030         if (asdefault) {
2031                 style = bparams.getTextClass().defaultLayout();
2032         } else {
2033                 style = layout();
2034         }
2035
2036         // Current base font for all inherited font changes, without any
2037         // change caused by an individual character, except for the language:
2038         // It is set to the language of the first character.
2039         // As long as we are in the label, this font is the base font of the
2040         // label. Before the first body character it is set to the base font
2041         // of the body.
2042         Font basefont;
2043
2044         // Maybe we have to create a optional argument.
2045         pos_type body_pos = beginOfBody();
2046         unsigned int column = 0;
2047
2048         if (body_pos > 0) {
2049                 // the optional argument is kept in curly brackets in
2050                 // case it contains a ']'
2051                 os << "[{";
2052                 column += 2;
2053                 basefont = getLabelFont(bparams, outerfont);
2054         } else {
2055                 basefont = getLayoutFont(bparams, outerfont);
2056         }
2057
2058         // Which font is currently active?
2059         Font running_font(basefont);
2060         // Do we have an open font change?
2061         bool open_font = false;
2062
2063         Change runningChange = Change(Change::UNCHANGED);
2064
2065         texrow.start(id(), 0);
2066
2067         // if the paragraph is empty, the loop will not be entered at all
2068         if (empty()) {
2069                 if (style->isCommand()) {
2070                         os << '{';
2071                         ++column;
2072                 }
2073                 if (!asdefault)
2074                         column += startTeXParParams(bparams, os, texrow,
2075                                                     runparams.moving_arg);
2076         }
2077
2078         for (pos_type i = 0; i < size(); ++i) {
2079                 // First char in paragraph or after label?
2080                 if (i == body_pos) {
2081                         if (body_pos > 0) {
2082                                 if (open_font) {
2083                                         column += running_font.latexWriteEndChanges(
2084                                                 os, bparams, runparams,
2085                                                 basefont, basefont);
2086                                         open_font = false;
2087                                 }
2088                                 basefont = getLayoutFont(bparams, outerfont);
2089                                 running_font = basefont;
2090
2091                                 column += Changes::latexMarkChange(os, bparams,
2092                                                 runningChange, Change(Change::UNCHANGED));
2093                                 runningChange = Change(Change::UNCHANGED);
2094
2095                                 os << "}] ";
2096                                 column +=3;
2097                         }
2098                         if (style->isCommand()) {
2099                                 os << '{';
2100                                 ++column;
2101                         }
2102
2103                         if (!asdefault)
2104                                 column += startTeXParParams(bparams, os,
2105                                                             texrow,
2106                                                             runparams.moving_arg);
2107                 }
2108
2109                 Change const & change = runparams.inDeletedInset ? runparams.changeOfDeletedInset
2110                                                                  : pimpl_->lookupChange(i);
2111
2112                 if (bparams.outputChanges && runningChange != change) {
2113                         if (open_font) {
2114                                 column += running_font.latexWriteEndChanges(
2115                                                 os, bparams, runparams, basefont, basefont);
2116                                 open_font = false;
2117                         }
2118                         basefont = getLayoutFont(bparams, outerfont);
2119                         running_font = basefont;
2120
2121                         column += Changes::latexMarkChange(os, bparams, runningChange, change);
2122                         runningChange = change;
2123                 }
2124
2125                 // do not output text which is marked deleted
2126                 // if change tracking output is disabled
2127                 if (!bparams.outputChanges && change.type == Change::DELETED) {
2128                         continue;
2129                 }
2130
2131                 ++column;
2132
2133                 value_type const c = getChar(i);
2134
2135                 // Fully instantiated font
2136                 Font const font = getFont(bparams, i, outerfont);
2137
2138                 Font const last_font = running_font;
2139
2140                 // Do we need to close the previous font?
2141                 if (open_font &&
2142                     (font != running_font ||
2143                      font.language() != running_font.language()))
2144                 {
2145                         column += running_font.latexWriteEndChanges(
2146                                         os, bparams, runparams, basefont,
2147                                         (i == body_pos-1) ? basefont : font);
2148                         running_font = basefont;
2149                         open_font = false;
2150                 }
2151
2152                 // Switch file encoding if necessary
2153                 if (runparams.encoding->package() == Encoding::inputenc &&
2154                     font.language()->encoding()->package() == Encoding::inputenc) {
2155                         std::pair<bool, int> const enc_switch = switchEncoding(os, bparams,
2156                                         runparams.moving_arg, *(runparams.encoding),
2157                                         *(font.language()->encoding()));
2158                         if (enc_switch.first) {
2159                                 column += enc_switch.second;
2160                                 runparams.encoding = font.language()->encoding();
2161                         }
2162                 }
2163
2164                 // Do we need to change font?
2165                 if ((font != running_font ||
2166                      font.language() != running_font.language()) &&
2167                         i != body_pos - 1)
2168                 {
2169                         odocstringstream ods;
2170                         column += font.latexWriteStartChanges(ods, bparams,
2171                                                               runparams, basefont,
2172                                                               last_font);
2173                         running_font = font;
2174                         open_font = true;
2175                         docstring fontchange = ods.str();
2176                         // check if the fontchange ends with a trailing blank
2177                         // (like "\small " (see bug 3382)
2178                         if (suffixIs(fontchange, ' ') && c == ' ')
2179                                 os << fontchange.substr(0, fontchange.size() - 1)
2180                                    << from_ascii("{}");
2181                         else
2182                                 os << fontchange;
2183                 }
2184
2185                 if (c == ' ') {
2186                         // Do not print the separation of the optional argument
2187                         // if style->pass_thru is false. This works because
2188                         // simpleTeXSpecialChars ignores spaces if
2189                         // style->pass_thru is false.
2190                         if (i != body_pos - 1) {
2191                                 if (pimpl_->simpleTeXBlanks(
2192                                                 *(runparams.encoding), os, texrow,
2193                                                 i, column, font, *style))
2194                                         // A surrogate pair was output. We
2195                                         // must not call simpleTeXSpecialChars
2196                                         // in this iteration, since
2197                                         // simpleTeXBlanks incremented i, and
2198                                         // simpleTeXSpecialChars would output
2199                                         // the combining character again.
2200                                         continue;
2201                         }
2202                 }
2203
2204                 OutputParams rp = runparams;
2205                 rp.free_spacing = style->free_spacing;
2206                 rp.local_font = &font;
2207                 rp.intitle = style->intitle;
2208                 pimpl_->simpleTeXSpecialChars(buf, bparams, os,
2209                                         texrow, rp, running_font,
2210                                         basefont, outerfont, open_font,
2211                                         runningChange, *style, i, column, c);
2212
2213                 // Set the encoding to that returned from simpleTeXSpecialChars (see
2214                 // comment for encoding member in OutputParams.h)
2215                 runparams.encoding = rp.encoding;
2216         }
2217
2218         // If we have an open font definition, we have to close it
2219         if (open_font) {
2220 #ifdef FIXED_LANGUAGE_END_DETECTION
2221                 if (next_) {
2222                         running_font
2223                                 .latexWriteEndChanges(os, bparams, runparams,
2224                                         basefont,
2225                                         next_->getFont(bparams, 0, outerfont));
2226                 } else {
2227                         running_font.latexWriteEndChanges(os, bparams,
2228                                         runparams, basefont, basefont);
2229                 }
2230 #else
2231 //FIXME: For now we ALWAYS have to close the foreign font settings if they are
2232 //FIXME: there as we start another \selectlanguage with the next paragraph if
2233 //FIXME: we are in need of this. This should be fixed sometime (Jug)
2234                 running_font.latexWriteEndChanges(os, bparams, runparams,
2235                                 basefont, basefont);
2236 #endif
2237         }
2238
2239         column += Changes::latexMarkChange(os, bparams, runningChange, Change(Change::UNCHANGED));
2240
2241         // Needed if there is an optional argument but no contents.
2242         if (body_pos > 0 && body_pos == size()) {
2243                 os << "}]~";
2244                 return_value = false;
2245         }
2246
2247         if (!asdefault) {
2248                 column += endTeXParParams(bparams, os, texrow,
2249                                           runparams.moving_arg);
2250         }
2251
2252         LYXERR(Debug::LATEX) << "SimpleTeXOnePar...done " << this << endl;
2253         return return_value;
2254 }
2255
2256
2257 namespace {
2258
2259 enum PAR_TAG {
2260         PAR_NONE=0,
2261         TT = 1,
2262         SF = 2,
2263         BF = 4,
2264         IT = 8,
2265         SL = 16,
2266         EM = 32
2267 };
2268
2269
2270 string tag_name(PAR_TAG const & pt) {
2271         switch (pt) {
2272         case PAR_NONE: return "!-- --";
2273         case TT: return "tt";
2274         case SF: return "sf";
2275         case BF: return "bf";
2276         case IT: return "it";
2277         case SL: return "sl";
2278         case EM: return "em";
2279         }
2280         return "";
2281 }
2282
2283
2284 inline
2285 void operator|=(PAR_TAG & p1, PAR_TAG const & p2)
2286 {
2287         p1 = static_cast<PAR_TAG>(p1 | p2);
2288 }
2289
2290
2291 inline
2292 void reset(PAR_TAG & p1, PAR_TAG const & p2)
2293 {
2294         p1 = static_cast<PAR_TAG>(p1 & ~p2);
2295 }
2296
2297 } // anon
2298
2299
2300 bool Paragraph::emptyTag() const
2301 {
2302         for (pos_type i = 0; i < size(); ++i) {
2303                 if (isInset(i)) {
2304                         Inset const * inset = getInset(i);
2305                         Inset::Code lyx_code = inset->lyxCode();
2306                         if (lyx_code != Inset::TOC_CODE &&
2307                             lyx_code != Inset::INCLUDE_CODE &&
2308                             lyx_code != Inset::GRAPHICS_CODE &&
2309                             lyx_code != Inset::ERT_CODE &&
2310                             lyx_code != Inset::LISTINGS_CODE &&
2311                             lyx_code != Inset::FLOAT_CODE &&
2312                             lyx_code != Inset::TABULAR_CODE) {
2313                                 return false;
2314                         }
2315                 } else {
2316                         value_type c = getChar(i);
2317                         if (c != ' ' && c != '\t')
2318                                 return false;
2319                 }
2320         }
2321         return true;
2322 }
2323
2324
2325 string Paragraph::getID(Buffer const & buf, OutputParams const & runparams) const
2326 {
2327         for (pos_type i = 0; i < size(); ++i) {
2328                 if (isInset(i)) {
2329                         Inset const * inset = getInset(i);
2330                         Inset::Code lyx_code = inset->lyxCode();
2331                         if (lyx_code == Inset::LABEL_CODE) {
2332                                 string const id = static_cast<InsetCommand const *>(inset)->getContents();
2333                                 return "id='" + to_utf8(sgml::cleanID(buf, runparams, from_utf8(id))) + "'";
2334                         }
2335                 }
2336
2337         }
2338         return string();
2339 }
2340
2341
2342 pos_type Paragraph::getFirstWord(Buffer const & buf, odocstream & os, OutputParams const & runparams) const
2343 {
2344         pos_type i;
2345         for (i = 0; i < size(); ++i) {
2346                 if (isInset(i)) {
2347                         Inset const * inset = getInset(i);
2348                         inset->docbook(buf, os, runparams);
2349                 } else {
2350                         value_type c = getChar(i);
2351                         if (c == ' ')
2352                                 break;
2353                         os << sgml::escapeChar(c);
2354                 }
2355         }
2356         return i;
2357 }
2358
2359
2360 bool Paragraph::onlyText(Buffer const & buf, Font const & outerfont, pos_type initial) const
2361 {
2362         Font font_old;
2363
2364         for (pos_type i = initial; i < size(); ++i) {
2365                 Font font = getFont(buf.params(), i, outerfont);
2366                 if (isInset(i))
2367                         return false;
2368                 if (i != initial && font != font_old)
2369                         return false;
2370                 font_old = font;
2371         }
2372
2373         return true;
2374 }
2375
2376
2377 void Paragraph::simpleDocBookOnePar(Buffer const & buf,
2378                                     odocstream & os,
2379                                     OutputParams const & runparams,
2380                                     Font const & outerfont,
2381                                     pos_type initial) const
2382 {
2383         bool emph_flag = false;
2384
2385         LayoutPtr const & style = layout();
2386         Font font_old =
2387                 style->labeltype == LABEL_MANUAL ? style->labelfont : style->font;
2388
2389         if (style->pass_thru && !onlyText(buf, outerfont, initial))
2390                 os << "]]>";
2391
2392         // parsing main loop
2393         for (pos_type i = initial; i < size(); ++i) {
2394                 Font font = getFont(buf.params(), i, outerfont);
2395
2396                 // handle <emphasis> tag
2397                 if (font_old.emph() != font.emph()) {
2398                         if (font.emph() == Font::ON) {
2399                                 os << "<emphasis>";
2400                                 emph_flag = true;
2401                         } else if (i != initial) {
2402                                 os << "</emphasis>";
2403                                 emph_flag = false;
2404                         }
2405                 }
2406
2407                 if (isInset(i)) {
2408                         Inset const * inset = getInset(i);
2409                         inset->docbook(buf, os, runparams);
2410                 } else {
2411                         value_type c = getChar(i);
2412
2413                         if (style->pass_thru)
2414                                 os.put(c);
2415                         else
2416                                 os << sgml::escapeChar(c);
2417                 }
2418                 font_old = font;
2419         }
2420
2421         if (emph_flag) {
2422                 os << "</emphasis>";
2423         }
2424
2425         if (style->free_spacing)
2426                 os << '\n';
2427         if (style->pass_thru && !onlyText(buf, outerfont, initial))
2428                 os << "<![CDATA[";
2429 }
2430
2431
2432 bool Paragraph::isNewline(pos_type pos) const
2433 {
2434         return isInset(pos)
2435                 && getInset(pos)->lyxCode() == Inset::NEWLINE_CODE;
2436 }
2437
2438
2439 bool Paragraph::isLineSeparator(pos_type pos) const
2440 {
2441         value_type const c = getChar(pos);
2442         return isLineSeparatorChar(c)
2443                 || (c == Paragraph::META_INSET && getInset(pos) &&
2444                 getInset(pos)->isLineSeparator());
2445 }
2446
2447
2448 /// Used by the spellchecker
2449 bool Paragraph::isLetter(pos_type pos) const
2450 {
2451         if (isInset(pos))
2452                 return getInset(pos)->isLetter();
2453         else {
2454                 value_type const c = getChar(pos);
2455                 return isLetterChar(c) || isDigit(c);
2456         }
2457 }
2458
2459
2460 Language const *
2461 Paragraph::getParLanguage(BufferParams const & bparams) const
2462 {
2463         if (!empty())
2464                 return getFirstFontSettings(bparams).language();
2465         // FIXME: we should check the prev par as well (Lgb)
2466         return bparams.language;
2467 }
2468
2469
2470 bool Paragraph::isRTL(BufferParams const & bparams) const
2471 {
2472         return lyxrc.rtl_support
2473                 && getParLanguage(bparams)->rightToLeft()
2474                 && ownerCode() != Inset::ERT_CODE
2475                 && ownerCode() != Inset::LISTINGS_CODE;
2476 }
2477
2478
2479 void Paragraph::changeLanguage(BufferParams const & bparams,
2480                                Language const * from, Language const * to)
2481 {
2482         // change language including dummy font change at the end
2483         for (pos_type i = 0; i <= size(); ++i) {
2484                 Font font = getFontSettings(bparams, i);
2485                 if (font.language() == from) {
2486                         font.setLanguage(to);
2487                         setFont(i, font);
2488                 }
2489         }
2490 }
2491
2492
2493 bool Paragraph::isMultiLingual(BufferParams const & bparams) const
2494 {
2495         Language const * doc_language = bparams.language;
2496         Pimpl::FontList::const_iterator cit = pimpl_->fontlist.begin();
2497         Pimpl::FontList::const_iterator end = pimpl_->fontlist.end();
2498
2499         for (; cit != end; ++cit)
2500                 if (cit->font().language() != ignore_language &&
2501                     cit->font().language() != latex_language &&
2502                     cit->font().language() != doc_language)
2503                         return true;
2504         return false;
2505 }
2506
2507
2508 // Convert the paragraph to a string.
2509 // Used for building the table of contents
2510 docstring const Paragraph::asString(Buffer const & buffer, bool label) const
2511 {
2512         return asString(buffer, 0, size(), label);
2513 }
2514
2515
2516 docstring const Paragraph::asString(Buffer const & buffer,
2517                                  pos_type beg, pos_type end, bool label) const
2518 {
2519
2520         odocstringstream os;
2521
2522         if (beg == 0 && label && !params().labelString().empty())
2523                 os << params().labelString() << ' ';
2524
2525         for (pos_type i = beg; i < end; ++i) {
2526                 value_type const c = getChar(i);
2527                 if (isPrintable(c))
2528                         os.put(c);
2529                 else if (c == META_INSET)
2530                         getInset(i)->textString(buffer, os);
2531         }
2532
2533         return os.str();
2534 }
2535
2536
2537 void Paragraph::setInsetOwner(Inset * inset)
2538 {
2539         pimpl_->inset_owner = inset;
2540 }
2541
2542
2543 Change const & Paragraph::lookupChange(pos_type pos) const
2544 {
2545         BOOST_ASSERT(pos <= size());
2546         return pimpl_->lookupChange(pos);
2547 }
2548
2549
2550 bool Paragraph::isChanged(pos_type start, pos_type end) const
2551 {
2552         return pimpl_->isChanged(start, end);
2553 }
2554
2555
2556 bool Paragraph::isMergedOnEndOfParDeletion(bool trackChanges) const
2557 {
2558         return pimpl_->isMergedOnEndOfParDeletion(trackChanges);
2559 }
2560
2561
2562 void Paragraph::setChange(Change const & change)
2563 {
2564         pimpl_->setChange(change);
2565 }
2566
2567
2568 void Paragraph::setChange(pos_type pos, Change const & change)
2569 {
2570         pimpl_->setChange(pos, change);
2571 }
2572
2573
2574 void Paragraph::acceptChanges(BufferParams const & bparams, pos_type start, pos_type end)
2575 {
2576         return pimpl_->acceptChanges(bparams, start, end);
2577 }
2578
2579
2580 void Paragraph::rejectChanges(BufferParams const & bparams, pos_type start, pos_type end)
2581 {
2582         return pimpl_->rejectChanges(bparams, start, end);
2583 }
2584
2585
2586 int Paragraph::id() const
2587 {
2588         return pimpl_->id_;
2589 }
2590
2591
2592 LayoutPtr const & Paragraph::layout() const
2593 {
2594         return layout_;
2595 }
2596
2597
2598 void Paragraph::layout(LayoutPtr const & new_layout)
2599 {
2600         layout_ = new_layout;
2601 }
2602
2603
2604 Inset * Paragraph::inInset() const
2605 {
2606         return pimpl_->inset_owner;
2607 }
2608
2609
2610 Inset::Code Paragraph::ownerCode() const
2611 {
2612         return pimpl_->inset_owner
2613                 ? pimpl_->inset_owner->lyxCode() : Inset::NO_CODE;
2614 }
2615
2616
2617 ParagraphParameters & Paragraph::params()
2618 {
2619         return pimpl_->params;
2620 }
2621
2622
2623 ParagraphParameters const & Paragraph::params() const
2624 {
2625         return pimpl_->params;
2626 }
2627
2628
2629 bool Paragraph::isFreeSpacing() const
2630 {
2631         if (layout()->free_spacing)
2632                 return true;
2633
2634         // for now we just need this, later should we need this in some
2635         // other way we can always add a function to Inset too.
2636         return ownerCode() == Inset::ERT_CODE || ownerCode() == Inset::LISTINGS_CODE;
2637 }
2638
2639
2640 bool Paragraph::allowEmpty() const
2641 {
2642         if (layout()->keepempty)
2643                 return true;
2644         return ownerCode() == Inset::ERT_CODE || ownerCode() == Inset::LISTINGS_CODE;
2645 }
2646
2647
2648 char_type Paragraph::transformChar(char_type c, pos_type pos) const
2649 {
2650         if (!Encodings::is_arabic(c))
2651                 return c;
2652
2653         value_type prev_char = ' ';
2654         value_type next_char = ' ';
2655
2656         for (pos_type i = pos - 1; i >= 0; --i) {
2657                 value_type const par_char = getChar(i);
2658                 if (!Encodings::isComposeChar_arabic(par_char)) {
2659                         prev_char = par_char;
2660                         break;
2661                 }
2662         }
2663
2664         for (pos_type i = pos + 1, end = size(); i < end; ++i) {
2665                 value_type const par_char = getChar(i);
2666                 if (!Encodings::isComposeChar_arabic(par_char)) {
2667                         next_char = par_char;
2668                         break;
2669                 }
2670         }
2671
2672         if (Encodings::is_arabic(next_char)) {
2673                 if (Encodings::is_arabic(prev_char) &&
2674                         !Encodings::is_arabic_special(prev_char))
2675                         return Encodings::transformChar(c, Encodings::FORM_MEDIAL);
2676                 else
2677                         return Encodings::transformChar(c, Encodings::FORM_INITIAL);
2678         } else {
2679                 if (Encodings::is_arabic(prev_char) &&
2680                         !Encodings::is_arabic_special(prev_char))
2681                         return Encodings::transformChar(c, Encodings::FORM_FINAL);
2682                 else
2683                         return Encodings::transformChar(c, Encodings::FORM_ISOLATED);
2684         }
2685 }
2686
2687
2688 int Paragraph::checkBiblio(bool track_changes)
2689 {
2690         //FIXME From JS:
2691         //This is getting more and more a mess. ...We really should clean
2692         //up this bibitem issue for 1.6. See also bug 2743.
2693
2694         // Add bibitem insets if necessary
2695         if (layout()->labeltype != LABEL_BIBLIO)
2696                 return 0;
2697
2698         bool hasbibitem = !insetlist.empty()
2699                 // Insist on it being in pos 0
2700                 && getChar(0) == Paragraph::META_INSET
2701                 && insetlist.begin()->inset->lyxCode() == Inset::BIBITEM_CODE;
2702
2703         docstring oldkey;
2704         docstring oldlabel;
2705
2706         // remove a bibitem in pos != 0
2707         // restore it later in pos 0 if necessary
2708         // (e.g. if a user inserts contents _before_ the item)
2709         // we're assuming there's only one of these, which there
2710         // should be.
2711         int erasedInsetPosition = -1;
2712         InsetList::iterator it = insetlist.begin();
2713         InsetList::iterator end = insetlist.end();
2714         for (; it != end; ++it)
2715                 if (it->inset->lyxCode() == Inset::BIBITEM_CODE
2716                     && it->pos > 0) {
2717                         InsetBibitem * olditem = static_cast<InsetBibitem *>(it->inset);
2718                         oldkey = olditem->getParam("key");
2719                         oldlabel = olditem->getParam("label");
2720                         erasedInsetPosition = it->pos;
2721                         eraseChar(erasedInsetPosition, track_changes);
2722                         break;
2723         }
2724
2725         //There was an InsetBibitem at the beginning, and we didn't
2726         //have to erase one.
2727         if (hasbibitem && erasedInsetPosition < 0)
2728                         return 0;
2729
2730         //There was an InsetBibitem at the beginning and we did have to
2731         //erase one. So we give its properties to the beginning inset.
2732         if (hasbibitem) {
2733                 InsetBibitem * inset =
2734                         static_cast<InsetBibitem *>(insetlist.begin()->inset);
2735                 if (!oldkey.empty())
2736                         inset->setParam("key", oldkey);
2737                 inset->setParam("label", oldlabel);
2738                 return -erasedInsetPosition;
2739         }
2740
2741         //There was no inset at the beginning, so we need to create one with
2742         //the key and label of the one we erased.
2743         InsetBibitem * inset(new InsetBibitem(InsetCommandParams("bibitem")));
2744         // restore values of previously deleted item in this par.
2745         if (!oldkey.empty())
2746                 inset->setParam("key", oldkey);
2747         inset->setParam("label", oldlabel);
2748         insertInset(0, static_cast<Inset *>(inset),
2749                     Change(track_changes ? Change::INSERTED : Change::UNCHANGED));
2750
2751         return 1;
2752 }
2753
2754
2755 void Paragraph::checkAuthors(AuthorList const & authorList)
2756 {
2757         pimpl_->changes_.checkAuthors(authorList);
2758 }
2759
2760 } // namespace lyx