src/Paragraph.cpp

   1 /**
   2  * \file Paragraph.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Asger Alstrup
   7  * \author Lars Gullik Bjønnes
   8  * \author Jean-Marc Lasgouttes
   9  * \author Angus Leeming
  10  * \author John Levon
  11  * \author André Pönitz
  12  * \author Dekel Tsur
  13  * \author Jürgen Vigna
  14  *
  15  * Full author contact details are available in file CREDITS.
  16  */
  17
  18 #include <config.h>
  19
  20 #include "Paragraph.h"
  21
  22 #include "Buffer.h"
  23 #include "BufferParams.h"
  24 #include "Changes.h"
  25 #include "Counters.h"
  26 #include "Encoding.h"
  27 #include "debug.h"
  28 #include "gettext.h"
  29 #include "InsetList.h"
  30 #include "Language.h"
  31 #include "LaTeXFeatures.h"
  32 #include "Color.h"
  33 #include "Layout.h"
  34 #include "Length.h"
  35 #include "Font.h"
  36 #include "LyXRC.h"
  37 #include "Messages.h"
  38 #include "OutputParams.h"
  39 #include "output_latex.h"
  40 #include "paragraph_funcs.h"
  41 #include "ParagraphParameters.h"
  42 #include "sgml.h"
  43 #include "TexRow.h"
  44 #include "VSpace.h"
  45
  46 #include "frontends/alert.h"
  47 #include "frontends/FontMetrics.h"
  48
  49 #include "insets/InsetBibitem.h"
  50 #include "insets/InsetOptArg.h"
  51
  52 #include "support/lstrings.h"
  53 #include "support/textutils.h"
  54 #include "support/convert.h"
  55 #include "support/unicode.h"
  56
  57 #include <boost/bind.hpp>
  58 #include <boost/next_prior.hpp>
  59
  60 #include <algorithm>
  61 #include <sstream>
  62
  63 using std::distance;
  64 using std::endl;
  65 using std::string;
  66 using std::ostream;
  67
  68 namespace lyx {
  69
  70 using support::contains;
  71 using support::prefixIs;
  72 using support::suffixIs;
  73 using support::rsplit;
  74 using support::rtrim;
  75
  76 namespace {
  77 /** A font entry covers a range of positions. Notice that the
  78     entries in the list are inserted in random order.
  79     I don't think it's worth the effort to implement a more effective
  80     datastructure, because the number of different fonts in a paragraph
  81     is limited. (Asger)
  82     Nevertheless, I decided to store fontlist_ using a sorted vector:
  83     fontlist_ = { {pos_1,font_1} , {pos_2,font_2} , ... } where
  84     pos_1 < pos_2 < ..., font_{i-1} != font_i for all i,
  85     and font_i covers the chars in positions pos_{i-1}+1,...,pos_i
  86     (font_1 covers the chars 0,...,pos_1) (Dekel)
  87 */
  88 class FontTable
  89 {
  90 public:
  91         ///
  92         FontTable(pos_type p, Font const & f)
  93                 : pos_(p), font_(f)
  94         {}
  95         ///
  96         pos_type pos() const { return pos_; }
  97         ///
  98         void pos(pos_type p) { pos_ = p; }
  99         ///
 100         Font const & font() const { return font_; }
 101         ///
 102         void font(Font const & f) { font_ = f;}
 103
 104 private:
 105         /// End position of paragraph this font attribute covers
 106         pos_type pos_;
 107         /** Font. Interpretation of the font values:
 108         If a value is Font::INHERIT_*, it means that the font
 109         attribute is inherited from either the layout of this
 110         paragraph or, in the case of nested paragraphs, from the
 111         layout in the environment one level up until completely
 112         resolved.
 113         The values Font::IGNORE_* and Font::TOGGLE are NOT
 114         allowed in these font tables.
 115         */
 116         Font font_;
 117 };
 118
 119
 120 class matchFT
 121 {
 122 public:
 123         /// used by lower_bound and upper_bound
 124         int operator()(FontTable const & a, FontTable const & b) const {
 125                 return a.pos() < b.pos();
 126         }
 127 };
 128
 129
 130 typedef std::vector<FontTable> FontList;
 131
 132 }
 133
 134 /////////////////////////////////////////////////////////////////////
 135 //
 136 // Paragraph::Private
 137 //
 138 /////////////////////////////////////////////////////////////////////
 139
 140 class Encoding;
 141 class Layout;
 142
 143
 144 class Paragraph::Private
 145 {
 146 public:
 147         ///
 148         Private(Paragraph * owner);
 149         /// "Copy constructor"
 150         Private(Private const &, Paragraph * owner);
 151
 152         ///
 153         value_type getChar(pos_type pos) const;
 154         ///
 155         void insertChar(pos_type pos, value_type c, Change const & change);
 156
 157         /// Output the surrogate pair formed by \p c and \p next to \p os.
 158         /// \return the number of characters written.
 159         int latexSurrogatePair(odocstream & os, value_type c, value_type next,
 160                                Encoding const &);
 161
 162         /// Output a space in appropriate formatting (or a surrogate pair
 163         /// if the next character is a combining character).
 164         /// \return whether a surrogate pair was output.
 165         bool simpleTeXBlanks(Encoding const &,
 166                              odocstream &, TexRow & texrow,
 167                              pos_type & i,
 168                              unsigned int & column,
 169                              Font const & font,
 170                              Layout const & style);
 171
 172         /// Output consecutive known unicode chars, belonging to the same
 173         /// language as specified by \p preamble, to \p os starting from \p c.
 174         /// \return the number of characters written.
 175         int knownLangChars(odocstream & os, value_type c, string & preamble,
 176                            Change &, Encoding const &, pos_type &);
 177         ///
 178         void simpleTeXSpecialChars(Buffer const &, BufferParams const &,
 179                                    odocstream &,
 180                                    TexRow & texrow, OutputParams &,
 181                                    Font & running_font,
 182                                    Font & basefont,
 183                                    Font const & outerfont,
 184                                    bool & open_font,
 185                                    Change & running_change,
 186                                    Layout const & style,
 187                                    pos_type & i,
 188                                    unsigned int & column, value_type const c);
 189
 190         ///
 191         void validate(LaTeXFeatures & features,
 192                       Layout const & layout) const;
 193
 194         ///
 195         pos_type size() const { return owner_->size(); }
 196
 197         /// match a string against a particular point in the paragraph
 198         bool isTextAt(std::string const & str, pos_type pos) const;
 199
 200         /// Which Paragraph owns us?
 201         Paragraph * owner_;
 202
 203         /// In which Inset?
 204         Inset * inset_owner_;
 205
 206         ///
 207         FontList fontlist_;
 208
 209         ///
 210         unsigned int id_;
 211         ///
 212         static unsigned int paragraph_id;
 213         ///
 214         ParagraphParameters params_;
 215
 216         /// for recording and looking up changes
 217         Changes changes_;
 218
 219         ///
 220         InsetList insetlist_;
 221 };
 222
 223
 224
 225
 226 using std::endl;
 227 using std::upper_bound;
 228 using std::lower_bound;
 229 using std::string;
 230
 231
 232 // Initialization of the counter for the paragraph id's,
 233 unsigned int Paragraph::Private::paragraph_id = 0;
 234
 235 namespace {
 236
 237 struct special_phrase {
 238         string phrase;
 239         docstring macro;
 240         bool builtin;
 241 };
 242
 243 special_phrase const special_phrases[] = {
 244         { "LyX", from_ascii("\\LyX{}"), false },
 245         { "TeX", from_ascii("\\TeX{}"), true },
 246         { "LaTeX2e", from_ascii("\\LaTeXe{}"), true },
 247         { "LaTeX", from_ascii("\\LaTeX{}"), true },
 248 };
 249
 250 size_t const phrases_nr = sizeof(special_phrases)/sizeof(special_phrase);
 251
 252 } // namespace anon
 253
 254
 255 Paragraph::Private::Private(Paragraph * owner)
 256         : owner_(owner)
 257 {
 258         inset_owner_ = 0;
 259         id_ = paragraph_id++;
 260 }
 261
 262
 263 Paragraph::Private::Private(Private const & p, Paragraph * owner)
 264         : owner_(owner), inset_owner_(p.inset_owner_), fontlist_(p.fontlist_),
 265           params_(p.params_), changes_(p.changes_), insetlist_(p.insetlist_)
 266 {
 267         id_ = paragraph_id++;
 268 }
 269
 270
 271 bool Paragraph::isChanged(pos_type start, pos_type end) const
 272 {
 273         BOOST_ASSERT(start >= 0 && start <= size());
 274         BOOST_ASSERT(end > start && end <= size() + 1);
 275
 276         return d->changes_.isChanged(start, end);
 277 }
 278
 279
 280 bool Paragraph::isMergedOnEndOfParDeletion(bool trackChanges) const {
 281         // keep the logic here in sync with the logic of eraseChars()
 282
 283         if (!trackChanges) {
 284                 return true;
 285         }
 286
 287         Change change = d->changes_.lookup(size());
 288
 289         return change.type == Change::INSERTED && change.author == 0;
 290 }
 291
 292
 293 void Paragraph::setChange(Change const & change)
 294 {
 295         // beware of the imaginary end-of-par character!
 296         d->changes_.set(change, 0, size() + 1);
 297
 298         /*
 299          * Propagate the change recursively - but not in case of DELETED!
 300          *
 301          * Imagine that your co-author makes changes in an existing inset. He
 302          * sends your document to you and you come to the conclusion that the
 303          * inset should go completely. If you erase it, LyX must not delete all
 304          * text within the inset. Otherwise, the change tracked insertions of
 305          * your co-author get lost and there is no way to restore them later.
 306          *
 307          * Conclusion: An inset's content should remain untouched if you delete it
 308          */
 309
 310         if (change.type != Change::DELETED) {
 311                 for (pos_type pos = 0; pos < size(); ++pos) {
 312                         if (isInset(pos))
 313                                 getInset(pos)->setChange(change);
 314                 }
 315         }
 316 }
 317
 318
 319 void Paragraph::setChange(pos_type pos, Change const & change)
 320 {
 321         BOOST_ASSERT(pos >= 0 && pos <= size());
 322
 323         d->changes_.set(change, pos);
 324
 325         // see comment in setChange(Change const &) above
 326
 327         if (change.type != Change::DELETED &&
 328             pos < size() && isInset(pos)) {
 329                 getInset(pos)->setChange(change);
 330         }
 331 }
 332
 333
 334 Change const & Paragraph::lookupChange(pos_type pos) const
 335 {
 336         BOOST_ASSERT(pos >= 0 && pos <= size());
 337
 338         return d->changes_.lookup(pos);
 339 }
 340
 341
 342 void Paragraph::acceptChanges(BufferParams const & bparams, pos_type start,
 343                 pos_type end)
 344 {
 345         BOOST_ASSERT(start >= 0 && start <= size());
 346         BOOST_ASSERT(end > start && end <= size() + 1);
 347
 348         for (pos_type pos = start; pos < end; ++pos) {
 349                 switch (lookupChange(pos).type) {
 350                         case Change::UNCHANGED:
 351                                 // accept changes in nested inset
 352                                 if (pos < size() && isInset(pos))
 353                                         getInset(pos)->acceptChanges(bparams);
 354
 355                                 break;
 356
 357                         case Change::INSERTED:
 358                                 d->changes_.set(Change(Change::UNCHANGED), pos);
 359                                 // also accept changes in nested inset
 360                                 if (pos < size() && isInset(pos)) {
 361                                         getInset(pos)->acceptChanges(bparams);
 362                                 }
 363                                 break;
 364
 365                         case Change::DELETED:
 366                                 // Suppress access to non-existent
 367                                 // "end-of-paragraph char"
 368                                 if (pos < size()) {
 369                                         eraseChar(pos, false);
 370                                         --end;
 371                                         --pos;
 372                                 }
 373                                 break;
 374                 }
 375
 376         }
 377 }
 378
 379
 380 void Paragraph::rejectChanges(BufferParams const & bparams,
 381                 pos_type start, pos_type end)
 382 {
 383         BOOST_ASSERT(start >= 0 && start <= size());
 384         BOOST_ASSERT(end > start && end <= size() + 1);
 385
 386         for (pos_type pos = start; pos < end; ++pos) {
 387                 switch (lookupChange(pos).type) {
 388                         case Change::UNCHANGED:
 389                                 // reject changes in nested inset
 390                                 if (pos < size() && isInset(pos)) {
 391                                         getInset(pos)->rejectChanges(bparams);
 392                                 }
 393                                 break;
 394
 395                         case Change::INSERTED:
 396                                 // Suppress access to non-existent
 397                                 // "end-of-paragraph char"
 398                                 if (pos < size()) {
 399                                         eraseChar(pos, false);
 400                                         --end;
 401                                         --pos;
 402                                 }
 403                                 break;
 404
 405                         case Change::DELETED:
 406                                 d->changes_.set(Change(Change::UNCHANGED), pos);
 407
 408                                 // Do NOT reject changes within a deleted inset!
 409                                 // There may be insertions of a co-author inside of it!
 410
 411                                 break;
 412                 }
 413         }
 414 }
 415
 416
 417 Paragraph::value_type Paragraph::Private::getChar(pos_type pos) const
 418 {
 419         BOOST_ASSERT(pos >= 0 && pos <= size());
 420
 421         return owner_->getChar(pos);
 422 }
 423
 424
 425 void Paragraph::Private::insertChar(pos_type pos, value_type c,
 426                 Change const & change)
 427 {
 428         BOOST_ASSERT(pos >= 0 && pos <= size());
 429
 430         // track change
 431         changes_.insert(change, pos);
 432
 433         // This is actually very common when parsing buffers (and
 434         // maybe inserting ascii text)
 435         if (pos == size()) {
 436                 // when appending characters, no need to update tables
 437                 owner_->text_.push_back(c);
 438                 return;
 439         }
 440
 441         owner_->text_.insert(owner_->text_.begin() + pos, c);
 442
 443         // Update the font table.
 444         FontTable search_font(pos, Font());
 445         for (FontList::iterator it
 446               = lower_bound(fontlist_.begin(), fontlist_.end(), search_font, matchFT());
 447              it != fontlist_.end(); ++it)
 448         {
 449                 it->pos(it->pos() + 1);
 450         }
 451
 452         // Update the insets
 453         insetlist_.increasePosAfterPos(pos);
 454 }
 455
 456
 457 void Paragraph::insertInset(pos_type pos, Inset * inset,
 458                                    Change const & change)
 459 {
 460         BOOST_ASSERT(inset);
 461         BOOST_ASSERT(pos >= 0 && pos <= size());
 462
 463         d->insertChar(pos, META_INSET, change);
 464         BOOST_ASSERT(d->owner_->text_[pos] == META_INSET);
 465
 466         // Add a new entry in the insetlist_.
 467         d->insetlist_.insert(inset, pos);
 468 }
 469
 470
 471 bool Paragraph::eraseChar(pos_type pos, bool trackChanges)
 472 {
 473         BOOST_ASSERT(pos >= 0 && pos <= size());
 474
 475         // keep the logic here in sync with the logic of isMergedOnEndOfParDeletion()
 476
 477         if (trackChanges) {
 478                 Change change = d->changes_.lookup(pos);
 479
 480                 // set the character to DELETED if
 481                 //  a) it was previously unchanged or
 482                 //  b) it was inserted by a co-author
 483
 484                 if (change.type == Change::UNCHANGED ||
 485                     (change.type == Change::INSERTED && change.author != 0)) {
 486                         setChange(pos, Change(Change::DELETED));
 487                         return false;
 488                 }
 489
 490                 if (change.type == Change::DELETED)
 491                         return false;
 492         }
 493
 494         // Don't physically access the imaginary end-of-paragraph character.
 495         // eraseChar() can only mark it as DELETED. A physical deletion of
 496         // end-of-par must be handled externally.
 497         if (pos == size()) {
 498                 return false;
 499         }
 500
 501         // track change
 502         d->changes_.erase(pos);
 503
 504         // if it is an inset, delete the inset entry
 505         if (text_[pos] == Paragraph::META_INSET)
 506                 d->insetlist_.erase(pos);
 507
 508         text_.erase(text_.begin() + pos);
 509
 510         // Erase entries in the tables.
 511         FontTable search_font(pos, Font());
 512
 513         FontList::iterator it =
 514                 lower_bound(d->fontlist_.begin(),
 515                             d->fontlist_.end(),
 516                             search_font, matchFT());
 517         FontList::iterator begin = d->fontlist_.begin();
 518         if (it != d->fontlist_.end() && it->pos() == pos &&
 519             (pos == 0 ||
 520              (it != begin
 521               && boost::prior(it)->pos() == pos - 1))) {
 522                 // If it is a multi-character font
 523                 // entry, we just make it smaller
 524                 // (see update below), otherwise we
 525                 // should delete it.
 526                 unsigned int const i = it - begin;
 527                 d->fontlist_.erase(begin + i);
 528                 it = begin + i;
 529                 if (i > 0 && i < d->fontlist_.size() &&
 530                     d->fontlist_[i - 1].font() == d->fontlist_[i].font()) {
 531                         d->fontlist_.erase(begin + i - 1);
 532                         it = begin + i - 1;
 533                 }
 534         }
 535
 536         // Update all other entries
 537         FontList::iterator fend = d->fontlist_.end();
 538         for (; it != fend; ++it)
 539                 it->pos(it->pos() - 1);
 540
 541         // Update the insetlist_
 542         d->insetlist_.decreasePosAfterPos(pos);
 543
 544         return true;
 545 }
 546
 547
 548 int Paragraph::eraseChars(pos_type start, pos_type end, bool trackChanges)
 549 {
 550         BOOST_ASSERT(start >= 0 && start <= size());
 551         BOOST_ASSERT(end >= start && end <= size() + 1);
 552
 553         pos_type i = start;
 554         for (pos_type count = end - start; count; --count) {
 555                 if (!eraseChar(i, trackChanges))
 556                         ++i;
 557         }
 558         return end - i;
 559 }
 560
 561
 562 int Paragraph::Private::latexSurrogatePair(odocstream & os, value_type c,
 563                 value_type next, Encoding const & encoding)
 564 {
 565         // Writing next here may circumvent a possible font change between
 566         // c and next. Since next is only output if it forms a surrogate pair
 567         // with c we can ignore this:
 568         // A font change inside a surrogate pair does not make sense and is
 569         // hopefully impossible to input.
 570         // FIXME: change tracking
 571         // Is this correct WRT change tracking?
 572         docstring const latex1 = encoding.latexChar(next);
 573         docstring const latex2 = encoding.latexChar(c);
 574         os << latex1 << '{' << latex2 << '}';
 575         return latex1.length() + latex2.length() + 2;
 576 }
 577
 578
 579 bool Paragraph::Private::simpleTeXBlanks(Encoding const & encoding,
 580                                        odocstream & os, TexRow & texrow,
 581                                        pos_type & i,
 582                                        unsigned int & column,
 583                                        Font const & font,
 584                                        Layout const & style)
 585 {
 586         if (style.pass_thru)
 587                 return false;
 588
 589         if (i + 1 < size()) {
 590                 char_type next = getChar(i + 1);
 591                 if (Encodings::isCombiningChar(next)) {
 592                         // This space has an accent, so we must always output it.
 593                         column += latexSurrogatePair(os, ' ', next, encoding) - 1;
 594                         ++i;
 595                         return true;
 596                 }
 597         }
 598
 599         if (lyxrc.plaintext_linelen > 0
 600             && column > lyxrc.plaintext_linelen
 601             && i
 602             && getChar(i - 1) != ' '
 603             && (i + 1 < size())
 604             // same in FreeSpacing mode
 605             && !owner_->isFreeSpacing()
 606             // In typewriter mode, we want to avoid
 607             // ! . ? : at the end of a line
 608             && !(font.family() == Font::TYPEWRITER_FAMILY
 609                  && (getChar(i - 1) == '.'
 610                      || getChar(i - 1) == '?'
 611                      || getChar(i - 1) == ':'
 612                      || getChar(i - 1) == '!'))) {
 613                 os << '\n';
 614                 texrow.newline();
 615                 texrow.start(owner_->id(), i + 1);
 616                 column = 0;
 617         } else if (style.free_spacing) {
 618                 os << '~';
 619         } else {
 620                 os << ' ';
 621         }
 622         return false;
 623 }
 624
 625
 626 int Paragraph::Private::knownLangChars(odocstream & os,
 627                                      value_type c,
 628                                      string & preamble,
 629                                      Change & runningChange,
 630                                      Encoding const & encoding,
 631                                      pos_type & i)
 632 {
 633         // When the character is marked by the proper language, we simply
 634         // get its code point in some encoding, otherwise we get the
 635         // translation specified in the unicodesymbols file, which is
 636         // something like "\textLANG{<spec>}". So, we have to retain
 637         // "\textLANG{<spec>" for the first char but only "<spec>" for
 638         // all subsequent chars.
 639         docstring const latex1 = rtrim(encoding.latexChar(c), "}");
 640         int length = latex1.length();
 641         os << latex1;
 642         while (i + 1 < size()) {
 643                 char_type next = getChar(i + 1);
 644                 // Stop here if next character belongs to another
 645                 // language or there is a change tracking status.
 646                 if (!Encodings::isKnownLangChar(next, preamble) ||
 647                     runningChange != owner_->lookupChange(i + 1))
 648                         break;
 649                 Font prev_font;
 650                 bool found = false;
 651                 FontList::const_iterator cit = fontlist_.begin();
 652                 FontList::const_iterator end = fontlist_.end();
 653                 for (; cit != end; ++cit) {
 654                         if (cit->pos() >= i && !found) {
 655                                 prev_font = cit->font();
 656                                 found = true;
 657                         }
 658                         if (cit->pos() >= i + 1)
 659                                 break;
 660                 }
 661                 // Stop here if there is a font attribute change.
 662                 if (found && cit != end && prev_font != cit->font())
 663                         break;
 664                 docstring const latex = rtrim(encoding.latexChar(next), "}");
 665                 docstring::size_type const j =
 666                                         latex.find_first_of(from_ascii("{"));
 667                 if (j == docstring::npos) {
 668                         os << latex;
 669                         length += latex.length();
 670                 } else {
 671                         os << latex.substr(j + 1);
 672                         length += latex.substr(j + 1).length();
 673                 }
 674                 ++i;
 675         }
 676         // When the proper language is set, we are simply passed a code
 677         // point, so we should not try to close the \textLANG command.
 678         if (prefixIs(latex1, from_ascii("\\" + preamble))) {
 679                 os << '}';
 680                 ++length;
 681         }
 682         return length;
 683 }
 684
 685
 686 bool Paragraph::Private::isTextAt(string const & str, pos_type pos) const
 687 {
 688         pos_type const len = str.length();
 689
 690         // is the paragraph large enough?
 691         if (pos + len > size())
 692                 return false;
 693
 694         // does the wanted text start at point?
 695         for (string::size_type i = 0; i < str.length(); ++i) {
 696                 // Caution: direct comparison of characters works only
 697                 // because str is pure ASCII.
 698                 if (str[i] != owner_->text_[pos + i])
 699                         return false;
 700         }
 701
 702         // is there a font change in middle of the word?
 703         FontList::const_iterator cit = fontlist_.begin();
 704         FontList::const_iterator end = fontlist_.end();
 705         for (; cit != end; ++cit) {
 706                 if (cit->pos() >= pos)
 707                         break;
 708         }
 709         if (cit != end && pos + len - 1 > cit->pos())
 710                 return false;
 711
 712         return true;
 713 }
 714
 715
 716 void Paragraph::Private::simpleTeXSpecialChars(Buffer const & buf,
 717                                              BufferParams const & bparams,
 718                                              odocstream & os,
 719                                              TexRow & texrow,
 720                                              OutputParams & runparams,
 721                                              Font & running_font,
 722                                              Font & basefont,
 723                                              Font const & outerfont,
 724                                              bool & open_font,
 725                                              Change & running_change,
 726                                              Layout const & style,
 727                                              pos_type & i,
 728                                              unsigned int & column,
 729                                              value_type const c)
 730 {
 731         if (style.pass_thru) {
 732                 if (c != Paragraph::META_INSET) {
 733                         if (c != '\0')
 734                                 // FIXME UNICODE: This can fail if c cannot
 735                                 // be encoded in the current encoding.
 736                                 os.put(c);
 737                 } else
 738                         owner_->getInset(i)->plaintext(buf, os, runparams);
 739                 return;
 740         }
 741
 742         // Two major modes:  LaTeX or plain
 743         // Handle here those cases common to both modes
 744         // and then split to handle the two modes separately.
 745         switch (c) {
 746         case Paragraph::META_INSET: {
 747                 Inset * inset = owner_->getInset(i);
 748
 749                 // FIXME: remove this check
 750                 if (!inset)
 751                         break;
 752
 753                 // FIXME: move this to InsetNewline::latex
 754                 if (inset->lyxCode() == NEWLINE_CODE) {
 755                         // newlines are handled differently here than
 756                         // the default in simpleTeXSpecialChars().
 757                         if (!style.newline_allowed) {
 758                                 os << '\n';
 759                         } else {
 760                                 if (open_font) {
 761                                         column += running_font.latexWriteEndChanges(
 762                                                 os, bparams, runparams,
 763                                                 basefont, basefont);
 764                                         open_font = false;
 765                                 }
 766
 767                                 if (running_font.family() == Font::TYPEWRITER_FAMILY)
 768                                         os << '~';
 769
 770                                 basefont = owner_->getLayoutFont(bparams, outerfont);
 771                                 running_font = basefont;
 772
 773                                 if (runparams.moving_arg)
 774                                         os << "\\protect ";
 775
 776                                 os << "\\\\\n";
 777                         }
 778                         texrow.newline();
 779                         texrow.start(owner_->id(), i + 1);
 780                         column = 0;
 781                         break;
 782                 }
 783
 784                 if (owner_->lookupChange(i).type == Change::DELETED) {
 785                         if( ++runparams.inDeletedInset == 1)
 786                                 runparams.changeOfDeletedInset = owner_->lookupChange(i);
 787                 }
 788
 789                 if (inset->canTrackChanges()) {
 790                         column += Changes::latexMarkChange(os, bparams, running_change,
 791                                 Change(Change::UNCHANGED));
 792                         running_change = Change(Change::UNCHANGED);
 793                 }
 794
 795                 bool close = false;
 796                 odocstream::pos_type const len = os.tellp();
 797
 798                 if ((inset->lyxCode() == GRAPHICS_CODE
 799                      || inset->lyxCode() == MATH_CODE
 800                      || inset->lyxCode() == HYPERLINK_CODE)
 801                     && running_font.isRightToLeft()) {
 802                         if (running_font.language()->lang() == "farsi")
 803                                 os << "\\beginL{}";
 804                         else
 805                                 os << "\\L{";
 806                         close = true;
 807                 }
 808
 809 // FIXME: Bug: we can have an empty font change here!
 810 // if there has just been a font change, we are going to close it
 811 // right now, which means stupid latex code like \textsf{}. AFAIK,
 812 // this does not harm dvi output. A minor bug, thus (JMarc)
 813                 // Some insets cannot be inside a font change command.
 814                 // However, even such insets *can* be placed in \L or \R
 815                 // or their equivalents (for RTL language switches), so we don't
 816                 // close the language in those cases.
 817                 // ArabTeX, though, cannot handle this special behavior, it seems.
 818                 bool arabtex = basefont.language()->lang() == "arabic_arabtex" ||
 819                                            running_font.language()->lang() == "arabic_arabtex";
 820                 if (open_font && inset->noFontChange()) {
 821                         bool closeLanguage = arabtex ||
 822                                 basefont.isRightToLeft() == running_font.isRightToLeft();
 823                         unsigned int count = running_font.latexWriteEndChanges(
 824                                         os, bparams, runparams,
 825                                                 basefont, basefont, closeLanguage);
 826                         column += count;
 827                         // if any font properties were closed, update the running_font,
 828                         // making sure, however, to leave the language as it was
 829                         if (count > 0) {
 830                                 // FIXME: probably a better way to keep track of the old
 831                                 // language, than copying the entire font?
 832                                 Font const copy_font(running_font);
 833                                 basefont = owner_->getLayoutFont(bparams, outerfont);
 834                                 running_font = basefont;
 835                                 if (!closeLanguage)
 836                                         running_font.setLanguage(copy_font.language());
 837                                 // leave font open if language is still open
 838                                 open_font = (running_font.language() == basefont.language());
 839                                 if (closeLanguage)
 840                                         runparams.local_font = &basefont;
 841                         }
 842                 }
 843
 844                 int tmp = inset->latex(buf, os, runparams);
 845
 846                 if (close) {
 847                         if (running_font.language()->lang() == "farsi")
 848                                 os << "\\endL{}";
 849                         else
 850                                 os << '}';
 851                 }
 852
 853                 if (tmp) {
 854                         for (int j = 0; j < tmp; ++j) {
 855                                 texrow.newline();
 856                         }
 857                         texrow.start(owner_->id(), i + 1);
 858                         column = 0;
 859                 } else {
 860                         column += os.tellp() - len;
 861                 }
 862
 863                 if (owner_->lookupChange(i).type == Change::DELETED) {
 864                         --runparams.inDeletedInset;
 865                 }
 866         }
 867         break;
 868
 869         default:
 870                 // And now for the special cases within each mode
 871
 872                 switch (c) {
 873                 case '\\':
 874                         os << "\\textbackslash{}";
 875                         column += 15;
 876                         break;
 877
 878                 case '|': case '<': case '>':
 879                         // In T1 encoding, these characters exist
 880                         if (lyxrc.fontenc == "T1") {
 881                                 os.put(c);
 882                                 //... but we should avoid ligatures
 883                                 if ((c == '>' || c == '<')
 884                                     && i <= size() - 2
 885                                     && getChar(i + 1) == c) {
 886                                         //os << "\\textcompwordmark{}";
 887                                         //column += 19;
 888                                         // Jean-Marc, have a look at
 889                                         // this. I think this works
 890                                         // equally well:
 891                                         os << "\\,{}";
 892                                         // Lgb
 893                                         column += 3;
 894                                 }
 895                                 break;
 896                         }
 897                         // Typewriter font also has them
 898                         if (running_font.family() == Font::TYPEWRITER_FAMILY) {
 899                                 os.put(c);
 900                                 break;
 901                         }
 902                         // Otherwise, we use what LaTeX
 903                         // provides us.
 904                         switch (c) {
 905                         case '<':
 906                                 os << "\\textless{}";
 907                                 column += 10;
 908                                 break;
 909                         case '>':
 910                                 os << "\\textgreater{}";
 911                                 column += 13;
 912                                 break;
 913                         case '|':
 914                                 os << "\\textbar{}";
 915                                 column += 9;
 916                                 break;
 917                         }
 918                         break;
 919
 920                 case '-': // "--" in Typewriter mode -> "-{}-"
 921                         if (i <= size() - 2 &&
 922                             getChar(i + 1) == '-' &&
 923                             running_font.family() == Font::TYPEWRITER_FAMILY) {
 924                                 os << "-{}";
 925                                 column += 2;
 926                         } else {
 927                                 os << '-';
 928                         }
 929                         break;
 930
 931                 case '\"':
 932                         os << "\\char`\\\"{}";
 933                         column += 9;
 934                         break;
 935
 936                 case '$': case '&':
 937                 case '%': case '#': case '{':
 938                 case '}': case '_':
 939                         os << '\\';
 940                         os.put(c);
 941                         column += 1;
 942                         break;
 943
 944                 case '~':
 945                         os << "\\textasciitilde{}";
 946                         column += 16;
 947                         break;
 948
 949                 case '^':
 950                         os << "\\textasciicircum{}";
 951                         column += 17;
 952                         break;
 953
 954                 case '*': case '[':
 955                         // avoid being mistaken for optional arguments
 956                         os << '{';
 957                         os.put(c);
 958                         os << '}';
 959                         column += 2;
 960                         break;
 961
 962                 case ' ':
 963                         // Blanks are printed before font switching.
 964                         // Sure? I am not! (try nice-latex)
 965                         // I am sure it's correct. LyX might be smarter
 966                         // in the future, but for now, nothing wrong is
 967                         // written. (Asger)
 968                         break;
 969
 970                 default:
 971
 972                         // I assume this is hack treating typewriter as verbatim
 973                         // FIXME UNICODE: This can fail if c cannot be encoded
 974                         // in the current encoding.
 975                         if (running_font.family() == Font::TYPEWRITER_FAMILY) {
 976                                 if (c != '\0') {
 977                                         os.put(c);
 978                                 }
 979                                 break;
 980                         }
 981
 982                         // LyX, LaTeX etc.
 983
 984                         // FIXME: if we have "LaTeX" with a font
 985                         // change in the middle (before the 'T', then
 986                         // the "TeX" part is still special cased.
 987                         // Really we should only operate this on
 988                         // "words" for some definition of word
 989
 990                         size_t pnr = 0;
 991
 992                         for (; pnr < phrases_nr; ++pnr) {
 993                                 if (isTextAt(special_phrases[pnr].phrase, i)) {
 994                                         os << special_phrases[pnr].macro;
 995                                         i += special_phrases[pnr].phrase.length() - 1;
 996                                         column += special_phrases[pnr].macro.length() - 1;
 997                                         break;
 998                                 }
 999                         }
1000
1001                         if (pnr == phrases_nr && c != '\0') {
1002                                 Encoding const & encoding = *(runparams.encoding);
1003                                 if (i + 1 < size()) {
1004                                         char_type next = getChar(i + 1);
1005                                         if (Encodings::isCombiningChar(next)) {
1006                                                 column += latexSurrogatePair(os, c, next, encoding) - 1;
1007                                                 ++i;
1008                                                 break;
1009                                         }
1010                                 }
1011                                 string preamble;
1012                                 if (Encodings::isKnownLangChar(c, preamble)) {
1013                                         column +=
1014                                                 knownLangChars(os, c, preamble,
1015                                                         running_change,
1016                                                         encoding, i) - 1;
1017                                         break;
1018                                 }
1019                                 docstring const latex = encoding.latexChar(c);
1020                                 if (latex.length() > 1 &&
1021                                     latex[latex.length() - 1] != '}') {
1022                                         // Prevent eating of a following
1023                                         // space or command corruption by
1024                                         // following characters
1025                                         column += latex.length() + 1;
1026                                         os << latex << "{}";
1027                                 } else {
1028                                         column += latex.length() - 1;
1029                                         os << latex;
1030                                 }
1031                         }
1032                         break;
1033                 }
1034         }
1035 }
1036
1037
1038 void Paragraph::Private::validate(LaTeXFeatures & features,
1039                                 Layout const & layout) const
1040 {
1041         BufferParams const & bparams = features.bufferParams();
1042
1043         // check the params.
1044         if (!params_.spacing().isDefault())
1045                 features.require("setspace");
1046
1047         // then the layouts
1048         features.useLayout(layout.name());
1049
1050         // then the fonts
1051         Language const * doc_language = bparams.language;
1052
1053         FontList::const_iterator fcit = fontlist_.begin();
1054         FontList::const_iterator fend = fontlist_.end();
1055         for (; fcit != fend; ++fcit) {
1056                 if (fcit->font().noun() == Font::ON) {
1057                         LYXERR(Debug::LATEX) << "font.noun: "
1058                                              << fcit->font().noun()
1059                                              << endl;
1060                         features.require("noun");
1061                         LYXERR(Debug::LATEX) << "Noun enabled. Font: "
1062                                              << to_utf8(fcit->font().stateText(0))
1063                                              << endl;
1064                 }
1065                 switch (fcit->font().color()) {
1066                 case Color::none:
1067                 case Color::inherit:
1068                 case Color::ignore:
1069                         // probably we should put here all interface colors used for
1070                         // font displaying! For now I just add this ones I know of (Jug)
1071                 case Color::latex:
1072                 case Color::note:
1073                         break;
1074                 default:
1075                         features.require("color");
1076                         LYXERR(Debug::LATEX) << "Color enabled. Font: "
1077                                              << to_utf8(fcit->font().stateText(0))
1078                                              << endl;
1079                 }
1080
1081                 Language const * language = fcit->font().language();
1082                 if (language->babel() != doc_language->babel() &&
1083                     language != ignore_language &&
1084                     language != latex_language)
1085                 {
1086                         features.useLanguage(language);
1087                         LYXERR(Debug::LATEX) << "Found language "
1088                                              << language->lang() << endl;
1089                 }
1090         }
1091
1092         if (!params_.leftIndent().zero())
1093                 features.require("ParagraphLeftIndent");
1094
1095         // then the insets
1096         InsetList::const_iterator icit = insetlist_.begin();
1097         InsetList::const_iterator iend = insetlist_.end();
1098         for (; icit != iend; ++icit) {
1099                 if (icit->inset) {
1100                         icit->inset->validate(features);
1101                         if (layout.needprotect &&
1102                             icit->inset->lyxCode() == FOOT_CODE)
1103                                 features.require("NeedLyXFootnoteCode");
1104                 }
1105         }
1106
1107         // then the contents
1108         for (pos_type i = 0; i < size() ; ++i) {
1109                 for (size_t pnr = 0; pnr < phrases_nr; ++pnr) {
1110                         if (!special_phrases[pnr].builtin
1111                             && isTextAt(special_phrases[pnr].phrase, i)) {
1112                                 features.require(special_phrases[pnr].phrase);
1113                                 break;
1114                         }
1115                 }
1116                 Encodings::validate(getChar(i), features);
1117         }
1118 }
1119
1120
1121 } // namespace lyx
1122
1123
1124 /////////////////////////////////////////////////////////////////////
1125 //
1126 // Paragraph
1127 //
1128 /////////////////////////////////////////////////////////////////////
1129
1130 namespace lyx {
1131
1132 Paragraph::Paragraph()
1133         : begin_of_body_(0), d(new Paragraph::Private(this))
1134 {
1135         itemdepth = 0;
1136         d->params_.clear();
1137 }
1138
1139
1140 Paragraph::Paragraph(Paragraph const & par)
1141         : itemdepth(par.itemdepth),
1142         layout_(par.layout_),
1143         text_(par.text_), begin_of_body_(par.begin_of_body_),
1144         d(new Paragraph::Private(*par.d, this))
1145 {
1146 }
1147
1148
1149 Paragraph & Paragraph::operator=(Paragraph const & par)
1150 {
1151         // needed as we will destroy the private part before copying it
1152         if (&par != this) {
1153                 itemdepth = par.itemdepth;
1154                 layout_ = par.layout();
1155                 text_ = par.text_;
1156                 begin_of_body_ = par.begin_of_body_;
1157
1158                 delete d;
1159                 d = new Private(*par.d, this);
1160         }
1161         return *this;
1162 }
1163
1164
1165 Paragraph::~Paragraph()
1166 {
1167         delete d;
1168 }
1169
1170
1171 void Paragraph::write(Buffer const & buf, ostream & os,
1172                           BufferParams const & bparams,
1173                           depth_type & dth) const
1174 {
1175         // The beginning or end of a deeper (i.e. nested) area?
1176         if (dth != params().depth()) {
1177                 if (params().depth() > dth) {
1178                         while (params().depth() > dth) {
1179                                 os << "\n\\begin_deeper";
1180                                 ++dth;
1181                         }
1182                 } else {
1183                         while (params().depth() < dth) {
1184                                 os << "\n\\end_deeper";
1185                                 --dth;
1186                         }
1187                 }
1188         }
1189
1190         // First write the layout
1191         os << "\n\\begin_layout " << to_utf8(layout()->name()) << '\n';
1192
1193         params().write(os);
1194
1195         Font font1(Font::ALL_INHERIT, bparams.language);
1196
1197         Change running_change = Change(Change::UNCHANGED);
1198
1199         int column = 0;
1200         for (pos_type i = 0; i <= size(); ++i) {
1201
1202                 Change change = lookupChange(i);
1203                 Changes::lyxMarkChange(os, column, running_change, change);
1204                 running_change = change;
1205
1206                 if (i == size())
1207                         break;
1208
1209                 // Write font changes
1210                 Font font2 = getFontSettings(bparams, i);
1211                 if (font2 != font1) {
1212                         font2.lyxWriteChanges(font1, os);
1213                         column = 0;
1214                         font1 = font2;
1215                 }
1216
1217                 value_type const c = getChar(i);
1218                 switch (c) {
1219                 case META_INSET:
1220                 {
1221                         Inset const * inset = getInset(i);
1222                         if (inset)
1223                                 if (inset->directWrite()) {
1224                                         // international char, let it write
1225                                         // code directly so it's shorter in
1226                                         // the file
1227                                         inset->write(buf, os);
1228                                 } else {
1229                                         if (i)
1230                                                 os << '\n';
1231                                         os << "\\begin_inset ";
1232                                         inset->write(buf, os);
1233                                         os << "\n\\end_inset\n\n";
1234                                         column = 0;
1235                                 }
1236                 }
1237                 break;
1238                 case '\\':
1239                         os << "\n\\backslash\n";
1240                         column = 0;
1241                         break;
1242                 case '.':
1243                         if (i + 1 < size() && getChar(i + 1) == ' ') {
1244                                 os << ".\n";
1245                                 column = 0;
1246                         } else
1247                                 os << '.';
1248                         break;
1249                 default:
1250                         if ((column > 70 && c == ' ')
1251                             || column > 79) {
1252                                 os << '\n';
1253                                 column = 0;
1254                         }
1255                         // this check is to amend a bug. LyX sometimes
1256                         // inserts '\0' this could cause problems.
1257                         if (c != '\0') {
1258                                 std::vector<char> tmp = ucs4_to_utf8(c);
1259                                 tmp.push_back('\0');
1260                                 os << &tmp[0];
1261                         } else
1262                                 lyxerr << "ERROR (Paragraph::writeFile):"
1263                                         " NULL char in structure." << endl;
1264                         ++column;
1265                         break;
1266                 }
1267         }
1268
1269         os << "\n\\end_layout\n";
1270 }
1271
1272
1273 void Paragraph::validate(LaTeXFeatures & features) const
1274 {
1275         d->validate(features, *layout());
1276 }
1277
1278
1279 void Paragraph::insert(pos_type start, docstring const & str,
1280                        Font const & font, Change const & change)
1281 {
1282         for (size_t i = 0, n = str.size(); i != n ; ++i)
1283                 insertChar(start + i, str[i], font, change);
1284 }
1285
1286
1287 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
1288                            bool trackChanges)
1289 {
1290         d->insertChar(pos, c, Change(trackChanges ?
1291                            Change::INSERTED : Change::UNCHANGED));
1292 }
1293
1294
1295 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
1296                            Font const & font, bool trackChanges)
1297 {
1298         d->insertChar(pos, c, Change(trackChanges ?
1299                            Change::INSERTED : Change::UNCHANGED));
1300         setFont(pos, font);
1301 }
1302
1303
1304 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
1305                            Font const & font, Change const & change)
1306 {
1307         d->insertChar(pos, c, change);
1308         setFont(pos, font);
1309 }
1310
1311
1312 void Paragraph::insertInset(pos_type pos, Inset * inset,
1313                             Font const & font, Change const & change)
1314 {
1315         insertInset(pos, inset, change);
1316         // Set the font/language of the inset...
1317         setFont(pos, font);
1318 }
1319
1320
1321 bool Paragraph::insetAllowed(InsetCode code)
1322 {
1323         return !d->inset_owner_ || d->inset_owner_->insetAllowed(code);
1324 }
1325
1326
1327 // Gets uninstantiated font setting at position.
1328 Font const Paragraph::getFontSettings(BufferParams const & bparams,
1329                                          pos_type pos) const
1330 {
1331         if (pos > size()) {
1332                 lyxerr << " pos: " << pos << " size: " << size() << endl;
1333                 BOOST_ASSERT(pos <= size());
1334         }
1335
1336         FontList::const_iterator cit = d->fontlist_.begin();
1337         FontList::const_iterator end = d->fontlist_.end();
1338         for (; cit != end; ++cit)
1339                 if (cit->pos() >= pos)
1340                         break;
1341
1342         if (cit != end)
1343                 return cit->font();
1344
1345         if (pos == size() && !empty())
1346                 return getFontSettings(bparams, pos - 1);
1347
1348         return Font(Font::ALL_INHERIT, getParLanguage(bparams));
1349 }
1350
1351
1352 FontSpan Paragraph::fontSpan(pos_type pos) const
1353 {
1354         BOOST_ASSERT(pos <= size());
1355         pos_type start = 0;
1356
1357         FontList::const_iterator cit = d->fontlist_.begin();
1358         FontList::const_iterator end = d->fontlist_.end();
1359         for (; cit != end; ++cit) {
1360                 if (cit->pos() >= pos) {
1361                         if (pos >= beginOfBody())
1362                                 return FontSpan(std::max(start, beginOfBody()),
1363                                                 cit->pos());
1364                         else
1365                                 return FontSpan(start,
1366                                                 std::min(beginOfBody() - 1,
1367                                                          cit->pos()));
1368                 }
1369                 start = cit->pos() + 1;
1370         }
1371
1372         // This should not happen, but if so, we take no chances.
1373         //lyxerr << "Paragraph::getEndPosOfFontSpan: This should not happen!"
1374         //      << endl;
1375         return FontSpan(pos, pos);
1376 }
1377
1378
1379 // Gets uninstantiated font setting at position 0
1380 Font const Paragraph::getFirstFontSettings(BufferParams const & bparams) const
1381 {
1382         if (!empty() && !d->fontlist_.empty())
1383                 return d->fontlist_[0].font();
1384
1385         return Font(Font::ALL_INHERIT, bparams.language);
1386 }
1387
1388
1389 // Gets the fully instantiated font at a given position in a paragraph
1390 // This is basically the same function as Text::GetFont() in text2.cpp.
1391 // The difference is that this one is used for generating the LaTeX file,
1392 // and thus cosmetic "improvements" are disallowed: This has to deliver
1393 // the true picture of the buffer. (Asger)
1394 Font const Paragraph::getFont(BufferParams const & bparams, pos_type pos,
1395                                  Font const & outerfont) const
1396 {
1397         BOOST_ASSERT(pos >= 0);
1398
1399         Font font = getFontSettings(bparams, pos);
1400
1401         pos_type const body_pos = beginOfBody();
1402         if (pos < body_pos)
1403                 font.realize(layout_->labelfont);
1404         else
1405                 font.realize(layout_->font);
1406
1407         font.realize(outerfont);
1408         font.realize(bparams.getFont());
1409
1410         return font;
1411 }
1412
1413
1414 Font const Paragraph::getLabelFont
1415         (BufferParams const & bparams, Font const & outerfont) const
1416 {
1417         Font tmpfont = layout()->labelfont;
1418         tmpfont.setLanguage(getParLanguage(bparams));
1419         tmpfont.realize(outerfont);
1420         tmpfont.realize(bparams.getFont());
1421         return tmpfont;
1422 }
1423
1424
1425 Font const Paragraph::getLayoutFont
1426         (BufferParams const & bparams, Font const & outerfont) const
1427 {
1428         Font tmpfont = layout()->font;
1429         tmpfont.setLanguage(getParLanguage(bparams));
1430         tmpfont.realize(outerfont);
1431         tmpfont.realize(bparams.getFont());
1432         return tmpfont;
1433 }
1434
1435
1436 /// Returns the height of the highest font in range
1437 Font_size Paragraph::highestFontInRange
1438         (pos_type startpos, pos_type endpos, Font_size def_size) const
1439 {
1440         if (d->fontlist_.empty())
1441                 return def_size;
1442
1443         FontList::const_iterator end_it = d->fontlist_.begin();
1444         FontList::const_iterator const end = d->fontlist_.end();
1445         for (; end_it != end; ++end_it) {
1446                 if (end_it->pos() >= endpos)
1447                         break;
1448         }
1449
1450         if (end_it != end)
1451                 ++end_it;
1452
1453         FontList::const_iterator cit = d->fontlist_.begin();
1454         for (; cit != end; ++cit) {
1455                 if (cit->pos() >= startpos)
1456                         break;
1457         }
1458
1459         Font::FONT_SIZE maxsize = Font::SIZE_TINY;
1460         for (; cit != end_it; ++cit) {
1461                 Font::FONT_SIZE size = cit->font().size();
1462                 if (size == Font::INHERIT_SIZE)
1463                         size = def_size;
1464                 if (size > maxsize && size <= Font::SIZE_HUGER)
1465                         maxsize = size;
1466         }
1467         return maxsize;
1468 }
1469
1470
1471 Paragraph::value_type
1472 Paragraph::getUChar(BufferParams const & bparams, pos_type pos) const
1473 {
1474         value_type c = getChar(pos);
1475         if (!lyxrc.rtl_support)
1476                 return c;
1477
1478         value_type uc = c;
1479         switch (c) {
1480         case '(':
1481                 uc = ')';
1482                 break;
1483         case ')':
1484                 uc = '(';
1485                 break;
1486         case '[':
1487                 uc = ']';
1488                 break;
1489         case ']':
1490                 uc = '[';
1491                 break;
1492         case '{':
1493                 uc = '}';
1494                 break;
1495         case '}':
1496                 uc = '{';
1497                 break;
1498         case '<':
1499                 uc = '>';
1500                 break;
1501         case '>':
1502                 uc = '<';
1503                 break;
1504         }
1505         if (uc != c && getFontSettings(bparams, pos).isRightToLeft())
1506                 return uc;
1507         else
1508                 return c;
1509 }
1510
1511
1512 void Paragraph::setFont(pos_type pos, Font const & font)
1513 {
1514         BOOST_ASSERT(pos <= size());
1515
1516         // First, reduce font against layout/label font
1517         // Update: The setCharFont() routine in text2.cpp already
1518         // reduces font, so we don't need to do that here. (Asger)
1519         // No need to simplify this because it will disappear
1520         // in a new kernel. (Asger)
1521         // Next search font table
1522
1523         FontList::iterator beg = d->fontlist_.begin();
1524         FontList::iterator it = beg;
1525         FontList::iterator endit = d->fontlist_.end();
1526         for (; it != endit; ++it) {
1527                 if (it->pos() >= pos)
1528                         break;
1529         }
1530         size_t const i = distance(beg, it);
1531         bool notfound = (it == endit);
1532
1533         if (!notfound && d->fontlist_[i].font() == font)
1534                 return;
1535
1536         bool begin = pos == 0 || notfound ||
1537                 (i > 0 && d->fontlist_[i - 1].pos() == pos - 1);
1538         // Is position pos is a beginning of a font block?
1539         bool end = !notfound && d->fontlist_[i].pos() == pos;
1540         // Is position pos is the end of a font block?
1541         if (begin && end) { // A single char block
1542                 if (i + 1 < d->fontlist_.size() &&
1543                     d->fontlist_[i + 1].font() == font) {
1544                         // Merge the singleton block with the next block
1545                         d->fontlist_.erase(d->fontlist_.begin() + i);
1546                         if (i > 0 && d->fontlist_[i - 1].font() == font)
1547                                 d->fontlist_.erase(d->fontlist_.begin() + i - 1);
1548                 } else if (i > 0 && d->fontlist_[i - 1].font() == font) {
1549                         // Merge the singleton block with the previous block
1550                         d->fontlist_[i - 1].pos(pos);
1551                         d->fontlist_.erase(d->fontlist_.begin() + i);
1552                 } else
1553                         d->fontlist_[i].font(font);
1554         } else if (begin) {
1555                 if (i > 0 && d->fontlist_[i - 1].font() == font)
1556                         d->fontlist_[i - 1].pos(pos);
1557                 else
1558                         d->fontlist_.insert(d->fontlist_.begin() + i,
1559                                         FontTable(pos, font));
1560         } else if (end) {
1561                 d->fontlist_[i].pos(pos - 1);
1562                 if (!(i + 1 < d->fontlist_.size() &&
1563                       d->fontlist_[i + 1].font() == font))
1564                         d->fontlist_.insert(d->fontlist_.begin() + i + 1,
1565                                         FontTable(pos, font));
1566         } else { // The general case. The block is splitted into 3 blocks
1567                 d->fontlist_.insert(d->fontlist_.begin() + i,
1568                                 FontTable(pos - 1, d->fontlist_[i].font()));
1569                 d->fontlist_.insert(d->fontlist_.begin() + i + 1,
1570                                 FontTable(pos, font));
1571         }
1572 }
1573
1574
1575 void Paragraph::makeSameLayout(Paragraph const & par)
1576 {
1577         layout(par.layout());
1578         // move to pimpl?
1579         d->params_ = par.params();
1580 }
1581
1582
1583 bool Paragraph::stripLeadingSpaces(bool trackChanges)
1584 {
1585         if (isFreeSpacing())
1586                 return false;
1587
1588         int pos = 0;
1589         int count = 0;
1590
1591         while (pos < size() && (isNewline(pos) || isLineSeparator(pos))) {
1592                 if (eraseChar(pos, trackChanges))
1593                         ++count;
1594                 else
1595                         ++pos;
1596         }
1597
1598         return count > 0 || pos > 0;
1599 }
1600
1601
1602 bool Paragraph::hasSameLayout(Paragraph const & par) const
1603 {
1604         return par.layout() == layout() && d->params_.sameLayout(par.params());
1605 }
1606
1607
1608 depth_type Paragraph::getDepth() const
1609 {
1610         return params().depth();
1611 }
1612
1613
1614 depth_type Paragraph::getMaxDepthAfter() const
1615 {
1616         if (layout()->isEnvironment())
1617                 return params().depth() + 1;
1618         else
1619                 return params().depth();
1620 }
1621
1622
1623 char Paragraph::getAlign() const
1624 {
1625         if (params().align() == LYX_ALIGN_LAYOUT)
1626                 return layout()->align;
1627         else
1628                 return params().align();
1629 }
1630
1631
1632 docstring const & Paragraph::getLabelstring() const
1633 {
1634         return params().labelString();
1635 }
1636
1637
1638 // the next two functions are for the manual labels
1639 docstring const Paragraph::getLabelWidthString() const
1640 {
1641         if (layout()->margintype == MARGIN_MANUAL)
1642                 return params().labelWidthString();
1643         else
1644                 return _("Senseless with this layout!");
1645 }
1646
1647
1648 void Paragraph::setLabelWidthString(docstring const & s)
1649 {
1650         params().labelWidthString(s);
1651 }
1652
1653
1654 docstring const Paragraph::translateIfPossible(docstring const & s,
1655                 BufferParams const & bparams) const
1656 {
1657         if (!support::isAscii(s) || s.empty()) {
1658                 // This must be a user defined layout. We cannot translate
1659                 // this, since gettext accepts only ascii keys.
1660                 return s;
1661         }
1662         // Probably standard layout, try to translate
1663         Messages & m = getMessages(getParLanguage(bparams)->code());
1664         return m.get(to_ascii(s));
1665 }
1666
1667
1668 docstring Paragraph::expandLabel(LayoutPtr const & layout,
1669                 BufferParams const & bparams, bool process_appendix) const
1670 {
1671         TextClass const & tclass = bparams.getTextClass();
1672
1673         docstring fmt;
1674         if (process_appendix && params().appendix())
1675                 fmt = translateIfPossible(layout->labelstring_appendix(),
1676                         bparams);
1677         else
1678                 fmt = translateIfPossible(layout->labelstring(), bparams);
1679
1680         if (fmt.empty() && layout->labeltype == LABEL_COUNTER
1681             && !layout->counter.empty())
1682                 fmt = "\\the" + layout->counter;
1683
1684         // handle 'inherited level parts' in 'fmt',
1685         // i.e. the stuff between '@' in   '@Section@.\arabic{subsection}'
1686         size_t const i = fmt.find('@', 0);
1687         if (i != docstring::npos) {
1688                 size_t const j = fmt.find('@', i + 1);
1689                 if (j != docstring::npos) {
1690                         docstring parent(fmt, i + 1, j - i - 1);
1691                         docstring label = from_ascii("??");
1692                         if (tclass.hasLayout(parent))
1693                                 docstring label = expandLabel(tclass[parent], bparams,
1694                                                       process_appendix);
1695                         fmt = docstring(fmt, 0, i) + label
1696                                 + docstring(fmt, j + 1, docstring::npos);
1697                 }
1698         }
1699
1700         return tclass.counters().counterLabel(fmt);
1701 }
1702
1703
1704 void Paragraph::applyLayout(LayoutPtr const & new_layout)
1705 {
1706         layout(new_layout);
1707         LyXAlignment const oldAlign = params().align();
1708
1709         if (!(oldAlign & layout()->alignpossible)) {
1710                 frontend::Alert::warning(_("Alignment not permitted"),
1711                         _("The new layout does not permit the alignment previously used.\nSetting to default."));
1712                 params().align(LYX_ALIGN_LAYOUT);
1713         }
1714 }
1715
1716
1717 pos_type Paragraph::beginOfBody() const
1718 {
1719         return begin_of_body_;
1720 }
1721
1722
1723 void Paragraph::setBeginOfBody()
1724 {
1725         if (layout()->labeltype != LABEL_MANUAL) {
1726                 begin_of_body_ = 0;
1727                 return;
1728         }
1729
1730         // Unroll the first two cycles of the loop
1731         // and remember the previous character to
1732         // remove unnecessary getChar() calls
1733         pos_type i = 0;
1734         pos_type end = size();
1735         if (i < end && !isNewline(i)) {
1736                 ++i;
1737                 char_type previous_char = 0;
1738                 char_type temp = 0;
1739                 if (i < end) {
1740                         previous_char = text_[i];
1741                         if (!isNewline(i)) {
1742                                 ++i;
1743                                 while (i < end && previous_char != ' ') {
1744                                         temp = text_[i];
1745                                         if (isNewline(i))
1746                                                 break;
1747                                         ++i;
1748                                         previous_char = temp;
1749                                 }
1750                         }
1751                 }
1752         }
1753
1754         begin_of_body_ = i;
1755 }
1756
1757
1758 // returns -1 if inset not found
1759 int Paragraph::getPositionOfInset(Inset const * inset) const
1760 {
1761         // Find the entry.
1762         InsetList::const_iterator it = d->insetlist_.begin();
1763         InsetList::const_iterator end = d->insetlist_.end();
1764         for (; it != end; ++it)
1765                 if (it->inset == inset)
1766                         return it->pos;
1767         return -1;
1768 }
1769
1770
1771 InsetBibitem * Paragraph::bibitem() const
1772 {
1773         if (!d->insetlist_.empty()) {
1774                 Inset * inset = d->insetlist_.begin()->inset;
1775                 if (inset->lyxCode() == BIBITEM_CODE)
1776                         return static_cast<InsetBibitem *>(inset);
1777         }
1778         return 0;
1779 }
1780
1781
1782 bool Paragraph::forceDefaultParagraphs() const
1783 {
1784         return inInset() && inInset()->forceDefaultParagraphs(0);
1785 }
1786
1787
1788 namespace {
1789
1790 // paragraphs inside floats need different alignment tags to avoid
1791 // unwanted space
1792
1793 bool noTrivlistCentering(InsetCode code)
1794 {
1795         return code == FLOAT_CODE || code == WRAP_CODE;
1796 }
1797
1798
1799 string correction(string const & orig)
1800 {
1801         if (orig == "flushleft")
1802                 return "raggedright";
1803         if (orig == "flushright")
1804                 return "raggedleft";
1805         if (orig == "center")
1806                 return "centering";
1807         return orig;
1808 }
1809
1810
1811 string const corrected_env(string const & suffix, string const & env,
1812         InsetCode code)
1813 {
1814         string output = suffix + "{";
1815         if (noTrivlistCentering(code))
1816                 output += correction(env);
1817         else
1818                 output += env;
1819         output += "}";
1820         if (suffix == "\\begin")
1821                 output += "\n";
1822         return output;
1823 }
1824
1825
1826 void adjust_row_column(string const & str, TexRow & texrow, int & column)
1827 {
1828         if (!contains(str, "\n"))
1829                 column += str.size();
1830         else {
1831                 string tmp;
1832                 texrow.newline();
1833                 column = rsplit(str, tmp, '\n').size();
1834         }
1835 }
1836
1837 } // namespace anon
1838
1839
1840 // This could go to ParagraphParameters if we want to
1841 int Paragraph::startTeXParParams(BufferParams const & bparams,
1842                                  odocstream & os, TexRow & texrow,
1843                                  bool moving_arg) const
1844 {
1845         int column = 0;
1846
1847         if (params().noindent()) {
1848                 os << "\\noindent ";
1849                 column += 10;
1850         }
1851
1852         LyXAlignment const curAlign = params().align();
1853
1854         if (curAlign == layout()->align)
1855                 return column;
1856
1857         switch (curAlign) {
1858         case LYX_ALIGN_NONE:
1859         case LYX_ALIGN_BLOCK:
1860         case LYX_ALIGN_LAYOUT:
1861         case LYX_ALIGN_SPECIAL:
1862                 break;
1863         case LYX_ALIGN_LEFT:
1864         case LYX_ALIGN_RIGHT:
1865         case LYX_ALIGN_CENTER:
1866                 if (moving_arg) {
1867                         os << "\\protect";
1868                         column += 8;
1869                 }
1870                 break;
1871         }
1872
1873         switch (curAlign) {
1874         case LYX_ALIGN_NONE:
1875         case LYX_ALIGN_BLOCK:
1876         case LYX_ALIGN_LAYOUT:
1877         case LYX_ALIGN_SPECIAL:
1878                 break;
1879         case LYX_ALIGN_LEFT: {
1880                 string output;
1881                 if (getParLanguage(bparams)->babel() != "hebrew")
1882                         output = corrected_env("\\begin", "flushleft", ownerCode());
1883                 else
1884                         output = corrected_env("\\begin", "flushright", ownerCode());
1885                 os << from_ascii(output);
1886                 adjust_row_column(output, texrow, column);
1887                 break;
1888         } case LYX_ALIGN_RIGHT: {
1889                 string output;
1890                 if (getParLanguage(bparams)->babel() != "hebrew")
1891                         output = corrected_env("\\begin", "flushright", ownerCode());
1892                 else
1893                         output = corrected_env("\\begin", "flushleft", ownerCode());
1894                 os << from_ascii(output);
1895                 adjust_row_column(output, texrow, column);
1896                 break;
1897         } case LYX_ALIGN_CENTER: {
1898                 string output;
1899                 output = corrected_env("\\begin", "center", ownerCode());
1900                 os << from_ascii(output);
1901                 adjust_row_column(output, texrow, column);
1902                 break;
1903         }
1904         }
1905
1906         return column;
1907 }
1908
1909
1910 // This could go to ParagraphParameters if we want to
1911 int Paragraph::endTeXParParams(BufferParams const & bparams,
1912                                odocstream & os, TexRow & texrow,
1913                                bool moving_arg) const
1914 {
1915         int column = 0;
1916
1917         switch (params().align()) {
1918         case LYX_ALIGN_NONE:
1919         case LYX_ALIGN_BLOCK:
1920         case LYX_ALIGN_LAYOUT:
1921         case LYX_ALIGN_SPECIAL:
1922                 break;
1923         case LYX_ALIGN_LEFT:
1924         case LYX_ALIGN_RIGHT:
1925         case LYX_ALIGN_CENTER:
1926                 if (moving_arg) {
1927                         os << "\\protect";
1928                         column = 8;
1929                 }
1930                 break;
1931         }
1932
1933         switch (params().align()) {
1934         case LYX_ALIGN_NONE:
1935         case LYX_ALIGN_BLOCK:
1936         case LYX_ALIGN_LAYOUT:
1937         case LYX_ALIGN_SPECIAL:
1938                 break;
1939         case LYX_ALIGN_LEFT: {
1940                 string output;
1941                 if (getParLanguage(bparams)->babel() != "hebrew")
1942                         output = corrected_env("\n\\par\\end", "flushleft", ownerCode());
1943                 else
1944                         output = corrected_env("\n\\par\\end", "flushright", ownerCode());
1945                 os << from_ascii(output);
1946                 adjust_row_column(output, texrow, column);
1947                 break;
1948         } case LYX_ALIGN_RIGHT: {
1949                 string output;
1950                 if (getParLanguage(bparams)->babel() != "hebrew")
1951                         output = corrected_env("\n\\par\\end", "flushright", ownerCode());
1952                 else
1953                         output = corrected_env("\n\\par\\end", "flushleft", ownerCode());
1954                 os << from_ascii(output);
1955                 adjust_row_column(output, texrow, column);
1956                 break;
1957         } case LYX_ALIGN_CENTER: {
1958                 string output;
1959                 output = corrected_env("\n\\par\\end", "center", ownerCode());
1960                 os << from_ascii(output);
1961                 adjust_row_column(output, texrow, column);
1962                 break;
1963         }
1964         }
1965
1966         return column;
1967 }
1968
1969
1970 // This one spits out the text of the paragraph
1971 bool Paragraph::simpleTeXOnePar(Buffer const & buf,
1972                                 BufferParams const & bparams,
1973                                 Font const & outerfont,
1974                                 odocstream & os, TexRow & texrow,
1975                                 OutputParams const & runparams) const
1976 {
1977         LYXERR(Debug::LATEX) << "SimpleTeXOnePar...     " << this << endl;
1978
1979         bool return_value = false;
1980
1981         LayoutPtr style;
1982
1983         // well we have to check if we are in an inset with unlimited
1984         // length (all in one row) if that is true then we don't allow
1985         // any special options in the paragraph and also we don't allow
1986         // any environment other than the default layout of the text class
1987         // to be valid!
1988         bool asdefault = forceDefaultParagraphs();
1989
1990         if (asdefault) {
1991                 style = bparams.getTextClass().defaultLayout();
1992         } else {
1993                 style = layout();
1994         }
1995
1996         // Current base font for all inherited font changes, without any
1997         // change caused by an individual character, except for the language:
1998         // It is set to the language of the first character.
1999         // As long as we are in the label, this font is the base font of the
2000         // label. Before the first body character it is set to the base font
2001         // of the body.
2002         Font basefont;
2003
2004         // Maybe we have to create a optional argument.
2005         pos_type body_pos = beginOfBody();
2006         unsigned int column = 0;
2007
2008         if (body_pos > 0) {
2009                 // the optional argument is kept in curly brackets in
2010                 // case it contains a ']'
2011                 os << "[{";
2012                 column += 2;
2013                 basefont = getLabelFont(bparams, outerfont);
2014         } else {
2015                 basefont = getLayoutFont(bparams, outerfont);
2016         }
2017
2018         // Which font is currently active?
2019         Font running_font(basefont);
2020         // Do we have an open font change?
2021         bool open_font = false;
2022
2023         Change runningChange = Change(Change::UNCHANGED);
2024
2025         texrow.start(id(), 0);
2026
2027         // if the paragraph is empty, the loop will not be entered at all
2028         if (empty()) {
2029                 if (style->isCommand()) {
2030                         os << '{';
2031                         ++column;
2032                 }
2033                 if (!asdefault)
2034                         column += startTeXParParams(bparams, os, texrow,
2035                                                     runparams.moving_arg);
2036         }
2037
2038         for (pos_type i = 0; i < size(); ++i) {
2039                 // First char in paragraph or after label?
2040                 if (i == body_pos) {
2041                         if (body_pos > 0) {
2042                                 if (open_font) {
2043                                         column += running_font.latexWriteEndChanges(
2044                                                 os, bparams, runparams,
2045                                                 basefont, basefont);
2046                                         open_font = false;
2047                                 }
2048                                 basefont = getLayoutFont(bparams, outerfont);
2049                                 running_font = basefont;
2050
2051                                 column += Changes::latexMarkChange(os, bparams,
2052                                                 runningChange, Change(Change::UNCHANGED));
2053                                 runningChange = Change(Change::UNCHANGED);
2054
2055                                 os << "}] ";
2056                                 column +=3;
2057                         }
2058                         if (style->isCommand()) {
2059                                 os << '{';
2060                                 ++column;
2061                         }
2062
2063                         if (!asdefault)
2064                                 column += startTeXParParams(bparams, os,
2065                                                             texrow,
2066                                                             runparams.moving_arg);
2067                 }
2068
2069                 Change const & change = runparams.inDeletedInset ? runparams.changeOfDeletedInset
2070                                                                  : lookupChange(i);
2071
2072                 if (bparams.outputChanges && runningChange != change) {
2073                         if (open_font) {
2074                                 column += running_font.latexWriteEndChanges(
2075                                                 os, bparams, runparams, basefont, basefont);
2076                                 open_font = false;
2077                         }
2078                         basefont = getLayoutFont(bparams, outerfont);
2079                         running_font = basefont;
2080
2081                         column += Changes::latexMarkChange(os, bparams, runningChange, change);
2082                         runningChange = change;
2083                 }
2084
2085                 // do not output text which is marked deleted
2086                 // if change tracking output is disabled
2087                 if (!bparams.outputChanges && change.type == Change::DELETED) {
2088                         continue;
2089                 }
2090
2091                 ++column;
2092
2093                 value_type const c = getChar(i);
2094
2095                 // Fully instantiated font
2096                 Font const font = getFont(bparams, i, outerfont);
2097
2098                 Font const last_font = running_font;
2099
2100                 // Do we need to close the previous font?
2101                 if (open_font &&
2102                     (font != running_font ||
2103                      font.language() != running_font.language()))
2104                 {
2105                         column += running_font.latexWriteEndChanges(
2106                                         os, bparams, runparams, basefont,
2107                                         (i == body_pos-1) ? basefont : font);
2108                         running_font = basefont;
2109                         open_font = false;
2110                 }
2111
2112                 // Switch file encoding if necessary
2113                 if (runparams.encoding->package() == Encoding::inputenc &&
2114                     font.language()->encoding()->package() == Encoding::inputenc) {
2115                         std::pair<bool, int> const enc_switch = switchEncoding(os, bparams,
2116                                         runparams.moving_arg, *(runparams.encoding),
2117                                         *(font.language()->encoding()));
2118                         if (enc_switch.first) {
2119                                 column += enc_switch.second;
2120                                 runparams.encoding = font.language()->encoding();
2121                         }
2122                 }
2123
2124                 // Do we need to change font?
2125                 if ((font != running_font ||
2126                      font.language() != running_font.language()) &&
2127                         i != body_pos - 1)
2128                 {
2129                         odocstringstream ods;
2130                         column += font.latexWriteStartChanges(ods, bparams,
2131                                                               runparams, basefont,
2132                                                               last_font);
2133                         running_font = font;
2134                         open_font = true;
2135                         docstring fontchange = ods.str();
2136                         // check if the fontchange ends with a trailing blank
2137                         // (like "\small " (see bug 3382)
2138                         if (suffixIs(fontchange, ' ') && c == ' ')
2139                                 os << fontchange.substr(0, fontchange.size() - 1)
2140                                    << from_ascii("{}");
2141                         else
2142                                 os << fontchange;
2143                 }
2144
2145                 if (c == ' ') {
2146                         // Do not print the separation of the optional argument
2147                         // if style->pass_thru is false. This works because
2148                         // simpleTeXSpecialChars ignores spaces if
2149                         // style->pass_thru is false.
2150                         if (i != body_pos - 1) {
2151                                 if (d->simpleTeXBlanks(
2152                                                 *(runparams.encoding), os, texrow,
2153                                                 i, column, font, *style))
2154                                         // A surrogate pair was output. We
2155                                         // must not call simpleTeXSpecialChars
2156                                         // in this iteration, since
2157                                         // simpleTeXBlanks incremented i, and
2158                                         // simpleTeXSpecialChars would output
2159                                         // the combining character again.
2160                                         continue;
2161                         }
2162                 }
2163
2164                 OutputParams rp = runparams;
2165                 rp.free_spacing = style->free_spacing;
2166                 rp.local_font = &font;
2167                 rp.intitle = style->intitle;
2168                 d->simpleTeXSpecialChars(buf, bparams, os,
2169                                         texrow, rp, running_font,
2170                                         basefont, outerfont, open_font,
2171                                         runningChange, *style, i, column, c);
2172
2173                 // Set the encoding to that returned from simpleTeXSpecialChars (see
2174                 // comment for encoding member in OutputParams.h)
2175                 runparams.encoding = rp.encoding;
2176         }
2177
2178         // If we have an open font definition, we have to close it
2179         if (open_font) {
2180 #ifdef FIXED_LANGUAGE_END_DETECTION
2181                 if (next_) {
2182                         running_font
2183                                 .latexWriteEndChanges(os, bparams, runparams,
2184                                         basefont,
2185                                         next_->getFont(bparams, 0, outerfont));
2186                 } else {
2187                         running_font.latexWriteEndChanges(os, bparams,
2188                                         runparams, basefont, basefont);
2189                 }
2190 #else
2191 //FIXME: For now we ALWAYS have to close the foreign font settings if they are
2192 //FIXME: there as we start another \selectlanguage with the next paragraph if
2193 //FIXME: we are in need of this. This should be fixed sometime (Jug)
2194                 running_font.latexWriteEndChanges(os, bparams, runparams,
2195                                 basefont, basefont);
2196 #endif
2197         }
2198
2199         column += Changes::latexMarkChange(os, bparams, runningChange, Change(Change::UNCHANGED));
2200
2201         // Needed if there is an optional argument but no contents.
2202         if (body_pos > 0 && body_pos == size()) {
2203                 os << "}]~";
2204                 return_value = false;
2205         }
2206
2207         if (!asdefault) {
2208                 column += endTeXParParams(bparams, os, texrow,
2209                                           runparams.moving_arg);
2210         }
2211
2212         LYXERR(Debug::LATEX) << "SimpleTeXOnePar...done " << this << endl;
2213         return return_value;
2214 }
2215
2216
2217 namespace {
2218
2219 enum PAR_TAG {
2220         PAR_NONE=0,
2221         TT = 1,
2222         SF = 2,
2223         BF = 4,
2224         IT = 8,
2225         SL = 16,
2226         EM = 32
2227 };
2228
2229
2230 string tag_name(PAR_TAG const & pt) {
2231         switch (pt) {
2232         case PAR_NONE: return "!-- --";
2233         case TT: return "tt";
2234         case SF: return "sf";
2235         case BF: return "bf";
2236         case IT: return "it";
2237         case SL: return "sl";
2238         case EM: return "em";
2239         }
2240         return "";
2241 }
2242
2243
2244 inline
2245 void operator|=(PAR_TAG & p1, PAR_TAG const & p2)
2246 {
2247         p1 = static_cast<PAR_TAG>(p1 | p2);
2248 }
2249
2250
2251 inline
2252 void reset(PAR_TAG & p1, PAR_TAG const & p2)
2253 {
2254         p1 = static_cast<PAR_TAG>(p1 & ~p2);
2255 }
2256
2257 } // anon
2258
2259
2260 bool Paragraph::emptyTag() const
2261 {
2262         for (pos_type i = 0; i < size(); ++i) {
2263                 if (isInset(i)) {
2264                         Inset const * inset = getInset(i);
2265                         InsetCode lyx_code = inset->lyxCode();
2266                         if (lyx_code != TOC_CODE &&
2267                             lyx_code != INCLUDE_CODE &&
2268                             lyx_code != GRAPHICS_CODE &&
2269                             lyx_code != ERT_CODE &&
2270                             lyx_code != LISTINGS_CODE &&
2271                             lyx_code != FLOAT_CODE &&
2272                             lyx_code != TABULAR_CODE) {
2273                                 return false;
2274                         }
2275                 } else {
2276                         value_type c = getChar(i);
2277                         if (c != ' ' && c != '\t')
2278                                 return false;
2279                 }
2280         }
2281         return true;
2282 }
2283
2284
2285 string Paragraph::getID(Buffer const & buf, OutputParams const & runparams) const
2286 {
2287         for (pos_type i = 0; i < size(); ++i) {
2288                 if (isInset(i)) {
2289                         Inset const * inset = getInset(i);
2290                         InsetCode lyx_code = inset->lyxCode();
2291                         if (lyx_code == LABEL_CODE) {
2292                                 string const id = static_cast<InsetCommand const *>(inset)->getContents();
2293                                 return "id='" + to_utf8(sgml::cleanID(buf, runparams, from_utf8(id))) + "'";
2294                         }
2295                 }
2296
2297         }
2298         return string();
2299 }
2300
2301
2302 pos_type Paragraph::getFirstWord(Buffer const & buf, odocstream & os, OutputParams const & runparams) const
2303 {
2304         pos_type i;
2305         for (i = 0; i < size(); ++i) {
2306                 if (isInset(i)) {
2307                         Inset const * inset = getInset(i);
2308                         inset->docbook(buf, os, runparams);
2309                 } else {
2310                         value_type c = getChar(i);
2311                         if (c == ' ')
2312                                 break;
2313                         os << sgml::escapeChar(c);
2314                 }
2315         }
2316         return i;
2317 }
2318
2319
2320 bool Paragraph::onlyText(Buffer const & buf, Font const & outerfont, pos_type initial) const
2321 {
2322         Font font_old;
2323
2324         for (pos_type i = initial; i < size(); ++i) {
2325                 Font font = getFont(buf.params(), i, outerfont);
2326                 if (isInset(i))
2327                         return false;
2328                 if (i != initial && font != font_old)
2329                         return false;
2330                 font_old = font;
2331         }
2332
2333         return true;
2334 }
2335
2336
2337 void Paragraph::simpleDocBookOnePar(Buffer const & buf,
2338                                     odocstream & os,
2339                                     OutputParams const & runparams,
2340                                     Font const & outerfont,
2341                                     pos_type initial) const
2342 {
2343         bool emph_flag = false;
2344
2345         LayoutPtr const & style = layout();
2346         Font font_old =
2347                 style->labeltype == LABEL_MANUAL ? style->labelfont : style->font;
2348
2349         if (style->pass_thru && !onlyText(buf, outerfont, initial))
2350                 os << "]]>";
2351
2352         // parsing main loop
2353         for (pos_type i = initial; i < size(); ++i) {
2354                 Font font = getFont(buf.params(), i, outerfont);
2355
2356                 // handle <emphasis> tag
2357                 if (font_old.emph() != font.emph()) {
2358                         if (font.emph() == Font::ON) {
2359                                 os << "<emphasis>";
2360                                 emph_flag = true;
2361                         } else if (i != initial) {
2362                                 os << "</emphasis>";
2363                                 emph_flag = false;
2364                         }
2365                 }
2366
2367                 if (isInset(i)) {
2368                         Inset const * inset = getInset(i);
2369                         inset->docbook(buf, os, runparams);
2370                 } else {
2371                         value_type c = getChar(i);
2372
2373                         if (style->pass_thru)
2374                                 os.put(c);
2375                         else
2376                                 os << sgml::escapeChar(c);
2377                 }
2378                 font_old = font;
2379         }
2380
2381         if (emph_flag) {
2382                 os << "</emphasis>";
2383         }
2384
2385         if (style->free_spacing)
2386                 os << '\n';
2387         if (style->pass_thru && !onlyText(buf, outerfont, initial))
2388                 os << "<![CDATA[";
2389 }
2390
2391
2392 bool Paragraph::isHfill(pos_type pos) const
2393 {
2394         return isInset(pos)
2395                 && getInset(pos)->lyxCode() == HFILL_CODE;
2396 }
2397
2398
2399 bool Paragraph::isNewline(pos_type pos) const
2400 {
2401         return isInset(pos)
2402                 && getInset(pos)->lyxCode() == NEWLINE_CODE;
2403 }
2404
2405
2406 bool Paragraph::isLineSeparator(pos_type pos) const
2407 {
2408         value_type const c = getChar(pos);
2409         return isLineSeparatorChar(c)
2410                 || (c == Paragraph::META_INSET && getInset(pos) &&
2411                 getInset(pos)->isLineSeparator());
2412 }
2413
2414
2415 /// Used by the spellchecker
2416 bool Paragraph::isLetter(pos_type pos) const
2417 {
2418         if (isInset(pos))
2419                 return getInset(pos)->isLetter();
2420         else {
2421                 value_type const c = getChar(pos);
2422                 return isLetterChar(c) || isDigit(c);
2423         }
2424 }
2425
2426
2427 Language const *
2428 Paragraph::getParLanguage(BufferParams const & bparams) const
2429 {
2430         if (!empty())
2431                 return getFirstFontSettings(bparams).language();
2432         // FIXME: we should check the prev par as well (Lgb)
2433         return bparams.language;
2434 }
2435
2436
2437 bool Paragraph::isRTL(BufferParams const & bparams) const
2438 {
2439         return lyxrc.rtl_support
2440                 && getParLanguage(bparams)->rightToLeft()
2441                 && ownerCode() != ERT_CODE
2442                 && ownerCode() != LISTINGS_CODE;
2443 }
2444
2445
2446 void Paragraph::changeLanguage(BufferParams const & bparams,
2447                                Language const * from, Language const * to)
2448 {
2449         // change language including dummy font change at the end
2450         for (pos_type i = 0; i <= size(); ++i) {
2451                 Font font = getFontSettings(bparams, i);
2452                 if (font.language() == from) {
2453                         font.setLanguage(to);
2454                         setFont(i, font);
2455                 }
2456         }
2457 }
2458
2459
2460 bool Paragraph::isMultiLingual(BufferParams const & bparams) const
2461 {
2462         Language const * doc_language = bparams.language;
2463         FontList::const_iterator cit = d->fontlist_.begin();
2464         FontList::const_iterator end = d->fontlist_.end();
2465
2466         for (; cit != end; ++cit)
2467                 if (cit->font().language() != ignore_language &&
2468                     cit->font().language() != latex_language &&
2469                     cit->font().language() != doc_language)
2470                         return true;
2471         return false;
2472 }
2473
2474
2475 // Convert the paragraph to a string.
2476 // Used for building the table of contents
2477 docstring const Paragraph::asString(Buffer const & buffer, bool label) const
2478 {
2479         return asString(buffer, 0, size(), label);
2480 }
2481
2482
2483 docstring const Paragraph::asString(Buffer const & buffer,
2484                                  pos_type beg, pos_type end, bool label) const
2485 {
2486
2487         odocstringstream os;
2488
2489         if (beg == 0 && label && !params().labelString().empty())
2490                 os << params().labelString() << ' ';
2491
2492         for (pos_type i = beg; i < end; ++i) {
2493                 value_type const c = getChar(i);
2494                 if (isPrintable(c))
2495                         os.put(c);
2496                 else if (c == META_INSET)
2497                         getInset(i)->textString(buffer, os);
2498         }
2499
2500         return os.str();
2501 }
2502
2503
2504 void Paragraph::setInsetOwner(Inset * inset)
2505 {
2506         d->inset_owner_ = inset;
2507 }
2508
2509
2510 int Paragraph::id() const
2511 {
2512         return d->id_;
2513 }
2514
2515
2516 LayoutPtr const & Paragraph::layout() const
2517 {
2518         return layout_;
2519 }
2520
2521
2522 void Paragraph::layout(LayoutPtr const & new_layout)
2523 {
2524         layout_ = new_layout;
2525 }
2526
2527
2528 Inset * Paragraph::inInset() const
2529 {
2530         return d->inset_owner_;
2531 }
2532
2533
2534 InsetCode Paragraph::ownerCode() const
2535 {
2536         return d->inset_owner_ ?
2537                 d->inset_owner_->lyxCode() : NO_CODE;
2538 }
2539
2540
2541 ParagraphParameters & Paragraph::params()
2542 {
2543         return d->params_;
2544 }
2545
2546
2547 ParagraphParameters const & Paragraph::params() const
2548 {
2549         return d->params_;
2550 }
2551
2552
2553 bool Paragraph::isFreeSpacing() const
2554 {
2555         if (layout()->free_spacing)
2556                 return true;
2557
2558         // for now we just need this, later should we need this in some
2559         // other way we can always add a function to Inset too.
2560         return ownerCode() == ERT_CODE || ownerCode() == LISTINGS_CODE;
2561 }
2562
2563
2564 bool Paragraph::allowEmpty() const
2565 {
2566         if (layout()->keepempty)
2567                 return true;
2568         return ownerCode() == ERT_CODE || ownerCode() == LISTINGS_CODE;
2569 }
2570
2571
2572 char_type Paragraph::transformChar(char_type c, pos_type pos) const
2573 {
2574         if (!Encodings::is_arabic(c))
2575                 return c;
2576
2577         value_type prev_char = ' ';
2578         value_type next_char = ' ';
2579
2580         for (pos_type i = pos - 1; i >= 0; --i) {
2581                 value_type const par_char = getChar(i);
2582                 if (!Encodings::isComposeChar_arabic(par_char)) {
2583                         prev_char = par_char;
2584                         break;
2585                 }
2586         }
2587
2588         for (pos_type i = pos + 1, end = size(); i < end; ++i) {
2589                 value_type const par_char = getChar(i);
2590                 if (!Encodings::isComposeChar_arabic(par_char)) {
2591                         next_char = par_char;
2592                         break;
2593                 }
2594         }
2595
2596         if (Encodings::is_arabic(next_char)) {
2597                 if (Encodings::is_arabic(prev_char) &&
2598                         !Encodings::is_arabic_special(prev_char))
2599                         return Encodings::transformChar(c, Encodings::FORM_MEDIAL);
2600                 else
2601                         return Encodings::transformChar(c, Encodings::FORM_INITIAL);
2602         } else {
2603                 if (Encodings::is_arabic(prev_char) &&
2604                         !Encodings::is_arabic_special(prev_char))
2605                         return Encodings::transformChar(c, Encodings::FORM_FINAL);
2606                 else
2607                         return Encodings::transformChar(c, Encodings::FORM_ISOLATED);
2608         }
2609 }
2610
2611
2612 int Paragraph::checkBiblio(bool track_changes)
2613 {
2614         //FIXME From JS:
2615         //This is getting more and more a mess. ...We really should clean
2616         //up this bibitem issue for 1.6. See also bug 2743.
2617
2618         // Add bibitem insets if necessary
2619         if (layout()->labeltype != LABEL_BIBLIO)
2620                 return 0;
2621
2622         bool hasbibitem = !d->insetlist_.empty()
2623                 // Insist on it being in pos 0
2624                 && getChar(0) == Paragraph::META_INSET
2625                 && d->insetlist_.begin()->inset->lyxCode() == BIBITEM_CODE;
2626
2627         docstring oldkey;
2628         docstring oldlabel;
2629
2630         // remove a bibitem in pos != 0
2631         // restore it later in pos 0 if necessary
2632         // (e.g. if a user inserts contents _before_ the item)
2633         // we're assuming there's only one of these, which there
2634         // should be.
2635         int erasedInsetPosition = -1;
2636         InsetList::iterator it = d->insetlist_.begin();
2637         InsetList::iterator end = d->insetlist_.end();
2638         for (; it != end; ++it)
2639                 if (it->inset->lyxCode() == BIBITEM_CODE
2640                     && it->pos > 0) {
2641                         InsetBibitem * olditem = static_cast<InsetBibitem *>(it->inset);
2642                         oldkey = olditem->getParam("key");
2643                         oldlabel = olditem->getParam("label");
2644                         erasedInsetPosition = it->pos;
2645                         eraseChar(erasedInsetPosition, track_changes);
2646                         break;
2647         }
2648
2649         //There was an InsetBibitem at the beginning, and we didn't
2650         //have to erase one.
2651         if (hasbibitem && erasedInsetPosition < 0)
2652                         return 0;
2653
2654         //There was an InsetBibitem at the beginning and we did have to
2655         //erase one. So we give its properties to the beginning inset.
2656         if (hasbibitem) {
2657                 InsetBibitem * inset =
2658                         static_cast<InsetBibitem *>(d->insetlist_.begin()->inset);
2659                 if (!oldkey.empty())
2660                         inset->setParam("key", oldkey);
2661                 inset->setParam("label", oldlabel);
2662                 return -erasedInsetPosition;
2663         }
2664
2665         //There was no inset at the beginning, so we need to create one with
2666         //the key and label of the one we erased.
2667         InsetBibitem * inset(new InsetBibitem(InsetCommandParams("bibitem")));
2668         // restore values of previously deleted item in this par.
2669         if (!oldkey.empty())
2670                 inset->setParam("key", oldkey);
2671         inset->setParam("label", oldlabel);
2672         insertInset(0, static_cast<Inset *>(inset),
2673                     Change(track_changes ? Change::INSERTED : Change::UNCHANGED));
2674
2675         return 1;
2676 }
2677
2678
2679 void Paragraph::checkAuthors(AuthorList const & authorList)
2680 {
2681         d->changes_.checkAuthors(authorList);
2682 }
2683
2684
2685 bool Paragraph::isUnchanged(pos_type pos) const
2686 {
2687         return lookupChange(pos).type == Change::UNCHANGED;
2688 }
2689
2690
2691 bool Paragraph::isInserted(pos_type pos) const
2692 {
2693         return lookupChange(pos).type == Change::INSERTED;
2694 }
2695
2696
2697 bool Paragraph::isDeleted(pos_type pos) const
2698 {
2699         return lookupChange(pos).type == Change::DELETED;
2700 }
2701
2702
2703 InsetList const & Paragraph::insetList() const
2704 {
2705         return d->insetlist_;
2706 }
2707
2708
2709 Inset * Paragraph::releaseInset(pos_type pos)
2710 {
2711         Inset * inset = d->insetlist_.release(pos);
2712         /// does not honour change tracking!
2713         eraseChar(pos, false);
2714         return inset;
2715 }
2716
2717
2718 Inset * Paragraph::getInset(pos_type pos)
2719 {
2720         return d->insetlist_.get(pos);
2721 }
2722
2723
2724 Inset const * Paragraph::getInset(pos_type pos) const
2725 {
2726         return d->insetlist_.get(pos);
2727 }
2728
2729 } // namespace lyx