]> git.lyx.org Git - lyx.git/blobdiff - src/paragraph.C
hopefully fix tex2lyx linking.
[lyx.git] / src / paragraph.C
index 28471576219e4313967b06f6ce01cfc9e86123f9..c0c53c3e524220eed2aefacf3aef80680facef39 100644 (file)
 #include "support/lstrings.h"
 #include "support/textutils.h"
 #include "support/convert.h"
+#include "support/unicode.h"
 
-#include <boost/tuple/tuple.hpp>
 #include <boost/bind.hpp>
 
+#include <algorithm>
 #include <list>
 #include <stack>
 #include <sstream>
 
-using lyx::pos_type;
 
-using lyx::support::subst;
+namespace lyx {
+
+using support::subst;
 
 using std::distance;
 using std::endl;
@@ -65,10 +67,6 @@ using std::ostream;
 using std::ostringstream;
 
 
-ParagraphList::ParagraphList()
-{}
-
-
 Paragraph::Paragraph()
        : begin_of_body_(0), pimpl_(new Paragraph::Pimpl(this))
 {
@@ -80,7 +78,8 @@ Paragraph::Paragraph()
 Paragraph::Paragraph(Paragraph const & par)
        :       itemdepth(par.itemdepth), insetlist(par.insetlist),
                dim_(par.dim_),
-               rows_(par.rows_), layout_(par.layout_),
+               rows_(par.rows_), rowSignature_(par.rowSignature_),
+               layout_(par.layout_),
                text_(par.text_), begin_of_body_(par.begin_of_body_),
          pimpl_(new Paragraph::Pimpl(*par.pimpl_, this))
 {
@@ -106,6 +105,7 @@ Paragraph & Paragraph::operator=(Paragraph const & par)
 
                rows_ = par.rows_;
                dim_ = par.dim_;
+               rowSignature_ = par.rowSignature_;
                layout_ = par.layout();
                text_ = par.text_;
                begin_of_body_ = par.begin_of_body_;
@@ -153,15 +153,17 @@ void Paragraph::write(Buffer const & buf, ostream & os,
        LyXFont font1(LyXFont::ALL_INHERIT, bparams.language);
 
        Change running_change = Change(Change::UNCHANGED);
-       lyx::time_type const curtime(lyx::current_time());
 
        int column = 0;
-       for (pos_type i = 0; i < size(); ++i) {
+       for (pos_type i = 0; i <= size(); ++i) {
 
-               Change change = pimpl_->lookupChangeFull(i);
-               Changes::lyxMarkChange(os, column, curtime, running_change, change);
+               Change change = pimpl_->lookupChange(i);
+               Changes::lyxMarkChange(os, column, running_change, change);
                running_change = change;
 
+               if (i == size())
+                       break;
+
                // Write font changes
                LyXFont font2 = getFontSettings(bparams, i);
                if (font2 != font1) {
@@ -210,9 +212,11 @@ void Paragraph::write(Buffer const & buf, ostream & os,
                        }
                        // this check is to amend a bug. LyX sometimes
                        // inserts '\0' this could cause problems.
-                       if (c != '\0')
-                               os << c;
-                       else
+                       if (c != '\0') {
+                               std::vector<char> tmp = ucs4_to_utf8(c);
+                               tmp.push_back('\0');
+                               os << &tmp[0];
+                       } else
                                lyxerr << "ERROR (Paragraph::writeFile):"
                                        " NULL char in structure." << endl;
                        ++column;
@@ -220,15 +224,6 @@ void Paragraph::write(Buffer const & buf, ostream & os,
                }
        }
 
-       // to make reading work properly
-       if (!size()) {
-               running_change = pimpl_->lookupChange(0);
-               Changes::lyxMarkChange(os, column, curtime,
-                       Change(Change::UNCHANGED), running_change);
-       }
-       Changes::lyxMarkChange(os, column, curtime,
-               running_change, Change(Change::UNCHANGED));
-
        os << "\n\\end_layout\n";
 }
 
@@ -239,62 +234,60 @@ void Paragraph::validate(LaTeXFeatures & features) const
 }
 
 
-void Paragraph::eraseIntern(lyx::pos_type pos)
+bool Paragraph::eraseChar(pos_type pos, bool trackChanges)
 {
-       pimpl_->eraseIntern(pos);
+       return pimpl_->eraseChar(pos, trackChanges);
 }
 
 
-bool Paragraph::erase(pos_type pos)
+int Paragraph::eraseChars(pos_type start, pos_type end, bool trackChanges)
 {
-       return pimpl_->erase(pos);
+       return pimpl_->eraseChars(start, end, trackChanges);
 }
 
 
-int Paragraph::erase(pos_type start, pos_type end)
+void Paragraph::insert(pos_type start, docstring const & str,
+                       LyXFont const & font, Change const & change)
 {
-       return pimpl_->erase(start, end);
+       for (size_t i = 0, n = str.size(); i != n ; ++i)
+               insertChar(start + i, str[i], font, change);
 }
 
 
-void Paragraph::insert(pos_type start, string const & str,
-                      LyXFont const & font)
-{
-       int size = str.size();
-       for (int i = 0 ; i < size ; ++i)
-               insertChar(start + i, str[i], font);
-}
-
-
-bool Paragraph::checkInsertChar(LyXFont &)
+void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
+                           bool trackChanges)
 {
-       return true;
+       pimpl_->insertChar(pos, c, Change(trackChanges ?
+                          Change::INSERTED : Change::UNCHANGED));
 }
 
 
 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
-                          Change change)
+                           LyXFont const & font, bool trackChanges)
 {
-       pimpl_->insertChar(pos, c, change);
+       pimpl_->insertChar(pos, c, Change(trackChanges ?
+                          Change::INSERTED : Change::UNCHANGED));
+       setFont(pos, font);
 }
 
 
 void Paragraph::insertChar(pos_type pos, Paragraph::value_type c,
-                          LyXFont const & font, Change change)
+                           LyXFont const & font, Change const & change)
 {
        pimpl_->insertChar(pos, c, change);
        setFont(pos, font);
 }
 
 
-void Paragraph::insertInset(pos_type pos, InsetBase * inset, Change change)
+void Paragraph::insertInset(pos_type pos, InsetBase * inset,
+                            Change const & change)
 {
        pimpl_->insertInset(pos, inset, change);
 }
 
 
 void Paragraph::insertInset(pos_type pos, InsetBase * inset,
-                           LyXFont const & font, Change change)
+                            LyXFont const & font, Change const & change)
 {
        pimpl_->insertInset(pos, inset, change);
        setFont(pos, font);
@@ -307,20 +300,6 @@ bool Paragraph::insetAllowed(InsetBase_code code)
 }
 
 
-InsetBase * Paragraph::getInset(pos_type pos)
-{
-       BOOST_ASSERT(pos < size());
-       return insetlist.get(pos);
-}
-
-
-InsetBase const * Paragraph::getInset(pos_type pos) const
-{
-       BOOST_ASSERT(pos < size());
-       return insetlist.get(pos);
-}
-
-
 // Gets uninstantiated font setting at position.
 LyXFont const Paragraph::getFontSettings(BufferParams const & bparams,
                                         pos_type pos) const
@@ -346,30 +325,40 @@ LyXFont const Paragraph::getFontSettings(BufferParams const & bparams,
 }
 
 
-lyx::pos_type Paragraph::getEndPosOfFontSpan(lyx::pos_type pos) const
+FontSpan Paragraph::fontSpan(pos_type pos) const
 {
        BOOST_ASSERT(pos <= size());
+       pos_type start = 0;
 
        Pimpl::FontList::const_iterator cit = pimpl_->fontlist.begin();
        Pimpl::FontList::const_iterator end = pimpl_->fontlist.end();
-       for (; cit != end; ++cit)
-               if (cit->pos() >= pos)
-                       return cit->pos();
+       for (; cit != end; ++cit) {
+               if (cit->pos() >= pos) {
+                       if (pos >= beginOfBody())
+                               return FontSpan(std::max(start, beginOfBody()),
+                                               cit->pos());
+                       else
+                               return FontSpan(start,
+                                               std::min(beginOfBody() - 1,
+                                                        cit->pos()));
+               }
+               start = cit->pos() + 1;
+       }
 
        // This should not happen, but if so, we take no chances.
        //lyxerr << "Paragraph::getEndPosOfFontSpan: This should not happen!"
        //      << endl;
-       return pos;
+       return FontSpan(pos, pos);
 }
 
 
 // Gets uninstantiated font setting at position 0
-LyXFont const Paragraph::getFirstFontSettings() const
+LyXFont const Paragraph::getFirstFontSettings(BufferParams const & bparams) const
 {
        if (!empty() && !pimpl_->fontlist.empty())
                return pimpl_->fontlist[0].font();
 
-       return LyXFont(LyXFont::ALL_INHERIT);
+       return LyXFont(LyXFont::ALL_INHERIT, bparams.language);
 }
 
 
@@ -396,7 +385,7 @@ LyXFont const Paragraph::getFont(BufferParams const & bparams, pos_type pos,
        LyXFont font = getFontSettings(bparams, pos);
        font.realize(layoutfont);
        font.realize(outerfont);
-       font.realize(bparams.getLyXTextClass().defaultfont());
+       font.realize(bparams.getFont());
 
        return font;
 }
@@ -408,7 +397,7 @@ LyXFont const Paragraph::getLabelFont
        LyXFont tmpfont = layout()->labelfont;
        tmpfont.setLanguage(getParLanguage(bparams));
        tmpfont.realize(outerfont);
-       tmpfont.realize(bparams.getLyXTextClass().defaultfont());
+       tmpfont.realize(bparams.getFont());
        return tmpfont;
 }
 
@@ -419,7 +408,7 @@ LyXFont const Paragraph::getLayoutFont
        LyXFont tmpfont = layout()->font;
        tmpfont.setLanguage(getParLanguage(bparams));
        tmpfont.realize(outerfont);
-       tmpfont.realize(bparams.getLyXTextClass().defaultfont());
+       tmpfont.realize(bparams.getFont());
        return tmpfont;
 }
 
@@ -518,7 +507,7 @@ void Paragraph::setFont(pos_type pos, LyXFont const & font)
                if (it->pos() >= pos)
                        break;
        }
-       unsigned int i = distance(beg, it);
+       size_t const i = distance(beg, it);
        bool notfound = (it == endit);
 
        if (!notfound && pimpl_->fontlist[i].font() == font)
@@ -577,10 +566,9 @@ int Paragraph::stripLeadingSpaces()
                return 0;
 
        int i = 0;
-       while (!empty() && (isNewline(0) || isLineSeparator(0))) {
-               // Set Change::Type to Change::INSERTED to quietly remove it
-               setChange(0, Change::INSERTED);
-               erase(0);
+       while (!empty() && (isNewline(0) || isLineSeparator(0))
+               && !isDeleted(0)) {
+               eraseChar(0, false); // no change tracking here
                ++i;
        }
 
@@ -594,13 +582,13 @@ bool Paragraph::hasSameLayout(Paragraph const & par) const
 }
 
 
-Paragraph::depth_type Paragraph::getDepth() const
+depth_type Paragraph::getDepth() const
 {
        return params().depth();
 }
 
 
-Paragraph::depth_type Paragraph::getMaxDepthAfter() const
+depth_type Paragraph::getMaxDepthAfter() const
 {
        if (layout()->isEnvironment())
                return params().depth() + 1;
@@ -615,14 +603,14 @@ char Paragraph::getAlign() const
 }
 
 
-string const & Paragraph::getLabelstring() const
+docstring const & Paragraph::getLabelstring() const
 {
        return params().labelString();
 }
 
 
 // the next two functions are for the manual labels
-string const Paragraph::getLabelWidthString() const
+docstring const Paragraph::getLabelWidthString() const
 {
        if (!params().labelWidthString().empty())
                return params().labelWidthString();
@@ -631,7 +619,7 @@ string const Paragraph::getLabelWidthString() const
 }
 
 
-void Paragraph::setLabelWidthString(string const & s)
+void Paragraph::setLabelWidthString(docstring const & s)
 {
        params().labelWidthString(s);
 }
@@ -640,7 +628,7 @@ void Paragraph::setLabelWidthString(string const & s)
 void Paragraph::applyLayout(LyXLayout_ptr const & new_layout)
 {
        layout(new_layout);
-       params().labelWidthString(string());
+       params().labelWidthString(docstring());
        params().align(LYX_ALIGN_LAYOUT);
        params().spacing(Spacing(Spacing::Default));
 }
@@ -666,8 +654,8 @@ void Paragraph::setBeginOfBody()
        pos_type end = size();
        if (i < end && !isNewline(i)) {
                ++i;
-               char previous_char = 0;
-               char temp = 0;
+               char_type previous_char = 0;
+               char_type temp = 0;
                if (i < end) {
                        previous_char = text_[i];
                        if (!isNewline(i)) {
@@ -704,7 +692,7 @@ InsetBibitem * Paragraph::bibitem() const
 {
        if (!insetlist.empty()) {
                InsetBase * inset = insetlist.begin()->inset;
-               if (inset->lyxCode() == InsetBase::BIBTEX_CODE)
+               if (inset->lyxCode() == InsetBase::BIBITEM_CODE)
                        return static_cast<InsetBibitem *>(inset);
        }
        return 0;
@@ -713,7 +701,7 @@ InsetBibitem * Paragraph::bibitem() const
 
 bool Paragraph::forceDefaultParagraphs() const
 {
-       return inInset() && inInset()->forceDefaultParagraphs(inInset());
+       return inInset() && inInset()->forceDefaultParagraphs(0);
 }
 
 
@@ -756,7 +744,7 @@ string const corrected_env(string const & suffix, string const & env,
 
 // This could go to ParagraphParameters if we want to
 int Paragraph::startTeXParParams(BufferParams const & bparams,
-                                ostream & os, bool moving_arg) const
+                                odocstream & os, bool moving_arg) const
 {
        int column = 0;
 
@@ -793,7 +781,7 @@ int Paragraph::startTeXParParams(BufferParams const & bparams,
                        output = corrected_env("\\begin", "flushleft", ownerCode());
                else
                        output = corrected_env("\\begin", "flushright", ownerCode());
-               os << output;
+               os << from_ascii(output);
                column += output.size();
                break;
        } case LYX_ALIGN_RIGHT: {
@@ -802,13 +790,13 @@ int Paragraph::startTeXParParams(BufferParams const & bparams,
                        output = corrected_env("\\begin", "flushright", ownerCode());
                else
                        output = corrected_env("\\begin", "flushleft", ownerCode());
-               os << output;
+               os << from_ascii(output);
                column += output.size();
                break;
        } case LYX_ALIGN_CENTER: {
                string output;
                output = corrected_env("\\begin", "center", ownerCode());
-               os << output;
+               os << from_ascii(output);
                column += output.size();
                break;
        }
@@ -820,7 +808,7 @@ int Paragraph::startTeXParParams(BufferParams const & bparams,
 
 // This could go to ParagraphParameters if we want to
 int Paragraph::endTeXParParams(BufferParams const & bparams,
-                              ostream & os, bool moving_arg) const
+                              odocstream & os, bool moving_arg) const
 {
        int column = 0;
 
@@ -852,7 +840,7 @@ int Paragraph::endTeXParParams(BufferParams const & bparams,
                        output = corrected_env("\\par\\end", "flushleft", ownerCode());
                else
                        output = corrected_env("\\par\\end", "flushright", ownerCode());
-               os << output;
+               os << from_ascii(output);
                column += output.size();
                break;
        } case LYX_ALIGN_RIGHT: {
@@ -861,13 +849,13 @@ int Paragraph::endTeXParParams(BufferParams const & bparams,
                        output = corrected_env("\\par\\end", "flushright", ownerCode());
                else
                        output = corrected_env("\\par\\end", "flushleft", ownerCode());
-               os << output;
+               os << from_ascii(output);
                column += output.size();
                break;
        } case LYX_ALIGN_CENTER: {
                string output;
                output = corrected_env("\\par\\end", "center", ownerCode());
-               os << output;
+               os << from_ascii(output);
                column += output.size();
                break;
        }
@@ -881,7 +869,7 @@ int Paragraph::endTeXParParams(BufferParams const & bparams,
 bool Paragraph::simpleTeXOnePar(Buffer const & buf,
                                BufferParams const & bparams,
                                LyXFont const & outerfont,
-                               ostream & os, TexRow & texrow,
+                               odocstream & os, TexRow & texrow,
                                OutputParams const & runparams) const
 {
        lyxerr[Debug::LATEX] << "SimpleTeXOnePar...     " << this << endl;
@@ -904,12 +892,12 @@ bool Paragraph::simpleTeXOnePar(Buffer const & buf,
 
        LyXFont basefont;
 
-       LaTeXFeatures features(buf, bparams, runparams.nice);
+       LaTeXFeatures features(buf, bparams, runparams);
 
        // output change tracking marks only if desired,
        // if dvipost is installed,
        // and with dvi/ps (other formats don't work)
-       bool const output = bparams.output_changes
+       bool const output = bparams.outputChanges
                && runparams.flavor == OutputParams::LATEX
                && features.isAvailable("dvipost");
 
@@ -932,7 +920,7 @@ bool Paragraph::simpleTeXOnePar(Buffer const & buf,
        // Do we have an open font change?
        bool open_font = false;
 
-       Change::Type running_change = Change::UNCHANGED;
+       Change::Type runningChangeType = Change::UNCHANGED;
 
        texrow.start(id(), 0);
 
@@ -1020,30 +1008,30 @@ bool Paragraph::simpleTeXOnePar(Buffer const & buf,
                        open_font = true;
                }
 
-               Change::Type change = pimpl_->lookupChange(i);
+               Change::Type changeType = pimpl_->lookupChange(i).type;
 
-               column += Changes::latexMarkChange(os, running_change,
-                       change, output);
-               running_change = change;
+               column += Changes::latexMarkChange(os, runningChangeType,
+                       changeType, output);
+               runningChangeType = changeType;
 
                // do not output text which is marked deleted
                // if change tracking output is not desired
-               if (output || running_change != Change::DELETED) {
+               if (output || runningChangeType != Change::DELETED) {
                        OutputParams rp = runparams;
                        rp.free_spacing = style->free_spacing;
-                       rp.local_language = font.language()->babel();
+                       rp.local_font = &font;
                        rp.intitle = style->intitle;
                        pimpl_->simpleTeXSpecialChars(buf, bparams,
                                                os, texrow, rp,
                                                font, running_font,
                                                basefont, outerfont, open_font,
-                                               running_change,
+                                               runningChangeType,
                                                *style, i, column, c);
                }
        }
 
        column += Changes::latexMarkChange(os,
-                       running_change, Change::UNCHANGED, output);
+                       runningChangeType, Change::UNCHANGED, output);
 
        // If we have an open font definition, we have to close it
        if (open_font) {
@@ -1069,7 +1057,7 @@ bool Paragraph::simpleTeXOnePar(Buffer const & buf,
 
        // Needed if there is an optional argument but no contents.
        if (body_pos > 0 && body_pos == size()) {
-               os << "]~";
+               os << "}]~";
                return_value = false;
        }
 
@@ -1137,201 +1125,6 @@ void reset(PAR_TAG & p1, PAR_TAG const & p2)
 } // anon
 
 
-// Handle internal paragraph parsing -- layout already processed.
-void Paragraph::simpleLinuxDocOnePar(Buffer const & buf,
-                                    ostream & os,
-                                    LyXFont const & outerfont,
-                                    OutputParams const & runparams,
-                                    lyx::depth_type /*depth*/) const
-{
-       LyXLayout_ptr const & style = layout();
-
-       string::size_type char_line_count = 5;     // Heuristic choice ;-)
-
-       // gets paragraph main font
-       LyXFont font_old;
-       bool desc_on;
-       if (style->labeltype == LABEL_MANUAL) {
-               font_old = style->labelfont;
-               desc_on = true;
-       } else {
-               font_old = style->font;
-               desc_on = false;
-       }
-
-       LyXFont::FONT_FAMILY family_type = LyXFont::ROMAN_FAMILY;
-       LyXFont::FONT_SERIES series_type = LyXFont::MEDIUM_SERIES;
-       LyXFont::FONT_SHAPE  shape_type  = LyXFont::UP_SHAPE;
-       bool is_em = false;
-
-       stack<PAR_TAG> tag_state;
-       // parsing main loop
-       for (pos_type i = 0; i < size(); ++i) {
-
-               PAR_TAG tag_close = PAR_NONE;
-               list < PAR_TAG > tag_open;
-
-               LyXFont const font = getFont(buf.params(), i, outerfont);
-
-               if (font_old.family() != font.family()) {
-                       switch (family_type) {
-                       case LyXFont::SANS_FAMILY:
-                               tag_close |= SF;
-                               break;
-                       case LyXFont::TYPEWRITER_FAMILY:
-                               tag_close |= TT;
-                               break;
-                       default:
-                               break;
-                       }
-
-                       family_type = font.family();
-
-                       switch (family_type) {
-                       case LyXFont::SANS_FAMILY:
-                               tag_open.push_back(SF);
-                               break;
-                       case LyXFont::TYPEWRITER_FAMILY:
-                               tag_open.push_back(TT);
-                               break;
-                       default:
-                               break;
-                       }
-               }
-
-               if (font_old.series() != font.series()) {
-                       switch (series_type) {
-                       case LyXFont::BOLD_SERIES:
-                               tag_close |= BF;
-                               break;
-                       default:
-                               break;
-                       }
-
-                       series_type = font.series();
-
-                       switch (series_type) {
-                       case LyXFont::BOLD_SERIES:
-                               tag_open.push_back(BF);
-                               break;
-                       default:
-                               break;
-                       }
-
-               }
-
-               if (font_old.shape() != font.shape()) {
-                       switch (shape_type) {
-                       case LyXFont::ITALIC_SHAPE:
-                               tag_close |= IT;
-                               break;
-                       case LyXFont::SLANTED_SHAPE:
-                               tag_close |= SL;
-                               break;
-                       default:
-                               break;
-                       }
-
-                       shape_type = font.shape();
-
-                       switch (shape_type) {
-                       case LyXFont::ITALIC_SHAPE:
-                               tag_open.push_back(IT);
-                               break;
-                       case LyXFont::SLANTED_SHAPE:
-                               tag_open.push_back(SL);
-                               break;
-                       default:
-                               break;
-                       }
-               }
-               // handle <em> tag
-               if (font_old.emph() != font.emph()) {
-                       if (font.emph() == LyXFont::ON) {
-                               tag_open.push_back(EM);
-                               is_em = true;
-                       }
-                       else if (is_em) {
-                               tag_close |= EM;
-                               is_em = false;
-                       }
-               }
-
-               list < PAR_TAG > temp;
-               while (!tag_state.empty() && tag_close) {
-                       PAR_TAG k =  tag_state.top();
-                       tag_state.pop();
-                       os << "</" << tag_name(k) << '>';
-                       if (tag_close & k)
-                               reset(tag_close,k);
-                       else
-                               temp.push_back(k);
-               }
-
-               for(list< PAR_TAG >::const_iterator j = temp.begin();
-                   j != temp.end(); ++j) {
-                       tag_state.push(*j);
-                       os << '<' << tag_name(*j) << '>';
-               }
-
-               for(list< PAR_TAG >::const_iterator j = tag_open.begin();
-                   j != tag_open.end(); ++j) {
-                       tag_state.push(*j);
-                       os << '<' << tag_name(*j) << '>';
-               }
-
-               value_type c = getChar(i);
-
-
-               if (c == Paragraph::META_INSET) {
-                       getInset(i)->linuxdoc(buf, os, runparams);
-                       font_old = font;
-                       continue;
-               }
-
-               if (style->latexparam() == "CDATA") {
-                       // "TeX"-Mode on == > SGML-Mode on.
-                       if (c != '\0')
-                               os << c;
-                       ++char_line_count;
-               } else {
-                       bool ws;
-                       string str;
-                       boost::tie(ws, str) = sgml::escapeChar(c);
-                       if (ws && !isFreeSpacing()) {
-                               // in freespacing mode, spaces are
-                               // non-breaking characters
-                               if (desc_on) { // if char is ' ' then...
-                                       ++char_line_count;
-                                       sgmlLineBreak(os, char_line_count, 6);
-                                       os << "</tag>";
-                                       desc_on = false;
-                               } else  {
-                                       sgmlLineBreak(os, char_line_count, 1);
-                                       os << c;
-                               }
-                       } else {
-                               os << str;
-                               char_line_count += str.length();
-                       }
-               }
-               font_old = font;
-       }
-
-       while (!tag_state.empty()) {
-               os << "</" << tag_name(tag_state.top()) << '>';
-               tag_state.pop();
-       }
-
-       // resets description flag correctly
-       if (desc_on) {
-               // <tag> not closed...
-               sgmlLineBreak(os, char_line_count, 6);
-               os << "</tag>";
-       }
-}
-
-
 bool Paragraph::emptyTag() const
 {
        for (pos_type i = 0; i < size(); ++i) {
@@ -1364,7 +1157,7 @@ string Paragraph::getID(Buffer const & buf, OutputParams const & runparams) cons
                        InsetBase::Code lyx_code = inset->lyxCode();
                        if (lyx_code == InsetBase::LABEL_CODE) {
                                string const id = static_cast<InsetCommand const *>(inset)->getContents();
-                               return "id=\"" + sgml::cleanID(buf, runparams, id) + "\"";
+                               return "id='" + to_utf8(sgml::cleanID(buf, runparams, from_utf8(id))) + "'";
                        }
                }
 
@@ -1373,7 +1166,7 @@ string Paragraph::getID(Buffer const & buf, OutputParams const & runparams) cons
 }
 
 
-pos_type Paragraph::getFirstWord(Buffer const & buf, ostream & os, OutputParams const & runparams) const
+pos_type Paragraph::getFirstWord(Buffer const & buf, odocstream & os, OutputParams const & runparams) const
 {
        pos_type i;
        for (i = 0; i < size(); ++i) {
@@ -1384,12 +1177,8 @@ pos_type Paragraph::getFirstWord(Buffer const & buf, ostream & os, OutputParams
                        value_type c = getChar(i);
                        if (c == ' ')
                                break;
-                       bool ws;
-                       string str;
-                       boost::tie(ws, str) = sgml::escapeChar(c);
-
-                       os << str;
-               }
+                       os << sgml::escapeChar(c);
+               }
        }
        return i;
 }
@@ -1413,7 +1202,7 @@ bool Paragraph::onlyText(Buffer const & buf, LyXFont const & outerfont, pos_type
 
 
 void Paragraph::simpleDocBookOnePar(Buffer const & buf,
-                                   ostream & os,
+                                   odocstream & os,
                                    OutputParams const & runparams,
                                    LyXFont const & outerfont,
                                    pos_type initial) const
@@ -1447,14 +1236,11 @@ void Paragraph::simpleDocBookOnePar(Buffer const & buf,
                        inset->docbook(buf, os, runparams);
                } else {
                        value_type c = getChar(i);
-                       bool ws;
-                       string str;
-                       boost::tie(ws, str) = sgml::escapeChar(c);
 
                        if (style->pass_thru)
-                               os << c;
+                                os.put(c);
                        else
-                               os << str;
+                                os << sgml::escapeChar(c);
                }
                font_old = font;
        }
@@ -1470,26 +1256,6 @@ void Paragraph::simpleDocBookOnePar(Buffer const & buf,
 }
 
 
-namespace {
-
-/// return true if the char is a meta-character for an inset
-inline
-bool IsInsetChar(char c)
-{
-       return (c == Paragraph::META_INSET);
-}
-
-} // namespace anon
-
-
-
-bool Paragraph::isHfill(pos_type pos) const
-{
-       return isInset(pos)
-               && getInset(pos)->lyxCode() == InsetBase::HFILL_CODE;
-}
-
-
 bool Paragraph::isNewline(pos_type pos) const
 {
        return isInset(pos)
@@ -1497,17 +1263,11 @@ bool Paragraph::isNewline(pos_type pos) const
 }
 
 
-bool Paragraph::isSeparator(pos_type pos) const
-{
-       return IsSeparatorChar(getChar(pos));
-}
-
-
 bool Paragraph::isLineSeparator(pos_type pos) const
 {
        value_type const c = getChar(pos);
-       return IsLineSeparatorChar(c)
-               || (IsInsetChar(c) && getInset(pos) &&
+       return isLineSeparatorChar(c)
+               || (c == Paragraph::META_INSET && getInset(pos) &&
                getInset(pos)->isLineSeparator());
 }
 
@@ -1519,7 +1279,7 @@ bool Paragraph::isLetter(pos_type pos) const
                return getInset(pos)->isLetter();
        else {
                value_type const c = getChar(pos);
-               return IsLetterChar(c) || IsDigit(c);
+               return isLetterChar(c) || isDigit(c);
        }
 }
 
@@ -1528,7 +1288,7 @@ Language const *
 Paragraph::getParLanguage(BufferParams const & bparams) const
 {
        if (!empty())
-               return getFirstFontSettings().language();
+               return getFirstFontSettings(bparams).language();
 #ifdef WITH_WARNINGS
 #warning FIXME we should check the prev par as well (Lgb)
 #endif
@@ -1539,7 +1299,7 @@ Paragraph::getParLanguage(BufferParams const & bparams) const
 bool Paragraph::isRightToLeftPar(BufferParams const & bparams) const
 {
        return lyxrc.rtl_support
-               && getParLanguage(bparams)->RightToLeft()
+               && getParLanguage(bparams)->rightToLeft()
                && ownerCode() != InsetBase::ERT_CODE;
 }
 
@@ -1547,7 +1307,8 @@ bool Paragraph::isRightToLeftPar(BufferParams const & bparams) const
 void Paragraph::changeLanguage(BufferParams const & bparams,
                               Language const * from, Language const * to)
 {
-       for (pos_type i = 0; i < size(); ++i) {
+       // change language including dummy font change at the end
+       for (pos_type i = 0; i <= size(); ++i) {
                LyXFont font = getFontSettings(bparams, i);
                if (font.language() == from) {
                        font.setLanguage(to);
@@ -1574,14 +1335,14 @@ bool Paragraph::isMultiLingual(BufferParams const & bparams) const
 
 // Convert the paragraph to a string.
 // Used for building the table of contents
-string const Paragraph::asString(Buffer const & buffer, bool label) const
+docstring const Paragraph::asString(Buffer const & buffer, bool label) const
 {
        OutputParams runparams;
        return asString(buffer, runparams, label);
 }
 
 
-string const Paragraph::asString(Buffer const & buffer,
+docstring const Paragraph::asString(Buffer const & buffer,
                                 OutputParams const & runparams,
                                 bool label) const
 {
@@ -1592,7 +1353,7 @@ string const Paragraph::asString(Buffer const & buffer,
 
        for (pos_type i = 0; i < size(); ++i) {
                value_type c = getChar(i);
-               if (IsPrintable(c))
+               if (isPrintable(c))
                        s += c;
                else if (c == META_INSET &&
                         getInset(i)->lyxCode() == InsetBase::MATH_CODE) {
@@ -1605,13 +1366,13 @@ string const Paragraph::asString(Buffer const & buffer,
        return s;
 #else
        // This should really be done by the caller and not here.
-       string ret = asString(buffer, runparams, 0, size(), label);
+       docstring ret = asString(buffer, runparams, 0, size(), label);
        return subst(ret, '\n', ' ');
 #endif
 }
 
 
-string const Paragraph::asString(Buffer const & buffer,
+docstring const Paragraph::asString(Buffer const & buffer,
                                 pos_type beg, pos_type end, bool label) const
 {
 
@@ -1620,104 +1381,67 @@ string const Paragraph::asString(Buffer const & buffer,
 }
 
 
-string const Paragraph::asString(Buffer const & buffer,
+docstring const Paragraph::asString(Buffer const & buffer,
                                 OutputParams const & runparams,
                                 pos_type beg, pos_type end, bool label) const
 {
-       ostringstream os;
+       lyx::odocstringstream os;
 
        if (beg == 0 && label && !params().labelString().empty())
                os << params().labelString() << ' ';
 
        for (pos_type i = beg; i < end; ++i) {
                value_type const c = getUChar(buffer.params(), i);
-               if (IsPrintable(c))
-                       os << c;
+               if (isPrintable(c))
+                       os.put(c);
                else if (c == META_INSET)
-                       getInset(i)->plaintext(buffer, os, runparams);
+                       getInset(i)->textString(buffer, os, runparams);
        }
 
        return os.str();
 }
 
 
-void Paragraph::setInsetOwner(UpdatableInset * inset)
+void Paragraph::setInsetOwner(InsetBase * inset)
 {
        pimpl_->inset_owner = inset;
 }
 
 
-void Paragraph::setContentsFromPar(Paragraph const & par)
+Change const Paragraph::lookupChange(pos_type pos) const
 {
-       pimpl_->setContentsFromPar(par);
-}
-
-
-void Paragraph::trackChanges(Change::Type type)
-{
-       pimpl_->trackChanges(type);
-}
-
-
-void Paragraph::untrackChanges()
-{
-       pimpl_->untrackChanges();
-}
-
-
-void Paragraph::cleanChanges()
-{
-       pimpl_->cleanChanges();
-}
-
-
-Change::Type Paragraph::lookupChange(lyx::pos_type pos) const
-{
-       BOOST_ASSERT(empty() || pos < size());
+       BOOST_ASSERT(pos <= size());
        return pimpl_->lookupChange(pos);
 }
 
 
-Change const Paragraph::lookupChangeFull(lyx::pos_type pos) const
-{
-       BOOST_ASSERT(empty() || pos < size());
-       return pimpl_->lookupChangeFull(pos);
-}
-
-
 bool Paragraph::isChanged(pos_type start, pos_type end) const
 {
        return pimpl_->isChanged(start, end);
 }
 
 
-bool Paragraph::isChangeEdited(pos_type start, pos_type end) const
+void Paragraph::setChange(Change const & change)
 {
-       return pimpl_->isChangeEdited(start, end);
+       pimpl_->setChange(change);
 }
 
 
-void Paragraph::setChange(lyx::pos_type pos, Change::Type type)
+void Paragraph::setChange(pos_type pos, Change const & change)
 {
-       pimpl_->setChange(pos, type);
+       pimpl_->setChange(pos, change);
 }
 
 
-void Paragraph::markErased()
+void Paragraph::acceptChanges(pos_type start, pos_type end)
 {
-       pimpl_->markErased();
+       return pimpl_->acceptChanges(start, end);
 }
 
 
-void Paragraph::acceptChange(pos_type start, pos_type end)
+void Paragraph::rejectChanges(pos_type start, pos_type end)
 {
-       return pimpl_->acceptChange(start, end);
-}
-
-
-void Paragraph::rejectChange(pos_type start, pos_type end)
-{
-       return pimpl_->rejectChange(start, end);
+       return pimpl_->rejectChanges(start, end);
 }
 
 
@@ -1739,7 +1463,7 @@ void Paragraph::layout(LyXLayout_ptr const & new_layout)
 }
 
 
-UpdatableInset * Paragraph::inInset() const
+InsetBase * Paragraph::inInset() const
 {
        return pimpl_->inset_owner;
 }
@@ -1758,12 +1482,6 @@ void Paragraph::clearContents()
 }
 
 
-void Paragraph::setChar(pos_type pos, value_type c)
-{
-       text_[pos] = c;
-}
-
-
 ParagraphParameters & Paragraph::params()
 {
        return pimpl_->params;
@@ -1795,10 +1513,15 @@ bool Paragraph::allowEmpty() const
 }
 
 
-Row & Paragraph::getRow(pos_type pos)
+Row & Paragraph::getRow(pos_type pos, bool boundary)
 {
        BOOST_ASSERT(!rows().empty());
 
+       // If boundary is set we should return the row on which
+       // the character before is inside.
+       if (pos > 0 && boundary)
+               --pos;
+
        RowList::iterator rit = rows_.end();
        RowList::iterator const begin = rows_.begin();
 
@@ -1809,10 +1532,15 @@ Row & Paragraph::getRow(pos_type pos)
 }
 
 
-Row const & Paragraph::getRow(pos_type pos) const
+Row const & Paragraph::getRow(pos_type pos, bool boundary) const
 {
        BOOST_ASSERT(!rows().empty());
 
+       // If boundary is set we should return the row on which
+       // the character before is inside.
+       if (pos > 0 && boundary)
+               --pos;
+
        RowList::const_iterator rit = rows_.end();
        RowList::const_iterator const begin = rows_.begin();
 
@@ -1837,20 +1565,21 @@ size_t Paragraph::pos2row(pos_type pos) const
 }
 
 
-unsigned char Paragraph::transformChar(unsigned char c, pos_type pos) const
+char_type Paragraph::transformChar(char_type c, pos_type pos) const
 {
        if (!Encodings::is_arabic(c))
-               if (lyxrc.font_norm_type == LyXRC::ISO_8859_6_8 && IsDigit(c))
+               if (lyxrc.font_norm_type == LyXRC::ISO_8859_6_8 && isDigit(c))
+                       // FIXME UNICODE What does this do?
                        return c + (0xb0 - '0');
                else
                        return c;
 
-       unsigned char const prev_char = pos > 0 ? getChar(pos - 1) : ' ';
-       unsigned char next_char = ' ';
+       value_type const prev_char = pos > 0 ? getChar(pos - 1) : ' ';
+       value_type next_char = ' ';
 
        for (pos_type i = pos + 1, end = size(); i < end; ++i) {
-               unsigned char const par_char = getChar(i);
-               if (!Encodings::IsComposeChar_arabic(par_char)) {
+               value_type const par_char = getChar(i);
+               if (!Encodings::isComposeChar_arabic(par_char)) {
                        next_char = par_char;
                        break;
                }
@@ -1859,15 +1588,15 @@ unsigned char Paragraph::transformChar(unsigned char c, pos_type pos) const
        if (Encodings::is_arabic(next_char)) {
                if (Encodings::is_arabic(prev_char) &&
                        !Encodings::is_arabic_special(prev_char))
-                       return Encodings::TransformChar(c, Encodings::FORM_MEDIAL);
+                       return Encodings::transformChar(c, Encodings::FORM_MEDIAL);
                else
-                       return Encodings::TransformChar(c, Encodings::FORM_INITIAL);
+                       return Encodings::transformChar(c, Encodings::FORM_INITIAL);
        } else {
                if (Encodings::is_arabic(prev_char) &&
                        !Encodings::is_arabic_special(prev_char))
-                       return Encodings::TransformChar(c, Encodings::FORM_FINAL);
+                       return Encodings::transformChar(c, Encodings::FORM_FINAL);
                else
-                       return Encodings::TransformChar(c, Encodings::FORM_ISOLATED);
+                       return Encodings::transformChar(c, Encodings::FORM_ISOLATED);
        }
 }
 
@@ -1880,3 +1609,38 @@ void Paragraph::dump() const
                rows_[i].dump();
        }
 }
+
+
+bool Paragraph::hfillExpansion(Row const & row, pos_type pos) const
+{
+       if (!isHfill(pos))
+               return false;
+
+       // at the end of a row it does not count
+       // unless another hfill exists on the line
+       if (pos >= row.endpos()) {
+               for (pos_type i = row.pos(); i < pos && !isHfill(i); ++i)
+                       return false;
+       }
+
+       // at the beginning of a row it does not count, if it is not
+       // the first row of a paragaph
+       if (row.pos() == 0)
+               return true;
+
+       // in some labels it does not count
+       if (layout()->margintype != MARGIN_MANUAL && pos < beginOfBody())
+               return false;
+
+       // if there is anything between the first char of the row and
+       // the specified position that is not a newline and not a hfill,
+       // the hfill will count, otherwise not
+       pos_type i = row.pos();
+       while (i < pos && (isNewline(i) || isHfill(i)))
+               ++i;
+
+       return i != pos;
+}
+
+
+} // namespace lyx