X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2FParagraph.cpp;h=e497a85ae60e3672baa1426c839727d9854ee8ed;hb=28be7d552f62cc02fa86d7f79201d089bfb2d7b5;hp=0570753fa9788a042bff406a175af3ebd48800d7;hpb=b627b8701b6598f5402cbab55a83026c9e3b1e86;p=lyx.git

diff --git a/src/Paragraph.cpp b/src/Paragraph.cpp
index 0570753fa9..e497a85ae6 100644
--- a/src/Paragraph.cpp
+++ b/src/Paragraph.cpp
@@ -40,6 +40,7 @@
 #include "ParagraphParameters.h"
 #include "SpellChecker.h"
 #include "sgml.h"
+#include "texstream.h"
 #include "TextClass.h"
 #include "TexRow.h"
 #include "Text.h"
@@ -51,6 +52,9 @@
 #include "insets/InsetBibitem.h"
 #include "insets/InsetLabel.h"
 #include "insets/InsetSpecialChar.h"
+#include "insets/InsetText.h"
+
+#include "mathed/InsetMathHull.h"
 
 #include "support/debug.h"
 #include "support/docstring_list.h"
@@ -60,12 +64,22 @@
 #include "support/lstrings.h"
 #include "support/textutils.h"
 
+#include <atomic>
 #include <sstream>
 #include <vector>
 
 using namespace std;
 using namespace lyx::support;
 
+// OSX clang, gcc < 4.8.0, and msvc < 2015 do not support C++11 thread_local
+#if defined(__APPLE__) || (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 8)
+#define THREAD_LOCAL_STATIC static __thread
+#elif defined(_MSC_VER) && (_MSC_VER < 1900)
+#define THREAD_LOCAL_STATIC static __declspec(thread)
+#else
+#define THREAD_LOCAL_STATIC thread_local static
+#endif
+
 namespace lyx {
 
 namespace {
@@ -282,10 +296,11 @@ private:
 
 class Paragraph::Private
 {
-	// Enforce our own "copy" constructor by declaring the standard one and
-	// the assignment operator private without implementing them.
-	Private(Private const &);
-	Private & operator=(Private const &);
+	// Enforce our own "copy" constructor
+	Private(Private const &) = delete;
+	Private & operator=(Private const &) = delete;
+	// Unique ID generator
+	static int make_id();
 public:
 	///
 	Private(Paragraph * owner, Layout const & layout);
@@ -358,24 +373,17 @@ public:
 		pos_type i,
 		unsigned int & column);
 	///
-	bool latexSpecialT3(
+	bool latexSpecialTU(
 		char_type const c,
 		otexstream & os,
 		pos_type i,
 		unsigned int & column);
 	///
-	bool latexSpecialTypewriter(
+	bool latexSpecialT3(
 		char_type const c,
 		otexstream & os,
 		pos_type i,
 		unsigned int & column);
-	///
-	bool latexSpecialPhrase(
-		otexstream & os,
-		pos_type & i,
-		pos_type end_pos,
-		unsigned int & column,
-		OutputParams const & runparams);
 
 	///
 	void validate(LaTeXFeatures & features) const;
@@ -384,9 +392,6 @@ public:
 	bool onlyText(Buffer const & buf, Font const & outerfont,
 		      pos_type initial) const;
 
-	/// match a string against a particular point in the paragraph
-	bool isTextAt(string const & str, pos_type pos) const;
-
 	/// a vector of speller skip positions
 	typedef vector<FontSpan> SkipPositions;
 	typedef SkipPositions::const_iterator SkipPositionsIterator;
@@ -514,26 +519,6 @@ public:
 };
 
 
-namespace {
-
-struct special_phrase {
-	string phrase;
-	docstring macro;
-	bool builtin;
-};
-
-special_phrase const special_phrases[] = {
-	{ "LyX", from_ascii("\\LyX{}"), false },
-	{ "TeX", from_ascii("\\TeX{}"), true },
-	{ "LaTeX2e", from_ascii("\\LaTeXe{}"), true },
-	{ "LaTeX", from_ascii("\\LaTeX{}"), true },
-};
-
-size_t const phrases_nr = sizeof(special_phrases)/sizeof(special_phrase);
-
-} // namespace anon
-
-
 Paragraph::Private::Private(Paragraph * owner, Layout const & layout)
 	: owner_(owner), inset_owner_(0), id_(-1), begin_of_body_(0), layout_(&layout)
 {
@@ -541,33 +526,37 @@ Paragraph::Private::Private(Paragraph * owner, Layout const & layout)
 }
 
 
-// Initialization of the counter for the paragraph id's,
-//
-// FIXME: There should be a more intelligent way to generate and use the
-// paragraph ids per buffer instead a global static counter for all InsetText
-// in the running program.
-static int paragraph_id = -1;
+//static
+int Paragraph::Private::make_id()
+{
+	// The id is unique per session across buffers because it is used in
+	// LFUN_PARAGRAPH_GOTO to switch to a different buffer, for instance in the
+	// outliner.
+	// (thread-safe)
+	static atomic_uint next_id(0);
+	return next_id++;
+}
+
 
 Paragraph::Private::Private(Private const & p, Paragraph * owner)
 	: owner_(owner), inset_owner_(p.inset_owner_), fontlist_(p.fontlist_),
+	  id_(make_id()),
 	  params_(p.params_), changes_(p.changes_), insetlist_(p.insetlist_),
 	  begin_of_body_(p.begin_of_body_), text_(p.text_), words_(p.words_),
 	  layout_(p.layout_)
 {
-	id_ = ++paragraph_id;
 	requestSpellCheck(p.text_.size());
 }
 
 
 Paragraph::Private::Private(Private const & p, Paragraph * owner,
 	pos_type beg, pos_type end)
-	: owner_(owner), inset_owner_(p.inset_owner_),
+	: owner_(owner), inset_owner_(p.inset_owner_), id_(make_id()),
 	  params_(p.params_), changes_(p.changes_),
 	  insetlist_(p.insetlist_, beg, end),
 	  begin_of_body_(p.begin_of_body_), words_(p.words_),
 	  layout_(p.layout_)
 {
-	id_ = ++paragraph_id;
 	if (beg >= pos_type(p.text_.size()))
 		return;
 	text_ = p.text_.substr(beg, end - beg);
@@ -589,10 +578,22 @@ Paragraph::Private::Private(Private const & p, Paragraph * owner,
 }
 
 
-void Paragraph::addChangesToToc(DocIterator const & cdit,
-	Buffer const & buf, bool output_active) const
+void Paragraph::addChangesToToc(DocIterator const & cdit, Buffer const & buf,
+                                bool output_active, TocBackend & backend) const
+{
+	d->changes_.addToToc(cdit, buf, output_active, backend);
+}
+
+
+void Paragraph::addChangesToBuffer(Buffer const & buf) const
+{
+	d->changes_.updateBuffer(buf);
+}
+
+
+bool Paragraph::isChangeUpdateRequired() const
 {
-	d->changes_.addToToc(cdit, buf, output_active);
+	return d->changes_.isUpdateRequired();
 }
 
 
@@ -951,10 +952,13 @@ int Paragraph::Private::writeScriptChars(otexstream & os,
 {
 	// FIXME: modifying i here is not very nice...
 
-	// We only arrive here when a proper language for character text_[i] has
-	// not been specified (i.e., it could not be translated in the current
-	// latex encoding) or its latex translation has been forced, and it
-	// belongs to a known script.
+	// We only arrive here when character text_[i] could not be translated
+	// into the current latex encoding (or its latex translation has been forced,)
+	// and it belongs to a known script.
+	// TODO: We need \textcyr and \textgreek wrappers also for characters
+	//       that can be encoded in the "LaTeX encoding" but not in the
+	//       current *font encoding*.
+	//       (See #9681 for details and test)
 	// Parameter ltx contains the latex translation of text_[i] as specified
 	// in the unicodesymbols file and is something like "\textXXX{<spec>}".
 	// The latex macro name "textXXX" specifies the script to which text_[i]
@@ -970,6 +974,10 @@ int Paragraph::Private::writeScriptChars(otexstream & os,
 	bool closing_brace = true;
 	if (script == "textgreek" && encoding.latexName() == "iso-8859-7") {
 		// Correct encoding is being used, so we can avoid \textgreek.
+		// TODO: wrong test: we need to check the *font encoding*
+		//       (i.e. the active language and its FontEncoding tag)
+	        // 	 instead of the LaTeX *input encoding*!
+		// 	 See #9637 for details and test-cases.
 		pos = brace1 + 1;
 		length -= pos;
 		closing_brace = false;
@@ -1016,26 +1024,6 @@ int Paragraph::Private::writeScriptChars(otexstream & os,
 }
 
 
-bool Paragraph::Private::isTextAt(string const & str, pos_type pos) const
-{
-	pos_type const len = str.length();
-
-	// is the paragraph large enough?
-	if (pos + len > int(text_.size()))
-		return false;
-
-	// does the wanted text start at point?
-	for (string::size_type i = 0; i < str.length(); ++i) {
-		// Caution: direct comparison of characters works only
-		// because str is pure ASCII.
-		if (str[i] != text_[pos + i])
-			return false;
-	}
-
-	return fontlist_.hasChangeInRange(pos, len);
-}
-
-
 void Paragraph::Private::latexInset(BufferParams const & bparams,
 				    otexstream & os,
 				    OutputParams & runparams,
@@ -1066,9 +1054,10 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 			os << '\n';
 		} else {
 			if (open_font) {
+				bool needPar = false;
 				column += running_font.latexWriteEndChanges(
 					os, bparams, runparams,
-					basefont, basefont);
+					basefont, basefont, needPar);
 				open_font = false;
 			}
 
@@ -1107,7 +1096,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 	    // decorations at all
 	    && inset->lyxCode() != ERT_CODE) {
 		if (running_font.language()->lang() == "farsi")
-			os << "\\beginL{}";
+			os << "\\beginL" << termcmd;
 		else
 			os << "\\L{";
 		close = true;
@@ -1126,10 +1115,12 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 	bool arabtex = basefont.language()->lang() == "arabic_arabtex"
 		|| running_font.language()->lang() == "arabic_arabtex";
 	if (open_font && !inset->inheritFont()) {
+		bool needPar = false;
 		bool closeLanguage = arabtex
 			|| basefont.isRightToLeft() == running_font.isRightToLeft();
 		unsigned int count = running_font.latexWriteEndChanges(os,
-			bparams, runparams, basefont, basefont, closeLanguage);
+					bparams, runparams, basefont, basefont,
+					needPar, closeLanguage);
 		column += count;
 		// if any font properties were closed, update the running_font,
 		// making sure, however, to leave the language as it was
@@ -1148,7 +1139,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 		}
 	}
 
-	int prev_rows = os.texrow().rows();
+	size_t const previous_row_count = os.texrow().rows();
 
 	try {
 		runparams.lastid = id_;
@@ -1163,12 +1154,12 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 
 	if (close) {
 		if (running_font.language()->lang() == "farsi")
-				os << "\\endL{}";
+				os << "\\endL" << termcmd;
 			else
 				os << '}';
 	}
 
-	if (os.texrow().rows() > prev_rows) {
+	if (os.texrow().rows() > previous_row_count) {
 		os.texrow().start(owner_->id(), i + 1);
 		column = 0;
 	} else {
@@ -1195,7 +1186,9 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 	char_type const c = (runparams.use_polyglossia) ?
 		owner_->getUChar(bparams, i) : text_[i];
 
-	if (style.pass_thru || runparams.pass_thru) {
+	if (style.pass_thru || runparams.pass_thru
+	    || contains(style.pass_thru_chars, c)
+	    || contains(runparams.pass_thru_chars, c)) {
 		if (c != '\0') {
 			Encoding const * const enc = runparams.encoding;
 			if (enc && !enc->encodable(c))
@@ -1210,41 +1203,51 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 		return;
 	// If T1 font encoding is used, use the special
 	// characters it provides.
-	// NOTE: some languages reset the font encoding
-	// internally
+	// NOTE: Some languages reset the font encoding internally to a
+	//       non-standard font encoding. If we are using such a language,
+	//       we do not output special T1 chars.
 	if (!runparams.inIPA && !running_font.language()->internalFontEncoding()
-	    && lyxrc.fontenc == "T1" && latexSpecialT1(c, os, i, column))
-		return;
-
-	// \tt font needs special treatment
-	if (!runparams.inIPA
-	     && running_font.fontInfo().family() == TYPEWRITER_FAMILY
-	     && latexSpecialTypewriter(c, os, i, column))
+	    && !runparams.isFullUnicode() && bparams.main_font_encoding() == "T1"
+	    && latexSpecialT1(c, os, i, column))
 		return;
+	// NOTE: XeTeX and LuaTeX use EU1/2 (pre 2017) or TU (as of 2017) encoding
+	else if (!runparams.inIPA && !running_font.language()->internalFontEncoding()
+		 && runparams.isFullUnicode() && latexSpecialTU(c, os, i, column))
+		     return;
 
 	// Otherwise, we use what LaTeX provides us.
 	switch (c) {
 	case '\\':
-		os << "\\textbackslash{}";
+		os << "\\textbackslash" << termcmd;
 		column += 15;
 		break;
 	case '<':
-		os << "\\textless{}";
+		os << "\\textless" << termcmd;
 		column += 10;
 		break;
 	case '>':
-		os << "\\textgreater{}";
+		os << "\\textgreater" << termcmd;
 		column += 13;
 		break;
 	case '|':
-		os << "\\textbar{}";
+		os << "\\textbar" << termcmd;
 		column += 9;
 		break;
 	case '-':
 		os << '-';
+		if (i + 1 < static_cast<pos_type>(text_.size()) &&
+		    (end_pos == -1 || i + 1 < end_pos) &&
+		    text_[i+1] == '-') {
+			// Prevent "--" becoming an endash and "---" becoming
+			// an emdash.
+			// Within \ttfamily, "--" is merged to "-" (no endash)
+			// so we avoid this rather irritating ligature as well
+			os << "{}";
+			column += 2;
+		}
 		break;
 	case '\"':
-		os << "\\char`\\\"{}";
+		os << "\\char34" << termcmd;
 		column += 9;
 		break;
 
@@ -1257,12 +1260,12 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 		break;
 
 	case '~':
-		os << "\\textasciitilde{}";
+		os << "\\textasciitilde" << termcmd;
 		column += 16;
 		break;
 
 	case '^':
-		os << "\\textasciicircum{}";
+		os << "\\textasciicircum" << termcmd;
 		column += 17;
 		break;
 
@@ -1284,11 +1287,22 @@ void Paragraph::Private::latexSpecialChar(otexstream & os,
 		// written. (Asger)
 		break;
 
+	case 0x2013:
+	case 0x2014:
+		if (bparams.use_dash_ligatures && !bparams.useNonTeXFonts) {
+			if (c == 0x2013) {
+				// en-dash
+				os << "--";
+				column +=2;
+			} else {
+				// em-dash
+				os << "---";
+				column +=3;
+			}
+			break;
+		}
+		// fall through
 	default:
-		// LyX, LaTeX etc.
-		if (latexSpecialPhrase(os, i, end_pos, column, runparams))
-			return;
-
 		if (c == '\0')
 			return;
 
@@ -1364,7 +1378,7 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os,
 		// but we should avoid ligatures
 		if (i + 1 >= int(text_.size()) || text_[i + 1] != c)
 			return true;
-		os << "\\textcompwordmark{}";
+		os << "\\textcompwordmark" << termcmd;
 		column += 19;
 		return true;
 	case '|':
@@ -1372,7 +1386,7 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os,
 		return true;
 	case '\"':
 		// soul.sty breaks with \char`\"
-		os << "\\textquotedbl{}";
+		os << "\\textquotedbl" << termcmd;
 		column += 14;
 		return true;
 	default:
@@ -1381,6 +1395,14 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os,
 }
 
 
+bool Paragraph::Private::latexSpecialTU(char_type const c, otexstream & os,
+	pos_type i, unsigned int & column)
+{
+	// TU encoding is currently on par with T1.
+	return latexSpecialT1(c, os, i, column);
+}
+
+
 bool Paragraph::Private::latexSpecialT3(char_type const c, otexstream & os,
 	pos_type /*i*/, unsigned int & column)
 {
@@ -1392,7 +1414,7 @@ bool Paragraph::Private::latexSpecialT3(char_type const c, otexstream & os,
 		os.put(c);
 		return true;
 	case '|':
-		os << "\\textvertline{}";
+		os << "\\textvertline" << termcmd;
 		column += 14;
 		return true;
 	default:
@@ -1401,55 +1423,6 @@ bool Paragraph::Private::latexSpecialT3(char_type const c, otexstream & os,
 }
 
 
-bool Paragraph::Private::latexSpecialTypewriter(char_type const c, otexstream & os,
-	pos_type i, unsigned int & column)
-{
-	switch (c) {
-	case '-':
-		// within \ttfamily, "--" is merged to "-" (no endash)
-		// so we avoid this rather irritating ligature
-		if (i + 1 < int(text_.size()) && text_[i + 1] == '-') {
-			os << "-{}";
-			column += 2;
-		} else
-			os << '-';
-		return true;
-
-	// everything else has to be checked separately
-	// (depending on the encoding)
-	default:
-		return false;
-	}
-}
-
-
-/// \param end_pos
-///   If [start_pos, end_pos) does not include entirely the special phrase, then
-///   do not apply the macro transformation.
-bool Paragraph::Private::latexSpecialPhrase(otexstream & os, pos_type & i, pos_type end_pos,
-	unsigned int & column, OutputParams const & runparams)
-{
-	// FIXME: if we have "LaTeX" with a font
-	// change in the middle (before the 'T', then
-	// the "TeX" part is still special cased.
-	// Really we should only operate this on
-	// "words" for some definition of word
-
-	for (size_t pnr = 0; pnr < phrases_nr; ++pnr) {
-		if (!isTextAt(special_phrases[pnr].phrase, i)
-		    || (end_pos != -1 && i + int(special_phrases[pnr].phrase.size()) > end_pos))
-			continue;
-		if (runparams.moving_arg)
-			os << "\\protect";
-		os << special_phrases[pnr].macro;
-		i += special_phrases[pnr].phrase.length() - 1;
-		column += special_phrases[pnr].macro.length() - 1;
-		return true;
-	}
-	return false;
-}
-
-
 void Paragraph::Private::validate(LaTeXFeatures & features) const
 {
 	if (layout_->inpreamble && inset_owner_) {
@@ -1458,13 +1431,12 @@ void Paragraph::Private::validate(LaTeXFeatures & features) const
 		BufferParams const & bp = features.runparams().is_child
 			? buf.masterParams() : buf.params();
 		Font f;
-		TexRow texrow;
 		// Using a string stream here circumvents the encoding
 		// switching machinery of odocstream. Therefore the
 		// output is wrong if this paragraph contains content
 		// that needs to switch encoding.
-		odocstringstream ods;
-		otexstream os(ods, texrow);
+		otexstringstream os;
+		os << layout_->preamble();
 		if (is_command) {
 			os << '\\' << from_ascii(layout_->latexname());
 			// we have to provide all the optional arguments here, even though
@@ -1477,20 +1449,19 @@ void Paragraph::Private::validate(LaTeXFeatures & features) const
 			}
 			os << from_ascii(layout_->latexparam());
 		}
-		docstring::size_type const length = ods.str().length();
+		size_t const length = os.length();
 		// this will output "{" at the beginning, but not at the end
 		owner_->latex(bp, f, os, features.runparams(), 0, -1, true);
-		if (ods.str().length() > length) {
+		if (os.length() > length) {
 			if (is_command) {
-				ods << '}';
+				os << '}';
 				if (!layout_->postcommandargs().empty()) {
 					OutputParams rp = features.runparams();
 					rp.local_font = &owner_->getFirstFontSettings(bp);
 					latexArgInsets(*owner_, os, rp, layout_->postcommandargs(), "post:");
 				}
 			}
-			string const snippet = to_utf8(ods.str());
-			features.addPreambleSnippet(snippet);
+			features.addPreambleSnippet(os.release(), true);
 		}
 	}
 
@@ -1518,22 +1489,17 @@ void Paragraph::Private::validate(LaTeXFeatures & features) const
 	InsetList::const_iterator iend = insetlist_.end();
 	for (; icit != iend; ++icit) {
 		if (icit->inset) {
+			features.inDeletedInset(owner_->isDeleted(icit->pos));
 			icit->inset->validate(features);
+			features.inDeletedInset(false);
 			if (layout_->needprotect &&
 			    icit->inset->lyxCode() == FOOT_CODE)
-				features.require("footmisc");
+				features.require("NeedLyXFootnoteCode");
 		}
 	}
 
 	// then the contents
 	for (pos_type i = 0; i < int(text_.size()) ; ++i) {
-		for (size_t pnr = 0; pnr < phrases_nr; ++pnr) {
-			if (!special_phrases[pnr].builtin
-			    && isTextAt(special_phrases[pnr].phrase, i)) {
-				features.require(special_phrases[pnr].phrase);
-				break;
-			}
-		}
 		BufferEncodings::validate(text_[i], features);
 	}
 }
@@ -1833,7 +1799,10 @@ Font const & Paragraph::getFontSettings(BufferParams const & bparams,
 
 FontSpan Paragraph::fontSpan(pos_type pos) const
 {
-	LBUFERR(pos < size());
+	LBUFERR(pos <= size());
+
+	if (pos == size())
+		return FontSpan(pos, pos);
 
 	pos_type start = 0;
 	FontList::const_iterator cit = d->fontlist_.begin();
@@ -1922,14 +1891,6 @@ Font const Paragraph::getLayoutFont
 }
 
 
-/// Returns the height of the highest font in range
-FontSize Paragraph::highestFontInRange
-	(pos_type startpos, pos_type endpos, FontSize def_size) const
-{
-	return d->fontlist_.highestInRange(startpos, endpos, def_size);
-}
-
-
 char_type Paragraph::getUChar(BufferParams const & bparams, pos_type pos) const
 {
 	char_type c = d->text_[pos];
@@ -2035,7 +1996,7 @@ depth_type Paragraph::getMaxDepthAfter() const
 }
 
 
-char Paragraph::getAlign() const
+LyXAlignment Paragraph::getAlign() const
 {
 	if (d->params_.align() == LYX_ALIGN_LAYOUT)
 		return d->layout_->align;
@@ -2101,7 +2062,7 @@ docstring Paragraph::expandParagraphLabel(Layout const & layout,
 			docstring parent(fmt, i + 1, j - i - 1);
 			docstring label = from_ascii("??");
 			if (tclass.hasLayout(parent))
-				docstring label = expandParagraphLabel(tclass[parent], bparams,
+				label = expandParagraphLabel(tclass[parent], bparams,
 						      process_appendix);
 			fmt = docstring(fmt, 0, i) + label
 				+ docstring(fmt, j + 1, docstring::npos);
@@ -2145,14 +2106,12 @@ void Paragraph::setBeginOfBody()
 	pos_type end = size();
 	if (i < end && !(isNewline(i) || isEnvSeparator(i))) {
 		++i;
-		char_type previous_char = 0;
-		char_type temp = 0;
 		if (i < end) {
-			previous_char = d->text_[i];
+			char_type previous_char = d->text_[i];
 			if (!(isNewline(i) || isEnvSeparator(i))) {
 				++i;
 				while (i < end && previous_char != ' ') {
-					temp = d->text_[i];
+					char_type temp = d->text_[i];
 					if (isNewline(i) || isEnvSeparator(i))
 						break;
 					++i;
@@ -2249,8 +2208,12 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
 {
 	int column = 0;
 
-	if (params_.noindent() && !layout_->pass_thru
-	    && (layout_->toggle_indent != ITOGGLE_NEVER)) {
+	bool canindent =
+		(bparams.paragraph_separation == BufferParams::ParagraphIndentSeparation) ?
+			(layout_->toggle_indent != ITOGGLE_NEVER) :
+			(layout_->toggle_indent == ITOGGLE_ALWAYS);
+
+	if (canindent && params_.noindent() && !layout_->pass_thru) {
 		os << "\\noindent ";
 		column += 10;
 	}
@@ -2295,7 +2258,6 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
 			corrected_env(os, begin_tag, "flushright", code, lastpar, column);
 		break;
 	} case LYX_ALIGN_RIGHT: {
-		string output;
 		if (owner_->getParLanguage(bparams)->babel() != "hebrew")
 			corrected_env(os, begin_tag, "flushright", code, lastpar, column);
 		else
@@ -2439,14 +2401,19 @@ void Paragraph::latex(BufferParams const & bparams,
 			column += d->startTeXParParams(bparams, os, runparams);
 	}
 
+	// Whether a \par can be issued for insets typeset inline with text.
+	// Yes if greater than 0. This has to be static.
+	THREAD_LOCAL_STATIC int parInline = 0;
+
 	for (pos_type i = 0; i < size(); ++i) {
 		// First char in paragraph or after label?
 		if (i == body_pos) {
 			if (body_pos > 0) {
 				if (open_font) {
+					bool needPar = false;
 					column += running_font.latexWriteEndChanges(
 						os, bparams, runparams,
-						basefont, basefont);
+						basefont, basefont, needPar);
 					open_font = false;
 				}
 				basefont = getLayoutFont(bparams, outerfont);
@@ -2475,18 +2442,38 @@ void Paragraph::latex(BufferParams const & bparams,
 							    runparams);
 		}
 
+		runparams.wasDisplayMath = runparams.inDisplayMath;
+		runparams.inDisplayMath = false;
+		bool deleted_display_math = false;
+
+		// Check whether a display math inset follows
+		if (d->text_[i] == META_INSET
+		    && i >= start_pos && (end_pos == -1 || i < end_pos)) {
+			InsetMath const * im = getInset(i)->asInsetMath();
+			if (im && im->asHullInset()
+			    && im->asHullInset()->outerDisplay()) {
+				runparams.inDisplayMath = true;
+				// runparams.inDeletedInset will be set by
+				// latexInset later, but we need this info
+				// before it is called. On the other hand, we
+				// cannot set it here because it is a counter.
+				deleted_display_math = isDeleted(i);
+			}
+		}
+
 		Change const & change = runparams.inDeletedInset
 			? runparams.changeOfDeletedInset : lookupChange(i);
 
 		if (bparams.output_changes && runningChange != change) {
 			if (open_font) {
+				bool needPar = false;
 				column += running_font.latexWriteEndChanges(
-						os, bparams, runparams, basefont, basefont);
+						os, bparams, runparams,
+						basefont, basefont, needPar);
 				open_font = false;
 			}
 			basefont = getLayoutFont(bparams, outerfont);
 			running_font = basefont;
-
 			column += Changes::latexMarkChange(os, bparams, runningChange,
 							   change, runparams);
 			runningChange = change;
@@ -2501,18 +2488,20 @@ void Paragraph::latex(BufferParams const & bparams,
 		++column;
 
 		// Fully instantiated font
-		Font const font = getFont(bparams, i, outerfont);
+		Font const current_font = getFont(bparams, i, outerfont);
 
 		Font const last_font = running_font;
 
 		// Do we need to close the previous font?
 		if (open_font &&
-		    (font != running_font ||
-		     font.language() != running_font.language()))
+		    (current_font != running_font ||
+		     current_font.language() != running_font.language()))
 		{
+			bool needPar = false;
 			column += running_font.latexWriteEndChanges(
-					os, bparams, runparams, basefont,
-					(i == body_pos-1) ? basefont : font);
+				    os, bparams, runparams, basefont,
+				    (i == body_pos-1) ? basefont : current_font,
+				    needPar);
 			running_font = basefont;
 			open_font = false;
 		}
@@ -2522,40 +2511,66 @@ void Paragraph::latex(BufferParams const & bparams,
 		// close babel's font environment before opening CJK.
 		string const lang_end_command = runparams.use_polyglossia ?
 			"\\end{$$lang}" : lyxrc.language_command_end;
+		bool const using_begin_end = runparams.use_polyglossia ||
+						!lang_end_command.empty();
 		if (!running_lang.empty() &&
-		    font.language()->encoding()->package() == Encoding::CJK) {
+		    current_font.language()->encoding()->package() == Encoding::CJK) {
 				string end_tag = subst(lang_end_command,
 							"$$lang",
 							running_lang);
 				os << from_ascii(end_tag);
 				column += end_tag.length();
+				if (using_begin_end)
+					popLanguageName();
 		}
 
 		// Switch file encoding if necessary (and allowed)
 		if (!runparams.pass_thru && !style.pass_thru &&
 		    runparams.encoding->package() != Encoding::none &&
-		    font.language()->encoding()->package() != Encoding::none) {
+		    current_font.language()->encoding()->package() != Encoding::none) {
 			pair<bool, int> const enc_switch =
 			    	switchEncoding(os.os(), bparams, runparams,
-					*(font.language()->encoding()));
+					*(current_font.language()->encoding()));
 			if (enc_switch.first) {
 				column += enc_switch.second;
-				runparams.encoding = font.language()->encoding();
+				runparams.encoding = current_font.language()->encoding();
 			}
 		}
 
 		char_type const c = d->text_[i];
 
+		// A display math inset inside an ulem command will be output
+		// as a box of width \columnwidth, so we have to either disable
+		// indentation if the inset starts a paragraph, or start a new
+		// line to accommodate such box. This has to be done before
+		// writing any font changing commands.
+		if (runparams.inDisplayMath && !deleted_display_math
+		    && runparams.inulemcmd) {
+			if (os.afterParbreak())
+				os << "\\noindent";
+			else
+				os << "\\\\\n";
+		}
+
 		// Do we need to change font?
-		if ((font != running_font ||
-		     font.language() != running_font.language()) &&
+		if ((current_font != running_font ||
+		     current_font.language() != running_font.language()) &&
 			i != body_pos - 1)
 		{
 			odocstringstream ods;
-			column += font.latexWriteStartChanges(ods, bparams,
+			column += current_font.latexWriteStartChanges(ods, bparams,
 							      runparams, basefont,
 							      last_font);
-			running_font = font;
+			// Check again for display math in ulem commands as a
+			// font change may also occur just before a math inset.
+			if (runparams.inDisplayMath && !deleted_display_math
+			    && runparams.inulemcmd) {
+				if (os.afterParbreak())
+					os << "\\noindent";
+				else
+					os << "\\\\\n";
+			}
+			running_font = current_font;
 			open_font = true;
 			docstring fontchange = ods.str();
 			// check whether the fontchange ends with a \\textcolor
@@ -2581,7 +2596,7 @@ void Paragraph::latex(BufferParams const & bparams,
 			// style.pass_thru is false.
 			if (i != body_pos - 1) {
 				if (d->simpleTeXBlanks(runparams, os,
-						i, column, font, style)) {
+						i, column, current_font, style)) {
 					// A surrogate pair was output. We
 					// must not call latexSpecialChar
 					// in this iteration, since it would output
@@ -2594,7 +2609,7 @@ void Paragraph::latex(BufferParams const & bparams,
 
 		OutputParams rp = runparams;
 		rp.free_spacing = style.free_spacing;
-		rp.local_font = &font;
+		rp.local_font = &current_font;
 		rp.intitle = style.intitle;
 
 		// Two major modes:  LaTeX or plain
@@ -2602,16 +2617,47 @@ void Paragraph::latex(BufferParams const & bparams,
 		// and then split to handle the two modes separately.
 		if (c == META_INSET) {
 			if (i >= start_pos && (end_pos == -1 || i < end_pos)) {
+				// Greyedout notes and, in general, all insets
+				// with InsetLayout::isDisplay() == false,
+				// are typeset inline with the text. So, we
+				// can add a \par to the last paragraph of
+				// such insets only if nothing else follows.
+				bool incremented = false;
+				Inset const * inset = getInset(i);
+				InsetText const * textinset = inset
+							? inset->asInsetText()
+							: 0;
+				if (i + 1 == size() && textinset
+				    && !inset->getLayout().isDisplay()) {
+					ParagraphList const & pars =
+						textinset->text().paragraphs();
+					pit_type const pit = pars.size() - 1;
+					Font const last_font =
+						pit < 0 || pars[pit].empty()
+						? pars[pit].getLayoutFont(
+								bparams,
+								outerfont)
+						: pars[pit].getFont(bparams,
+							pars[pit].size() - 1,
+							outerfont);
+					if (last_font.fontInfo().size() !=
+					    basefont.fontInfo().size()) {
+						++parInline;
+						incremented = true;
+					}
+				}
 				d->latexInset(bparams, os, rp, running_font,
 						basefont, outerfont, open_font,
 						runningChange, style, i, column);
+				if (incremented)
+					--parInline;
 			}
-		} else {
-			if (i >= start_pos && (end_pos == -1 || i < end_pos)) {
-				try {
-					d->latexSpecialChar(os, bparams, rp, running_font, runningChange,
-							    style, i, end_pos, column);
-				} catch (EncodingException & e) {
+		} else if (i >= start_pos && (end_pos == -1 || i < end_pos)) {
+			try {
+				d->latexSpecialChar(os, bparams, rp,
+						    running_font, runningChange,
+						    style, i, end_pos, column);
+			} catch (EncodingException & e) {
 				if (runparams.dryrun) {
 					os << "<" << _("LyX Warning: ")
 					   << _("uncodable character") << " '";
@@ -2625,31 +2671,75 @@ void Paragraph::latex(BufferParams const & bparams,
 				}
 			}
 		}
-		}
 
 		// Set the encoding to that returned from latexSpecialChar (see
 		// comment for encoding member in OutputParams.h)
 		runparams.encoding = rp.encoding;
+
+		// Also carry on the info on a closed ulem command for insets
+		// such as Note that do not produce any output, so that no
+		// command is ever executed but its opening was recorded.
+		runparams.inulemcmd = rp.inulemcmd;
 	}
 
 	// If we have an open font definition, we have to close it
 	if (open_font) {
+		// Make sure that \\par is done with the font of the last
+		// character if this has another size as the default.
+		// This is necessary because LaTeX (and LyX on the screen)
+		// calculates the space between the baselines according
+		// to this font. (Matthias)
+		//
+		// We must not change the font for the last paragraph
+		// of non-multipar insets, tabular cells or commands,
+		// since this produces unwanted whitespace.
+
+		Font const font = empty()
+			? getLayoutFont(bparams, outerfont)
+			: getFont(bparams, size() - 1, outerfont);
+
+		InsetText const * textinset = inInset().asInsetText();
+
+		bool const maintext = textinset
+			? textinset->text().isMainText()
+			: false;
+
+		size_t const numpars = textinset
+			? textinset->text().paragraphs().size()
+			: 0;
+
+		bool needPar = false;
+
+		if (style.resfont.size() != font.fontInfo().size()
+		    && (!runparams.isLastPar || maintext
+			|| (numpars > 1 && d->ownerCode() != CELL_CODE
+			    && (inInset().getLayout().isDisplay()
+				|| parInline)))
+		    && !style.isCommand()) {
+			needPar = true;
+		}
 #ifdef FIXED_LANGUAGE_END_DETECTION
 		if (next_) {
 			running_font.latexWriteEndChanges(os, bparams,
 					runparams, basefont,
-					next_->getFont(bparams, 0, outerfont));
+					next_->getFont(bparams, 0, outerfont),
+						       needPar);
 		} else {
 			running_font.latexWriteEndChanges(os, bparams,
-					runparams, basefont, basefont);
+					runparams, basefont, basefont, needPar);
 		}
 #else
 //FIXME: For now we ALWAYS have to close the foreign font settings if they are
 //FIXME: there as we start another \selectlanguage with the next paragraph if
 //FIXME: we are in need of this. This should be fixed sometime (Jug)
 		running_font.latexWriteEndChanges(os, bparams, runparams,
-				basefont, basefont);
+				basefont, basefont, needPar);
 #endif
+		if (needPar) {
+			// The \par could not be inserted at the same nesting
+			// level of the font size change, so do it now.
+			os << "{\\" << font.latexSize() << "\\par}";
+		}
 	}
 
 	column += Changes::latexMarkChange(os, bparams, runningChange,
@@ -2668,8 +2758,7 @@ void Paragraph::latex(BufferParams const & bparams,
 	if (allowcust && d->endTeXParParams(bparams, os, runparams)
 	    && runparams.encoding != prev_encoding) {
 		runparams.encoding = prev_encoding;
-		if (!runparams.isFullUnicode())
-			os << setEncoding(prev_encoding->iconvName());
+		os << setEncoding(prev_encoding->iconvName());
 	}
 
 	LYXERR(Debug::LATEX, "Paragraph::latex... done " << this);
@@ -2846,6 +2935,7 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 				    XHTMLStream & xs,
 				    OutputParams const & runparams,
 				    Font const & outerfont,
+				    bool start_paragraph, bool close_paragraph,
 				    pos_type initial) const
 {
 	docstring retval;
@@ -2857,6 +2947,7 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 	bool ubar_flag = false;
 	bool dbar_flag = false;
 	bool sout_flag = false;
+	bool xout_flag = false;
 	bool wave_flag = false;
 	// shape tags
 	bool shap_flag = false;
@@ -2867,7 +2958,8 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 
 	Layout const & style = *d->layout_;
 
-	xs.startParagraph(allowEmpty());
+	if (start_paragraph)
+		xs.startDivision(allowEmpty());
 
 	FontInfo font_old =
 		style.labeltype == LABEL_MANUAL ? style.labelfont : style.font;
@@ -2910,6 +3002,11 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 		if (font_old.strikeout() != curstate)
 			doFontSwitch(tagsToOpen, tagsToClose, sout_flag, curstate, html::FT_SOUT);
 
+		// xout
+		curstate = font.fontInfo().xout();
+		if (font_old.xout() != curstate)
+			doFontSwitch(tagsToOpen, tagsToClose, xout_flag, curstate, html::FT_XOUT);
+
 		// double underbar
 		curstate = font.fontInfo().uuline();
 		if (font_old.uuline() != curstate)
@@ -3153,43 +3250,26 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 			if (!runparams.for_toc || inset->isInToc()) {
 				OutputParams np = runparams;
 				np.local_font = &font;
-				if (!inset->getLayout().htmlisblock())
+				// If the paragraph has size 1, then we are in the "special
+				// case" where we do not output the containing paragraph info
+				if (!inset->getLayout().htmlisblock() && size() != 1)
 					np.html_in_par = true;
 				retval += inset->xhtml(xs, np);
 			}
 		} else {
 			char_type c = getUChar(buf.masterBuffer()->params(), i);
-
-			if (style.pass_thru || runparams.pass_thru)
-				xs << c;
-			else if (c == '-' && !runparams.inIPA &&
-			         font.fontInfo().family() != TYPEWRITER_FAMILY) {
-				docstring str;
-				int j = i + 1;
-				if (j < size() && d->text_[j] == '-') {
-					j += 1;
-					if (j < size() && d->text_[j] == '-') {
-						str += from_ascii("&mdash;");
-						i += 2;
-					} else {
-						str += from_ascii("&ndash;");
-						i += 1;
-					}
-				}
-				else
-					str += c;
-				// We don't want to escape the entities. Note that
-				// it is safe to do this, since str can otherwise
-				// only be "-". E.g., it can't be "<".
-				xs << XHTMLStream::ESCAPE_NONE << str;
-			} else
-				xs << c;
+			xs << c;
 		}
 		font_old = font.fontInfo();
 	}
 
+	// FIXME XHTML
+	// I'm worried about what happens if a branch, say, is itself
+	// wrapped in some font stuff. I think that will not work.
 	xs.closeFontTags();
-	xs.endParagraph();
+	if (close_paragraph)
+		xs.endDivision();
+
 	return retval;
 }
 
@@ -3203,6 +3283,11 @@ bool Paragraph::isHfill(pos_type pos) const
 
 bool Paragraph::isNewline(pos_type pos) const
 {
+	// U+2028 LINE SEPARATOR
+	// U+2029 PARAGRAPH SEPARATOR
+	char_type const c = d->text_[pos];
+	if (c == 0x2028 || c == 0x2029)
+		return true;
 	Inset const * inset = getInset(pos);
 	return inset && inset->lyxCode() == NEWLINE_CODE;
 }
@@ -3258,16 +3343,13 @@ bool Paragraph::isHardHyphenOrApostrophe(pos_type pos) const
 	if ((nextpos == psize || isSpace(nextpos))
 		&& (pos == 0 || isSpace(prevpos)))
 		return false;
-	return c == '\''
-		|| ((nextpos == psize || d->text_[nextpos] != '-')
-		&& (pos == 0 || d->text_[prevpos] != '-'));
+	return true;
 }
 
 
-bool Paragraph::isSameSpellRange(pos_type pos1, pos_type pos2) const
+FontSpan const & Paragraph::getSpellRange(pos_type pos) const
 {
-	return pos1 == pos2
-		|| d->speller_state_.getRange(pos1) == d->speller_state_.getRange(pos2);
+	return d->speller_state_.getRange(pos);
 }
 
 
@@ -3386,21 +3468,23 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options, const Out
 }
 
 
-void Paragraph::forOutliner(docstring & os, size_t maxlen) const
+void Paragraph::forOutliner(docstring & os, size_t const maxlen,
+                            bool const shorten, bool const label) const
 {
-	if (!d->params_.labelString().empty())
-		os += d->params_.labelString() + ' ';
-	for (pos_type i = 0; i < size() && os.length() < maxlen; ++i) {
+	size_t tmplen = shorten ? maxlen + 1 : maxlen;
+	if (label && !labelString().empty())
+		os += labelString() + ' ';
+	for (pos_type i = 0; i < size() && os.length() < tmplen; ++i) {
 		if (isDeleted(i))
 			continue;
 		char_type const c = d->text_[i];
 		if (isPrintable(c))
 			os += c;
-		else if (c == '\t' || c == '\n')
-			os += ' ';
 		else if (c == META_INSET)
-			getInset(i)->forOutliner(os, maxlen);
+			getInset(i)->forOutliner(os, tmplen, false);
 	}
+	if (shorten)
+		Text::shortenForOutliner(os, maxlen);
 }
 
 
@@ -3586,6 +3670,12 @@ void Paragraph::setBuffer(Buffer & b)
 }
 
 
+void Paragraph::resetBuffer()
+{
+	d->insetlist_.resetBuffer();
+}
+
+
 Inset * Paragraph::releaseInset(pos_type pos)
 {
 	Inset * inset = d->insetlist_.release(pos);
@@ -3751,11 +3841,11 @@ void Paragraph::deregisterWords()
 	Private::LangWordsMap::const_iterator itl = d->words_.begin();
 	Private::LangWordsMap::const_iterator ite = d->words_.end();
 	for (; itl != ite; ++itl) {
-		WordList * wl = theWordList(itl->first);
+		WordList & wl = theWordList(itl->first);
 		Private::Words::const_iterator it = (itl->second).begin();
 		Private::Words::const_iterator et = (itl->second).end();
 		for (; it != et; ++it)
-			wl->remove(*it);
+			wl.remove(*it);
 	}
 	d->words_.clear();
 }
@@ -3831,11 +3921,11 @@ void Paragraph::registerWords()
 	Private::LangWordsMap::const_iterator itl = d->words_.begin();
 	Private::LangWordsMap::const_iterator ite = d->words_.end();
 	for (; itl != ite; ++itl) {
-		WordList * wl = theWordList(itl->first);
+		WordList & wl = theWordList(itl->first);
 		Private::Words::const_iterator it = (itl->second).begin();
 		Private::Words::const_iterator et = (itl->second).end();
 		for (; it != et; ++it)
-			wl->insert(*it);
+			wl.insert(*it);
 	}
 }