X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2FParagraph.cpp;h=df8de4c39f09ced1bece8559b5f0405d4eb102b3;hb=dfdcc56cbaf77b67048cfd7e2ad877b7687f4888;hp=0b2aa87175c1c60881ac4621f3cf472e6b5b9a85;hpb=47cf32b67e0e994efb30b68fa4dd99fda3eda80e;p=lyx.git

diff --git a/src/Paragraph.cpp b/src/Paragraph.cpp
index 0b2aa87175..df8de4c39f 100644
--- a/src/Paragraph.cpp
+++ b/src/Paragraph.cpp
@@ -43,7 +43,6 @@
 #include "TextClass.h"
 #include "TexRow.h"
 #include "Text.h"
-#include "VSpace.h"
 #include "WordLangTuple.h"
 #include "WordList.h"
 
@@ -51,6 +50,7 @@
 
 #include "insets/InsetBibitem.h"
 #include "insets/InsetLabel.h"
+#include "insets/InsetSpecialChar.h"
 
 #include "support/debug.h"
 #include "support/docstring_list.h"
@@ -71,6 +71,53 @@ namespace lyx {
 namespace {
 /// Inset identifier (above 0x10ffff, for ucs-4)
 char_type const META_INSET = 0x200001;
+}
+
+
+/////////////////////////////////////////////////////////////////////
+//
+// SpellResultRange
+//
+/////////////////////////////////////////////////////////////////////
+
+class SpellResultRange {
+public:
+	SpellResultRange(FontSpan range, SpellChecker::Result result)
+	: range_(range), result_(result)
+	{}
+	///
+	FontSpan const & range() const { return range_; }
+	///
+	void range(FontSpan const & r) { range_ = r; }
+	///
+	SpellChecker::Result result() const { return result_; }
+	///
+	void result(SpellChecker::Result r) { result_ = r; }
+	///
+	bool inside(pos_type pos) const { return range_.inside(pos); }
+	///
+	bool covered(FontSpan const & r) const
+	{
+		// 1. first of new range inside current range or
+		// 2. last of new range inside current range or
+		// 3. first of current range inside new range or
+		// 4. last of current range inside new range
+		return range_.inside(r.first) || range_.inside(r.last) ||
+			r.inside(range_.first) || r.inside(range_.last);
+	}
+	///
+	void shift(pos_type pos, int offset)
+	{
+		if (range_.first > pos) {
+			range_.first += offset;
+			range_.last += offset;
+		} else if (range_.last >= pos) {
+			range_.last += offset;
+		}
+	}
+private:
+	FontSpan range_ ;
+	SpellChecker::Result result_ ;
 };
 
 
@@ -89,9 +136,32 @@ public:
 
 	void setRange(FontSpan const fp, SpellChecker::Result state)
 	{
-		eraseCoveredRanges(fp);
+		Ranges result;
+		RangesIterator et = ranges_.end();
+		RangesIterator it = ranges_.begin();
+		for (; it != et; ++it) {
+			if (!it->covered(fp))
+				result.push_back(SpellResultRange(it->range(), it->result()));
+			else if (state == SpellChecker::WORD_OK) {
+				// trim or split the current misspelled range
+				// store misspelled ranges only
+				FontSpan range = it->range();
+				if (fp.first > range.first) {
+					// misspelled area in front of WORD_OK
+					range.last = fp.first - 1;
+					result.push_back(SpellResultRange(range, it->result()));
+					range = it->range();
+				}
+				if (fp.last < range.last) {
+					// misspelled area after WORD_OK range
+					range.first = fp.last + 1;
+					result.push_back(SpellResultRange(range, it->result()));
+				}
+			}
+		}
+		ranges_ = result;
 		if (state != SpellChecker::WORD_OK)
-			ranges_[fp] = state;
+			ranges_.push_back(SpellResultRange(fp, state));
 	}
 
 	void increasePosAfterPos(pos_type pos)
@@ -106,21 +176,39 @@ public:
 		needsRefresh(pos);
 	}
 
+	void refreshLast(pos_type pos)
+	{
+		if (pos < refresh_.last)
+			refresh_.last = pos;
+	}
+
 	SpellChecker::Result getState(pos_type pos) const
 	{
 		SpellChecker::Result result = SpellChecker::WORD_OK;
 		RangesIterator et = ranges_.end();
 		RangesIterator it = ranges_.begin();
 		for (; it != et; ++it) {
-			FontSpan fc = it->first;
-			if(fc.first <= pos && pos <= fc.last) {
-				result = it->second;
-				break;
+			if(it->inside(pos)) {
+				return it->result();
 			}
 		}
 		return result;
 	}
 
+	FontSpan const & getRange(pos_type pos) const
+	{
+		/// empty span to indicate mismatch
+		static FontSpan empty_;
+		RangesIterator et = ranges_.end();
+		RangesIterator it = ranges_.begin();
+		for (; it != et; ++it) {
+			if(it->inside(pos)) {
+				return it->range();
+			}
+		}
+		return empty_;
+	}
+
 	bool needsRefresh() const {
 		return needs_refresh_;
 	}
@@ -141,8 +229,10 @@ public:
 			if (pos > refresh_.last)
 				refresh_.last = pos;
 		} else if (pos != -1) {
-			refresh_.first = pos;
-			refresh_.last = pos;
+			// init request check for neighbour positions too
+			refresh_.first = pos > 0 ? pos - 1 : 0;
+			// no need for special end of paragraph check
+			refresh_.last = pos + 1;
 		}
 		needs_refresh_ = pos != -1;
 	}
@@ -155,52 +245,23 @@ public:
 	}
 
 private:
-	/// store the ranges as map of FontSpan and spell result pairs
-	typedef map<FontSpan, SpellChecker::Result> Ranges;
+	typedef vector<SpellResultRange> Ranges;
 	typedef Ranges::const_iterator RangesIterator;
 	Ranges ranges_;
-	///
+	/// the area of the paragraph with pending spell check
 	FontSpan refresh_;
 	bool needs_refresh_;
+	/// spell state cache version number
 	SpellChecker::ChangeNumber current_change_number_;
 
-	void eraseCoveredRanges(FontSpan const fp)
-	{
-		Ranges result;
-		RangesIterator et = ranges_.end();
-		RangesIterator it = ranges_.begin();
-		for (; it != et; ++it) {
-			FontSpan fc = it->first;
-			// 1. first of new range inside current range or
-			// 2. last of new range inside current range or
-			// 3. first of current range inside new range or
-			// 4. last of current range inside new range
-			if (fc.inside(fp.first) || fc.inside(fp.last) ||
-				fp.inside(fc.first) || fp.inside(fc.last))
-			{
-				continue;
-			}
-			result[fc] = it->second;
-		}
-		ranges_ = result;
-	}
 
 	void correctRangesAfterPos(pos_type pos, int offset)
 	{
-		Ranges result;
 		RangesIterator et = ranges_.end();
-		RangesIterator it = ranges_.begin();
+		Ranges::iterator it = ranges_.begin();
 		for (; it != et; ++it) {
-			FontSpan m = it->first;
-			if (m.first > pos) {
-				m.first += offset;
-				m.last += offset;
-			} else if (m.last > pos) {
-				m.last += offset;
-			}
-			result[m] = it->second;
+			it->shift(pos, offset);
 		}
-		ranges_ = result;
 	}
 
 };
@@ -226,14 +287,14 @@ public:
 
 	/// Output the surrogate pair formed by \p c and \p next to \p os.
 	/// \return the number of characters written.
-	int latexSurrogatePair(odocstream & os, char_type c, char_type next,
+	int latexSurrogatePair(otexstream & os, char_type c, char_type next,
 			       OutputParams const &);
 
 	/// Output a space in appropriate formatting (or a surrogate pair
 	/// if the next character is a combining character).
 	/// \return whether a surrogate pair was output.
 	bool simpleTeXBlanks(OutputParams const &,
-			     odocstream &, TexRow & texrow,
+			     otexstream &,
 			     pos_type i,
 			     unsigned int & column,
 			     Font const & font,
@@ -242,21 +303,21 @@ public:
 	/// Output consecutive unicode chars, belonging to the same script as
 	/// specified by the latex macro \p ltx, to \p os starting from \p i.
 	/// \return the number of characters written.
-	int writeScriptChars(odocstream & os, docstring const & ltx,
+	int writeScriptChars(otexstream & os, docstring const & ltx,
 			   Change const &, Encoding const &, pos_type & i);
 
 	/// This could go to ParagraphParameters if we want to.
-	int startTeXParParams(BufferParams const &, odocstream &, TexRow &,
+	int startTeXParParams(BufferParams const &, otexstream &,
 			      OutputParams const &) const;
 
 	/// This could go to ParagraphParameters if we want to.
-	int endTeXParParams(BufferParams const &, odocstream &, TexRow &,
-			    OutputParams const &) const;
+	bool endTeXParParams(BufferParams const &, otexstream &,
+			     OutputParams const &) const;
 
 	///
 	void latexInset(BufferParams const &,
-				   odocstream &,
-				   TexRow & texrow, OutputParams &,
+				   otexstream &,
+				   OutputParams &,
 				   Font & running_font,
 				   Font & basefont,
 				   Font const & outerfont,
@@ -268,30 +329,32 @@ public:
 
 	///
 	void latexSpecialChar(
-				   odocstream & os,
+				   otexstream & os,
 				   OutputParams const & runparams,
 				   Font const & running_font,
 				   Change const & running_change,
 				   Layout const & style,
 				   pos_type & i,
+				   pos_type end_pos,
 				   unsigned int & column);
 
 	///
 	bool latexSpecialT1(
 		char_type const c,
-		odocstream & os,
+		otexstream & os,
 		pos_type i,
 		unsigned int & column);
 	///
 	bool latexSpecialTypewriter(
 		char_type const c,
-		odocstream & os,
+		otexstream & os,
 		pos_type i,
 		unsigned int & column);
 	///
 	bool latexSpecialPhrase(
-		odocstream & os,
+		otexstream & os,
 		pos_type & i,
+		pos_type end_pos,
 		unsigned int & column,
 		OutputParams const & runparams);
 
@@ -305,14 +368,16 @@ public:
 	/// match a string against a particular point in the paragraph
 	bool isTextAt(string const & str, pos_type pos) const;
 
-	/// a vector of inset positions
-	typedef vector<pos_type> Positions;
-	typedef Positions::const_iterator PositionsIterator;
+	/// a vector of speller skip positions
+	typedef vector<FontSpan> SkipPositions;
+	typedef SkipPositions::const_iterator SkipPositionsIterator;
 
+	void appendSkipPosition(SkipPositions & skips, pos_type const pos) const;
+	
 	Language * getSpellLanguage(pos_type const from) const;
 
 	Language * locateSpellRange(pos_type & from, pos_type & to,
-								Positions & softbreaks) const;
+								SkipPositions & skips) const;
 
 	bool hasSpellerChange() const {
 		SpellChecker::ChangeNumber speller_change_number = 0;
@@ -321,21 +386,23 @@ public:
 		return speller_change_number > speller_state_.currentChangeNumber();
 	}
 
+	bool ignoreWord(docstring const & word) const ;
+	
 	void setMisspelled(pos_type from, pos_type to, SpellChecker::Result state)
 	{
 		pos_type textsize = owner_->size();
 		// check for sane arguments
-		if (to < from || from >= textsize)
+		if (to <= from || from >= textsize)
 			return;
-		FontSpan fp = FontSpan(from, to);
-		// don't mark end of paragraph
-		if (fp.last >= textsize)
-			fp.last = textsize - 1;
+		FontSpan fp = FontSpan(from, to - 1);
 		speller_state_.setRange(fp, state);
 	}
 
 	void requestSpellCheck(pos_type pos) {
-		speller_state_.needsRefresh(pos);
+		if (pos == -1)
+			speller_state_.needsCompleteRefresh(speller_state_.currentChangeNumber());
+		else
+			speller_state_.needsRefresh(pos);
 	}
 
 	void readySpellCheck() {
@@ -363,22 +430,23 @@ public:
 		last = endpos;
 	}
 
-	int countSoftbreaks(PositionsIterator & it, PositionsIterator const et,
+	int countSkips(SkipPositionsIterator & it, SkipPositionsIterator const et,
 			    int & start) const
 	{
-		int numbreaks = 0;
-		while (it != et && *it < start) {
-			++start;
-			++numbreaks;
+		int numskips = 0;
+		while (it != et && it->first < start) {
+			int skip = it->last - it->first + 1;
+			start += skip;
+			numskips += skip;
 			++it;
 		}
-		return numbreaks;
+		return numskips;
 	}
 
 	void markMisspelledWords(pos_type const & first, pos_type const & last,
 							 SpellChecker::Result result,
 							 docstring const & word,
-							 Positions const & softbreaks);
+							 SkipPositions const & skips);
 
 	InsetCode ownerCode() const
 	{
@@ -462,7 +530,7 @@ Paragraph::Private::Private(Private const & p, Paragraph * owner)
 	: owner_(owner), inset_owner_(p.inset_owner_), fontlist_(p.fontlist_),
 	  params_(p.params_), changes_(p.changes_), insetlist_(p.insetlist_),
 	  begin_of_body_(p.begin_of_body_), text_(p.text_), words_(p.words_),
-	  layout_(p.layout_), speller_state_(p.speller_state_)
+	  layout_(p.layout_)
 {
 	id_ = ++paragraph_id;
 	requestSpellCheck(p.text_.size());
@@ -475,7 +543,7 @@ Paragraph::Private::Private(Private const & p, Paragraph * owner,
 	  params_(p.params_), changes_(p.changes_),
 	  insetlist_(p.insetlist_, beg, end),
 	  begin_of_body_(p.begin_of_body_), words_(p.words_),
-	  layout_(p.layout_), speller_state_(p.speller_state_)
+	  layout_(p.layout_)
 {
 	id_ = ++paragraph_id;
 	if (beg >= pos_type(p.text_.size()))
@@ -720,6 +788,8 @@ bool Paragraph::eraseChar(pos_type pos, bool trackChanges)
 		if (!change.changed() ||
 		      (change.inserted() && !change.currentAuthor())) {
 			setChange(pos, Change(Change::DELETED));
+			// request run of spell checker
+			requestSpellCheck(pos);
 			return false;
 		}
 
@@ -751,6 +821,7 @@ bool Paragraph::eraseChar(pos_type pos, bool trackChanges)
 
 	// Update list of misspelled positions
 	d->speller_state_.decreasePosAfterPos(pos);
+	d->speller_state_.refreshLast(size());
 
 	return true;
 }
@@ -770,7 +841,7 @@ int Paragraph::eraseChars(pos_type start, pos_type end, bool trackChanges)
 }
 
 
-int Paragraph::Private::latexSurrogatePair(odocstream & os, char_type c,
+int Paragraph::Private::latexSurrogatePair(otexstream & os, char_type c,
 		char_type next, OutputParams const & runparams)
 {
 	// Writing next here may circumvent a possible font change between
@@ -799,7 +870,7 @@ int Paragraph::Private::latexSurrogatePair(odocstream & os, char_type c,
 
 
 bool Paragraph::Private::simpleTeXBlanks(OutputParams const & runparams,
-				       odocstream & os, TexRow & texrow,
+				       otexstream & os,
 				       pos_type i,
 				       unsigned int & column,
 				       Font const & font,
@@ -832,8 +903,7 @@ bool Paragraph::Private::simpleTeXBlanks(OutputParams const & runparams,
 		     || text_[i - 1] == ':'
 		     || text_[i - 1] == '!'))) {
 		os << '\n';
-		texrow.newline();
-		texrow.start(owner_->id(), i + 1);
+		os.texrow().start(owner_->id(), i + 1);
 		column = 0;
 	} else if (style.free_spacing) {
 		os << '~';
@@ -844,7 +914,7 @@ bool Paragraph::Private::simpleTeXBlanks(OutputParams const & runparams,
 }
 
 
-int Paragraph::Private::writeScriptChars(odocstream & os,
+int Paragraph::Private::writeScriptChars(otexstream & os,
 					 docstring const & ltx,
 					 Change const & runningChange,
 					 Encoding const & encoding,
@@ -938,8 +1008,7 @@ bool Paragraph::Private::isTextAt(string const & str, pos_type pos) const
 
 
 void Paragraph::Private::latexInset(BufferParams const & bparams,
-				    odocstream & os,
-				    TexRow & texrow,
+				    otexstream & os,
 				    OutputParams & runparams,
 				    Font & running_font,
 				    Font & basefont,
@@ -954,7 +1023,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 	LASSERT(inset, /**/);
 
 	if (style.pass_thru) {
-		inset->plaintext(os, runparams);
+		inset->plaintext(os.os(), runparams);
 		return;
 	}
 
@@ -982,8 +1051,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 				os << "\\protect ";
 
 		}
-		texrow.newline();
-		texrow.start(owner_->id(), i + 1);
+		os.texrow().start(owner_->id(), i + 1);
 		column = 0;
 	}
 
@@ -999,7 +1067,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 	}
 
 	bool close = false;
-	odocstream::pos_type const len = os.tellp();
+	odocstream::pos_type const len = os.os().tellp();
 
 	if (inset->forceLTR()
 	    && running_font.isRightToLeft()
@@ -1025,7 +1093,7 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 	// ArabTeX, though, cannot handle this special behavior, it seems.
 	bool arabtex = basefont.language()->lang() == "arabic_arabtex"
 		|| running_font.language()->lang() == "arabic_arabtex";
-	if (open_font && inset->noFontChange()) {
+	if (open_font && !inset->inheritFont()) {
 		bool closeLanguage = arabtex
 			|| basefont.isRightToLeft() == running_font.isRightToLeft();
 		unsigned int count = running_font.latexWriteEndChanges(os,
@@ -1048,10 +1116,12 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 		}
 	}
 
-	int tmp;
+	int prev_rows = os.texrow().rows();
 
 	try {
-		tmp = inset->latex(os, runparams);
+		runparams.lastid = id_;
+		runparams.lastpos = i;
+		inset->latex(os, runparams);
 	} catch (EncodingException & e) {
 		// add location information and throw again.
 		e.par_id = id_;
@@ -1066,12 +1136,11 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 				os << '}';
 	}
 
-	if (tmp) {
-		texrow.newlines(tmp);
-		texrow.start(owner_->id(), i + 1);
+	if (os.texrow().rows() > prev_rows) {
+		os.texrow().start(owner_->id(), i + 1);
 		column = 0;
 	} else {
-		column += os.tellp() - len;
+		column += (unsigned int)(os.os().tellp() - len);
 	}
 
 	if (owner_->isDeleted(i))
@@ -1079,22 +1148,24 @@ void Paragraph::Private::latexInset(BufferParams const & bparams,
 }
 
 
-void Paragraph::Private::latexSpecialChar(
-					     odocstream & os,
-					     OutputParams const & runparams,
-					     Font const & running_font,
-					     Change const & running_change,
-					     Layout const & style,
-					     pos_type & i,
-					     unsigned int & column)
+void Paragraph::Private::latexSpecialChar(otexstream & os,
+					  OutputParams const & runparams,
+					  Font const & running_font,
+					  Change const & running_change,
+					  Layout const & style,
+					  pos_type & i,
+					  pos_type end_pos,
+					  unsigned int & column)
 {
 	char_type const c = text_[i];
 
 	if (style.pass_thru || runparams.pass_thru) {
-		if (c != '\0')
-			// FIXME UNICODE: This can fail if c cannot
-			// be encoded in the current encoding.
+		if (c != '\0') {
+			Encoding const * const enc = runparams.encoding;
+			if (enc && enc->latexChar(c, true).empty())
+				throw EncodingException(c);
 			os.put(c);
+		}
 		return;
 	}
 
@@ -1175,7 +1246,7 @@ void Paragraph::Private::latexSpecialChar(
 
 	default:
 		// LyX, LaTeX etc.
-		if (latexSpecialPhrase(os, i, column, runparams))
+		if (latexSpecialPhrase(os, i, end_pos, column, runparams))
 			return;
 
 		if (c == '\0')
@@ -1211,7 +1282,7 @@ void Paragraph::Private::latexSpecialChar(
 }
 
 
-bool Paragraph::Private::latexSpecialT1(char_type const c, odocstream & os,
+bool Paragraph::Private::latexSpecialT1(char_type const c, otexstream & os,
 	pos_type i, unsigned int & column)
 {
 	switch (c) {
@@ -1239,7 +1310,7 @@ bool Paragraph::Private::latexSpecialT1(char_type const c, odocstream & os,
 }
 
 
-bool Paragraph::Private::latexSpecialTypewriter(char_type const c, odocstream & os,
+bool Paragraph::Private::latexSpecialTypewriter(char_type const c, otexstream & os,
 	pos_type i, unsigned int & column)
 {
 	switch (c) {
@@ -1261,7 +1332,10 @@ bool Paragraph::Private::latexSpecialTypewriter(char_type const c, odocstream &
 }
 
 
-bool Paragraph::Private::latexSpecialPhrase(odocstream & os, pos_type & i,
+/// \param end_pos
+///   If [start_pos, end_pos) does not include entirely the special phrase, then
+///   do not apply the macro transformation.
+bool Paragraph::Private::latexSpecialPhrase(otexstream & os, pos_type & i, pos_type end_pos,
 	unsigned int & column, OutputParams const & runparams)
 {
 	// FIXME: if we have "LaTeX" with a font
@@ -1271,7 +1345,8 @@ bool Paragraph::Private::latexSpecialPhrase(odocstream & os, pos_type & i,
 	// "words" for some definition of word
 
 	for (size_t pnr = 0; pnr < phrases_nr; ++pnr) {
-		if (!isTextAt(special_phrases[pnr].phrase, i))
+		if (!isTextAt(special_phrases[pnr].phrase, i)
+		    || (end_pos != -1 && i + int(special_phrases[pnr].phrase.size()) > end_pos))
 			continue;
 		if (runparams.moving_arg)
 			os << "\\protect";
@@ -1287,20 +1362,36 @@ bool Paragraph::Private::latexSpecialPhrase(odocstream & os, pos_type & i,
 void Paragraph::Private::validate(LaTeXFeatures & features) const
 {
 	if (layout_->inpreamble && inset_owner_) {
+		bool const is_command = layout_->latextype == LATEX_COMMAND;
 		Buffer const & buf = inset_owner_->buffer();
 		BufferParams const & bp = buf.params();
 		Font f;
-		TexRow tr;
+		TexRow texrow;
+		// Using a string stream here circumvents the encoding
+		// switching machinery of odocstream. Therefore the
+		// output is wrong if this paragraph contains content
+		// that needs to switch encoding.
 		odocstringstream ods;
-		// we have to provide all the optional arguments here, even though
-		// the last one is the only one we care about.
-		owner_->latex(bp, f, ods, tr, features.runparams(), 0, -1, true);
-		docstring const d = ods.str();
-		if (!d.empty()) {
-			// this will have "{" at the beginning, but not at the end
-			string const content = to_utf8(d);
-			string const cmd = layout_->latexname();
-			features.addPreambleSnippet("\\" + cmd + content + "}");
+		otexstream os(ods, texrow);
+		if (is_command) {
+			os << '\\' << from_ascii(layout_->latexname());
+			// we have to provide all the optional arguments here, even though
+			// the last one is the only one we care about.
+			// Separate handling of optional argument inset.
+			if (layout_->optargs != 0 || layout_->reqargs != 0)
+				latexArgInsets(*owner_, os, features.runparams(),
+					layout_->reqargs, layout_->optargs);
+			else
+				os << from_ascii(layout_->latexparam());
+		}
+		docstring::size_type const length = ods.str().length();
+		// this will output "{" at the beginning, but not at the end
+		owner_->latex(bp, f, os, features.runparams(), 0, -1, true);
+		if (ods.str().length() > length) {
+			if (is_command)
+				ods << '}';
+			string const snippet = to_utf8(ods.str());
+			features.addPreambleSnippet(snippet);
 		}
 	}
 
@@ -1546,6 +1637,7 @@ void Paragraph::appendChar(char_type c, Font const & font,
 	// when appending characters, no need to update tables
 	d->text_.push_back(c);
 	setFont(d->text_.size() - 1, font);
+	d->requestSpellCheck(d->text_.size() - 1);
 }
 
 
@@ -1566,6 +1658,7 @@ void Paragraph::appendString(docstring const & s, Font const & font,
 	for (size_t i = oldsize; i != newsize; ++i) {
 		// track change
 		d->changes_.insert(change, i);
+		d->requestSpellCheck(i);
 	}
 	d->fontlist_.set(oldsize, font);
 	d->fontlist_.set(newsize - 1, font);
@@ -1985,6 +2078,11 @@ bool Paragraph::usePlainLayout() const
 }
 
 
+bool Paragraph::isPassThru() const
+{
+	return inInset().getLayout().isPassThru() || d->layout_->pass_thru;
+}
+
 namespace {
 
 // paragraphs inside floats need different alignment tags to avoid
@@ -2032,13 +2130,12 @@ string const corrected_env(string const & suffix, string const & env,
 }
 
 
-void adjust_row_column(string const & str, TexRow & texrow, int & column)
+void adjust_column(string const & str, int & column)
 {
 	if (!contains(str, "\n"))
 		column += str.size();
 	else {
 		string tmp;
-		texrow.newline();
 		column = rsplit(str, tmp, '\n').size();
 	}
 }
@@ -2047,12 +2144,11 @@ void adjust_row_column(string const & str, TexRow & texrow, int & column)
 
 
 int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
-				 odocstream & os, TexRow & texrow,
-				 OutputParams const & runparams) const
+			otexstream & os, OutputParams const & runparams) const
 {
 	int column = 0;
 
-	if (params_.noindent()) {
+	if (params_.noindent() && !layout_->pass_thru) {
 		os << "\\noindent ";
 		column += 10;
 	}
@@ -2097,7 +2193,7 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
 		else
 			output = corrected_env(begin_tag, "flushright", code, lastpar);
 		os << from_ascii(output);
-		adjust_row_column(output, texrow, column);
+		adjust_column(output, column);
 		break;
 	} case LYX_ALIGN_RIGHT: {
 		string output;
@@ -2106,13 +2202,13 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
 		else
 			output = corrected_env(begin_tag, "flushleft", code, lastpar);
 		os << from_ascii(output);
-		adjust_row_column(output, texrow, column);
+		adjust_column(output, column);
 		break;
 	} case LYX_ALIGN_CENTER: {
 		string output;
 		output = corrected_env(begin_tag, "center", code, lastpar);
 		os << from_ascii(output);
-		adjust_row_column(output, texrow, column);
+		adjust_column(output, column);
 		break;
 	}
 	}
@@ -2121,16 +2217,13 @@ int Paragraph::Private::startTeXParParams(BufferParams const & bparams,
 }
 
 
-int Paragraph::Private::endTeXParParams(BufferParams const & bparams,
-			       odocstream & os, TexRow & texrow,
-			       OutputParams const & runparams) const
+bool Paragraph::Private::endTeXParParams(BufferParams const & bparams,
+			otexstream & os, OutputParams const & runparams) const
 {
-	int column = 0;
-
 	LyXAlignment const curAlign = params_.align();
 
 	if (curAlign == layout_->align)
-		return column;
+		return false;
 
 	switch (curAlign) {
 	case LYX_ALIGN_NONE:
@@ -2142,13 +2235,12 @@ int Paragraph::Private::endTeXParParams(BufferParams const & bparams,
 	case LYX_ALIGN_LEFT:
 	case LYX_ALIGN_RIGHT:
 	case LYX_ALIGN_CENTER:
-		if (runparams.moving_arg) {
+		if (runparams.moving_arg)
 			os << "\\protect";
-			column = 8;
-		}
 		break;
 	}
 
+	string output;
 	string const end_tag = "\n\\par\\end";
 	InsetCode code = ownerCode();
 	bool const lastpar = runparams.isLastPar;
@@ -2161,40 +2253,34 @@ int Paragraph::Private::endTeXParParams(BufferParams const & bparams,
 	case LYX_ALIGN_DECIMAL:
 		break;
 	case LYX_ALIGN_LEFT: {
-		string output;
 		if (owner_->getParLanguage(bparams)->babel() != "hebrew")
 			output = corrected_env(end_tag, "flushleft", code, lastpar);
 		else
 			output = corrected_env(end_tag, "flushright", code, lastpar);
 		os << from_ascii(output);
-		adjust_row_column(output, texrow, column);
 		break;
 	} case LYX_ALIGN_RIGHT: {
-		string output;
 		if (owner_->getParLanguage(bparams)->babel() != "hebrew")
 			output = corrected_env(end_tag, "flushright", code, lastpar);
 		else
 			output = corrected_env(end_tag, "flushleft", code, lastpar);
 		os << from_ascii(output);
-		adjust_row_column(output, texrow, column);
 		break;
 	} case LYX_ALIGN_CENTER: {
-		string output;
 		output = corrected_env(end_tag, "center", code, lastpar);
 		os << from_ascii(output);
-		adjust_row_column(output, texrow, column);
 		break;
 	}
 	}
 
-	return column;
+	return !output.empty() || lastpar;
 }
 
 
 // This one spits out the text of the paragraph
 void Paragraph::latex(BufferParams const & bparams,
 	Font const & outerfont,
-	odocstream & os, TexRow & texrow,
+	otexstream & os,
 	OutputParams const & runparams,
 	int start_pos, int end_pos, bool force) const
 {
@@ -2225,6 +2311,10 @@ void Paragraph::latex(BufferParams const & bparams,
 	if (body_pos > 0) {
 		// the optional argument is kept in curly brackets in
 		// case it contains a ']'
+		// This is not strictly needed, but if this is changed it
+		// would be a file format change, and tex2lyx would need
+		// to be adjusted, since it unconditionally removes the
+		// braces when it parses \item.
 		os << "[{";
 		column += 2;
 		basefont = getLabelFont(bparams, outerfont);
@@ -2241,7 +2331,7 @@ void Paragraph::latex(BufferParams const & bparams,
 
 	Encoding const * const prev_encoding = runparams.encoding;
 
-	texrow.start(id(), 0);
+	os.texrow().start(id(), 0);
 
 	// if the paragraph is empty, the loop will not be entered at all
 	if (empty()) {
@@ -2250,8 +2340,7 @@ void Paragraph::latex(BufferParams const & bparams,
 			++column;
 		}
 		if (allowcust)
-			column += d->startTeXParParams(bparams, os, texrow,
-						    runparams);
+			column += d->startTeXParParams(bparams, os, runparams);
 	}
 
 	for (pos_type i = 0; i < size(); ++i) {
@@ -2282,12 +2371,11 @@ void Paragraph::latex(BufferParams const & bparams,
 
 			if (allowcust)
 				column += d->startTeXParParams(bparams, os,
-							    texrow,
 							    runparams);
 		}
 
-		Change const & change = runparams.inDeletedInset ? runparams.changeOfDeletedInset
-								 : lookupChange(i);
+		Change const & change = runparams.inDeletedInset
+			? runparams.changeOfDeletedInset : lookupChange(i);
 
 		if (bparams.outputChanges && runningChange != change) {
 			if (open_font) {
@@ -2328,12 +2416,16 @@ void Paragraph::latex(BufferParams const & bparams,
 			open_font = false;
 		}
 
+		string const running_lang = runparams.use_polyglossia ?
+			running_font.language()->polyglossia() : running_font.language()->babel();
 		// close babel's font environment before opening CJK.
-		if (!running_font.language()->babel().empty() &&
+		string const lang_end_command = runparams.use_polyglossia ?
+			"\\end{$$lang}" : lyxrc.language_command_end;
+		if (!running_lang.empty() &&
 		    font.language()->encoding()->package() == Encoding::CJK) {
-				string end_tag = subst(lyxrc.language_command_end,
+				string end_tag = subst(lang_end_command,
 							"$$lang",
-							running_font.language()->babel());
+							running_lang);
 				os << from_ascii(end_tag);
 				column += end_tag.length();
 		}
@@ -2342,8 +2434,9 @@ void Paragraph::latex(BufferParams const & bparams,
 		if (!runparams.pass_thru && !style.pass_thru &&
 		    runparams.encoding->package() != Encoding::none &&
 		    font.language()->encoding()->package() != Encoding::none) {
-			pair<bool, int> const enc_switch = switchEncoding(os, bparams,
-					runparams, *(font.language()->encoding()));
+			pair<bool, int> const enc_switch =
+			    	switchEncoding(os.os(), bparams, runparams,
+					*(font.language()->encoding()));
 			if (enc_switch.first) {
 				column += enc_switch.second;
 				runparams.encoding = font.language()->encoding();
@@ -2386,8 +2479,7 @@ void Paragraph::latex(BufferParams const & bparams,
 			// latexSpecialChar ignores spaces if
 			// style.pass_thru is false.
 			if (i != body_pos - 1) {
-				if (d->simpleTeXBlanks(
-						runparams, os, texrow,
+				if (d->simpleTeXBlanks(runparams, os,
 						i, column, font, style)) {
 					// A surrogate pair was output. We
 					// must not call latexSpecialChar
@@ -2409,8 +2501,7 @@ void Paragraph::latex(BufferParams const & bparams,
 		// and then split to handle the two modes separately.
 		if (c == META_INSET) {
 			if (i >= start_pos && (end_pos == -1 || i < end_pos)) {
-				d->latexInset(bparams, os,
-						texrow, rp, running_font,
+				d->latexInset(bparams, os, rp, running_font,
 						basefont, outerfont, open_font,
 						runningChange, style, i, column);
 			}
@@ -2418,7 +2509,7 @@ void Paragraph::latex(BufferParams const & bparams,
 			if (i >= start_pos && (end_pos == -1 || i < end_pos)) {
 				try {
 					d->latexSpecialChar(os, rp, running_font, runningChange,
-						style, i, column);
+							    style, i, end_pos, column);
 				} catch (EncodingException & e) {
 				if (runparams.dryrun) {
 					os << "<" << _("LyX Warning: ")
@@ -2468,10 +2559,10 @@ void Paragraph::latex(BufferParams const & bparams,
 		os << "}]~";
 	}
 
-	if (allowcust && d->endTeXParParams(bparams, os, texrow, runparams)
+	if (allowcust && d->endTeXParParams(bparams, os, runparams)
 	    && runparams.encoding != prev_encoding) {
 		runparams.encoding = prev_encoding;
-		if (!bparams.useXetex)
+		if (!runparams.isFullUnicode())
 			os << setEncoding(prev_encoding->iconvName());
 	}
 
@@ -2639,10 +2730,11 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 
 	bool emph_flag = false;
 	bool bold_flag = false;
-	string closing_tag;
 
 	Layout const & style = *d->layout_;
 
+	xs.startParagraph(allowEmpty());
+
 	if (!runparams.for_toc && runparams.html_make_pars) {
 		// generate a magic label for this paragraph
 		string const attr = "id='" + magicLabel() + "'";
@@ -2685,13 +2777,9 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 
 		Inset const * inset = getInset(i);
 		if (inset) {
-			InsetCommand const * ic = inset->asInsetCommand();
-			InsetLayout const & il = inset->getLayout();
-			InsetMath const * im = inset->asInsetMath();
-			if (!runparams.for_toc
-			    || im || il.isInToc() || (ic && ic->isInToc())) {
+			if (!runparams.for_toc || inset->isInToc()) {
 				OutputParams np = runparams;
-				if (!il.htmlisblock())
+				if (!inset->getLayout().htmlisblock())
 					np.html_in_par = true;
 				retval += inset->xhtml(xs, np);
 			}
@@ -2718,7 +2806,7 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 				// We don't want to escape the entities. Note that
 				// it is safe to do this, since str can otherwise
 				// only be "-". E.g., it can't be "<".
-				xs << XHTMLStream::NextRaw() << str;
+				xs << XHTMLStream::ESCAPE_NONE << str;
 			} else
 				xs << c;
 		}
@@ -2726,6 +2814,7 @@ docstring Paragraph::simpleLyXHTMLOnePar(Buffer const & buf,
 	}
 
 	xs.closeFontTags();
+	xs.endParagraph();
 	return retval;
 }
 
@@ -2757,13 +2846,47 @@ bool Paragraph::isLineSeparator(pos_type pos) const
 
 bool Paragraph::isWordSeparator(pos_type pos) const
 {
+	if (pos == size())
+		return true;
 	if (Inset const * inset = getInset(pos))
 		return !inset->isLetter();
+	// if we have a hard hyphen (no en- or emdash) or apostrophe
+	// we pass this to the spell checker
+	// FIXME: this method is subject to change, visit
+	// https://bugzilla.mozilla.org/show_bug.cgi?id=355178
+	// to get an impression how complex this is.
+	if (isHardHyphenOrApostrophe(pos))
+		return false;
 	char_type const c = d->text_[pos];
-	// We want to pass the ' and escape chars to the spellchecker
-	static docstring const quote = from_utf8(lyxrc.spellchecker_esc_chars + '\'');
-	return (!isLetterChar(c) && !isDigit(c) && !contains(quote, c))
-		|| pos == size();
+	// We want to pass the escape chars to the spellchecker
+	docstring const escape_chars = from_utf8(lyxrc.spellchecker_esc_chars);
+	return !isLetterChar(c) && !isDigitASCII(c) && !contains(escape_chars, c);
+}
+
+
+bool Paragraph::isHardHyphenOrApostrophe(pos_type pos) const
+{
+	pos_type const psize = size();
+	if (pos >= psize)
+		return false;
+	char_type const c = d->text_[pos];
+	if (c != '-' && c != '\'')
+		return false;
+	int nextpos = pos + 1;
+	int prevpos = pos > 0 ? pos - 1 : 0;
+	if ((nextpos == psize || isSpace(nextpos))
+		&& (pos == 0 || isSpace(prevpos)))
+		return false;
+	return c == '\''
+		|| ((nextpos == psize || d->text_[nextpos] != '-')
+		&& (pos == 0 || d->text_[prevpos] != '-'));
+}
+
+
+bool Paragraph::isSameSpellRange(pos_type pos1, pos_type pos2) const
+{
+	return pos1 == pos2
+		|| d->speller_state_.getRange(pos1) == d->speller_state_.getRange(pos2);
 }
 
 
@@ -2772,7 +2895,7 @@ bool Paragraph::isChar(pos_type pos) const
 	if (Inset const * inset = getInset(pos))
 		return inset->isChar();
 	char_type const c = d->text_[pos];
-	return !isLetterChar(c) && !isDigit(c) && !lyx::isSpace(c);
+	return !isLetterChar(c) && !isDigitASCII(c) && !lyx::isSpace(c);
 }
 
 
@@ -2812,9 +2935,9 @@ void Paragraph::changeLanguage(BufferParams const & bparams,
 		if (font.language() == from) {
 			font.setLanguage(to);
 			setFont(i, font);
+			d->requestSpellCheck(i);
 		}
 	}
-	d->requestSpellCheck(size());
 }
 
 
@@ -2863,12 +2986,14 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options) const
 		os << d->params_.labelString() << ' ';
 
 	for (pos_type i = beg; i < end; ++i) {
+		if ((options & AS_STR_SKIPDELETE) && isDeleted(i))
+			continue;
 		char_type const c = d->text_[i];
 		if (isPrintable(c) || c == '\t'
 		    || (c == '\n' && (options & AS_STR_NEWLINES)))
 			os.put(c);
 		else if (c == META_INSET && (options & AS_STR_INSETS)) {
-			getInset(i)->tocString(os);
+			getInset(i)->toString(os);
 			if (getInset(i)->asInsetMath())
 				os << " ";
 		}
@@ -2878,6 +3003,24 @@ docstring Paragraph::asString(pos_type beg, pos_type end, int options) const
 }
 
 
+void Paragraph::forToc(docstring & os, size_t maxlen) const
+{
+	if (!d->params_.labelString().empty())
+		os += d->params_.labelString() + ' ';
+	for (pos_type i = 0; i < size() && os.length() < maxlen; ++i) {
+		if (isDeleted(i))
+			continue;
+		char_type const c = d->text_[i];
+		if (isPrintable(c))
+			os += c;
+		else if (c == '\t' || c == '\n')
+			os += ' ';
+		else if (c == META_INSET)
+			getInset(i)->forToc(os, maxlen);
+	}
+}
+
+
 docstring Paragraph::stringify(pos_type beg, pos_type end, int options, OutputParams & runparams) const
 {
 	odocstringstream os;
@@ -3032,7 +3175,7 @@ int Paragraph::checkBiblio(Buffer const & buffer)
 {
 	// FIXME From JS:
 	// This is getting more and more a mess. ...We really should clean
-	// up this bibitem issue for 1.6. See also bug 2743.
+	// up this bibitem issue for 1.6.
 
 	// Add bibitem insets if necessary
 	if (d->layout_->labeltype != LABEL_BIBLIO)
@@ -3058,8 +3201,8 @@ int Paragraph::checkBiblio(Buffer const & buffer)
 	InsetList::iterator end = d->insetlist_.end();
 	for (; it != end; ++it)
 		if (it->inset->lyxCode() == BIBITEM_CODE
-		    && it->pos > 0) {
-			InsetBibitem * olditem = static_cast<InsetBibitem *>(it->inset);
+		      && it->pos > 0) {
+			InsetCommand * olditem = it->inset->asInsetCommand();
 			oldkey = olditem->getParam("key");
 			oldlabel = olditem->getParam("label");
 			erasedInsetPosition = it->pos;
@@ -3075,8 +3218,7 @@ int Paragraph::checkBiblio(Buffer const & buffer)
 	// There was an InsetBibitem at the beginning and we did have to
 	// erase one. So we give its properties to the beginning inset.
 	if (hasbibitem) {
-		InsetBibitem * inset =
-			static_cast<InsetBibitem *>(d->insetlist_.begin()->inset);
+		InsetCommand * inset = d->insetlist_.begin()->inset->asInsetCommand();
 		if (!oldkey.empty())
 			inset->setParam("key", oldkey);
 		inset->setParam("label", oldlabel);
@@ -3091,7 +3233,7 @@ int Paragraph::checkBiblio(Buffer const & buffer)
 	if (!oldkey.empty())
 		inset->setParam("key", oldkey);
 	inset->setParam("label", oldlabel);
-	insertInset(0, static_cast<Inset *>(inset),
+	insertInset(0, inset,
 		    Change(track_changes ? Change::INSERTED : Change::UNCHANGED));
 
 	return 1;
@@ -3221,36 +3363,44 @@ void Paragraph::changeCase(BufferParams const & bparams, pos_type pos,
 }
 
 
-bool Paragraph::find(docstring const & str, bool cs, bool mw,
-		pos_type pos, bool del) const
+int Paragraph::find(docstring const & str, bool cs, bool mw,
+		pos_type start_pos, bool del) const
 {
+	pos_type pos = start_pos;
 	int const strsize = str.length();
 	int i = 0;
 	pos_type const parsize = d->text_.size();
-	for (i = 0; pos + i < parsize; ++i) {
-		if (i >= strsize)
-			break;
-		if (cs && str[i] != d->text_[pos + i])
+	for (i = 0; i < strsize && pos < parsize; ++i, ++pos) {
+		// Ignore "invisible" letters such as ligature breaks
+		// and hyphenation chars while searching
+		while (pos < parsize - 1 && isInset(pos)) {
+			odocstringstream os;
+			getInset(pos)->toString(os);
+			if (!getInset(pos)->isLetter() || !os.str().empty())
+				break;
+			pos++;
+		}
+		if (cs && str[i] != d->text_[pos])
 			break;
-		if (!cs && uppercase(str[i]) != uppercase(d->text_[pos + i]))
+		if (!cs && uppercase(str[i]) != uppercase(d->text_[pos]))
 			break;
-		if (!del && isDeleted(pos + i))
+		if (!del && isDeleted(pos))
 			break;
 	}
 
 	if (i != strsize)
-		return false;
+		return 0;
 
 	// if necessary, check whether string matches word
 	if (mw) {
-		if (pos > 0 && !isWordSeparator(pos - 1))
-			return false;
-		if (pos + strsize < parsize
-			&& !isWordSeparator(pos + strsize))
-			return false;
+		if (start_pos > 0 && !isWordSeparator(start_pos - 1))
+			return 0;
+		if (pos < parsize
+			&& !isWordSeparator(pos))
+			return 0;
 	}
 
-	return true;
+	return pos - start_pos;
 }
 
 
@@ -3382,9 +3532,24 @@ void Paragraph::updateWords()
 }
 
 
+void Paragraph::Private::appendSkipPosition(SkipPositions & skips, pos_type const pos) const
+{
+	SkipPositionsIterator begin = skips.begin();
+	SkipPositions::iterator end = skips.end();
+	if (pos > 0 && begin < end) {
+		--end;
+		if (end->last == pos - 1) {
+			end->last = pos;
+			return;
+		}
+	}
+	skips.insert(end, FontSpan(pos, pos));
+}
+
+
 Language * Paragraph::Private::locateSpellRange(
 	pos_type & from, pos_type & to,
-	Positions & softbreaks) const
+	SkipPositions & skips) const
 {
 	// skip leading white space
 	while (from < to && owner_->isWordSeparator(from))
@@ -3401,7 +3566,9 @@ Language * Paragraph::Private::locateSpellRange(
 		// hop to end of word
 		while (last < to && !owner_->isWordSeparator(last)) {
 			if (owner_->getInset(last)) {
-				softbreaks.insert(softbreaks.end(), last);
+				appendSkipPosition(skips, last);
+			} else if (owner_->isDeleted(last)) {
+				appendSkipPosition(skips, last);
 			}
 			++last;
 		}
@@ -3409,6 +3576,9 @@ Language * Paragraph::Private::locateSpellRange(
 		while (sameinset && last < to && owner_->isWordSeparator(last)) {
 			if (Inset const * inset = owner_->getInset(last))
 				sameinset = inset->isChar() && inset->isLetter();
+			if (sameinset && owner_->isDeleted(last)) {
+				appendSkipPosition(skips, last);
+			}
 			if (sameinset)
 				last++;
 		}
@@ -3444,7 +3614,7 @@ Language * Paragraph::Private::getSpellLanguage(pos_type const from) const
 
 void Paragraph::requestSpellCheck(pos_type pos)
 {
-	d->requestSpellCheck(pos == -1 ? size() : pos);
+	d->requestSpellCheck(pos);
 }
 
 
@@ -3460,6 +3630,21 @@ bool Paragraph::needsSpellCheck() const
 }
 
 
+bool Paragraph::Private::ignoreWord(docstring const & word) const
+{
+	// Ignore words with digits
+	// FIXME: make this customizable
+	// (note that some checkers ignore words with digits by default)
+	docstring::const_iterator cit = word.begin();
+	docstring::const_iterator const end = word.end();
+	for (; cit != end; ++cit) {
+		if (isNumber((*cit)))
+			return true;
+	}
+	return false;
+}
+
+
 SpellChecker::Result Paragraph::spellCheck(pos_type & from, pos_type & to,
 	WordLangTuple & wl, docstring_list & suggestions,
 	bool do_suggestion, bool check_learned) const
@@ -3473,10 +3658,10 @@ SpellChecker::Result Paragraph::spellCheck(pos_type & from, pos_type & to,
 		return result;
 
 	locateWord(from, to, WHOLE_WORD);
-	if (from == to || from >= pos_type(d->text_.size()))
+	if (from == to || from >= size())
 		return result;
 
-	docstring word = asString(from, to, AS_STR_INSETS);
+	docstring word = asString(from, to, AS_STR_INSETS | AS_STR_SKIPDELETE);
 	Language * lang = d->getSpellLanguage(from);
 
 	wl = WordLangTuple(word, lang);
@@ -3485,26 +3670,45 @@ SpellChecker::Result Paragraph::spellCheck(pos_type & from, pos_type & to,
 		return result;
 
 	if (needsSpellCheck() || check_learned) {
-		// Ignore words with digits
-		// FIXME: make this customizable
-		// (note that some checkers ignore words with digits by default)
-		if (!hasDigit(word))
+		pos_type end = to;
+		if (!d->ignoreWord(word)) {
+			bool const trailing_dot = to < size() && d->text_[to] == '.';
 			result = speller->check(wl);
-		d->setMisspelled(from, to, result);
+			if (SpellChecker::misspelled(result) && trailing_dot) {
+				wl = WordLangTuple(word.append(from_ascii(".")), lang);
+				result = speller->check(wl);
+				if (!SpellChecker::misspelled(result)) {
+					LYXERR(Debug::GUI, "misspelled word is correct with dot: \"" <<
+					   word << "\" [" <<
+					   from << ".." << to << "]");
+				} else {
+					// spell check with dot appended failed too
+					// restore original word/lang value
+					word = asString(from, to, AS_STR_INSETS | AS_STR_SKIPDELETE);
+					wl = WordLangTuple(word, lang);
+				}
+			}
+		}
+		if (!SpellChecker::misspelled(result)) {
+			// area up to the begin of the next word is not misspelled
+			while (end < size() && isWordSeparator(end))
+				++end;
+		}
+		d->setMisspelled(from, end, result);
 	} else {
 		result = d->speller_state_.getState(from);
 	}
 
-	bool const misspelled_ = SpellChecker::misspelled(result) ;
-	if (misspelled_ && do_suggestion)
-		speller->suggest(wl, suggestions);
-	else if (misspelled_)
+	if (do_suggestion)
+		suggestions.clear();
+
+	if (SpellChecker::misspelled(result)) {
 		LYXERR(Debug::GUI, "misspelled word: \"" <<
 			   word << "\" [" <<
 			   from << ".." << to << "]");
-	else
-		suggestions.clear();
-
+		if (do_suggestion)
+			speller->suggest(wl, suggestions);
+	}
 	return result;
 }
 
@@ -3513,7 +3717,7 @@ void Paragraph::Private::markMisspelledWords(
 	pos_type const & first, pos_type const & last,
 	SpellChecker::Result result,
 	docstring const & word,
-	Positions const & softbreaks)
+	SkipPositions const & skips)
 {
 	if (!SpellChecker::misspelled(result)) {
 		setMisspelled(first, last, SpellChecker::WORD_OK);
@@ -3523,9 +3727,9 @@ void Paragraph::Private::markMisspelledWords(
 	SpellChecker * speller = theSpellChecker();
 	// locate and enumerate the error positions
 	int nerrors = speller->numMisspelledWords();
-	int numbreaks = 0;
-	PositionsIterator it = softbreaks.begin();
-	PositionsIterator et = softbreaks.end();
+	int numskipped = 0;
+	SkipPositionsIterator it = skips.begin();
+	SkipPositionsIterator et = skips.end();
 	for (int index = 0; index < nerrors; ++index) {
 		int wstart;
 		int wlen = 0;
@@ -3533,15 +3737,15 @@ void Paragraph::Private::markMisspelledWords(
 		/// should not happen if speller supports range checks
 		if (!wlen) continue;
 		docstring const misspelled = word.substr(wstart, wlen);
-		wstart += first + numbreaks;
+		wstart += first + numskipped;
 		if (snext < wstart) {
 			/// mark the range of correct spelling
-			numbreaks += countSoftbreaks(it, et, wstart);
+			numskipped += countSkips(it, et, wstart);
 			setMisspelled(snext,
 				wstart - 1, SpellChecker::WORD_OK);
 		}
 		snext = wstart + wlen;
-		numbreaks += countSoftbreaks(it, et, snext);
+		numskipped += countSkips(it, et, snext);
 		/// mark the range of misspelling
 		setMisspelled(wstart, snext, result);
 		LYXERR(Debug::GUI, "misspelled word: \"" <<
@@ -3568,16 +3772,16 @@ void Paragraph::spellCheck() const
 		// loop until we leave the range
 		for (pos_type first = start; first < endpos; ) {
 			pos_type last = endpos;
-			Private::Positions softbreaks;
-			Language * lang = d->locateSpellRange(first, last, softbreaks);
+			Private::SkipPositions skips;
+			Language * lang = d->locateSpellRange(first, last, skips);
 			if (first >= endpos)
 				break;
 			// start the spell checker on the unit of meaning
-			docstring word = asString(first, last, AS_STR_INSETS);
+			docstring word = asString(first, last, AS_STR_INSETS + AS_STR_SKIPDELETE);
 			WordLangTuple wl = WordLangTuple(word, lang);
 			SpellChecker::Result result = word.size() ?
 				speller->check(wl) : SpellChecker::WORD_OK;
-			d->markMisspelledWords(first, last, result, word, softbreaks);
+			d->markMisspelledWords(first, last, result, word, skips);
 			first = ++last;
 		}
 	} else {
@@ -3593,9 +3797,14 @@ void Paragraph::spellCheck() const
 }
 
 
-bool Paragraph::isMisspelled(pos_type pos) const
+bool Paragraph::isMisspelled(pos_type pos, bool check_boundary) const
 {
-	return SpellChecker::misspelled(d->speller_state_.getState(pos));
+	bool result = SpellChecker::misspelled(d->speller_state_.getState(pos));
+	if (result || pos <= 0 || pos > size())
+		return result;
+	if (check_boundary && (pos == size() || isWordSeparator(pos)))
+		result = SpellChecker::misspelled(d->speller_state_.getState(pos - 1));
+	return result;
 }