X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2FLexer.cpp;h=e99875b28660286882eea04638e53b0848df2b3a;hb=4b7f1b3c3918cd32070c72b6d8e95a888981c7a2;hp=b5f72e51a2c65383087a2b66d95a738084d7e085;hpb=c88e7a7ef66a880b5fb6166ff9151001269bc7cd;p=lyx.git

diff --git a/src/Lexer.cpp b/src/Lexer.cpp
index b5f72e51a2..e99875b286 100644
--- a/src/Lexer.cpp
+++ b/src/Lexer.cpp
@@ -4,7 +4,7 @@
  * Licence details can be found in the file COPYING.
  *
  * \author Alejandro Aguilar Sierra
- * \author Lars Gullik Bjønnes
+ * \author Lars Gullik Bjønnes
  * \author Jean-Marc Lasgouttes
  * \author John Levon
  *
@@ -20,6 +20,7 @@
 #include "support/FileName.h"
 #include "support/filetools.h"
 #include "support/gzstream.h"
+#include "support/lassert.h"
 #include "support/lstrings.h"
 #include "support/lyxalgo.h"
 #include "support/types.h"
@@ -102,6 +103,8 @@ public:
 	string pushTok;
 	///
 	char commentChar;
+	/// used for error messages
+	string context;
 private:
 	/// non-copyable
 	Pimpl(Pimpl const &);
@@ -171,7 +174,9 @@ void Lexer::Pimpl::printError(string const & message) const
 {
 	string const tmpmsg = subst(message, "$$Token", getString());
 	lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
-		<< " of file " << to_utf8(makeDisplayPath(name)) << ']' << endl;
+		<< " of file " << to_utf8(makeDisplayPath(name))
+		<< " current token: '" << getString() << "'"
+		<< " context: '" << context << "']" << endl;
 }
 
 
@@ -247,7 +252,8 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
 		is.rdbuf(&gz_);
 		name = filename.absFilename();
 		lineno = 0;
-		return gz_.is_open() && is.good();
+		if (!gz_.is_open() || !is.good())
+			return false;
 	} else {
 		LYXERR(Debug::LYXLEX, "lyxlex: UNcompressed");
 
@@ -262,8 +268,21 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
 		is.rdbuf(&fb_);
 		name = filename.absFilename();
 		lineno = 0;
-		return fb_.is_open() && is.good();
+		if (!fb_.is_open() || !is.good())
+			return false;
 	}
+
+	// Skip byte order mark.
+	if (is.peek() == 0xef) {
+		int c = is.get();
+		if (is.peek() == 0xbb) {
+			c = is.get();
+			LASSERT(is.get() == 0xbf, /**/);
+		} else
+			is.unget();
+	}
+
+	return true;
 }
 
 
@@ -309,11 +328,6 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
 		is.get(cc);
 		c = cc;
 
-		// skip ','s
-		if (esc && c == ',')
-			continue;
-
-		
 		if (c == commentChar) {
 			// Read rest of line (fast :-)
 #if 1
@@ -378,13 +392,13 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
 			break;
 		}
 
-		if (!esc && c == ',')
+		if (c == ',')
 			continue;              /* Skip ','s */
 
-			// using relational operators with chars other
-			// than == and != is not safe. And if it is done
-			// the type _have_ to be unsigned. It usually a
-			// lot better to use the functions from cctype
+		// using relational operators with chars other
+		// than == and != is not safe. And if it is done
+		// the type _have_ to be unsigned. It usually a
+		// lot better to use the functions from cctype
 		if (c > ' ' && is)  {
 			buff.clear();
 
@@ -402,7 +416,7 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
 			status = LEX_TOKEN;
 		}
 
-		if (!esc && c == '\r' && is) {
+		if (c == '\r' && is) {
 			// The Windows support has lead to the
 			// possibility of "\r\n" at the end of
 			// a line.  This will stop LyX choking
@@ -500,7 +514,7 @@ bool Lexer::Pimpl::nextToken()
 		char cc = 0;
 		is.get(cc);
 		c = cc;
-		if (c >= ' ' && is) {
+		if ((c >= ' ' || c == '\t') && is) {
 			buff.clear();
 
 			if (c == '\\') { // first char == '\\'
@@ -514,7 +528,7 @@ bool Lexer::Pimpl::nextToken()
 					buff.push_back(c);
 					is.get(cc);
 					c = cc;
-				} while (c >= ' ' && c != '\\' && is);
+				} while ((c >= ' ' || c == '\t') && c != '\\' && is);
 			}
 
 			if (c == '\\')
@@ -637,6 +651,7 @@ void Lexer::setCommentChar(char c)
 	pimpl_->setCommentChar(c);
 }
 
+
 int Lexer::lex()
 {
 	return pimpl_->lex();
@@ -751,15 +766,18 @@ string const Lexer::getLongString(string const & endtoken)
 
 bool Lexer::getBool() const
 {
-	if (pimpl_->getString() == "true") {
+	string const s = pimpl_->getString();	
+	if (s == "false" || s == "0") {
+		lastReadOk_ = true;
+		return false;
+	}
+	if (s == "true" || s == "1") {
 		lastReadOk_ = true;
 		return true;
-	} else if (pimpl_->getString() != "false") {
-		pimpl_->printError("Bad boolean `$$Token'. "
-				   "Use \"false\" or \"true\"");
-		lastReadOk_ = false;
 	}
-	lastReadOk_ = true;
+	pimpl_->printError("Bad boolean `$$Token'. "
+				 "Use \"false\" or \"true\"");
+	lastReadOk_ = false;
 	return false;
 }
 
@@ -876,9 +894,19 @@ Lexer & Lexer::operator>>(bool & s)
 }
 
 
+Lexer & Lexer::operator>>(char & c)
+{
+	string s;
+	operator>>(s);
+	if (!s.empty())
+		c = s[0];
+	return *this;
+}
+
+
 // quotes a string, e.g. for use in preferences files or as an argument
 // of the "log" dialog
-string const Lexer::quoteString(string const & arg)
+string Lexer::quoteString(string const & arg)
 {
 	string res;
 	res += '"';
@@ -888,4 +916,46 @@ string const Lexer::quoteString(string const & arg)
 }
 
 
+// same for docstring
+docstring Lexer::quoteString(docstring const & arg)
+{
+	docstring res;
+	res += '"';
+	res += subst(subst(arg, from_ascii("\\"), from_ascii("\\\\")), 
+		     from_ascii("\""), from_ascii("\\\""));
+	res += '"';
+	return res;
+}
+
+
+Lexer & Lexer::operator>>(char const * required)
+{
+	string token;
+	*this >> token;
+	if (token != required) {
+		LYXERR0("Missing '" << required << "'-tag in " << pimpl_->context 
+			<< ". Got " << token << " instead. Line: " << lineNumber());
+		pushToken(token);
+	}
+	return *this;
+}
+
+
+bool Lexer::checkFor(char const * required)
+{
+	string token;
+	*this >> token;
+	if (token == required)
+		return true;
+	pushToken(token);
+	return false;
+}
+
+
+void Lexer::setContext(std::string const & str)
+{
+	pimpl_->context = str;
+}
+
+
 } // namespace lyx