X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2Flyxlex_pimpl.C;h=6351ec523ba8b933e36aead8a4aa5e780264ca38;hb=4e4ebe5d4d675348bbd7f75b9ca6424b6727a253;hp=ba0d7d2aab5c92027611d6c02f0b7b9128e17a5c;hpb=43b967349298437e051e9a783fbcde4f2c46d945;p=lyx.git

diff --git a/src/lyxlex_pimpl.C b/src/lyxlex_pimpl.C
index ba0d7d2aab..6351ec523b 100644
--- a/src/lyxlex_pimpl.C
+++ b/src/lyxlex_pimpl.C
@@ -8,6 +8,7 @@
 #include "lyxlex_pimpl.h"
 #include "support/lyxalgo.h"
 #include "support/filetools.h"
+#include "support/lstrings.h"
 #include "debug.h"
 
 using std::sort;
@@ -22,7 +23,10 @@ struct compare_tags {
 	// used by lower_bound, sort and sorted
 	inline
 	int operator()(keyword_item const & a, keyword_item const & b) const {
-		return compare_no_case(a.tag, b.tag) < 0;
+		// we use the ascii version, because in turkish, 'i'
+		// is not the lowercase version of 'I', and thus
+		// turkish locale breaks parsing of tags.
+		return compare_ascii_no_case(a.tag, b.tag) < 0;
 	}
 };
 // } // end of anon namespace
@@ -30,13 +34,13 @@ struct compare_tags {
 
 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num) 
 	: is(&fb__), table(tab), no_items(num),
-	  status(0), lineno(0)
+	  status(0), lineno(0), commentChar('#')
 {
 	verifyTable();
 }
 
 
-string const LyXLex::Pimpl::GetString() const
+string const LyXLex::Pimpl::getString() const
 {
 	return string(buff);
 }
@@ -44,7 +48,7 @@ string const LyXLex::Pimpl::GetString() const
 
 void LyXLex::Pimpl::printError(string const & message) const
 {
-	string tmpmsg = subst(message, "$$Token", GetString());
+	string const tmpmsg = subst(message, "$$Token", getString());
 	lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
 	       << " of file " << MakeDisplayPath(name) << ']' << endl;
 }
@@ -53,7 +57,7 @@ void LyXLex::Pimpl::printError(string const & message) const
 void LyXLex::Pimpl::printTable(ostream & os)
 {
 	os << "\nNumber of tags: " << no_items << '\n';
-	for(int i= 0; i < no_items; ++i)
+	for (int i= 0; i < no_items; ++i)
 		os << "table[" << i
 		   << "]:  tag: `" << table[i].tag
 		   << "'  code:" << table[i].code << '\n';
@@ -65,7 +69,7 @@ void LyXLex::Pimpl::verifyTable()
 {
 	// Check if the table is sorted and if not, sort it.
 	if (table
-	    && !sorted(table, table + no_items, compare_tags())) {
+	    && !lyx::sorted(table, table + no_items, compare_tags())) {
 		lyxerr << "The table passed to LyXLex is not sorted!\n"
 		       << "Tell the developers to fix it!" << endl;
 		// We sort it anyway to avoid problems.
@@ -130,14 +134,30 @@ void LyXLex::Pimpl::setStream(istream & i)
 	lineno = 0;
 }
 
+void LyXLex::Pimpl::setCommentChar(char c)
+{
+	commentChar = c;
+}
+
 
 bool LyXLex::Pimpl::next(bool esc /* = false */)
 {
 	if (!pushTok.empty()) {
-		pushTok.copy(buff, string::npos);
-		buff[pushTok.length()] = '\0';
-		pushTok.erase();
-		return true;
+		// There can have been a whole line pushed so
+		// we extract the first word and leaves the rest
+		// in pushTok. (Lgb)
+		if (pushTok.find(' ') != string::npos) {
+			string tmp;
+			pushTok = split(pushTok, tmp, ' ');
+			tmp.copy(buff, string::npos);
+			buff[tmp.length()] = '\0';
+			return true;
+		} else {
+			pushTok.copy(buff, string::npos);
+			buff[pushTok.length()] = '\0';
+			pushTok.erase();
+			return true;
+		}     
 	}
 	if (!esc) {
 		unsigned char c = 0; // getc() returns an int
@@ -146,11 +166,17 @@ bool LyXLex::Pimpl::next(bool esc /* = false */)
 		while (is && !status) {
 			is.get(cc);
 			c = cc;
-			if (c == '#') {
+			if (c == commentChar) {
 				// Read rest of line (fast :-)
+				// That is not fast... (Lgb)
+#if 1
 				is.getline(buff, sizeof(buff));
 				lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 						      << buff << "'" << endl;
+#else
+				// unfortunately ignore is buggy (Lgb)
+				is.ignore(100, '\n');
+#endif
 				++lineno;
 				continue;
 			}
@@ -256,11 +282,18 @@ bool LyXLex::Pimpl::next(bool esc /* = false */)
 				continue;
 			}
 			
-			if (c == '#') {
+			if (c == commentChar) {
 				// Read rest of line (fast :-)
+				// That is still not fast... (Lgb)
+#if 1
 				is.getline(buff, sizeof(buff));
 				lyxerr[Debug::LYXLEX] << "Comment read: `" << c
 						      << buff << "'" << endl;
+#else
+				// but ignore is also still buggy (Lgb)
+				// This is fast (Lgb)
+				is.ignore(100, '\n');
+#endif
 				++lineno;
 				continue;
 			}
@@ -360,7 +393,7 @@ int LyXLex::Pimpl::lex()
 }
 
 	
-bool LyXLex::Pimpl::EatLine()
+bool LyXLex::Pimpl::eatLine()
 {
 	int i = 0;
 	unsigned char c = '\0';
@@ -368,8 +401,8 @@ bool LyXLex::Pimpl::EatLine()
 	while(is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
 		is.get(cc);
 		c = cc;
-		lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
-				      << c << "'" << endl;
+		//lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
+		//		      << c << "'" << endl;
 		if (c != '\r')
 			buff[i++] = c;
 	}
@@ -394,10 +427,21 @@ bool LyXLex::Pimpl::EatLine()
 bool LyXLex::Pimpl::nextToken()
 {
 	if (!pushTok.empty()) {
-		pushTok.copy(buff, string::npos);
-		buff[pushTok.length()] = '\0';
-		pushTok.erase();
-		return true;
+		// There can have been a whole line pushed so
+		// we extract the first word and leaves the rest
+		// in pushTok. (Lgb)
+		if (pushTok.find(' ') != string::npos) {
+			string tmp;
+			pushTok = split(pushTok, tmp, ' ');
+			tmp.copy(buff, string::npos);
+			buff[tmp.length()] = '\0';
+			return true;
+		} else {
+			pushTok.copy(buff, string::npos);
+			buff[pushTok.length()] = '\0';
+			pushTok.erase();
+			return true;
+		}
 	}
 
 	status = 0;