From fa3ea10057b4fa7549f5e1abc9a3042748e67275 Mon Sep 17 00:00:00 2001
From: Jean-Marc Lasgouttes <lasgouttes@lyx.org>
Date: Sat, 15 Nov 2008 14:38:27 +0000
Subject: [PATCH] break parser::tokenize in parts

git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@27454 a592a061-630c-0410-9148-cb99ea01b6c8
---
 src/tex2lyx/Parser.cpp | 149 +++++++++++++++++++++--------------------
 src/tex2lyx/Parser.h   |   2 +
 2 files changed, 80 insertions(+), 71 deletions(-)

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 5759cd544d..7ad85af3dc 100644
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -351,86 +351,93 @@ string const Parser::verbatimEnvironment(string const & name)
 }
 
 
-void Parser::tokenize(istream & is)
+void Parser::tokenize_one(istream & is)
 {
-	static bool init_done = false;
-
-	if (!init_done) {
-		catInit();
-		init_done = true;
-	}
-
 	char c;
-	while (is.get(c)) {
-		//cerr << "reading c: " << c << "\n";
-
-		switch (catcode(c)) {
-			case catSpace: {
-				string s(1, c);
-				while (is.get(c) && catcode(c) == catSpace)
+	if (!is.get(c)) 
+		return;
+	//cerr << "reading c: " << c << "\n";
+
+	switch (catcode(c)) {
+	case catSpace: {
+		string s(1, c);
+		while (is.get(c) && catcode(c) == catSpace)
+			s += c;
+		if (catcode(c) != catSpace)
+			is.putback(c);
+		push_back(Token(s, catSpace));
+		break;
+	}
+		
+	case catNewline: {
+		++lineno_;
+		string s(1, getNewline(is, c));
+		while (is.get(c) && catcode(c) == catNewline) {
+			++lineno_;
+			s += getNewline(is, c);
+		}
+		if (catcode(c) != catNewline)
+			is.putback(c);
+		push_back(Token(s, catNewline));
+		break;
+	}
+		
+	case catComment: {
+		// We don't treat "%\n" combinations here specially because
+		// we want to preserve them in the preamble
+		string s;
+		while (is.get(c) && catcode(c) != catNewline)
+			s += c;
+		// handle possible DOS line ending
+		if (catcode(c) == catNewline)
+			c = getNewline(is, c);
+		// Note: The '%' at the beginning and the '\n' at the end
+		// of the comment are not stored.
+		++lineno_;
+		push_back(Token(s, catComment));
+		break;
+	}
+		
+	case catEscape: {
+		is.get(c);
+		if (!is) {
+			error("unexpected end of input");
+		} else {
+			string s(1, c);
+			if (catcode(c) == catLetter) {
+				// collect letters
+				while (is.get(c) && catcode(c) == catLetter)
 					s += c;
-				if (catcode(c) != catSpace)
-					is.putback(c);
-				push_back(Token(s, catSpace));
-				break;
-			}
-
-			case catNewline: {
-				++lineno_;
-				string s(1, getNewline(is, c));
-				while (is.get(c) && catcode(c) == catNewline) {
-					++lineno_;
-					s += getNewline(is, c);
-				}
-				if (catcode(c) != catNewline)
+				if (catcode(c) != catLetter)
 					is.putback(c);
-				push_back(Token(s, catNewline));
-				break;
-			}
-
-			case catComment: {
-				// We don't treat "%\n" combinations here specially because
-				// we want to preserve them in the preamble
-				string s;
-				while (is.get(c) && catcode(c) != catNewline)
-					s += c;
-				// handle possible DOS line ending
-				if (catcode(c) == catNewline)
-					c = getNewline(is, c);
-				// Note: The '%' at the beginning and the '\n' at the end
-				// of the comment are not stored.
-				++lineno_;
-				push_back(Token(s, catComment));
-				break;
 			}
+			push_back(Token(s, catEscape));
+		}
+		break;
+	}
+		
+	case catIgnore: {
+		cerr << "ignoring a char: " << int(c) << "\n";
+		break;
+	}
+		
+	default:
+		push_back(Token(c, catcode(c)));
+	}
+}
 
-			case catEscape: {
-				is.get(c);
-				if (!is) {
-					error("unexpected end of input");
-				} else {
-					string s(1, c);
-					if (catcode(c) == catLetter) {
-						// collect letters
-						while (is.get(c) && catcode(c) == catLetter)
-							s += c;
-						if (catcode(c) != catLetter)
-							is.putback(c);
-					}
-					push_back(Token(s, catEscape));
-				}
-				break;
-			}
 
-			case catIgnore: {
-				cerr << "ignoring a char: " << int(c) << "\n";
-				break;
-			}
+void Parser::tokenize(istream & is)
+{
+	static bool init_done = false;
 
-			default:
-				push_back(Token(c, catcode(c)));
-		}
+	if (!init_done) {
+		catInit();
+		init_done = true;
 	}
+
+	while (is) 
+		tokenize_one(is);
 }
 
 
diff --git a/src/tex2lyx/Parser.h b/src/tex2lyx/Parser.h
index 1d0fa87685..daebca8365 100644
--- a/src/tex2lyx/Parser.h
+++ b/src/tex2lyx/Parser.h
@@ -174,6 +174,8 @@ public:
 	char getChar();
 	///
 	void error(std::string const & msg);
+	/// Parses one token from \p is 
+	void tokenize_one(std::istream & is);
 	/// Parses \p is into tokens
 	void tokenize(std::istream & is);
 	///
-- 
2.39.2