break parser::tokenize in parts

git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@27454 a592a061-630c-0410-9148-cb99ea01b6c8

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 5759cd544d310a38626fb808924332642160cf09..7ad85af3dc2a65317e23e59188f9df190fb6ecc4 100644 (file)
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -351,86 +351,93 @@ string const Parser::verbatimEnvironment(string const & name)
 }
 
 
-void Parser::tokenize(istream & is)
+void Parser::tokenize_one(istream & is)
 {
-       static bool init_done = false;
-
-       if (!init_done) {
-               catInit();
-               init_done = true;
-       }
-
        char c;
-       while (is.get(c)) {
-               //cerr << "reading c: " << c << "\n";
-
-               switch (catcode(c)) {
-                       case catSpace: {
-                               string s(1, c);
-                               while (is.get(c) && catcode(c) == catSpace)
+       if (!is.get(c)) 
+               return;
+       //cerr << "reading c: " << c << "\n";
+
+       switch (catcode(c)) {
+       case catSpace: {
+               string s(1, c);
+               while (is.get(c) && catcode(c) == catSpace)
+                       s += c;
+               if (catcode(c) != catSpace)
+                       is.putback(c);
+               push_back(Token(s, catSpace));
+               break;
+       }
+               
+       case catNewline: {
+               ++lineno_;
+               string s(1, getNewline(is, c));
+               while (is.get(c) && catcode(c) == catNewline) {
+                       ++lineno_;
+                       s += getNewline(is, c);
+               }
+               if (catcode(c) != catNewline)
+                       is.putback(c);
+               push_back(Token(s, catNewline));
+               break;
+       }
+               
+       case catComment: {
+               // We don't treat "%\n" combinations here specially because
+               // we want to preserve them in the preamble
+               string s;
+               while (is.get(c) && catcode(c) != catNewline)
+                       s += c;
+               // handle possible DOS line ending
+               if (catcode(c) == catNewline)
+                       c = getNewline(is, c);
+               // Note: The '%' at the beginning and the '\n' at the end
+               // of the comment are not stored.
+               ++lineno_;
+               push_back(Token(s, catComment));
+               break;
+       }
+               
+       case catEscape: {
+               is.get(c);
+               if (!is) {
+                       error("unexpected end of input");
+               } else {
+                       string s(1, c);
+                       if (catcode(c) == catLetter) {
+                               // collect letters
+                               while (is.get(c) && catcode(c) == catLetter)
                                        s += c;
-                               if (catcode(c) != catSpace)
-                                       is.putback(c);
-                               push_back(Token(s, catSpace));
-                               break;
-                       }
-
-                       case catNewline: {
-                               ++lineno_;
-                               string s(1, getNewline(is, c));
-                               while (is.get(c) && catcode(c) == catNewline) {
-                                       ++lineno_;
-                                       s += getNewline(is, c);
-                               }
-                               if (catcode(c) != catNewline)
+                               if (catcode(c) != catLetter)
                                        is.putback(c);
-                               push_back(Token(s, catNewline));
-                               break;
-                       }
-
-                       case catComment: {
-                               // We don't treat "%\n" combinations here specially because
-                               // we want to preserve them in the preamble
-                               string s;
-                               while (is.get(c) && catcode(c) != catNewline)
-                                       s += c;
-                               // handle possible DOS line ending
-                               if (catcode(c) == catNewline)
-                                       c = getNewline(is, c);
-                               // Note: The '%' at the beginning and the '\n' at the end
-                               // of the comment are not stored.
-                               ++lineno_;
-                               push_back(Token(s, catComment));
-                               break;
                        }
+                       push_back(Token(s, catEscape));
+               }
+               break;
+       }
+               
+       case catIgnore: {
+               cerr << "ignoring a char: " << int(c) << "\n";
+               break;
+       }
+               
+       default:
+               push_back(Token(c, catcode(c)));
+       }
+}
 
-                       case catEscape: {
-                               is.get(c);
-                               if (!is) {
-                                       error("unexpected end of input");
-                               } else {
-                                       string s(1, c);
-                                       if (catcode(c) == catLetter) {
-                                               // collect letters
-                                               while (is.get(c) && catcode(c) == catLetter)
-                                                       s += c;
-                                               if (catcode(c) != catLetter)
-                                                       is.putback(c);
-                                       }
-                                       push_back(Token(s, catEscape));
-                               }
-                               break;
-                       }
 
-                       case catIgnore: {
-                               cerr << "ignoring a char: " << int(c) << "\n";
-                               break;
-                       }
+void Parser::tokenize(istream & is)
+{
+       static bool init_done = false;
 
-                       default:
-                               push_back(Token(c, catcode(c)));
-               }
+       if (!init_done) {
+               catInit();
+               init_done = true;
        }
+
+       while (is) 
+               tokenize_one(is);
 }
 
 

diff --git a/src/tex2lyx/Parser.h b/src/tex2lyx/Parser.h
index 1d0fa87685d11540bd31110a65496dd1d34b9be8..daebca83659c5a76d9a82cd0e7f87cef39f7df92 100644 (file)
--- a/src/tex2lyx/Parser.h
+++ b/src/tex2lyx/Parser.h
@@ -174,6 +174,8 @@ public:
        char getChar();
        ///
        void error(std::string const & msg);
+       /// Parses one token from \p is 
+       void tokenize_one(std::istream & is);
        /// Parses \p is into tokens
        void tokenize(std::istream & is);
        ///