Fix bug in paragraph detection (can be seen in test case of bug #5187):

If we want to look at the token after the next token, it may be needed
to call tokenize_one() twice and not only once as done in good().

git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@40234 a592a061-630c-0410-9148-cb99ea01b6c8

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 5fafef9944c6f7ebde6ba693bfdf8bd50f16483b..c48301207a447ec0e5593c64fea0e2b39881a2c0 100644 (file)
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -220,6 +220,20 @@ Token const Parser::next_token()
 }
 
 
+// We return a copy here because the tokens_ vector may get reallocated
+Token const Parser::next_next_token()
+{
+       static const Token dummy;
+       // If good() has not been called after the last get_token() we need
+       // to tokenize two more tokens.
+       if (pos_ + 1 >= tokens_.size()) {
+               tokenize_one();
+               tokenize_one();
+       }
+       return pos_ + 1 < tokens_.size() ? tokens_[pos_ + 1] : dummy;
+}
+
+
 // We return a copy here because the tokens_ vector may get reallocated
 Token const Parser::get_token()
 {
@@ -238,8 +252,7 @@ bool Parser::isParagraph()
        if (curr_token().cat() == catNewline &&
            (curr_token().cs().size() > 1 ||
             (next_token().cat() == catSpace &&
-             pos_ < tokens_.size() - 1 &&
-             tokens_[pos_ + 1].cat() == catNewline)))
+             next_next_token().cat() == catNewline)))
                return true;
        if (curr_token().cat() == catEscape && curr_token().cs() == "par")
                return true;

diff --git a/src/tex2lyx/Parser.h b/src/tex2lyx/Parser.h
index dbb202ddf008c17c7a1cef10af2fa6779e9c4c89..5e749e3019f40d8be6897664d0e68fed87ae94cb 100644 (file)
--- a/src/tex2lyx/Parser.h
+++ b/src/tex2lyx/Parser.h
@@ -213,6 +213,8 @@ public:
        Token const curr_token() const;
        /// The next token.
        Token const next_token();
+       /// The next but one token.
+       Token const next_next_token();
        /// Make the next token current and return that.
        Token const get_token();
        /// \return whether the current token starts a new paragraph