Implement deparsing of tokens

When switching catcodes (for example to verbatim), there are in general some tokens that have already been parsed according to the old catcodes. It is therefore needed to "forget" those token and reparse them.

The basic idea is to use idocstream::putback() to feed the characters back to the stream, but this does not work (probably because we disable buffering). Therefore, we implement a simple stream-like class that implements putback().

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 7139d1e3fa5b9b685beadc35626a3d3ab7ca0cc6..d7d3144b4b0013a10bfbe2407d7c93b07e1d7896 100644 (file)
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -12,6 +12,7 @@
 
 #include "Encoding.h"
 #include "Parser.h"
+#include "support/foreach.h"
 #include "support/lstrings.h"
 #include "support/textutils.h"
 
@@ -29,7 +30,7 @@ namespace {
  * \p c must have catcode catNewline, and it must be the last character read
  * from \p is.
  */
-char_type getNewline(idocstream & is, char_type c)
+char_type getNewline(iparserdocstream & is, char_type c)
 {
        // we have to handle 3 different line endings:
        // - UNIX (\n)
@@ -140,6 +141,20 @@ Parser::~Parser()
 }
 
 
+void Parser::deparse()
+{
+       for(size_type i = pos_ ; i < tokens_.size() ; ++i) {
+               docstring const s = from_utf8(tokens_[i].asInput());
+               //cerr << "deparsing [" << to_utf8(s) << "]" <<endl;
+               foreach(char_type c, s)
+                       is_.putback(c);
+       }
+       tokens_.erase(tokens_.begin() + pos_, tokens_.end());
+       // make sure that next token is read
+       tokenize_one();
+}
+
+
 void Parser::setEncoding(std::string const & e, int const & p)
 {
        // We may (and need to) use unsafe encodings here: Since the text is
@@ -197,19 +212,21 @@ CatCode Parser::catcode(char_type c) const
 void Parser::setCatcode(char c, CatCode cat)
 {
        theCatcode_[(unsigned char)c] = cat;
+       deparse();
 }
 
 
 void Parser::setCatcodes(cat_type t)
 {
        theCatcodesType_ = t;
+       deparse();
 }
 
 
 void Parser::setEncoding(std::string const & e)
 {
        //cerr << "setting encoding to " << e << std::endl;
-       is_ << lyx::setEncoding(e);
+       is_.docstream() << lyx::setEncoding(e);
        encoding_iconv_ = e;
 }
 
@@ -262,7 +279,9 @@ Token const Parser::next_next_token()
 Token const Parser::get_token()
 {
        static const Token dummy;
-       //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
+       // if (good()) 
+       //      cerr << "looking at token " << tokens_[pos_] 
+       //           << " pos: " << pos_ << '\n';
        return good() ? tokens_[pos_++] : dummy;
 }
 

diff --git a/src/tex2lyx/Parser.h b/src/tex2lyx/Parser.h
index 6511fb7b2c760dabb59cec66dfaa3298ce59754b..c6b7a72bb3d5d0b725593cb310ed8946ef2c84c0 100644 (file)
--- a/src/tex2lyx/Parser.h
+++ b/src/tex2lyx/Parser.h
@@ -117,6 +117,35 @@ std::ostream & operator<<(std::ostream & os, Token const & t);
 extern void debugToken(std::ostream & os, Token const & t, unsigned int flags);
 #endif
 
+// A docstream version that supports putback even when not buffered
+class iparserdocstream
+{
+public:
+       iparserdocstream(idocstream & is) : is_(is) {};
+
+       operator bool() const { return is_; };
+
+       idocstream & docstream() { return is_; };
+
+       void putback(char_type c) { s_ += c; };
+
+       iparserdocstream & get(char_type &c)
+       {
+               if (s_.empty())
+                       is_.get(c);
+               else {
+                       c = s_[0];
+                       s_.erase(0,1);
+               }
+               return *this;
+       };
+private:
+       ///
+       idocstream & is_;
+       ///
+       docstring s_;
+};
+
 
 /*!
  * Actual parser class
@@ -141,6 +170,11 @@ public:
        ///
        ~Parser();
 
+       /** forget already parsed next tokens and put the
+        * corresponding characters into the input stream for
+        * re-reading. Useful when changing catcodes. */
+       void deparse();
+
        ///
        CatCode catcode(char_type c) const;
        ///
@@ -298,7 +332,7 @@ private:
        ///
        idocstringstream * iss_;
        ///
-       idocstream & is_;
+       iparserdocstream is_;
        /// iconv name of the current encoding
        std::string encoding_iconv_;
        ///