From 47b8ab14db71c6932ba5a8f159fc71a745792aca Mon Sep 17 00:00:00 2001
From: Jean-Marc Lasgouttes <lasgouttes@lyx.org>
Date: Fri, 15 Feb 2013 17:03:18 +0100
Subject: [PATCH] Implement deparsing of tokens

When switching catcodes (for example to verbatim), there are in general some tokens that have already been parsed according to the old catcodes. It is therefore needed to "forget" those token and reparse them.

The basic idea is to use idocstream::putback() to feed the characters back to the stream, but this does not work (probably because we disable buffering). Therefore, we implement a simple stream-like class that implements putback().
---
 src/tex2lyx/Parser.cpp | 25 ++++++++++++++++++++++---
 src/tex2lyx/Parser.h   | 36 +++++++++++++++++++++++++++++++++++-
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 7139d1e3fa..d7d3144b4b 100644
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -12,6 +12,7 @@
 
 #include "Encoding.h"
 #include "Parser.h"
+#include "support/foreach.h"
 #include "support/lstrings.h"
 #include "support/textutils.h"
 
@@ -29,7 +30,7 @@ namespace {
  * \p c must have catcode catNewline, and it must be the last character read
  * from \p is.
  */
-char_type getNewline(idocstream & is, char_type c)
+char_type getNewline(iparserdocstream & is, char_type c)
 {
 	// we have to handle 3 different line endings:
 	// - UNIX (\n)
@@ -140,6 +141,20 @@ Parser::~Parser()
 }
 
 
+void Parser::deparse()
+{
+	for(size_type i = pos_ ; i < tokens_.size() ; ++i) {
+		docstring const s = from_utf8(tokens_[i].asInput());
+		//cerr << "deparsing [" << to_utf8(s) << "]" <<endl;
+		foreach(char_type c, s)
+			is_.putback(c);
+	}
+	tokens_.erase(tokens_.begin() + pos_, tokens_.end());
+	// make sure that next token is read
+	tokenize_one();
+}
+
+
 void Parser::setEncoding(std::string const & e, int const & p)
 {
 	// We may (and need to) use unsafe encodings here: Since the text is
@@ -197,19 +212,21 @@ CatCode Parser::catcode(char_type c) const
 void Parser::setCatcode(char c, CatCode cat)
 {
 	theCatcode_[(unsigned char)c] = cat;
+	deparse();
 }
 
 
 void Parser::setCatcodes(cat_type t)
 {
 	theCatcodesType_ = t;
+	deparse();
 }
 
 
 void Parser::setEncoding(std::string const & e)
 {
 	//cerr << "setting encoding to " << e << std::endl;
-	is_ << lyx::setEncoding(e);
+	is_.docstream() << lyx::setEncoding(e);
 	encoding_iconv_ = e;
 }
 
@@ -262,7 +279,9 @@ Token const Parser::next_next_token()
 Token const Parser::get_token()
 {
 	static const Token dummy;
-	//cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
+	// if (good()) 
+	// 	cerr << "looking at token " << tokens_[pos_] 
+	// 	     << " pos: " << pos_ << '\n';
 	return good() ? tokens_[pos_++] : dummy;
 }
 
diff --git a/src/tex2lyx/Parser.h b/src/tex2lyx/Parser.h
index 6511fb7b2c..c6b7a72bb3 100644
--- a/src/tex2lyx/Parser.h
+++ b/src/tex2lyx/Parser.h
@@ -117,6 +117,35 @@ std::ostream & operator<<(std::ostream & os, Token const & t);
 extern void debugToken(std::ostream & os, Token const & t, unsigned int flags);
 #endif
 
+// A docstream version that supports putback even when not buffered
+class iparserdocstream
+{
+public:
+	iparserdocstream(idocstream & is) : is_(is) {};
+
+	operator bool() const { return is_; };
+
+	idocstream & docstream() { return is_; };
+
+	void putback(char_type c) { s_ += c; };
+
+	iparserdocstream & get(char_type &c)
+	{
+		if (s_.empty())
+			is_.get(c);
+		else {
+			c = s_[0];
+			s_.erase(0,1);
+		}
+		return *this;
+	};
+private:
+	///
+	idocstream & is_;
+	///
+	docstring s_;
+};
+
 
 /*!
  * Actual parser class
@@ -141,6 +170,11 @@ public:
 	///
 	~Parser();
 
+	/** forget already parsed next tokens and put the
+	 * corresponding characters into the input stream for
+	 * re-reading. Useful when changing catcodes. */
+	void deparse();
+
 	///
 	CatCode catcode(char_type c) const;
 	///
@@ -298,7 +332,7 @@ private:
 	///
 	idocstringstream * iss_;
 	///
-	idocstream & is_;
+	iparserdocstream is_;
 	/// iconv name of the current encoding
 	std::string encoding_iconv_;
 	///
-- 
2.39.5