Handle properly insets which have the PassThru property

[lyx.git] / src / tex2lyx / Parser.cpp

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 7139d1e3fa5b9b685beadc35626a3d3ab7ca0cc6..11ecfe12ec410698ec1caa56d4f14ba80ff137a1 100644 (file)
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -12,6 +12,7 @@
 
 #include "Encoding.h"
 #include "Parser.h"
+#include "support/foreach.h"
 #include "support/lstrings.h"
 #include "support/textutils.h"
 
@@ -29,7 +30,7 @@ namespace {
  * \p c must have catcode catNewline, and it must be the last character read
  * from \p is.
  */
-char_type getNewline(idocstream & is, char_type c)
+char_type getNewline(iparserdocstream & is, char_type c)
 {
        // we have to handle 3 different line endings:
        // - UNIX (\n)
@@ -113,6 +114,45 @@ void debugToken(std::ostream & os, Token const & t, unsigned int flags)
 #endif
 
 
+//
+// Wrapper
+//
+
+bool iparserdocstream::setEncoding(std::string const & e)
+{
+       is_ << lyx::setEncoding(e);
+       if (s_.empty())
+               return true;
+       cerr << "Setting encoding " << e << " too late. The encoding of `"
+            << to_utf8(s_) << "´ is wrong." << std::endl;
+       return false;
+}
+
+
+void iparserdocstream::putback(char_type c)
+{
+       s_ += c;
+}
+
+
+void iparserdocstream::put_almost_back(docstring s)
+{
+       s_ = s + s_;
+}
+
+
+iparserdocstream & iparserdocstream::get(char_type &c)
+{
+       if (s_.empty())
+               is_.get(c);
+       else {
+               c = s_[0];
+               s_.erase(0,1);
+       }
+       return *this;
+}
+
+
 //
 // Parser
 //
@@ -140,7 +180,20 @@ Parser::~Parser()
 }
 
 
-void Parser::setEncoding(std::string const & e, int const & p)
+void Parser::deparse()
+{
+       string s;
+       for(size_type i = pos_ ; i < tokens_.size() ; ++i) {
+               s += tokens_[i].asInput();
+       }
+       is_.put_almost_back(from_utf8(s));
+       tokens_.erase(tokens_.begin() + pos_, tokens_.end());
+       // make sure that next token is read
+       tokenize_one();
+}
+
+
+bool Parser::setEncoding(std::string const & e, int const & p)
 {
        // We may (and need to) use unsafe encodings here: Since the text is
        // converted to unicode while reading from is_, we never see text in
@@ -149,9 +202,9 @@ void Parser::setEncoding(std::string const & e, int const & p)
        Encoding const * const enc = encodings.fromLaTeXName(e, p, true);
        if (!enc) {
                cerr << "Unknown encoding " << e << ". Ignoring." << std::endl;
-               return;
+               return false;
        }
-       setEncoding(enc->iconvName());
+       return setEncoding(enc->iconvName());
 }
 
 
@@ -197,20 +250,22 @@ CatCode Parser::catcode(char_type c) const
 void Parser::setCatcode(char c, CatCode cat)
 {
        theCatcode_[(unsigned char)c] = cat;
+       deparse();
 }
 
 
 void Parser::setCatcodes(cat_type t)
 {
        theCatcodesType_ = t;
+       deparse();
 }
 
 
-void Parser::setEncoding(std::string const & e)
+bool Parser::setEncoding(std::string const & e)
 {
        //cerr << "setting encoding to " << e << std::endl;
-       is_ << lyx::setEncoding(e);
        encoding_iconv_ = e;
+       return is_.setEncoding(e);
 }
 
 
@@ -240,7 +295,11 @@ Token const Parser::curr_token() const
 Token const Parser::next_token()
 {
        static const Token dummy;
-       return good() ? tokens_[pos_] : dummy;
+       if (!good())
+               return dummy;
+       if (pos_ >= tokens_.size())
+               tokenize_one();
+       return pos_ < tokens_.size() ? tokens_[pos_] : dummy;
 }
 
 
@@ -248,11 +307,14 @@ Token const Parser::next_token()
 Token const Parser::next_next_token()
 {
        static const Token dummy;
-       // If good() has not been called after the last get_token() we need
-       // to tokenize two more tokens.
-       if (pos_ + 1 >= tokens_.size()) {
-               tokenize_one();
+       if (!good())
+               return dummy;
+       // If tokenize_one() has not been called after the last get_token() we
+       // need to tokenize two more tokens.
+       if (pos_ >= tokens_.size()) {
                tokenize_one();
+               if (pos_ + 1 >= tokens_.size())
+                       tokenize_one();
        }
        return pos_ + 1 < tokens_.size() ? tokens_[pos_ + 1] : dummy;
 }
@@ -262,8 +324,16 @@ Token const Parser::next_next_token()
 Token const Parser::get_token()
 {
        static const Token dummy;
-       //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
-       return good() ? tokens_[pos_++] : dummy;
+       if (!good())
+               return dummy;
+       if (pos_ >= tokens_.size()) {
+               tokenize_one();
+               if (pos_ >= tokens_.size())
+                       return dummy;
+       }
+       // cerr << "looking at token " << tokens_[pos_] 
+       //      << " pos: " << pos_ << '\n';
+       return tokens_[pos_++];
 }
 
 
@@ -362,16 +432,9 @@ bool Parser::good()
 {
        if (pos_ < tokens_.size())
                return true;
-       tokenize_one();
-       return pos_ < tokens_.size();
-}
-
-
-char Parser::getChar()
-{
-       if (!good())
-               error("The input stream is not well...");
-       return get_token().character();
+       if (!is_.good())
+               return false;
+       return is_.peek() != idocstream::traits_type::eof();
 }
 
 
@@ -424,7 +487,8 @@ Parser::Arg Parser::getFullArg(char left, char right, bool allow_escaping)
                putback();
                return make_pair(false, string());
        } else {
-               for (t = get_token(); good(); t = get_token()) {
+               while (good()) {
+                       t = get_token();
                        // Ignore comments
                        if (t.cat() == catComment) {
                                if (!t.cs().empty())