tex2lyx/preamble.cpp: support Vietnamese (since format 291)

[lyx.git] / src / tex2lyx / Parser.cpp

diff --git a/src/tex2lyx/Parser.cpp b/src/tex2lyx/Parser.cpp
index 8e14406642492760628e6a1c92e052cb14636450..d387c6fcba89cfc2520a401ef1346794070fc08e 100644 (file)
--- a/src/tex2lyx/Parser.cpp
+++ b/src/tex2lyx/Parser.cpp
@@ -59,7 +59,7 @@ void catInit()
  * \p c must have catcode catNewline, and it must be the last character read
  * from \p is.
  */
-char getNewline(idocstream & is, char c)
+char_type getNewline(idocstream & is, char_type c)
 {
        // we have to handle 3 different line endings:
        // - UNIX (\n)
@@ -110,12 +110,6 @@ ostream & operator<<(ostream & os, Token const & t)
 }
 
 
-string Token::asString() const
-{
-       return cs_;
-}
-
-
 string Token::asInput() const
 {
        if (cat_ == catComment)
@@ -132,14 +126,15 @@ string Token::asInput() const
 
 
 Parser::Parser(idocstream & is)
-       : lineno_(0), pos_(0), iss_(0), is_(is)
+       : lineno_(0), pos_(0), iss_(0), is_(is), encoding_latex_("utf8")
 {
 }
 
 
 Parser::Parser(string const & s)
        : lineno_(0), pos_(0), 
-         iss_(new idocstringstream(from_utf8(s))), is_(*iss_)
+         iss_(new idocstringstream(from_utf8(s))), is_(*iss_), 
+         encoding_latex_("utf8")
 {
 }
 
@@ -153,8 +148,13 @@ Parser::~Parser()
 void Parser::setEncoding(std::string const & e)
 {
        Encoding const * enc = encodings.fromLaTeXName(e);
-       cerr << "setting encoding to " << enc->iconvName();
+       if (!enc) {
+               cerr << "Unknown encoding " << e << ". Ignoring." << std::endl;
+               return;
+       }
+       //cerr << "setting encoding to " << enc->iconvName() << std::endl;
        is_ << lyx::setEncoding(enc->iconvName());
+       encoding_latex_ = e;
 }
 
 
@@ -164,28 +164,32 @@ void Parser::push_back(Token const & t)
 }
 
 
-Token const & Parser::prev_token() const
+// We return a copy here because the tokens_ vector may get reallocated
+Token const Parser::prev_token() const
 {
        static const Token dummy;
        return pos_ > 1 ? tokens_[pos_ - 2] : dummy;
 }
 
 
-Token const & Parser::curr_token() const
+// We return a copy here because the tokens_ vector may get reallocated
+Token const Parser::curr_token() const
 {
        static const Token dummy;
        return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 }
 
 
-Token const & Parser::next_token()
+// We return a copy here because the tokens_ vector may get reallocated
+Token const Parser::next_token()
 {
        static const Token dummy;
        return good() ? tokens_[pos_] : dummy;
 }
 
 
-Token const & Parser::get_token()
+// We return a copy here because the tokens_ vector may get reallocated
+Token const Parser::get_token()
 {
        static const Token dummy;
        //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
@@ -211,20 +215,24 @@ bool Parser::isParagraph()
 }
 
 
-void Parser::skip_spaces(bool skip_comments)
+bool Parser::skip_spaces(bool skip_comments)
 {
        // We just silently return if we have no more tokens.
        // skip_spaces() should be callable at any time,
        // the caller must check p::good() anyway.
+       bool skipped = false;
        while (good()) {
                get_token();
                if (isParagraph()) {
                        putback();
                        break;
                }
-               if ( curr_token().cat() == catSpace ||
-                    curr_token().cat() == catNewline ||
-                   (curr_token().cat() == catComment && curr_token().cs().empty()))
+               if (curr_token().cat() == catSpace ||
+                   curr_token().cat() == catNewline) {
+                       skipped = true;
+                       continue;
+               }
+               if ((curr_token().cat() == catComment && curr_token().cs().empty()))
                        continue;
                if (skip_comments && curr_token().cat() == catComment)
                        cerr << "  Ignoring comment: " << curr_token().asInput();
@@ -233,6 +241,7 @@ void Parser::skip_spaces(bool skip_comments)
                        break;
                }
        }
+       return skipped;
 }
 
 
@@ -321,10 +330,23 @@ string Parser::getFullOpt()
 }
 
 
-string Parser::getOpt()
+string Parser::getOpt(bool keepws)
+{
+       string const res = getArg('[', ']');
+       if (res.empty()) {
+               if (keepws)
+                       unskip_spaces(true);
+               return string();
+       }
+       return '[' + res + ']';
+}
+
+
+string Parser::getOptContent()
+// the same as getOpt but without the brackets
 {
        string const res = getArg('[', ']');
-       return res.empty() ? string() : '[' + res + ']';
+       return res.empty() ? string() : res;
 }
 
 
@@ -474,7 +496,7 @@ string Parser::verbatimOption()
                                putback();
                                res += '{' + verbatim_item() + '}';
                        } else
-                               res += t.asString();
+                               res += t.cs();
                }
        }
        return res;