prepare Qt 5.6 builds

[lyx.git] / src / Lexer.cpp

diff --git a/src/Lexer.cpp b/src/Lexer.cpp
index 08b2f08077048585d44a09c37fcaefcfaaa7d937..25d337a0793287e3ac6ba8e532dc74ca821b076f 100644 (file)
--- a/src/Lexer.cpp
+++ b/src/Lexer.cpp
@@ -4,7 +4,7 @@
  * Licence details can be found in the file COPYING.
  *
  * \author Alejandro Aguilar Sierra
- * \author Lars Gullik Bjønnes
+ * \author Lars Gullik Bjønnes
  * \author Jean-Marc Lasgouttes
  * \author John Levon
  *
@@ -14,12 +14,14 @@
 #include <config.h>
 
 #include "Lexer.h"
+#include "Format.h"
 
 #include "support/convert.h"
 #include "support/debug.h"
 #include "support/FileName.h"
 #include "support/filetools.h"
 #include "support/gzstream.h"
+#include "support/lassert.h"
 #include "support/lstrings.h"
 #include "support/lyxalgo.h"
 #include "support/types.h"
@@ -237,9 +239,7 @@ void Lexer::Pimpl::popTable()
 bool Lexer::Pimpl::setFile(FileName const & filename)
 {
        // Check the format of the file.
-       string const format = filename.guessFormatFromContents();
-
-       if (format == "gzip" || format == "zip" || format == "compress") {
+       if (formats.isZippedFile(filename)) {
                LYXERR(Debug::LYXLEX, "lyxlex: compressed");
                // The check only outputs a debug message, because it triggers
                // a bug in compaq cxx 6.2, where is_open() returns 'true' for
@@ -249,9 +249,10 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
                                "file or stream already set.");
                gz_.open(filename.toFilesystemEncoding().c_str(), ios::in);
                is.rdbuf(&gz_);
-               name = filename.absFilename();
+               name = filename.absFileName();
                lineno = 0;
-               return gz_.is_open() && is.good();
+               if (!gz_.is_open() || !is.good())
+                       return false;
        } else {
                LYXERR(Debug::LYXLEX, "lyxlex: UNcompressed");
 
@@ -262,12 +263,25 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
                        LYXERR(Debug::LYXLEX, "Error in Lexer::setFile: "
                                "file or stream already set.");
                }
-               fb_.open(filename.toFilesystemEncoding().c_str(), ios::in);
+               fb_.open(filename.toSafeFilesystemEncoding().c_str(), ios::in);
                is.rdbuf(&fb_);
-               name = filename.absFilename();
+               name = filename.absFileName();
                lineno = 0;
-               return fb_.is_open() && is.good();
+               if (!fb_.is_open() || !is.good())
+                       return false;
+       }
+
+       // Skip byte order mark.
+       if (is.peek() == 0xef) {
+               is.get();
+               if (is.peek() == 0xbb) {
+                       is.get();
+                       LASSERT(is.get() == 0xbf, /**/);
+               } else
+                       is.unget();
        }
+
+       return true;
 }
 
 
@@ -306,12 +320,11 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
        }
 
 
-       unsigned char c = 0; // getc() returns an int
        char cc = 0;
        status = 0;
        while (is && !status) {
                is.get(cc);
-               c = cc;
+               unsigned char c = cc;
 
                if (c == commentChar) {
                        // Read rest of line (fast :-)
@@ -334,9 +347,8 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
 
                        if (esc) {
 
-                               bool escaped = false;
                                do {
-                                       escaped = false;
+                                       bool escaped = false;
                                        is.get(cc);
                                        c = cc;
                                        if (c == '\r') continue;
@@ -458,7 +470,7 @@ bool Lexer::Pimpl::eatLine()
                is.get(cc);
                c = cc;
                //LYXERR(Debug::LYXLEX, "Lexer::EatLine read char: `" << c << '\'');
-               if (c != '\r')
+               if (c != '\r' && is)
                        buff.push_back(c);
        }
 
@@ -499,7 +511,7 @@ bool Lexer::Pimpl::nextToken()
                char cc = 0;
                is.get(cc);
                c = cc;
-               if (c >= ' ' && is) {
+               if ((c >= ' ' || c == '\t') && is) {
                        buff.clear();
 
                        if (c == '\\') { // first char == '\\'
@@ -513,7 +525,7 @@ bool Lexer::Pimpl::nextToken()
                                        buff.push_back(c);
                                        is.get(cc);
                                        c = cc;
-                               } while (c >= ' ' && c != '\\' && is);
+                               } while ((c >= ' ' || c == '\t') && c != '\\' && is);
                        }
 
                        if (c == '\\')
@@ -555,7 +567,7 @@ void Lexer::Pimpl::pushToken(string const & pt)
 //////////////////////////////////////////////////////////////////////
 
 Lexer::Lexer()
-       : pimpl_(new Pimpl(0, 0))
+       : pimpl_(new Pimpl(0, 0)), lastReadOk_(false)
 {}
 
 
@@ -636,6 +648,7 @@ void Lexer::setCommentChar(char c)
        pimpl_->setCommentChar(c);
 }
 
+
 int Lexer::lex()
 {
        return pimpl_->lex();
@@ -680,23 +693,23 @@ double Lexer::getFloat() const
 }
 
 
-string const Lexer::getString() const
+string const Lexer::getString(bool trim) const
 {
        lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
 
        if (lastReadOk_)
-       return pimpl_->getString();
+               return trim ? support::trim(pimpl_->getString(), "\t ") : pimpl_->getString();
 
        return string();
 }
 
 
-docstring const Lexer::getDocString() const
+docstring const Lexer::getDocString(bool trim) const
 {
        lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
 
        if (lastReadOk_)
-               return pimpl_->getDocString();
+               return trim ? support::trim(pimpl_->getDocString(), "\t ") : pimpl_->getDocString();
 
        return docstring();
 }
@@ -900,6 +913,18 @@ string Lexer::quoteString(string const & arg)
 }
 
 
+// same for docstring
+docstring Lexer::quoteString(docstring const & arg)
+{
+       docstring res;
+       res += '"';
+       res += subst(subst(arg, from_ascii("\\"), from_ascii("\\\\")), 
+                    from_ascii("\""), from_ascii("\\\""));
+       res += '"';
+       return res;
+}
+
+
 Lexer & Lexer::operator>>(char const * required)
 {
        string token;