Account for old versions of Pygments

[lyx.git] / src / Lexer.cpp

diff --git a/src/Lexer.cpp b/src/Lexer.cpp
index b0fabc29ad68ef673fc840e32c1562f8975b228e..bbae12418c0bd1da6abfd542a6cdbc63b2e1976b 100644 (file)
--- a/src/Lexer.cpp
+++ b/src/Lexer.cpp
@@ -14,12 +14,14 @@
 #include <config.h>
 
 #include "Lexer.h"
+#include "Format.h"
 
 #include "support/convert.h"
 #include "support/debug.h"
 #include "support/FileName.h"
 #include "support/filetools.h"
 #include "support/gzstream.h"
+#include "support/lassert.h"
 #include "support/lstrings.h"
 #include "support/lyxalgo.h"
 #include "support/types.h"
@@ -237,9 +239,7 @@ void Lexer::Pimpl::popTable()
 bool Lexer::Pimpl::setFile(FileName const & filename)
 {
        // Check the format of the file.
-       string const format = filename.guessFormatFromContents();
-
-       if (format == "gzip" || format == "zip" || format == "compress") {
+       if (theFormats().isZippedFile(filename)) {
                LYXERR(Debug::LYXLEX, "lyxlex: compressed");
                // The check only outputs a debug message, because it triggers
                // a bug in compaq cxx 6.2, where is_open() returns 'true' for
@@ -249,9 +249,10 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
                                "file or stream already set.");
                gz_.open(filename.toFilesystemEncoding().c_str(), ios::in);
                is.rdbuf(&gz_);
-               name = filename.absFilename();
+               name = filename.absFileName();
                lineno = 0;
-               return gz_.is_open() && is.good();
+               if (!gz_.is_open() || !is.good())
+                       return false;
        } else {
                LYXERR(Debug::LYXLEX, "lyxlex: UNcompressed");
 
@@ -262,12 +263,25 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
                        LYXERR(Debug::LYXLEX, "Error in Lexer::setFile: "
                                "file or stream already set.");
                }
-               fb_.open(filename.toFilesystemEncoding().c_str(), ios::in);
+               fb_.open(filename.toSafeFilesystemEncoding().c_str(), ios::in);
                is.rdbuf(&fb_);
-               name = filename.absFilename();
+               name = filename.absFileName();
                lineno = 0;
-               return fb_.is_open() && is.good();
+               if (!fb_.is_open() || !is.good())
+                       return false;
+       }
+
+       // Skip byte order mark.
+       if (is.peek() == 0xef) {
+               is.get();
+               if (is.peek() == 0xbb) {
+                       is.get();
+                       LASSERT(is.get() == 0xbf, /**/);
+               } else
+                       is.unget();
        }
+
+       return true;
 }
 
 
@@ -306,12 +320,11 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
        }
 
 
-       unsigned char c = 0; // getc() returns an int
        char cc = 0;
        status = 0;
        while (is && !status) {
                is.get(cc);
-               c = cc;
+               unsigned char c = cc;
 
                if (c == commentChar) {
                        // Read rest of line (fast :-)
@@ -334,9 +347,8 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
 
                        if (esc) {
 
-                               bool escaped = false;
                                do {
-                                       escaped = false;
+                                       bool escaped = false;
                                        is.get(cc);
                                        c = cc;
                                        if (c == '\r') continue;
@@ -458,7 +470,7 @@ bool Lexer::Pimpl::eatLine()
                is.get(cc);
                c = cc;
                //LYXERR(Debug::LYXLEX, "Lexer::EatLine read char: `" << c << '\'');
-               if (c != '\r')
+               if (c != '\r' && is)
                        buff.push_back(c);
        }
 
@@ -555,7 +567,7 @@ void Lexer::Pimpl::pushToken(string const & pt)
 //////////////////////////////////////////////////////////////////////
 
 Lexer::Lexer()
-       : pimpl_(new Pimpl(0, 0))
+       : pimpl_(new Pimpl(0, 0)), lastReadOk_(false)
 {}
 
 
@@ -681,23 +693,23 @@ double Lexer::getFloat() const
 }
 
 
-string const Lexer::getString() const
+string const Lexer::getString(bool trim) const
 {
        lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
 
        if (lastReadOk_)
-       return pimpl_->getString();
+               return trim ? support::trim(pimpl_->getString(), "\t ") : pimpl_->getString();
 
        return string();
 }
 
 
-docstring const Lexer::getDocString() const
+docstring const Lexer::getDocString(bool trim) const
 {
        lastReadOk_ = pimpl_->status == LEX_DATA || pimpl_->status == LEX_TOKEN;
 
        if (lastReadOk_)
-               return pimpl_->getDocString();
+               return trim ? support::trim(pimpl_->getDocString(), "\t ") : pimpl_->getDocString();
 
        return docstring();
 }
@@ -706,28 +718,27 @@ docstring const Lexer::getDocString() const
 // I would prefer to give a tag number instead of an explicit token
 // here, but it is not possible because Buffer::readDocument uses
 // explicit tokens (JMarc)
-string const Lexer::getLongString(string const & endtoken)
+docstring Lexer::getLongString(docstring const & endtoken)
 {
-       string str;
-       string prefix;
+       docstring str;
+       docstring prefix;
        bool firstline = true;
 
        while (pimpl_->is) { //< eatLine only reads from is, not from pushTok
                if (!eatLine())
                        // blank line in the file being read
                        continue;
+               docstring tmpstr = getDocString();
+               docstring const token = trim(tmpstr, " \t");
 
-               string const token = trim(getString(), " \t");
-
-               LYXERR(Debug::PARSER, "LongString: `" << getString() << '\'');
+               LYXERR(Debug::PARSER, "LongString: `" << tmpstr << '\'');
 
                // We do a case independent comparison, like searchKeyword does.
-               if (compare_ascii_no_case(token, endtoken) == 0)
+               if (compare_no_case(token, endtoken) == 0)
                        break;
 
-               string tmpstr = getString();
                if (firstline) {
-                       size_t i = tmpstr.find_first_not_of(' ');
+                       size_t i = tmpstr.find_first_not_of(char_type(' '));
                        if (i != string::npos)
                                prefix = tmpstr.substr(0, i);
                        firstline = false;
@@ -743,7 +754,7 @@ string const Lexer::getLongString(string const & endtoken)
        }
 
        if (!pimpl_->is)
-               printError("Long string not ended by `" + endtoken + '\'');
+               printError("Long string not ended by `" + to_utf8(endtoken) + '\'');
 
        return str;
 }
@@ -901,6 +912,18 @@ string Lexer::quoteString(string const & arg)
 }
 
 
+// same for docstring
+docstring Lexer::quoteString(docstring const & arg)
+{
+       docstring res;
+       res += '"';
+       res += subst(subst(arg, from_ascii("\\"), from_ascii("\\\\")), 
+                    from_ascii("\""), from_ascii("\\\""));
+       res += '"';
+       return res;
+}
+
+
 Lexer & Lexer::operator>>(char const * required)
 {
        string token;