Initialize BibTeXInfo modifier to zero, always.

[lyx.git] / src / Lexer.cpp

diff --git a/src/Lexer.cpp b/src/Lexer.cpp
index 5136c672073d6dc451360a22f6b088915c1a66f3..06a3408f41e9591d77afabfbb3074d2f3b18d9de 100644 (file)
--- a/src/Lexer.cpp
+++ b/src/Lexer.cpp
@@ -4,7 +4,7 @@
  * Licence details can be found in the file COPYING.
  *
  * \author Alejandro Aguilar Sierra
- * \author Lars Gullik Bjønnes
+ * \author Lars Gullik Bjønnes
  * \author Jean-Marc Lasgouttes
  * \author John Levon
  *
@@ -14,12 +14,14 @@
 #include <config.h>
 
 #include "Lexer.h"
+#include "Format.h"
 
 #include "support/convert.h"
 #include "support/debug.h"
 #include "support/FileName.h"
 #include "support/filetools.h"
 #include "support/gzstream.h"
+#include "support/lassert.h"
 #include "support/lstrings.h"
 #include "support/lyxalgo.h"
 #include "support/types.h"
@@ -173,7 +175,9 @@ void Lexer::Pimpl::printError(string const & message) const
 {
        string const tmpmsg = subst(message, "$$Token", getString());
        lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
-               << " of file " << to_utf8(makeDisplayPath(name)) << ']' << endl;
+               << " of file " << to_utf8(makeDisplayPath(name))
+               << " current token: '" << getString() << "'"
+               << " context: '" << context << "']" << endl;
 }
 
 
@@ -234,10 +238,14 @@ void Lexer::Pimpl::popTable()
 
 bool Lexer::Pimpl::setFile(FileName const & filename)
 {
+#ifdef TEX2LYX
+       // tex2lyx does not read lyxrc and therefore can't really check for
+       // zipped formats.
+       if (false) {
+#else
        // Check the format of the file.
-       string const format = filename.guessFormatFromContents();
-
-       if (format == "gzip" || format == "zip" || format == "compress") {
+       if (formats.isZippedFile(filename)) {
+#endif
                LYXERR(Debug::LYXLEX, "lyxlex: compressed");
                // The check only outputs a debug message, because it triggers
                // a bug in compaq cxx 6.2, where is_open() returns 'true' for
@@ -247,9 +255,10 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
                                "file or stream already set.");
                gz_.open(filename.toFilesystemEncoding().c_str(), ios::in);
                is.rdbuf(&gz_);
-               name = filename.absFilename();
+               name = filename.absFileName();
                lineno = 0;
-               return gz_.is_open() && is.good();
+               if (!gz_.is_open() || !is.good())
+                       return false;
        } else {
                LYXERR(Debug::LYXLEX, "lyxlex: UNcompressed");
 
@@ -260,12 +269,25 @@ bool Lexer::Pimpl::setFile(FileName const & filename)
                        LYXERR(Debug::LYXLEX, "Error in Lexer::setFile: "
                                "file or stream already set.");
                }
-               fb_.open(filename.toFilesystemEncoding().c_str(), ios::in);
+               fb_.open(filename.toSafeFilesystemEncoding().c_str(), ios::in);
                is.rdbuf(&fb_);
-               name = filename.absFilename();
+               name = filename.absFileName();
                lineno = 0;
-               return fb_.is_open() && is.good();
+               if (!fb_.is_open() || !is.good())
+                       return false;
+       }
+
+       // Skip byte order mark.
+       if (is.peek() == 0xef) {
+               is.get();
+               if (is.peek() == 0xbb) {
+                       is.get();
+                       LASSERT(is.get() == 0xbf, /**/);
+               } else
+                       is.unget();
        }
+
+       return true;
 }
 
 
@@ -311,11 +333,6 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
                is.get(cc);
                c = cc;
 
-               // skip ','s
-               if (esc && c == ',')
-                       continue;
-
-               
                if (c == commentChar) {
                        // Read rest of line (fast :-)
 #if 1
@@ -380,13 +397,13 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
                        break;
                }
 
-               if (!esc && c == ',')
+               if (c == ',')
                        continue;              /* Skip ','s */
 
-                       // using relational operators with chars other
-                       // than == and != is not safe. And if it is done
-                       // the type _have_ to be unsigned. It usually a
-                       // lot better to use the functions from cctype
+               // using relational operators with chars other
+               // than == and != is not safe. And if it is done
+               // the type _have_ to be unsigned. It usually a
+               // lot better to use the functions from cctype
                if (c > ' ' && is)  {
                        buff.clear();
 
@@ -404,7 +421,7 @@ bool Lexer::Pimpl::next(bool esc /* = false */)
                        status = LEX_TOKEN;
                }
 
-               if (!esc && c == '\r' && is) {
+               if (c == '\r' && is) {
                        // The Windows support has lead to the
                        // possibility of "\r\n" at the end of
                        // a line.  This will stop LyX choking
@@ -461,7 +478,7 @@ bool Lexer::Pimpl::eatLine()
                is.get(cc);
                c = cc;
                //LYXERR(Debug::LYXLEX, "Lexer::EatLine read char: `" << c << '\'');
-               if (c != '\r')
+               if (c != '\r' && is)
                        buff.push_back(c);
        }
 
@@ -502,7 +519,7 @@ bool Lexer::Pimpl::nextToken()
                char cc = 0;
                is.get(cc);
                c = cc;
-               if (c >= ' ' && is) {
+               if ((c >= ' ' || c == '\t') && is) {
                        buff.clear();
 
                        if (c == '\\') { // first char == '\\'
@@ -516,7 +533,7 @@ bool Lexer::Pimpl::nextToken()
                                        buff.push_back(c);
                                        is.get(cc);
                                        c = cc;
-                               } while (c >= ' ' && c != '\\' && is);
+                               } while ((c >= ' ' || c == '\t') && c != '\\' && is);
                        }
 
                        if (c == '\\')
@@ -639,6 +656,7 @@ void Lexer::setCommentChar(char c)
        pimpl_->setCommentChar(c);
 }
 
+
 int Lexer::lex()
 {
        return pimpl_->lex();
@@ -765,6 +783,7 @@ bool Lexer::getBool() const
        pimpl_->printError("Bad boolean `$$Token'. "
                                 "Use \"false\" or \"true\"");
        lastReadOk_ = false;
+       return false;
 }
 
 
@@ -902,6 +921,18 @@ string Lexer::quoteString(string const & arg)
 }
 
 
+// same for docstring
+docstring Lexer::quoteString(docstring const & arg)
+{
+       docstring res;
+       res += '"';
+       res += subst(subst(arg, from_ascii("\\"), from_ascii("\\\\")), 
+                    from_ascii("\""), from_ascii("\\\""));
+       res += '"';
+       return res;
+}
+
+
 Lexer & Lexer::operator>>(char const * required)
 {
        string token;