Fix fuer #209

[lyx.git] / src / lyxlex_pimpl.C

diff --git a/src/lyxlex_pimpl.C b/src/lyxlex_pimpl.C
index 94a268f1713fbe0868ecaac0c4e02f00b9b2d29b..1dc807724e00ff4470132b768b7386a1ee3fdb63 100644 (file)
--- a/src/lyxlex_pimpl.C
+++ b/src/lyxlex_pimpl.C
@@ -8,6 +8,7 @@
 #include "lyxlex_pimpl.h"
 #include "support/lyxalgo.h"
 #include "support/filetools.h"
+#include "support/lstrings.h"
 #include "debug.h"
 
 using std::sort;
@@ -22,7 +23,10 @@ struct compare_tags {
        // used by lower_bound, sort and sorted
        inline
        int operator()(keyword_item const & a, keyword_item const & b) const {
-               return compare_no_case(a.tag, b.tag) < 0;
+               // we use the ascii version, because in turkish, 'i'
+               // is not the lowercase version of 'I', and thus
+               // turkish locale breaks parsing of tags.
+               return compare_ascii_no_case(a.tag, b.tag) < 0;
        }
 };
 // } // end of anon namespace
@@ -36,7 +40,7 @@ LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
 }
 
 
-string const LyXLex::Pimpl::GetString() const
+string const LyXLex::Pimpl::getString() const
 {
        return string(buff);
 }
@@ -44,7 +48,7 @@ string const LyXLex::Pimpl::GetString() const
 
 void LyXLex::Pimpl::printError(string const & message) const
 {
-       string tmpmsg = subst(message, "$$Token", GetString());
+       string const tmpmsg = subst(message, "$$Token", getString());
        lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
               << " of file " << MakeDisplayPath(name) << ']' << endl;
 }
@@ -65,7 +69,7 @@ void LyXLex::Pimpl::verifyTable()
 {
        // Check if the table is sorted and if not, sort it.
        if (table
-           && !sorted(table, table + no_items, compare_tags())) {
+           && !lyx::sorted(table, table + no_items, compare_tags())) {
                lyxerr << "The table passed to LyXLex is not sorted!\n"
                       << "Tell the developers to fix it!" << endl;
                // We sort it anyway to avoid problems.
@@ -139,10 +143,21 @@ void LyXLex::Pimpl::setCommentChar(char c)
 bool LyXLex::Pimpl::next(bool esc /* = false */)
 {
        if (!pushTok.empty()) {
-               pushTok.copy(buff, string::npos);
-               buff[pushTok.length()] = '\0';
-               pushTok.erase();
-               return true;
+               // There can have been a whole line pushed so
+               // we extract the first word and leaves the rest
+               // in pushTok. (Lgb)
+               if (pushTok.find(' ') != string::npos) {
+                       string tmp;
+                       pushTok = split(pushTok, tmp, ' ');
+                       tmp.copy(buff, string::npos);
+                       buff[tmp.length()] = '\0';
+                       return true;
+               } else {
+                       pushTok.copy(buff, string::npos);
+                       buff[pushTok.length()] = '\0';
+                       pushTok.erase();
+                       return true;
+               }     
        }
        if (!esc) {
                unsigned char c = 0; // getc() returns an int
@@ -159,7 +174,7 @@ bool LyXLex::Pimpl::next(bool esc /* = false */)
                                lyxerr[Debug::LYXLEX] << "Comment read: `" << c
                                                      << buff << "'" << endl;
 #else
-                               // unfortunately is ignore buggy (Lgb)
+                               // unfortunately ignore is buggy (Lgb)
                                is.ignore(100, '\n');
 #endif
                                ++lineno;
@@ -296,7 +311,10 @@ bool LyXLex::Pimpl::next(bool esc /* = false */)
                                                // escape the next char
                                                is.get(cc);
                                                c = cc;
-                                               escaped = true;
+                                               if (c == '\"' || c == '\\')
+                                                       escaped = true;
+                                               else
+                                                       buff[++i] = '\\';
                                        }
                                        buff[++i] = c;
                                        
@@ -378,16 +396,16 @@ int LyXLex::Pimpl::lex()
 }
 
        
-bool LyXLex::Pimpl::EatLine()
+bool LyXLex::Pimpl::eatLine()
 {
        int i = 0;
        unsigned char c = '\0';
        char cc = 0;
-       while(is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
+       while (is && c != '\n' && i != (LEX_MAX_BUFF - 1)) {
                is.get(cc);
                c = cc;
-               lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
-                                     << c << "'" << endl;
+               //lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
+               //                    << c << "'" << endl;
                if (c != '\r')
                        buff[i++] = c;
        }
@@ -412,10 +430,21 @@ bool LyXLex::Pimpl::EatLine()
 bool LyXLex::Pimpl::nextToken()
 {
        if (!pushTok.empty()) {
-               pushTok.copy(buff, string::npos);
-               buff[pushTok.length()] = '\0';
-               pushTok.erase();
-               return true;
+               // There can have been a whole line pushed so
+               // we extract the first word and leaves the rest
+               // in pushTok. (Lgb)
+               if (pushTok.find(' ') != string::npos) {
+                       string tmp;
+                       pushTok = split(pushTok, tmp, ' ');
+                       tmp.copy(buff, string::npos);
+                       buff[tmp.length()] = '\0';
+                       return true;
+               } else {
+                       pushTok.copy(buff, string::npos);
+                       buff[pushTok.length()] = '\0';
+                       pushTok.erase();
+                       return true;
+               }
        }
 
        status = 0;