georg baum:

- add encoding arg to idocfstream
- open .bib files with idocfstream
- add ascii_lowercase that takes docstring

git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@16281 a592a061-630c-0410-9148-cb99ea01b6c8

diff --git a/src/insets/insetbibtex.C b/src/insets/insetbibtex.C
index 09ba80701a377147ee54dc8e4e6c19ff1294067d..9d8d81732cd355ef357d4d148d4ab3a645c186c8 100644 (file)
--- a/src/insets/insetbibtex.C
+++ b/src/insets/insetbibtex.C
@@ -16,6 +16,7 @@
 #include "bufferparams.h"
 #include "dispatchresult.h"
 #include "debug.h"
+#include "encoding.h"
 #include "funcrequest.h"
 #include "gettext.h"
 #include "LaTeXFeatures.h"
@@ -32,9 +33,6 @@
 
 #include <boost/tokenizer.hpp>
 
-#include <fstream>
-#include <sstream>
-
 
 namespace lyx {
 
@@ -66,7 +64,6 @@ namespace os = support::os;
 using std::endl;
 using std::getline;
 using std::string;
-using std::ifstream;
 using std::ostream;
 using std::pair;
 using std::vector;
@@ -342,26 +339,36 @@ void InsetBibtex::fillWithBibKeys(Buffer const & buffer,
                // files. All it does is to look for lines starting
                // in @ and not being @preamble and @string entries.
                // It does NOT do any syntax checking!
-               ifstream ifs(it->toFilesystemEncoding().c_str());
-               string linebuf0;
+
+               // Officially bibtex does only support ASCII, but in practice
+               // you can use the encoding of the main document as long as
+               // some elements like keys and names are pure ASCII. Therefore
+               // we convert the file from the buffer encoding.
+               idocfstream ifs(it->toFilesystemEncoding().c_str(),
+                               std::ios_base::in,
+                               buffer.params().encoding().iconvName());
+               docstring linebuf0;
                while (getline(ifs, linebuf0)) {
-                       string linebuf = trim(linebuf0);
+                       docstring linebuf = trim(linebuf0);
                        if (linebuf.empty()) continue;
-                       if (prefixIs(linebuf, "@")) {
+                       if (prefixIs(linebuf, from_ascii("@"))) {
                                linebuf = subst(linebuf, '{', '(');
-                               string tmp;
+                               docstring tmp;
                                linebuf = split(linebuf, tmp, '(');
                                tmp = ascii_lowercase(tmp);
-                               if (!prefixIs(tmp, "@string")
-                                   && !prefixIs(tmp, "@preamble")) {
+                               if (!prefixIs(tmp, from_ascii("@string")) &&
+                                   !prefixIs(tmp, from_ascii("@preamble"))) {
                                        linebuf = split(linebuf, tmp, ',');
                                        tmp = ltrim(tmp, " \t");
                                        if (!tmp.empty()) {
-                                               keys.push_back(pair<string,string>(tmp,string()));
+                                               // to_ascii because bibtex keys may
+                                               // only consist of ASCII characters
+                                               keys.push_back(pair<string, string>(to_ascii(tmp), string()));
                                        }
                                }
                        } else if (!keys.empty()) {
-                               keys.back().second += linebuf + "\n";
+                               // FIXME UNICODE
+                               keys.back().second += to_utf8(linebuf + '\n');
                        }
                }
        }

diff --git a/src/support/docstream.C b/src/support/docstream.C
index 525f3673e0a9fa7e2fd43a9d149d99976f0c2951..cec49561563f1af43fbc43c7effea12e01c9b883 100644 (file)
--- a/src/support/docstream.C
+++ b/src/support/docstream.C
@@ -26,8 +26,6 @@ using std::string;
 
 namespace {
 
-char const * utf8_codeset = "UTF-8";
-
 // We use C IO throughout this file, because the facets might be used with
 // lyxerr in the future.
 
@@ -199,20 +197,21 @@ const char * iconv_codecvt_facet_exception::what() const throw()
 }
 
 
-idocfstream::idocfstream() : base()
+idocfstream::idocfstream(string const & encoding) : base()
 {
        std::locale global;
-       std::locale locale(global, new iconv_codecvt_facet(utf8_codeset, in));
+       std::locale locale(global, new iconv_codecvt_facet(encoding, in));
        imbue(locale);
 }
 
        
-idocfstream::idocfstream(const char* s, std::ios_base::openmode mode)
+idocfstream::idocfstream(const char* s, std::ios_base::openmode mode,
+                         string const & encoding)
        : base()
 {
        // We must imbue the stream before openening the file
        std::locale global;
-       std::locale locale(global, new iconv_codecvt_facet(utf8_codeset, in));
+       std::locale locale(global, new iconv_codecvt_facet(encoding, in));
        imbue(locale);
        open(s, mode);
 }

diff --git a/src/support/docstream.h b/src/support/docstream.h
index f5245ec52175e7b05c151a348414baa54273b010..1783fd4271b92fcde6db53dc56ee74ab2e12bbd3 100644 (file)
--- a/src/support/docstream.h
+++ b/src/support/docstream.h
@@ -45,9 +45,10 @@ typedef std::basic_ostream<char_type> odocstream;
 class idocfstream : public std::basic_ifstream<char_type> {
        typedef std::basic_ifstream<char_type> base;
 public:
-       idocfstream();
+       idocfstream(std::string const & encoding = "UTF-8");
        explicit idocfstream(const char* s,
-               std::ios_base::openmode mode = std::ios_base::in);
+               std::ios_base::openmode mode = std::ios_base::in,
+               std::string const & encoding = "UTF-8");
        ~idocfstream() {}
 };
 

diff --git a/src/support/lstrings.C b/src/support/lstrings.C
index ecc12352aa00d4fb878acdd92dcb765bcd6cf5a5..43dab89fae15b58cc3cac4debf755df31ccf9e99 100644 (file)
--- a/src/support/lstrings.C
+++ b/src/support/lstrings.C
@@ -373,8 +373,8 @@ struct local_uppercase {
        }
 };
 
-struct local_ascii_lowercase {
-       char operator()(char c) const {
+template<typename Char> struct local_ascii_lowercase {
+       Char operator()(Char c) const {
                return ascii_tolower(c);
        }
 };
@@ -409,7 +409,16 @@ string const ascii_lowercase(string const & a)
 {
        string tmp(a);
        transform(tmp.begin(), tmp.end(), tmp.begin(),
-                 local_ascii_lowercase());
+                 local_ascii_lowercase<char>());
+       return tmp;
+}
+
+
+docstring const ascii_lowercase(docstring const & a)
+{
+       docstring tmp(a);
+       transform(tmp.begin(), tmp.end(), tmp.begin(),
+                 local_ascii_lowercase<char_type>());
        return tmp;
 }
 

diff --git a/src/support/lstrings.h b/src/support/lstrings.h
index 25ceb527ca9a7e36c4b68f5e9cb21f8bda201cff..82346d0b459c0ce7ca0b8ca78da8f595cfdb27ee 100644 (file)
--- a/src/support/lstrings.h
+++ b/src/support/lstrings.h
@@ -89,6 +89,7 @@ char_type uppercase(char_type c);
 
 /// same as lowercase(), but ignores locale
 std::string const ascii_lowercase(std::string const &);
+docstring const ascii_lowercase(docstring const &);
 
 ///
 std::string const lowercase(std::string const &);