Merge branch 'master' of git.lyx.org:lyx

[lyx.git] / src / support / unicode.cpp

diff --git a/src/support/unicode.cpp b/src/support/unicode.cpp
index 09bf8c450eb4e44fad2593c5072d089ee5e0ff56..b48f10609f7fe9315e257575aec432b9d8fcb1dc 100644 (file)
--- a/src/support/unicode.cpp
+++ b/src/support/unicode.cpp
@@ -14,6 +14,7 @@
 
 #include "support/unicode.h"
 #include "support/debug.h"
+#include "support/mutex.h"
 
 #include <iconv.h>
 
@@ -21,14 +22,12 @@
 
 #include <cerrno>
 #include <iomanip>
-#include <ostream>
 #include <map>
+#include <ostream>
+#include <string>
+
 
-using std::endl;
-using std::map;
-using std::make_pair;
-using std::string;
-using std::vector;
+using namespace std;
 
 namespace {
 
@@ -54,7 +53,7 @@ static const iconv_t invalid_cd = (iconv_t)(-1);
 
 struct IconvProcessor::Impl
 {
-       Impl(std::string const & to, std::string const & from)
+       Impl(string const & to, string const & from)
                : cd(invalid_cd), tocode_(to), fromcode_(from)
        {}
 
@@ -65,8 +64,10 @@ struct IconvProcessor::Impl
        }
 
        iconv_t cd;
-       std::string tocode_;
-       std::string fromcode_;
+       string tocode_;
+       string fromcode_;
+
+       Mutex mutex_; // iconv() is not thread save, see #7240
 };
 
 
@@ -123,6 +124,8 @@ bool IconvProcessor::init()
 int IconvProcessor::convert(char const * buf, size_t buflen,
                char * outbuf, size_t maxoutsize)
 {
+       Mutex::Locker lock(&pimpl_->mutex_);
+
        if (buflen == 0)
                return 0;
 
@@ -137,7 +140,12 @@ int IconvProcessor::convert(char const * buf, size_t buflen,
 
        int res = iconv(pimpl_->cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
 
-       //lyxerr << std::dec;
+       // flush out remaining data. This is needed because iconv sometimes
+       // holds back chars in the stream, waiting for a combination character
+       // (see e.g. http://sources.redhat.com/bugzilla/show_bug.cgi?id=1124)
+       iconv(pimpl_->cd, NULL, NULL, &outbuf, &outbytesleft);
+
+       //lyxerr << dec;
        //lyxerr << "Inbytesleft: " << inbytesleft << endl;
        //lyxerr << "Outbytesleft: " << outbytesleft << endl;
 
@@ -156,7 +164,7 @@ int IconvProcessor::convert(char const * buf, size_t buflen,
                                << " has been encountered in the input.\n"
                                << "When converting from " << pimpl_->fromcode_
                                << " to " << pimpl_->tocode_ << ".\n";
-                       lyxerr << "Input:" << std::hex;
+                       lyxerr << "Input:" << hex;
                        for (size_t i = 0; i < buflen; ++i) {
                                // char may be signed, avoid output of
                                // something like 0xffffffc2
@@ -164,14 +172,14 @@ int IconvProcessor::convert(char const * buf, size_t buflen,
                                        *reinterpret_cast<unsigned char const *>(buf + i);
                                lyxerr << " 0x" << (unsigned int)b;
                        }
-                       lyxerr << std::dec << endl;
+                       lyxerr << dec << endl;
                        break;
                case EINVAL:
                        lyxerr << "EINVAL An incomplete multibyte sequence"
                                << " has been encountered in the input.\n"
                                << "When converting from " << pimpl_->fromcode_
                                << " to " << pimpl_->tocode_ << ".\n";
-                       lyxerr << "Input:" << std::hex;
+                       lyxerr << "Input:" << hex;
                        for (size_t i = 0; i < buflen; ++i) {
                                // char may be signed, avoid output of
                                // something like 0xffffffc2
@@ -179,7 +187,7 @@ int IconvProcessor::convert(char const * buf, size_t buflen,
                                        *reinterpret_cast<unsigned char const *>(buf + i);
                                lyxerr << " 0x" << (unsigned int)b;
                        }
-                       lyxerr << std::dec << endl;
+                       lyxerr << dec << endl;
                        break;
                default:
                        lyxerr << "\tSome other error: " << errno << endl;
@@ -195,6 +203,18 @@ int IconvProcessor::convert(char const * buf, size_t buflen,
 }
 
 
+std::string IconvProcessor::from() const
+{
+       return pimpl_->fromcode_;
+}
+
+
+std::string IconvProcessor::to() const
+{
+       return pimpl_->tocode_;
+}
+
+
 namespace {
 
 
@@ -208,17 +228,21 @@ iconv_convert(IconvProcessor & processor, InType const * buf, size_t buflen)
        char const * inbuf = reinterpret_cast<char const *>(buf);
        size_t inbytesleft = buflen * sizeof(InType);
 
-       size_t const outsize = 32768;
-       static char out[outsize];
-       char * outbuf = out;
+       static std::vector<char> outbuf(32768);
+       // The number of UCS4 code points in buf is at most inbytesleft.
+       // The output encoding will use at most
+       // max_encoded_bytes(pimpl_->tocode_) per UCS4 code point.
+       size_t maxoutbufsize = max_encoded_bytes(processor.to()) * inbytesleft;
+       if (outbuf.size() < maxoutbufsize)
+               outbuf.resize(maxoutbufsize);
 
-       int bytes = processor.convert(inbuf, inbytesleft, outbuf, outsize);
+       int bytes = processor.convert(inbuf, inbytesleft, &outbuf[0], outbuf.size());
        if (bytes <= 0)
                // Conversion failed
                // FIXME Maybe throw an exception and handle that in the caller?
                return vector<RetType>();
 
-       RetType const * tmp = reinterpret_cast<RetType const *>(out);
+       RetType const * tmp = reinterpret_cast<RetType const *>(&outbuf[0]);
        return vector<RetType>(tmp, tmp + bytes / sizeof(RetType));
 }
 
@@ -343,4 +367,35 @@ void ucs4_to_multibytes(char_type ucs4, vector<char> & out,
                out.clear();
 }
 
+int max_encoded_bytes(std::string const & encoding)
+{
+       // FIXME: this information should be transferred to lib/encodings
+       // UTF8 uses at most 4 bytes to represent one UCS4 code point
+       // (see RFC 3629). RFC 2279 specifies 6 bytes, but that
+       // information is outdated, and RFC 2279 has been superseded by
+       // RFC 3629.
+       // The CJK encodings use (different) multibyte representation as well.
+       // All other encodings encode one UCS4 code point in one byte
+       // (and can therefore only encode a subset of UCS4)
+       // Note that BIG5 and SJIS do not work with LaTeX (see lib/encodings).
+       // Furthermore, all encodings that use shifting (like SJIS) do not work with
+       // iconv_codecvt_facet.
+       if (encoding == "UTF-8" ||
+           encoding == "GB" ||
+           encoding == "EUC-TW")
+               return 4;
+       else if (encoding == "EUC-JP")
+               return 3;
+       else if (encoding == "ISO-2022-JP")
+               return 8;
+       else if (encoding == "BIG5" ||
+                encoding == "EUC-KR" ||
+                encoding == "EUC-CN" ||
+                encoding == "SJIS" ||
+                encoding == "GBK")
+               return 2;
+       else
+               return 1;
+}
+
 } // namespace lyx