X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=src%2Fsupport%2Fdocstream.cpp;h=04b57045b24d2fed98bea7c4bb1aa52359512deb;hb=41a8994da902031a743373c1c57d028b7c900797;hp=481fd8555af396a1a104d832225502d013ffbfee;hpb=974f44ea79acb58808f45e9a417e651b764de2ba;p=lyx.git diff --git a/src/support/docstream.cpp b/src/support/docstream.cpp index 481fd8555a..04b57045b2 100644 --- a/src/support/docstream.cpp +++ b/src/support/docstream.cpp @@ -11,17 +11,41 @@ #include #include "support/docstream.h" +#include "support/lstrings.h" #include "support/unicode.h" +#include #include #include +#include #include #include +using namespace std; using lyx::ucs4_codeset; -using std::string; + +#if defined(_MSC_VER) && (_MSC_VER >= 1600) +std::locale::id numpunct::id; + +namespace std { +// Implementation of numpunct defined in numpunct_lyx_char_type.h +typedef basic_string string_type; + +string_type numpunct::truename() const +{ + return lyx::from_ascii(numpunct::truename()); +} + +string_type numpunct::falsename() const +{ + return lyx::from_ascii(numpunct::falsename()); +} + +} // namespace std + +#endif // _MSC_VER >= 1600 namespace { @@ -32,18 +56,18 @@ namespace { /// codecvt facet for conversion of UCS4 (internal representation) to UTF8 /// (external representation) or vice versa -class iconv_codecvt_facet : public std::codecvt +class iconv_codecvt_facet : public codecvt { - typedef std::codecvt base; + typedef codecvt base; public: /// Constructor. You have to specify with \p inout whether you want /// to use this facet only for input, only for output or for both. explicit iconv_codecvt_facet(string const & encoding = "UTF-8", - std::ios_base::openmode inout = std::ios_base::in | std::ios_base::out, + ios_base::openmode inout = ios_base::in | ios_base::out, size_t refs = 0) : base(refs), encoding_(encoding) { - if (inout & std::ios_base::in) { + if (inout & ios_base::in) { in_cd_ = iconv_open(ucs4_codeset, encoding.c_str()); if (in_cd_ == (iconv_t)(-1)) { fprintf(stderr, "Error %d returned from iconv_open(in_cd_): %s\n", @@ -53,7 +77,7 @@ public: } } else in_cd_ = (iconv_t)(-1); - if (inout & std::ios_base::out) { + if (inout & ios_base::out) { out_cd_ = iconv_open(encoding.c_str(), ucs4_codeset); if (out_cd_ == (iconv_t)(-1)) { fprintf(stderr, "Error %d returned from iconv_open(out_cd_): %s\n", @@ -64,6 +88,7 @@ public: } else out_cd_ = (iconv_t)(-1); } + string const & encoding() const { return encoding_; } protected: virtual ~iconv_codecvt_facet() { @@ -85,13 +110,50 @@ protected: extern_type * to, extern_type * to_end, extern_type *& to_next) const { +#define WORKAROUND_ICONV_BUG 1 +#if WORKAROUND_ICONV_BUG + // Due to a bug in some iconv versions, when the last char + // in the buffer is a wide char and the output encoding is + // ISO-2022-JP and we are going to switch to another encoding, + // the appropriate escape sequence for changing the character + // set is not output (see bugs 5216, 5280, and also 5489). + // As a workaround, we append a nul char in order to force + // a switch to ASCII, and then remove it from output after + // the conversion. + intern_type * from_new = 0; + intern_type const * from_old = from; + size_t extra = 0; + if (*(from_end - 1) >= 0x80 && encoding_ == "ISO-2022-JP") { + size_t len = from_end - from; + from_new = new intern_type[len + 1]; + memcpy(from_new, from, len * sizeof(intern_type)); + from_new[len] = 0; + from_end = from_new + len + 1; + from = from_new; + extra = 1; + } +#endif size_t inbytesleft = (from_end - from) * sizeof(intern_type); size_t outbytesleft = (to_end - to) * sizeof(extern_type); +#if WORKAROUND_ICONV_BUG + outbytesleft += extra * sizeof(extern_type); +#endif from_next = from; to_next = to; result const retval = do_iconv(out_cd_, reinterpret_cast(&from_next), &inbytesleft, &to_next, &outbytesleft); +#if WORKAROUND_ICONV_BUG + // Remove from output the nul char that we inserted at the end + // of the input buffer in order to circumvent an iconv bug. + if (from_new) { + --to_next; + --from_next; + from_next = from_old + (from_next - from); + from = from_old; + delete[] from_new; + } +#endif if (retval == base::error) { fprintf(stderr, "Error %d returned from iconv when converting from %s to %s: %s\n", @@ -201,37 +263,12 @@ protected: return to_next - to; #else size_t const length = end - from; - return std::min(length, max); + return min(length, max); #endif } virtual int do_max_length() const throw() { - // FIXME: this information should be transferred to lib/encodings - // UTF8 uses at most 4 bytes to represent one UCS4 code point - // (see RFC 3629). RFC 2279 specifies 6 bytes, but that - // information is outdated, and RFC 2279 has been superseded by - // RFC 3629. - // The CJK encodings use (different) multibyte representation as well. - // All other encodings encode one UCS4 code point in one byte - // (and can therefore only encode a subset of UCS4) - // Note that BIG5 and SJIS do not work with LaTeX (see lib/encodings). - // Furthermore, all encodings that use shifting (like SJIS) do not work with - // iconv_codecvt_facet. - if (encoding_ == "UTF-8" || - encoding_ == "GB" || - encoding_ == "EUC-TW") - return 4; - else if (encoding_ == "EUC-JP") - return 3; - else if (encoding_ == "BIG5" || - encoding_ == "EUC-KR" || - encoding_ == "EUC-CN" || - encoding_ == "SJIS" || - encoding_ == "GBK" || - encoding_ == "JIS" ) - return 2; - else - return 1; + return lyx::max_encoded_bytes(encoding_); } private: /// Do the actual conversion. The interface is equivalent to that of @@ -244,6 +281,14 @@ private: inbytesleft, to, outbytesleft); if (converted == (size_t)(-1)) { switch(errno) { + case 0: + // As strange as it may seem, this + // does happen on windows when parsing + // comments with accented chars in + // tex2lyx. See the following thread + // for details + // http://thread.gmane.org/gmane.editors.lyx.devel/117636 + break; case EINVAL: case E2BIG: return base::partial; @@ -259,20 +304,20 @@ private: iconv_t in_cd_; iconv_t out_cd_; /// The narrow encoding - std::string encoding_; + string encoding_; }; -} // namespace anon +} // namespace namespace lyx { template -void setEncoding(Ios & ios, string const & encoding, std::ios_base::openmode mode) +void setEncoding(Ios & ios, string const & encoding, ios_base::openmode mode) { // We must imbue the stream before openening the file - std::locale global; - std::locale locale(global, new iconv_codecvt_facet(encoding, mode)); + locale global; + locale locale(global, new iconv_codecvt_facet(encoding, mode)); ios.imbue(locale); } @@ -283,13 +328,19 @@ const char * iconv_codecvt_facet_exception::what() const throw() } -idocfstream::idocfstream(string const & encoding) : base() +ifdocstream::ifdocstream() : base() { - setEncoding(*this, encoding, in); + setEncoding(*this, "UTF-8", in); } -idocfstream::idocfstream(const char* s, std::ios_base::openmode mode, +ifdocstream::ifdocstream(SetEnc const & enc) : base() +{ + setEncoding(*this, enc.encoding, in); +} + + +ifdocstream::ifdocstream(const char* s, ios_base::openmode mode, string const & encoding) : base() { @@ -298,18 +349,19 @@ idocfstream::idocfstream(const char* s, std::ios_base::openmode mode, } -odocfstream::odocfstream(): base() +ofdocstream::ofdocstream(): base() { + setEncoding(*this, "UTF-8", out); } -odocfstream::odocfstream(string const & encoding) : base() +ofdocstream::ofdocstream(SetEnc const & enc) : base() { - setEncoding(*this, encoding, out); + setEncoding(*this, enc.encoding, out); } -odocfstream::odocfstream(const char* s, std::ios_base::openmode mode, +ofdocstream::ofdocstream(const char* s, ios_base::openmode mode, string const & encoding) : base() { @@ -318,7 +370,7 @@ odocfstream::odocfstream(const char* s, std::ios_base::openmode mode, } -void odocfstream::reset(string const & encoding) +void ofdocstream::reset(string const & encoding) { setEncoding(*this, encoding, out); } @@ -333,24 +385,60 @@ SetEnc setEncoding(string const & encoding) odocstream & operator<<(odocstream & os, SetEnc e) { - if (std::has_facet(os.rdbuf()->getloc())) { + if (has_facet(os.rdbuf()->getloc())) { // This stream must be a file stream, since we never imbue // any other stream with a locale having a iconv_codecvt_facet. + iconv_codecvt_facet const & facet = + use_facet(os.rdbuf()->getloc()); + + // FIXME Changing the codecvt facet of an open file is allowed, + // but unsafe for facets that use internal state (see the thread + // "iostreams: Does imbue() need to be called before open()?" + // in comp.std.c++. + // Currently it seems to work with gcc and MSVC, but not with + // clang on OS X. + // Avoid imbueing with the same encoding again if possible. + if (facet.encoding() == e.encoding) + return os; + // Flush the stream so that all pending output is written // with the old encoding. os.flush(); - std::locale locale(os.rdbuf()->getloc(), - new iconv_codecvt_facet(e.encoding, std::ios_base::out)); - // FIXME Does changing the codecvt facet of an open file - // stream always work? It does with gcc 4.1, but I have read - // somewhere that it does not with MSVC. - // What does the standard say? + + locale locale(os.rdbuf()->getloc(), + new iconv_codecvt_facet(e.encoding, ios_base::out)); os.imbue(locale); } return os; } +idocstream & operator<<(idocstream & is, SetEnc e) +{ + if (has_facet(is.rdbuf()->getloc())) { + // This stream must be a file stream, since we never imbue + // any other stream with a locale having a iconv_codecvt_facet. + iconv_codecvt_facet const & facet = + use_facet(is.rdbuf()->getloc()); + + // FIXME Changing the codecvt facet of an open file is allowed, + // but unsafe for facets that use internal state (see the thread + // "iostreams: Does imbue() need to be called before open()?" + // in comp.std.c++. + // Currently it seems to work with gcc and MSVC, but not with + // clang on OS X. + // Avoid imbueing with the same encoding again if possible. + if (facet.encoding() == e.encoding) + return is; + + locale locale(is.rdbuf()->getloc(), + new iconv_codecvt_facet(e.encoding, ios_base::in)); + is.imbue(locale); + } + return is; +} + + #if ! defined(USE_WCHAR_T) odocstream & operator<<(odocstream & os, char c) { @@ -359,7 +447,8 @@ odocstream & operator<<(odocstream & os, char c) } #endif -} +} // namespace lyx + #if ! defined(USE_WCHAR_T) && defined(__GNUC__) // We get undefined references to these virtual methods. This looks like @@ -406,17 +495,6 @@ bool codecvt::do_always_noconv() const throw() return true; } -#if __GNUC__ == 3 && __GNUC_MINOR__ < 4 - -template<> -int codecvt::do_length( - mbstate_t const &, const char *, const char *, size_t) const -{ - return 1; -} - -#else - template<> int codecvt::do_length( mbstate_t &, const char *, const char *, size_t) const @@ -424,8 +502,6 @@ int codecvt::do_length( return 1; } -#endif - template<> int codecvt::do_max_length() const throw() {