#include <config.h>
-#include "docstream.h"
-#include "unicode.h"
+#include "support/docstream.h"
+#include "support/lstrings.h"
+#include "support/unicode.h"
+#include <algorithm>
#include <cerrno>
#include <cstdio>
+#include <cstring>
#include <iconv.h>
#include <locale>
+using namespace std;
using lyx::ucs4_codeset;
-using std::string;
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1600)
+std::locale::id numpunct<lyx::char_type>::id;
+
+namespace std {
+// Implementation of numpunct<lyx::char_type> defined in numpunct_lyx_char_type.h
+typedef basic_string<lyx::char_type> string_type;
+
+string_type numpunct<lyx::char_type>::truename() const
+{
+ return lyx::from_ascii(numpunct<char>::truename());
+}
+
+string_type numpunct<lyx::char_type>::falsename() const
+{
+ return lyx::from_ascii(numpunct<char>::falsename());
+}
+
+} // namespace std
+
+#endif // _MSC_VER >= 1600
namespace {
/// codecvt facet for conversion of UCS4 (internal representation) to UTF8
/// (external representation) or vice versa
-class iconv_codecvt_facet : public std::codecvt<lyx::char_type, char, std::mbstate_t>
+class iconv_codecvt_facet : public codecvt<lyx::char_type, char, mbstate_t>
{
- typedef std::codecvt<lyx::char_type, char, std::mbstate_t> base;
+ typedef codecvt<lyx::char_type, char, mbstate_t> base;
public:
/// Constructor. You have to specify with \p inout whether you want
/// to use this facet only for input, only for output or for both.
explicit iconv_codecvt_facet(string const & encoding = "UTF-8",
- std::ios_base::openmode inout = std::ios_base::in | std::ios_base::out,
+ ios_base::openmode inout = ios_base::in | ios_base::out,
size_t refs = 0)
: base(refs), encoding_(encoding)
{
- if (inout & std::ios_base::in) {
+ if (inout & ios_base::in) {
in_cd_ = iconv_open(ucs4_codeset, encoding.c_str());
if (in_cd_ == (iconv_t)(-1)) {
fprintf(stderr, "Error %d returned from iconv_open(in_cd_): %s\n",
}
} else
in_cd_ = (iconv_t)(-1);
- if (inout & std::ios_base::out) {
+ if (inout & ios_base::out) {
out_cd_ = iconv_open(encoding.c_str(), ucs4_codeset);
if (out_cd_ == (iconv_t)(-1)) {
fprintf(stderr, "Error %d returned from iconv_open(out_cd_): %s\n",
} else
out_cd_ = (iconv_t)(-1);
}
+ string const & encoding() const { return encoding_; }
protected:
virtual ~iconv_codecvt_facet()
{
extern_type * to, extern_type * to_end,
extern_type *& to_next) const
{
+#define WORKAROUND_ICONV_BUG 1
+#if WORKAROUND_ICONV_BUG
+ // Due to a bug in some iconv versions, when the last char
+ // in the buffer is a wide char and the output encoding is
+ // ISO-2022-JP and we are going to switch to another encoding,
+ // the appropriate escape sequence for changing the character
+ // set is not output (see bugs 5216, 5280, and also 5489).
+ // As a workaround, we append a nul char in order to force
+ // a switch to ASCII, and then remove it from output after
+ // the conversion.
+ intern_type * from_new = 0;
+ intern_type const * from_old = from;
+ size_t extra = 0;
+ if (*(from_end - 1) >= 0x80 && encoding_ == "ISO-2022-JP") {
+ size_t len = from_end - from;
+ from_new = new intern_type[len + 1];
+ memcpy(from_new, from, len * sizeof(intern_type));
+ from_new[len] = 0;
+ from_end = from_new + len + 1;
+ from = from_new;
+ extra = 1;
+ }
+#endif
size_t inbytesleft = (from_end - from) * sizeof(intern_type);
size_t outbytesleft = (to_end - to) * sizeof(extern_type);
+#if WORKAROUND_ICONV_BUG
+ outbytesleft += extra * sizeof(extern_type);
+#endif
from_next = from;
to_next = to;
result const retval = do_iconv(out_cd_,
reinterpret_cast<char const **>(&from_next),
&inbytesleft, &to_next, &outbytesleft);
+#if WORKAROUND_ICONV_BUG
+ // Remove from output the nul char that we inserted at the end
+ // of the input buffer in order to circumvent an iconv bug.
+ if (from_new) {
+ --to_next;
+ --from_next;
+ from_next = from_old + (from_next - from);
+ from = from_old;
+ delete[] from_new;
+ }
+#endif
if (retval == base::error) {
fprintf(stderr,
"Error %d returned from iconv when converting from %s to %s: %s\n",
return to_next - to;
#else
size_t const length = end - from;
- return std::min(length, max);
+ return min(length, max);
#endif
}
virtual int do_max_length() const throw()
{
- // FIXME: this information should be transferred to lib/encodings
- // UTF8 uses at most 4 bytes to represent one UCS4 code point
- // (see RFC 3629). RFC 2279 specifies 6 bytes, but that
- // information is outdated, and RFC 2279 has been superseded by
- // RFC 3629.
- // The CJK encodings use (different) multibyte representation as well.
- // All other encodings encode one UCS4 code point in one byte
- // (and can therefore only encode a subset of UCS4)
- // Note that BIG5 and SJIS do not work with LaTeX (see lib/encodings).
- // Furthermore, all encodings that use shifting (like SJIS) do not work with
- // iconv_codecvt_facet.
- if (encoding_ == "UTF-8" ||
- encoding_ == "GB" ||
- encoding_ == "EUC-TW")
- return 4;
- else if (encoding_ == "EUC-JP")
- return 3;
- else if (encoding_ == "BIG5" ||
- encoding_ == "EUC-KR" ||
- encoding_ == "EUC-CN" ||
- encoding_ == "SJIS" ||
- encoding_ == "GBK" ||
- encoding_ == "JIS" )
- return 2;
- else
- return 1;
+ return lyx::max_encoded_bytes(encoding_);
}
private:
/// Do the actual conversion. The interface is equivalent to that of
inbytesleft, to, outbytesleft);
if (converted == (size_t)(-1)) {
switch(errno) {
+ case 0:
+ // As strange as it may seem, this
+ // does happen on windows when parsing
+ // comments with accented chars in
+ // tex2lyx. See the following thread
+ // for details
+ // http://thread.gmane.org/gmane.editors.lyx.devel/117636
+ break;
case EINVAL:
case E2BIG:
return base::partial;
iconv_t in_cd_;
iconv_t out_cd_;
/// The narrow encoding
- std::string encoding_;
+ string encoding_;
};
-} // namespace anon
+} // namespace
namespace lyx {
+template<class Ios>
+void setEncoding(Ios & ios, string const & encoding, ios_base::openmode mode)
+{
+ // We must imbue the stream before openening the file
+ locale global;
+ locale locale(global, new iconv_codecvt_facet(encoding, mode));
+ ios.imbue(locale);
+}
+
const char * iconv_codecvt_facet_exception::what() const throw()
{
}
-idocfstream::idocfstream(string const & encoding) : base()
+ifdocstream::ifdocstream() : base()
{
- std::locale global;
- std::locale locale(global, new iconv_codecvt_facet(encoding, in));
- imbue(locale);
+ setEncoding(*this, "UTF-8", in);
}
-idocfstream::idocfstream(const char* s, std::ios_base::openmode mode,
+ifdocstream::ifdocstream(SetEnc const & enc) : base()
+{
+ setEncoding(*this, enc.encoding, in);
+}
+
+
+ifdocstream::ifdocstream(const char* s, ios_base::openmode mode,
string const & encoding)
: base()
{
- // We must imbue the stream before openening the file
- std::locale global;
- std::locale locale(global, new iconv_codecvt_facet(encoding, in));
- imbue(locale);
+ setEncoding(*this, encoding, in);
open(s, mode);
}
-odocfstream::odocfstream(string const & encoding) : base()
+ofdocstream::ofdocstream(): base()
{
- std::locale global;
- std::locale locale(global, new iconv_codecvt_facet(encoding, out));
- imbue(locale);
+ setEncoding(*this, "UTF-8", out);
}
-odocfstream::odocfstream(const char* s, std::ios_base::openmode mode,
+ofdocstream::ofdocstream(SetEnc const & enc) : base()
+{
+ setEncoding(*this, enc.encoding, out);
+}
+
+
+ofdocstream::ofdocstream(const char* s, ios_base::openmode mode,
string const & encoding)
: base()
{
- // We must imbue the stream before openening the file
- std::locale global;
- std::locale locale(global, new iconv_codecvt_facet(encoding, out));
- imbue(locale);
+ setEncoding(*this, encoding, out);
open(s, mode);
}
+void ofdocstream::reset(string const & encoding)
+{
+ setEncoding(*this, encoding, out);
+}
+
+
+
SetEnc setEncoding(string const & encoding)
{
return SetEnc(encoding);
odocstream & operator<<(odocstream & os, SetEnc e)
{
- if (std::has_facet<iconv_codecvt_facet>(os.rdbuf()->getloc())) {
+ if (has_facet<iconv_codecvt_facet>(os.rdbuf()->getloc())) {
// This stream must be a file stream, since we never imbue
// any other stream with a locale having a iconv_codecvt_facet.
+ iconv_codecvt_facet const & facet =
+ use_facet<iconv_codecvt_facet>(os.rdbuf()->getloc());
+
+ // FIXME Changing the codecvt facet of an open file is allowed,
+ // but unsafe for facets that use internal state (see the thread
+ // "iostreams: Does imbue() need to be called before open()?"
+ // in comp.std.c++.
+ // Currently it seems to work with gcc and MSVC, but not with
+ // clang on OS X.
+ // Avoid imbueing with the same encoding again if possible.
+ if (facet.encoding() == e.encoding)
+ return os;
+
// Flush the stream so that all pending output is written
// with the old encoding.
os.flush();
- std::locale locale(os.rdbuf()->getloc(),
- new iconv_codecvt_facet(e.encoding, std::ios_base::out));
- // FIXME Does changing the codecvt facet of an open file
- // stream always work? It does with gcc 4.1, but I have read
- // somewhere that it does not with MSVC.
- // What does the standard say?
+
+ locale locale(os.rdbuf()->getloc(),
+ new iconv_codecvt_facet(e.encoding, ios_base::out));
os.imbue(locale);
}
return os;
}
+
+idocstream & operator<<(idocstream & is, SetEnc e)
+{
+ if (has_facet<iconv_codecvt_facet>(is.rdbuf()->getloc())) {
+ // This stream must be a file stream, since we never imbue
+ // any other stream with a locale having a iconv_codecvt_facet.
+ iconv_codecvt_facet const & facet =
+ use_facet<iconv_codecvt_facet>(is.rdbuf()->getloc());
+
+ // FIXME Changing the codecvt facet of an open file is allowed,
+ // but unsafe for facets that use internal state (see the thread
+ // "iostreams: Does imbue() need to be called before open()?"
+ // in comp.std.c++.
+ // Currently it seems to work with gcc and MSVC, but not with
+ // clang on OS X.
+ // Avoid imbueing with the same encoding again if possible.
+ if (facet.encoding() == e.encoding)
+ return is;
+
+ locale locale(is.rdbuf()->getloc(),
+ new iconv_codecvt_facet(e.encoding, ios_base::in));
+ is.imbue(locale);
+ }
+ return is;
+}
+
+
+#if ! defined(USE_WCHAR_T)
+odocstream & operator<<(odocstream & os, char c)
+{
+ os.put(c);
+ return os;
}
+#endif
+
+} // namespace lyx
+
#if ! defined(USE_WCHAR_T) && defined(__GNUC__)
// We get undefined references to these virtual methods. This looks like
// a bug in gcc. The implementation here does not do anything useful, since
// it is overriden in iconv_codecvt_facet.
namespace std {
+
template<> codecvt<lyx::char_type, char, mbstate_t>::result
-codecvt<lyx::char_type, char, mbstate_t>::do_out(mbstate_t &, const lyx::char_type *, const lyx::char_type *, const lyx::char_type *&,
- char *, char *, char *&) const { return error; }
+codecvt<lyx::char_type, char, mbstate_t>::do_out(
+ mbstate_t &, const lyx::char_type *, const lyx::char_type *,
+ const lyx::char_type *&, char *, char *, char *&) const
+{
+ return error;
+}
+
+
template<> codecvt<lyx::char_type, char, mbstate_t>::result
-codecvt<lyx::char_type, char, mbstate_t>::do_unshift(mbstate_t &, char *, char *, char *&) const { return error; }
+codecvt<lyx::char_type, char, mbstate_t>::do_unshift(
+ mbstate_t &, char *, char *, char *&) const
+{
+ return error;
+}
+
+
template<> codecvt<lyx::char_type, char, mbstate_t>::result
-codecvt<lyx::char_type, char, mbstate_t>::do_in(mbstate_t &, const char *, const char *, const char *&,
- lyx::char_type *, lyx::char_type *, lyx::char_type *&) const { return error; }
-template<> int codecvt<lyx::char_type, char, mbstate_t>::do_encoding() const throw() { return 0; }
-template<> bool codecvt<lyx::char_type, char, mbstate_t>::do_always_noconv() const throw() { return true; }
-#if __GNUC__ == 3 && __GNUC_MINOR__ < 4
-template<> int codecvt<lyx::char_type, char, mbstate_t>::do_length(mbstate_t const &, const char *, const char *, size_t) const { return 1; }
-#else
-template<> int codecvt<lyx::char_type, char, mbstate_t>::do_length(mbstate_t &, const char *, const char *, size_t) const { return 1; }
-#endif
-template<> int codecvt<lyx::char_type, char, mbstate_t>::do_max_length() const throw() { return 4; }
+codecvt<lyx::char_type, char, mbstate_t>::do_in(
+ mbstate_t &, const char *, const char *, const char *&,
+ lyx::char_type *, lyx::char_type *, lyx::char_type *&) const
+{
+ return error;
+}
+
+
+template<>
+int codecvt<lyx::char_type, char, mbstate_t>::do_encoding() const throw()
+{
+ return 0;
+}
+
+
+template<>
+bool codecvt<lyx::char_type, char, mbstate_t>::do_always_noconv() const throw()
+{
+ return true;
+}
+
+template<>
+int codecvt<lyx::char_type, char, mbstate_t>::do_length(
+ mbstate_t &, const char *, const char *, size_t) const
+{
+ return 1;
}
+
+template<>
+int codecvt<lyx::char_type, char, mbstate_t>::do_max_length() const throw()
+{
+ return 4;
+}
+
+} // namespace std
#endif
projects / lyx.git / blobdiff