src/support/unicode.C

   1 /**
   2  * \file unicode.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  *
  10  * A collection of unicode conversion functions, using iconv.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "unicode.h"
  16
  17 #include "debug.h"
  18
  19 #include <iconv.h>
  20
  21 #include <cerrno>
  22 #include <iomanip>
  23
  24
  25 namespace lyx {
  26
  27 using std::endl;
  28
  29 #ifdef WORDS_BIGENDIAN
  30         char const * ucs4_codeset = "UCS-4BE";
  31         char const * ucs2_codeset = "UCS-2BE";
  32 #else
  33         char const * ucs4_codeset = "UCS-4LE";
  34         char const * ucs2_codeset = "UCS-2LE";
  35 #endif
  36
  37 namespace {
  38
  39 template<typename RetType, typename InType>
  40 std::vector<RetType>
  41 iconv_convert(iconv_t * cd,
  42               char const * tocode,
  43               char const * fromcode,
  44               InType const * buf,
  45               size_t buflen)
  46 {
  47         if (buflen == 0)
  48                 return std::vector<RetType>();
  49
  50         if (*cd == (iconv_t)(-1)) {
  51                 *cd = iconv_open(tocode, fromcode);
  52                 if (*cd == (iconv_t)(-1)) {
  53                         lyxerr << "Error returned from iconv_open" << endl;
  54                         switch (errno) {
  55                         case EINVAL:
  56                                 lyxerr << "EINVAL The conversion from " << fromcode
  57                                        << " to " << tocode
  58                                        << " is not supported by the implementation."
  59                                        << endl;
  60                                 break;
  61                         default:
  62                                 lyxerr << "\tSome other error: " << errno << endl;
  63                                 break;
  64                         }
  65                 }
  66         }
  67
  68         char ICONV_CONST * inbuf = const_cast<char ICONV_CONST *>(reinterpret_cast<char const *>(buf));
  69         size_t inbytesleft = buflen * sizeof(InType);
  70         // The preamble of the user guide is more than 11.500 characters, so we go for 32kb
  71         size_t const outsize = 32768;
  72         static char out[outsize];
  73         char * outbuf = out;
  74         size_t outbytesleft = outsize;
  75
  76         size_t res = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
  77
  78         if (res == (size_t)(-1)) {
  79                 lyxerr << "Error returned from iconv" << endl;
  80                 switch (errno) {
  81                 case E2BIG:
  82                         lyxerr << "E2BIG  There is not sufficient room at *outbuf." << endl;
  83                         break;
  84                 case EILSEQ:
  85                         lyxerr << "EILSEQ An invalid multibyte sequence"
  86                                << " has been encountered in the input.\n"
  87                                << "When converting from " << fromcode
  88                                << " to " << tocode << ".\n";
  89                         lyxerr << "Input: " << std::hex;
  90                         for (size_t i = 0; i < buflen; ++i) {
  91                                 boost::uint32_t const b = buf[i];
  92                                 lyxerr << "0x" << b << " ";
  93                         }
  94                         lyxerr << endl;
  95                         break;
  96                 case EINVAL:
  97                         lyxerr << "EINVAL An incomplete multibyte sequence"
  98                                << " has been encountered in the input.\n"
  99                                << "When converting from " << fromcode
 100                                << " to " << tocode << ".\n";
 101                         lyxerr << "Input: " << std::hex;
 102                         for (size_t i = 0; i < buflen; ++i) {
 103                                 boost::uint32_t const b = buf[i];
 104                                 lyxerr << "0x" << b << " ";
 105                         }
 106                         lyxerr << endl;
 107                         break;
 108                 default:
 109                         lyxerr << "\tSome other error: " << errno << endl;
 110                         break;
 111                 }
 112                 // We got an error so we close down the conversion engine
 113                 if (iconv_close(*cd) == -1) {
 114                         lyxerr << "Error returned from iconv_close("
 115                                << errno << ")" << endl;
 116                 }
 117                 *cd = (iconv_t)(-1);
 118         }
 119
 120         //lyxerr << std::dec;
 121         //lyxerr << "Inbytesleft: " << inbytesleft << endl;
 122         //lyxerr << "Outbytesleft: " << outbytesleft << endl;
 123         int bytes = outsize - outbytesleft;
 124
 125         RetType const * tmp = reinterpret_cast<RetType const *>(out);
 126         return std::vector<RetType>(tmp, tmp + bytes / sizeof(RetType));
 127 }
 128
 129 } // anon namespace
 130
 131
 132 std::vector<lyx::char_type> utf8_to_ucs4(std::vector<char> const & utf8str)
 133 {
 134         if (utf8str.empty())
 135                 return std::vector<lyx::char_type>();
 136
 137         return utf8_to_ucs4(&utf8str[0], utf8str.size());
 138 }
 139
 140
 141 std::vector<lyx::char_type>
 142 utf8_to_ucs4(char const * utf8str, size_t ls)
 143 {
 144         static iconv_t cd = (iconv_t)(-1);
 145         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, "UTF-8",
 146                                               utf8str, ls);
 147 }
 148
 149
 150 lyx::char_type
 151 ucs2_to_ucs4(unsigned short c)
 152 {
 153         return ucs2_to_ucs4(&c, 1)[0];
 154 }
 155
 156
 157 std::vector<lyx::char_type>
 158 ucs2_to_ucs4(std::vector<unsigned short> const & ucs2str)
 159 {
 160         if (ucs2str.empty())
 161                 return std::vector<lyx::char_type>();
 162
 163         return ucs2_to_ucs4(&ucs2str[0], ucs2str.size());
 164 }
 165
 166
 167 std::vector<lyx::char_type>
 168 ucs2_to_ucs4(unsigned short const * ucs2str, size_t ls)
 169 {
 170         static iconv_t cd = (iconv_t)(-1);
 171         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, ucs2_codeset,
 172                                               ucs2str, ls);
 173 }
 174
 175
 176 unsigned short
 177 ucs4_to_ucs2(lyx::char_type c)
 178 {
 179         return ucs4_to_ucs2(&c, 1)[0];
 180 }
 181
 182
 183 std::vector<unsigned short>
 184 ucs4_to_ucs2(std::vector<lyx::char_type> const & ucs4str)
 185 {
 186         if (ucs4str.empty())
 187                 return std::vector<unsigned short>();
 188
 189         return ucs4_to_ucs2(&ucs4str[0], ucs4str.size());
 190 }
 191
 192
 193 std::vector<unsigned short>
 194 ucs4_to_ucs2(lyx::char_type const * s, size_t ls)
 195 {
 196         static iconv_t cd = (iconv_t)(-1);
 197         return iconv_convert<unsigned short>(&cd, ucs2_codeset, ucs4_codeset,
 198                                              s, ls);
 199 }
 200
 201
 202 std::vector<char>
 203 ucs4_to_utf8(lyx::char_type c)
 204 {
 205         static iconv_t cd = (iconv_t)(-1);
 206         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset, &c, 1);
 207 }
 208
 209
 210 std::vector<char>
 211 ucs4_to_utf8(std::vector<lyx::char_type> const & ucs4str)
 212 {
 213         if (ucs4str.empty())
 214                 return std::vector<char>();
 215
 216         return ucs4_to_utf8(&ucs4str[0], ucs4str.size());
 217 }
 218
 219
 220 std::vector<char>
 221 ucs4_to_utf8(lyx::char_type const * ucs4str, size_t ls)
 222 {
 223         static iconv_t cd = (iconv_t)(-1);
 224         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset,
 225                                    ucs4str, ls);
 226 }
 227
 228
 229 } // namespace lyx