src/support/unicode.C

   1 /**
   2  * \file unicode.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  *
  10  * A collection of unicode conversion functions, using iconv.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "unicode.h"
  16
  17 #include "debug.h"
  18
  19 #include <iconv.h>
  20
  21 #include <cerrno>
  22 #include <iomanip>
  23
  24 using std::endl;
  25
  26 #ifdef WORDS_BIGENDIAN
  27         char const * ucs4_codeset = "UCS-4BE";
  28         char const * ucs2_codeset = "UCS-2BE";
  29 #else
  30         char const * ucs4_codeset = "UCS-4LE";
  31         char const * ucs2_codeset = "UCS-2LE";
  32 #endif
  33
  34 namespace {
  35
  36 template<typename RetType, typename InType>
  37 std::vector<RetType>
  38 iconv_convert(iconv_t * cd,
  39               char const * tocode,
  40               char const * fromcode,
  41               InType const * buf,
  42               size_t buflen)
  43 {
  44         if (buflen == 0)
  45                 return std::vector<RetType>();
  46
  47         if (*cd == (iconv_t)(-1)) {
  48                 *cd = iconv_open(tocode, fromcode);
  49                 if (*cd == (iconv_t)(-1)) {
  50                         lyxerr << "Error returned from iconv_open" << endl;
  51                         switch (errno) {
  52                         case EINVAL:
  53                                 lyxerr << "EINVAL The conversion from " << fromcode
  54                                        << " to " << tocode
  55                                        << " is not supported by the implementation."
  56                                        << endl;
  57                                 break;
  58                         default:
  59                                 lyxerr << "\tSome other error: " << errno << endl;
  60                                 break;
  61                         }
  62                 }
  63         }
  64
  65         char ICONV_CONST * inbuf = const_cast<char ICONV_CONST *>(reinterpret_cast<char const *>(buf));
  66         size_t inbytesleft = buflen * sizeof(InType);
  67         size_t const outsize = 1000;
  68         static char out[outsize];
  69         char * outbuf = out;
  70         size_t outbytesleft = outsize;
  71
  72         size_t res = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
  73
  74         if (res == (size_t)(-1)) {
  75                 lyxerr << "Error returned from iconv" << endl;
  76                 switch (errno) {
  77                 case E2BIG:
  78                         lyxerr << "E2BIG  There is not sufficient room at *outbuf." << endl;
  79                         break;
  80                 case EILSEQ:
  81                         lyxerr << "EILSEQ An invalid multibyte sequence"
  82                                << " has been encountered in the input.\n"
  83                                << "When converting from " << fromcode
  84                                << " to " << tocode << ".\n";
  85                         lyxerr << "Input: " << std::hex;
  86                         for (size_t i = 0; i < buflen; ++i) {
  87                                 boost::uint32_t const b = buf[i];
  88                                 lyxerr << "0x" << b << " ";
  89                         }
  90                         lyxerr << endl;
  91                         break;
  92                 case EINVAL:
  93                         lyxerr << "EINVAL An incomplete multibyte sequence"
  94                                << " has been encountered in the input.\n"
  95                                << "When converting from " << fromcode
  96                                << " to " << tocode << ".\n";
  97                         lyxerr << "Input: " << std::hex;
  98                         for (size_t i = 0; i < buflen; ++i) {
  99                                 boost::uint32_t const b = buf[i];
 100                                 lyxerr << "0x" << b << " ";
 101                         }
 102                         lyxerr << endl;
 103                         break;
 104                 default:
 105                         lyxerr << "\tSome other error: " << errno << endl;
 106                         break;
 107                 }
 108                 // We got an error so we close down the conversion engine
 109                 if (iconv_close(*cd) == -1) {
 110                         lyxerr << "Error returned from iconv_close("
 111                                << errno << ")" << endl;
 112                 }
 113                 *cd = (iconv_t)(-1);
 114         }
 115
 116         //lyxerr << std::dec;
 117         //lyxerr << "Inbytesleft: " << inbytesleft << endl;
 118         //lyxerr << "Outbytesleft: " << outbytesleft << endl;
 119         int bytes = outsize - outbytesleft;
 120
 121         RetType const * tmp = reinterpret_cast<RetType const *>(out);
 122         return std::vector<RetType>(tmp, tmp + bytes / sizeof(RetType));
 123 }
 124
 125 } // anon namespace
 126
 127
 128 std::vector<lyx::char_type> utf8_to_ucs4(std::vector<char> const & utf8str)
 129 {
 130         return utf8_to_ucs4(&utf8str[0], utf8str.size());
 131 }
 132
 133
 134 std::vector<lyx::char_type>
 135 utf8_to_ucs4(char const * utf8str, size_t ls)
 136 {
 137         static iconv_t cd = (iconv_t)(-1);
 138         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, "UTF-8",
 139                                               utf8str, ls);
 140 }
 141
 142
 143 lyx::char_type
 144 ucs2_to_ucs4(unsigned short c)
 145 {
 146         return ucs2_to_ucs4(&c, 1)[0];
 147 }
 148
 149
 150 std::vector<lyx::char_type>
 151 ucs2_to_ucs4(std::vector<unsigned short> const & ucs2str)
 152 {
 153         return ucs2_to_ucs4(&ucs2str[0], ucs2str.size());
 154 }
 155
 156
 157 std::vector<lyx::char_type>
 158 ucs2_to_ucs4(unsigned short const * ucs2str, size_t ls)
 159 {
 160         static iconv_t cd = (iconv_t)(-1);
 161         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, ucs2_codeset,
 162                                               ucs2str, ls);
 163 }
 164
 165
 166 unsigned short
 167 ucs4_to_ucs2(lyx::char_type c)
 168 {
 169         return ucs4_to_ucs2(&c, 1)[0];
 170 }
 171
 172
 173 std::vector<unsigned short>
 174 ucs4_to_ucs2(std::vector<lyx::char_type> const & ucs4str)
 175 {
 176         return ucs4_to_ucs2(&ucs4str[0], ucs4str.size());
 177 }
 178
 179
 180 std::vector<unsigned short>
 181 ucs4_to_ucs2(lyx::char_type const * s, size_t ls)
 182 {
 183         static iconv_t cd = (iconv_t)(-1);
 184         return iconv_convert<unsigned short>(&cd, ucs2_codeset, ucs4_codeset,
 185                                              s, ls);
 186 }
 187
 188
 189 std::vector<char>
 190 ucs4_to_utf8(lyx::char_type c)
 191 {
 192         static iconv_t cd = (iconv_t)(-1);
 193         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset, &c, 1);
 194 }
 195
 196
 197 std::vector<char>
 198 ucs4_to_utf8(std::vector<lyx::char_type> const & ucs4str)
 199 {
 200         return ucs4_to_utf8(&ucs4str[0], ucs4str.size());
 201 }
 202
 203
 204 std::vector<char>
 205 ucs4_to_utf8(lyx::char_type const * ucs4str, size_t ls)
 206 {
 207         static iconv_t cd = (iconv_t)(-1);
 208         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset,
 209                                    ucs4str, ls);
 210 }