src/support/unicode.C

   1 /**
   2  * \file unicode.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  *
  10  * A collection of unicode conversion functions, using iconv.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "unicode.h"
  16
  17 #include "debug.h"
  18
  19 #include <iconv.h>
  20
  21 #include <cerrno>
  22 #include <iomanip>
  23
  24 using std::endl;
  25
  26 namespace {
  27
  28 #ifdef WORDS_BIGENDIAN
  29         char const * ucs4_codeset = "UCS-4BE";
  30         char const * ucs2_codeset = "UCS-2BE";
  31 #else
  32         char const * ucs4_codeset = "UCS-4LE";
  33         char const * ucs2_codeset = "UCS-2LE";
  34 #endif
  35
  36 template<typename RetType, typename InType>
  37 std::vector<RetType>
  38 iconv_convert(iconv_t * cd,
  39               char const * tocode,
  40               char const * fromcode,
  41               InType const * buf,
  42               size_t buflen)
  43 {
  44         if (buflen == 0)
  45                 return std::vector<RetType>();
  46
  47         if (*cd == (iconv_t)(-1)) {
  48                 *cd = iconv_open(tocode, fromcode);
  49                 if (*cd == (iconv_t)(-1)) {
  50                         lyxerr << "Error returned from iconv_open" << endl;
  51                         switch (errno) {
  52                         case EINVAL:
  53                                 lyxerr << "EINVAL The conversion from " << fromcode
  54                                        << " to " << tocode
  55                                        << " is not supported by the implementation."
  56                                        << endl;
  57                                 break;
  58                         default:
  59                                 lyxerr << "\tSome other error: " << errno << endl;
  60                                 break;
  61                         }
  62                 }
  63         }
  64
  65         char ICONV_CONST * inbuf = const_cast<char ICONV_CONST *>(reinterpret_cast<char const *>(buf));
  66         size_t inbytesleft = buflen * sizeof(InType);
  67         static char out[1000];
  68         char * outbuf = out;
  69         size_t outbytesleft = 1000;
  70
  71         size_t res = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
  72
  73         if (res == (size_t)(-1)) {
  74                 lyxerr << "Error returned from iconv" << endl;
  75                 switch (errno) {
  76                 case E2BIG:
  77                         lyxerr << "E2BIG  There is not sufficient room at *outbuf." << endl;
  78                         break;
  79                 case EILSEQ:
  80                         lyxerr << "EILSEQ An invalid multibyte sequence"
  81                                << " has been encountered in the input.\n"
  82                                << "When converting from " << fromcode
  83                                << " to " << tocode << ".\n";
  84                         lyxerr << "Input: " << std::hex;
  85                         for (size_t i = 0; i < buflen; ++i) {
  86                                 unsigned char const b = buf[i];
  87                                 lyxerr << "0x" << int(b) << " ";
  88                         }
  89                         lyxerr << endl;
  90                         break;
  91                 case EINVAL:
  92                         lyxerr << "EINVAL An incomplete multibyte sequence"
  93                                << " has been encountered in the input.\n"
  94                                << "When converting from " << fromcode
  95                                << " to " << tocode << ".\n";
  96                         lyxerr << "Input: " << std::hex;
  97                         for (size_t i = 0; i < buflen; ++i) {
  98                                 unsigned char const b = buf[i];
  99                                 lyxerr << "0x" << int(b) << " ";
 100                         }
 101                         lyxerr << endl;
 102                         break;
 103                 default:
 104                         lyxerr << "\tSome other error: " << errno << endl;
 105                         break;
 106                 }
 107                 // We got an error so we close down the conversion engine
 108                 if (iconv_close(*cd) == -1) {
 109                         lyxerr << "Error returned from iconv_close("
 110                                << errno << ")" << endl;
 111                 }
 112                 *cd = (iconv_t)(-1);
 113         }
 114
 115         //lyxerr << std::dec;
 116         //lyxerr << "Inbytesleft: " << inbytesleft << endl;
 117         //lyxerr << "Outbytesleft: " << outbytesleft << endl;
 118         int bytes = 1000 - outbytesleft;
 119
 120         RetType const * tmp = reinterpret_cast<RetType const *>(out);
 121         return std::vector<RetType>(tmp, tmp + bytes / sizeof(RetType));
 122 }
 123
 124 } // anon namespace
 125
 126
 127 std::vector<lyx::char_type> utf8_to_ucs4(std::vector<char> const & utf8str)
 128 {
 129         return utf8_to_ucs4(&utf8str[0], utf8str.size());
 130 }
 131
 132
 133 std::vector<lyx::char_type>
 134 utf8_to_ucs4(char const * utf8str, size_t ls)
 135 {
 136         static iconv_t cd = (iconv_t)(-1);
 137         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, "UTF-8",
 138                                               utf8str, ls);
 139 }
 140
 141
 142 lyx::char_type
 143 ucs2_to_ucs4(unsigned short c)
 144 {
 145         return ucs2_to_ucs4(&c, 1)[0];
 146 }
 147
 148
 149 std::vector<lyx::char_type>
 150 ucs2_to_ucs4(std::vector<unsigned short> const & ucs2str)
 151 {
 152         return ucs2_to_ucs4(&ucs2str[0], ucs2str.size());
 153 }
 154
 155
 156 std::vector<lyx::char_type>
 157 ucs2_to_ucs4(unsigned short const * ucs2str, size_t ls)
 158 {
 159         static iconv_t cd = (iconv_t)(-1);
 160         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, ucs2_codeset,
 161                                               ucs2str, ls);
 162 }
 163
 164
 165 unsigned short
 166 ucs4_to_ucs2(lyx::char_type c)
 167 {
 168         return ucs4_to_ucs2(&c, 1)[0];
 169 }
 170
 171
 172 std::vector<unsigned short>
 173 ucs4_to_ucs2(std::vector<lyx::char_type> const & ucs4str)
 174 {
 175         return ucs4_to_ucs2(&ucs4str[0], ucs4str.size());
 176 }
 177
 178
 179 std::vector<unsigned short>
 180 ucs4_to_ucs2(lyx::char_type const * s, size_t ls)
 181 {
 182         static iconv_t cd = (iconv_t)(-1);
 183         return iconv_convert<unsigned short>(&cd, ucs2_codeset, ucs4_codeset,
 184                                              s, ls);
 185 }
 186
 187
 188 std::vector<char>
 189 ucs4_to_utf8(lyx::char_type c)
 190 {
 191         static iconv_t cd = (iconv_t)(-1);
 192         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset, &c, 1);
 193 }
 194
 195
 196 std::vector<char>
 197 ucs4_to_utf8(std::vector<lyx::char_type> const & ucs4str)
 198 {
 199         return ucs4_to_utf8(&ucs4str[0], ucs4str.size());
 200 }
 201
 202
 203 std::vector<char>
 204 ucs4_to_utf8(lyx::char_type const * ucs4str, size_t ls)
 205 {
 206         static iconv_t cd = (iconv_t)(-1);
 207         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset,
 208                                    ucs4str, ls);
 209 }