src/support/unicode.C

   1 /**
   2  * \file unicode.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  *
  10  * A collection of unicode conversion functions, using iconv.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "unicode.h"
  16
  17 #include "debug.h"
  18
  19 #include <iconv.h>
  20
  21 #include <cerrno>
  22 #include <iomanip>
  23
  24 using std::endl;
  25
  26 #ifdef WORDS_BIGENDIAN
  27         char const * ucs4_codeset = "UCS-4BE";
  28         char const * ucs2_codeset = "UCS-2BE";
  29 #else
  30         char const * ucs4_codeset = "UCS-4LE";
  31         char const * ucs2_codeset = "UCS-2LE";
  32 #endif
  33
  34 namespace {
  35
  36 template<typename RetType, typename InType>
  37 std::vector<RetType>
  38 iconv_convert(iconv_t * cd,
  39               char const * tocode,
  40               char const * fromcode,
  41               InType const * buf,
  42               size_t buflen)
  43 {
  44         if (buflen == 0)
  45                 return std::vector<RetType>();
  46
  47         if (*cd == (iconv_t)(-1)) {
  48                 *cd = iconv_open(tocode, fromcode);
  49                 if (*cd == (iconv_t)(-1)) {
  50                         lyxerr << "Error returned from iconv_open" << endl;
  51                         switch (errno) {
  52                         case EINVAL:
  53                                 lyxerr << "EINVAL The conversion from " << fromcode
  54                                        << " to " << tocode
  55                                        << " is not supported by the implementation."
  56                                        << endl;
  57                                 break;
  58                         default:
  59                                 lyxerr << "\tSome other error: " << errno << endl;
  60                                 break;
  61                         }
  62                 }
  63         }
  64
  65         char ICONV_CONST * inbuf = const_cast<char ICONV_CONST *>(reinterpret_cast<char const *>(buf));
  66         size_t inbytesleft = buflen * sizeof(InType);
  67         // The preamble of the user guide is more than 11.500 characters, so we go for 32kb
  68         size_t const outsize = 32768;
  69         static char out[outsize];
  70         char * outbuf = out;
  71         size_t outbytesleft = outsize;
  72
  73         size_t res = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
  74
  75         if (res == (size_t)(-1)) {
  76                 lyxerr << "Error returned from iconv" << endl;
  77                 switch (errno) {
  78                 case E2BIG:
  79                         lyxerr << "E2BIG  There is not sufficient room at *outbuf." << endl;
  80                         break;
  81                 case EILSEQ:
  82                         lyxerr << "EILSEQ An invalid multibyte sequence"
  83                                << " has been encountered in the input.\n"
  84                                << "When converting from " << fromcode
  85                                << " to " << tocode << ".\n";
  86                         lyxerr << "Input: " << std::hex;
  87                         for (size_t i = 0; i < buflen; ++i) {
  88                                 boost::uint32_t const b = buf[i];
  89                                 lyxerr << "0x" << b << " ";
  90                         }
  91                         lyxerr << endl;
  92                         break;
  93                 case EINVAL:
  94                         lyxerr << "EINVAL An incomplete multibyte sequence"
  95                                << " has been encountered in the input.\n"
  96                                << "When converting from " << fromcode
  97                                << " to " << tocode << ".\n";
  98                         lyxerr << "Input: " << std::hex;
  99                         for (size_t i = 0; i < buflen; ++i) {
 100                                 boost::uint32_t const b = buf[i];
 101                                 lyxerr << "0x" << b << " ";
 102                         }
 103                         lyxerr << endl;
 104                         break;
 105                 default:
 106                         lyxerr << "\tSome other error: " << errno << endl;
 107                         break;
 108                 }
 109                 // We got an error so we close down the conversion engine
 110                 if (iconv_close(*cd) == -1) {
 111                         lyxerr << "Error returned from iconv_close("
 112                                << errno << ")" << endl;
 113                 }
 114                 *cd = (iconv_t)(-1);
 115         }
 116
 117         //lyxerr << std::dec;
 118         //lyxerr << "Inbytesleft: " << inbytesleft << endl;
 119         //lyxerr << "Outbytesleft: " << outbytesleft << endl;
 120         int bytes = outsize - outbytesleft;
 121
 122         RetType const * tmp = reinterpret_cast<RetType const *>(out);
 123         return std::vector<RetType>(tmp, tmp + bytes / sizeof(RetType));
 124 }
 125
 126 } // anon namespace
 127
 128
 129 std::vector<lyx::char_type> utf8_to_ucs4(std::vector<char> const & utf8str)
 130 {
 131         if (utf8str.empty())
 132                 return std::vector<lyx::char_type>();
 133
 134         return utf8_to_ucs4(&utf8str[0], utf8str.size());
 135 }
 136
 137
 138 std::vector<lyx::char_type>
 139 utf8_to_ucs4(char const * utf8str, size_t ls)
 140 {
 141         static iconv_t cd = (iconv_t)(-1);
 142         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, "UTF-8",
 143                                               utf8str, ls);
 144 }
 145
 146
 147 lyx::char_type
 148 ucs2_to_ucs4(unsigned short c)
 149 {
 150         return ucs2_to_ucs4(&c, 1)[0];
 151 }
 152
 153
 154 std::vector<lyx::char_type>
 155 ucs2_to_ucs4(std::vector<unsigned short> const & ucs2str)
 156 {
 157         if (ucs2str.empty())
 158                 return std::vector<lyx::char_type>();
 159
 160         return ucs2_to_ucs4(&ucs2str[0], ucs2str.size());
 161 }
 162
 163
 164 std::vector<lyx::char_type>
 165 ucs2_to_ucs4(unsigned short const * ucs2str, size_t ls)
 166 {
 167         static iconv_t cd = (iconv_t)(-1);
 168         return iconv_convert<lyx::char_type>(&cd, ucs4_codeset, ucs2_codeset,
 169                                               ucs2str, ls);
 170 }
 171
 172
 173 unsigned short
 174 ucs4_to_ucs2(lyx::char_type c)
 175 {
 176         return ucs4_to_ucs2(&c, 1)[0];
 177 }
 178
 179
 180 std::vector<unsigned short>
 181 ucs4_to_ucs2(std::vector<lyx::char_type> const & ucs4str)
 182 {
 183         if (ucs4str.empty())
 184                 return std::vector<unsigned short>();
 185
 186         return ucs4_to_ucs2(&ucs4str[0], ucs4str.size());
 187 }
 188
 189
 190 std::vector<unsigned short>
 191 ucs4_to_ucs2(lyx::char_type const * s, size_t ls)
 192 {
 193         static iconv_t cd = (iconv_t)(-1);
 194         return iconv_convert<unsigned short>(&cd, ucs2_codeset, ucs4_codeset,
 195                                              s, ls);
 196 }
 197
 198
 199 std::vector<char>
 200 ucs4_to_utf8(lyx::char_type c)
 201 {
 202         static iconv_t cd = (iconv_t)(-1);
 203         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset, &c, 1);
 204 }
 205
 206
 207 std::vector<char>
 208 ucs4_to_utf8(std::vector<lyx::char_type> const & ucs4str)
 209 {
 210         if (ucs4str.empty())
 211                 return std::vector<char>();
 212
 213         return ucs4_to_utf8(&ucs4str[0], ucs4str.size());
 214 }
 215
 216
 217 std::vector<char>
 218 ucs4_to_utf8(lyx::char_type const * ucs4str, size_t ls)
 219 {
 220         static iconv_t cd = (iconv_t)(-1);
 221         return iconv_convert<char>(&cd, "UTF-8", ucs4_codeset,
 222                                    ucs4str, ls);
 223 }