src/support/unicode.C

   1 /**
   2  * \file unicode.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author Lars Gullik Bjønnes
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  *
  10  * A collection of unicode conversion functions, using iconv.
  11  */
  12
  13 #include <config.h>
  14
  15 #include "unicode.h"
  16
  17 #include "debug.h"
  18
  19 #include <iconv.h>
  20
  21 #include <cerrno>
  22 #include <iomanip>
  23 #include <map>
  24
  25 using std::endl;
  26
  27 namespace lyx {
  28
  29 #ifdef WORDS_BIGENDIAN
  30         char const * ucs4_codeset = "UCS-4BE";
  31         char const * ucs2_codeset = "UCS-2BE";
  32 #else
  33         char const * ucs4_codeset = "UCS-4LE";
  34         char const * ucs2_codeset = "UCS-2LE";
  35 #endif
  36
  37 static const iconv_t invalid_cd = (iconv_t)(-1);
  38
  39
  40 struct IconvProcessor::Private {
  41         Private(): cd(invalid_cd) {}
  42         iconv_t cd;
  43 };
  44
  45
  46 IconvProcessor::IconvProcessor(char const * tocode,
  47                 char const * fromcode): tocode_(tocode), fromcode_(fromcode),
  48                 pimpl_(new IconvProcessor::Private)
  49 {
  50 }
  51
  52
  53 IconvProcessor::~IconvProcessor()
  54 {
  55         if (iconv_close(pimpl_->cd) == -1) {
  56                 lyxerr << "Error returned from iconv_close("
  57                         << errno << ")" << endl;
  58         }
  59         delete pimpl_;
  60 }
  61
  62
  63 bool IconvProcessor::init()
  64 {
  65         if (pimpl_->cd != invalid_cd)
  66                 return true;
  67
  68         pimpl_->cd = iconv_open(tocode_.c_str(), fromcode_.c_str());
  69         if (pimpl_->cd != invalid_cd)
  70                 return true;
  71
  72         lyxerr << "Error returned from iconv_open" << endl;
  73         switch (errno) {
  74                 case EINVAL:
  75                         lyxerr << "EINVAL The conversion from " << fromcode_
  76                                 << " to " << tocode_
  77                                 << " is not supported by the implementation."
  78                                 << endl;
  79                         break;
  80                 default:
  81                         lyxerr << "\tSome other error: " << errno << endl;
  82                         break;
  83         }
  84         return false;
  85 }
  86
  87
  88 int IconvProcessor::convert(char const * buf, size_t buflen,
  89                 char * outbuf, size_t maxoutsize)
  90 {
  91         if (buflen == 0)
  92                 return 0;
  93
  94         if (pimpl_->cd == invalid_cd) {
  95                 if (!init())
  96                         return -1;
  97         }
  98
  99         char ICONV_CONST * inbuf = const_cast<char ICONV_CONST *>(buf);
 100         size_t inbytesleft = buflen;
 101         size_t outbytesleft = maxoutsize;
 102
 103         int res = iconv(pimpl_->cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
 104
 105         //lyxerr << std::dec;
 106         //lyxerr << "Inbytesleft: " << inbytesleft << endl;
 107         //lyxerr << "Outbytesleft: " << outbytesleft << endl;
 108
 109         if (res != -1)
 110                 // Everything went well.
 111                 return maxoutsize - outbytesleft;
 112
 113         // There are some errors in the conversion
 114         lyxerr << "Error returned from iconv" << endl;
 115         switch (errno) {
 116                 case E2BIG:
 117                         lyxerr << "E2BIG  There is not sufficient room at *outbuf." << endl;
 118                         break;
 119                 case EILSEQ:
 120                         lyxerr << "EILSEQ An invalid multibyte sequence"
 121                                 << " has been encountered in the input.\n"
 122                                 << "When converting from " << fromcode_
 123                                 << " to " << tocode_ << ".\n";
 124                         lyxerr << "Input: " << std::hex;
 125                         for (size_t i = 0; i < buflen; ++i) {
 126                                 boost::uint32_t const b = buf[i];
 127                                 lyxerr << "0x" << b << " ";
 128                         }
 129                         lyxerr << endl;
 130                         break;
 131                 case EINVAL:
 132                         lyxerr << "EINVAL An incomplete multibyte sequence"
 133                                 << " has been encountered in the input.\n"
 134                                 << "When converting from " << fromcode_
 135                                 << " to " << tocode_ << ".\n";
 136                         lyxerr << "Input: " << std::hex;
 137                         for (size_t i = 0; i < buflen; ++i) {
 138                                 boost::uint32_t const b = buf[i];
 139                                 lyxerr << "0x" << b << " ";
 140                         }
 141                         lyxerr << endl;
 142                         break;
 143                 default:
 144                         lyxerr << "\tSome other error: " << errno << endl;
 145                         break;
 146         }
 147         // We got an error so we close down the conversion engine
 148         if (iconv_close(pimpl_->cd) == -1) {
 149                 lyxerr << "Error returned from iconv_close("
 150                         << errno << ")" << endl;
 151         }
 152         pimpl_->cd = invalid_cd;
 153         return -1;
 154 }
 155
 156
 157 namespace {
 158
 159
 160 template<typename RetType, typename InType>
 161 std::vector<RetType>
 162 iconv_convert(IconvProcessor & processor,
 163               InType const * buf,
 164               size_t buflen)
 165 {
 166         if (buflen == 0)
 167                 return std::vector<RetType>();
 168
 169         char const * inbuf = reinterpret_cast<char const *>(buf);
 170         size_t inbytesleft = buflen * sizeof(InType);
 171
 172         size_t const outsize = 32768;
 173         static char out[outsize];
 174         char * outbuf = out;
 175
 176         int bytes = processor.convert(inbuf, inbytesleft, outbuf, outsize);
 177
 178         RetType const * tmp = reinterpret_cast<RetType const *>(out);
 179         return std::vector<RetType>(tmp, tmp + bytes / sizeof(RetType));
 180 }
 181
 182 } // anon namespace
 183
 184
 185 std::vector<lyx::char_type> utf8_to_ucs4(std::vector<char> const & utf8str)
 186 {
 187         if (utf8str.empty())
 188                 return std::vector<lyx::char_type>();
 189
 190         return utf8_to_ucs4(&utf8str[0], utf8str.size());
 191 }
 192
 193
 194 std::vector<lyx::char_type>
 195 utf8_to_ucs4(char const * utf8str, size_t ls)
 196 {
 197         static IconvProcessor processor(ucs4_codeset, "UTF-8");
 198         return iconv_convert<lyx::char_type>(processor, utf8str, ls);
 199 }
 200
 201
 202 lyx::char_type
 203 ucs2_to_ucs4(unsigned short c)
 204 {
 205         return ucs2_to_ucs4(&c, 1)[0];
 206 }
 207
 208
 209 std::vector<lyx::char_type>
 210 ucs2_to_ucs4(std::vector<unsigned short> const & ucs2str)
 211 {
 212         if (ucs2str.empty())
 213                 return std::vector<lyx::char_type>();
 214
 215         return ucs2_to_ucs4(&ucs2str[0], ucs2str.size());
 216 }
 217
 218
 219 std::vector<lyx::char_type>
 220 ucs2_to_ucs4(unsigned short const * ucs2str, size_t ls)
 221 {
 222         static IconvProcessor processor(ucs4_codeset, ucs2_codeset);
 223         return iconv_convert<lyx::char_type>(processor, ucs2str, ls);
 224 }
 225
 226
 227 unsigned short
 228 ucs4_to_ucs2(lyx::char_type c)
 229 {
 230         return ucs4_to_ucs2(&c, 1)[0];
 231 }
 232
 233
 234 std::vector<unsigned short>
 235 ucs4_to_ucs2(std::vector<lyx::char_type> const & ucs4str)
 236 {
 237         if (ucs4str.empty())
 238                 return std::vector<unsigned short>();
 239
 240         return ucs4_to_ucs2(&ucs4str[0], ucs4str.size());
 241 }
 242
 243
 244 std::vector<unsigned short>
 245 ucs4_to_ucs2(lyx::char_type const * s, size_t ls)
 246 {
 247         static IconvProcessor processor(ucs2_codeset, ucs4_codeset);
 248         return iconv_convert<unsigned short>(processor, s, ls);
 249 }
 250
 251
 252 std::vector<char>
 253 ucs4_to_utf8(lyx::char_type c)
 254 {
 255         static IconvProcessor processor("UTF-8", ucs4_codeset);
 256         return iconv_convert<char>(processor, &c, 1);
 257 }
 258
 259
 260 std::vector<char>
 261 ucs4_to_utf8(std::vector<lyx::char_type> const & ucs4str)
 262 {
 263         if (ucs4str.empty())
 264                 return std::vector<char>();
 265
 266         return ucs4_to_utf8(&ucs4str[0], ucs4str.size());
 267 }
 268
 269
 270 std::vector<char>
 271 ucs4_to_utf8(lyx::char_type const * ucs4str, size_t ls)
 272 {
 273         static IconvProcessor processor("UTF-8", ucs4_codeset);
 274         return iconv_convert<char>(processor, ucs4str, ls);
 275 }
 276
 277
 278 std::vector<lyx::char_type>
 279 eightbit_to_ucs4(char const * s, size_t ls, std::string const & encoding)
 280 {
 281         static std::map<std::string, IconvProcessor> processors;
 282         if (processors.find(encoding) == processors.end()) {
 283                 IconvProcessor processor(ucs4_codeset, encoding.c_str());
 284                 processors.insert(std::make_pair(encoding, processor));
 285         }
 286         return iconv_convert<char_type>(processors[encoding], s, ls);
 287 }
 288
 289
 290 std::vector<char>
 291 ucs4_to_eightbit(lyx::char_type const * ucs4str, size_t ls, std::string const & encoding)
 292 {
 293         static std::map<std::string, IconvProcessor> processors;
 294         if (processors.find(encoding) == processors.end()) {
 295                 IconvProcessor processor(encoding.c_str(), ucs4_codeset);
 296                 processors.insert(std::make_pair(encoding, processor));
 297         }
 298         return iconv_convert<char>(processors[encoding], ucs4str, ls);
 299 }
 300
 301 } // namespace lyx