3 * This file is part of LyX, the document processor.
4 * Licence details can be found in the file COPYING.
6 * \author Lars Gullik Bjønnes
8 * Full author contact details are available in file CREDITS.
10 * A collection of unicode conversion functions, using iconv.
13 #ifndef LYX_SUPPORT_UNICODE_H
14 #define LYX_SUPPORT_UNICODE_H
16 #include "support/strfwd.h"
25 * Wrapper for iconv(3).
27 * According to the POSIX standard, all specified functions are thread-safe,
28 * with some exceptions. The iconv() function is not listed as an exception:
29 * http://pubs.opengroup.org/onlinepubs/9699919799/xrat/V4_xsh_chap02.html#tag_22_02_09_09
30 * http://man7.org/linux/man-pages/man7/pthreads.7.html
32 * Therefore, you can use as many instances of this class in parallel as you
33 * like. However, you need to ensure that each instance is only used by one
34 * thread at any given time. If this condition is not met you get nasty
35 * mixtures of different thread data as in bug 7240.
37 * From a performance point of view it is best to use one static instance
38 * per thread for each in/out encoding pair. This can e.g. be achieved by
39 * using helpers for thread-local storage such as QThreadStorage or
40 * boost::thread_specific_ptr. A single static instance protected by a mutex
41 * would work as well, and might be preferrable for exotic encoding pairs.
42 * Creating local IconvProcessor instances should be avoided because of the
43 * overhead in iconv_open().
48 IconvProcessor(char const * tocode = "", char const * fromcode = "");
49 /// copy constructor needed because of pimpl_
50 IconvProcessor(IconvProcessor const &);
51 /// assignment operator needed because of pimpl_
52 IconvProcessor & operator=(IconvProcessor const &);
56 /// convert any data from \c fromcode to \c tocode unicode format.
57 /// \return the number of bytes of the converted output buffer.
58 int convert(char const * in_buffer, size_t in_size,
59 char * out_buffer, size_t max_out_size);
62 std::string from() const;
64 std::string to() const;
68 /// \return true if the processor is ready to use.
75 /// Get the global IconvProcessor instance of the current thread for
76 /// utf8->ucs4 conversions
77 IconvProcessor & utf8ToUcs4();
79 // A single codepoint conversion for utf8_to_ucs4 does not make
80 // sense, so that function is left out.
82 std::vector<char_type> utf8_to_ucs4(std::vector<char> const & utf8str);
84 std::vector<char_type> utf8_to_ucs4(char const * utf8str, size_t ls);
88 std::vector<char_type> utf16_to_ucs4(unsigned short const * s, size_t ls);
92 std::vector<unsigned short> ucs4_to_utf16(char_type const * s, size_t ls);
94 /// Get the global IconvProcessor instance of the current thread for
95 /// ucs4->utf8 conversions
96 IconvProcessor & ucs4ToUtf8();
100 std::vector<char> ucs4_to_utf8(char_type c);
102 std::vector<char> ucs4_to_utf8(std::vector<char_type> const & ucs4str);
104 std::vector<char> ucs4_to_utf8(char_type const * ucs4str, size_t ls);
106 /// convert \p s from encoding \p encoding to ucs4.
107 /// \p encoding must be a valid iconv 8bit encoding
108 std::vector<char_type>
109 eightbit_to_ucs4(char const * s, size_t ls, std::string const & encoding);
111 /// convert \p s from ucs4 to encoding \p encoding.
112 /// \p encoding must be a valid iconv 8bit encoding
113 std::vector<char> ucs4_to_eightbit(char_type const * ucs4str,
114 size_t ls, std::string const & encoding);
116 /// convert ucs4 character \p c to encoding \p encoding.
117 /// \p encoding must be a valid iconv 8bit encoding
118 char ucs4_to_eightbit(char_type c, std::string const & encoding);
121 void ucs4_to_multibytes(char_type ucs4, std::vector<char> & out,
122 std::string const & encoding);
124 extern char const * ucs4_codeset;
126 /// How many bytes does one UCS4 code point use at most in encoding \p encoding?
127 int max_encoded_bytes(std::string const & encoding);