#include <config.h>
#include "support/docstream.h"
+#include "support/lstrings.h"
#include "support/unicode.h"
+#include <algorithm>
#include <cerrno>
#include <cstdio>
#include <cstring>
using namespace std;
using lyx::ucs4_codeset;
+using lyx::support::contains;
+using lyx::support::split;
+
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1600)
+std::locale::id numpunct<lyx::char_type>::id;
+#endif
+
namespace {
{
#define WORKAROUND_ICONV_BUG 1
#if WORKAROUND_ICONV_BUG
- // Due to a bug in some iconv versions, when the last char in
- // the buffer is a wide char, it gets truncated (see bugs 5216,
- // 5280, and also 5489). As a workaround, we append a null
- // char and then remove it from output after the conversion.
+ // Due to a bug in some iconv versions, when the last char
+ // in the buffer is a wide char and the output encoding is
+ // ISO-2022-JP and we are going to switch to another encoding,
+ // the appropriate escape sequence for changing the character
+ // set is not output (see bugs 5216, 5280, and also 5489).
+ // As a workaround, we append a nul char in order to force
+ // a switch to ASCII, and then remove it from output after
+ // the conversion.
intern_type * from_new = 0;
- if (*(from_end - 1) >= 0x80) {
+ intern_type const * from_old = from;
+ size_t extra = 0;
+ if (*(from_end - 1) >= 0x80 && encoding_ == "ISO-2022-JP") {
size_t len = from_end - from;
from_new = new intern_type[len + 1];
memcpy(from_new, from, len * sizeof(intern_type));
from_new[len] = 0;
from_end = from_new + len + 1;
from = from_new;
+ extra = 1;
}
#endif
size_t inbytesleft = (from_end - from) * sizeof(intern_type);
size_t outbytesleft = (to_end - to) * sizeof(extern_type);
+#if WORKAROUND_ICONV_BUG
+ outbytesleft += extra * sizeof(extern_type);
+#endif
from_next = from;
to_next = to;
result const retval = do_iconv(out_cd_,
reinterpret_cast<char const **>(&from_next),
&inbytesleft, &to_next, &outbytesleft);
#if WORKAROUND_ICONV_BUG
- // Remove from output the null char that we inserted at the end
+ // Remove from output the nul char that we inserted at the end
// of the input buffer in order to circumvent an iconv bug.
- if (from_new)
+ if (from_new) {
--to_next;
+ --from_next;
+ from_next = from_old + (from_next - from);
+ from = from_old;
+ delete[] from_new;
+ }
#endif
if (retval == base::error) {
fprintf(stderr,
fputc('\n', stderr);
fflush(stderr);
}
-#if WORKAROUND_ICONV_BUG
- delete[] from_new;
-#endif
return retval;
}
virtual result do_unshift(state_type &, extern_type * to,
}
virtual int do_max_length() const throw()
{
- // FIXME: this information should be transferred to lib/encodings
- // UTF8 uses at most 4 bytes to represent one UCS4 code point
- // (see RFC 3629). RFC 2279 specifies 6 bytes, but that
- // information is outdated, and RFC 2279 has been superseded by
- // RFC 3629.
- // The CJK encodings use (different) multibyte representation as well.
- // All other encodings encode one UCS4 code point in one byte
- // (and can therefore only encode a subset of UCS4)
- // Note that BIG5 and SJIS do not work with LaTeX (see lib/encodings).
- // Furthermore, all encodings that use shifting (like SJIS) do not work with
- // iconv_codecvt_facet.
- if (encoding_ == "UTF-8" ||
- encoding_ == "GB" ||
- encoding_ == "EUC-TW")
- return 4;
- else if (encoding_ == "EUC-JP" ||
- encoding_ == "ISO-2022-JP")
- return 3;
- else if (encoding_ == "BIG5" ||
- encoding_ == "EUC-KR" ||
- encoding_ == "EUC-CN" ||
- encoding_ == "SJIS" ||
- encoding_ == "GBK")
- return 2;
- else
- return 1;
+ return lyx::max_encoded_bytes(encoding_);
}
private:
/// Do the actual conversion. The interface is equivalent to that of
inbytesleft, to, outbytesleft);
if (converted == (size_t)(-1)) {
switch(errno) {
+ case 0:
+ // As strange as it may seem, this
+ // does happen on windows when parsing
+ // comments with accented chars in
+ // tex2lyx. See the following thread
+ // for details
+ // http://thread.gmane.org/gmane.editors.lyx.devel/117636
+ break;
case EINVAL:
case E2BIG:
return base::partial;
}
-ifdocstream::ifdocstream(string const & encoding) : base()
+ifdocstream::ifdocstream() : base()
{
- setEncoding(*this, encoding, in);
+ setEncoding(*this, "UTF-8", in);
}
}
#endif
+
+void otexstream::put(char_type const & c)
+{
+ if (protectspace_) {
+ if (!canbreakline_ && c == ' ')
+ os_ << "{}";
+ protectspace_ = false;
+ }
+ os_.put(c);
+ lastchar_ = c;
+ if (c == '\n') {
+ texrow_.newline();
+ canbreakline_ = false;
+ } else
+ canbreakline_ = true;
+}
+
+
+BreakLine breakln;
+SafeBreakLine safebreakln;
+
+
+otexstream & operator<<(otexstream & ots, BreakLine)
+{
+ if (ots.canBreakLine()) {
+ ots.os().put('\n');
+ ots.lastChar('\n');
+ ots.canBreakLine(false);
+ ots.texrow().newline();
+ }
+ ots.protectSpace(false);
+ return ots;
+}
+
+
+otexstream & operator<<(otexstream & ots, SafeBreakLine)
+{
+ if (ots.canBreakLine()) {
+ ots.os() << "%\n";
+ ots.lastChar('\n');
+ ots.canBreakLine(false);
+ ots.texrow().newline();
+ }
+ ots.protectSpace(false);
+ return ots;
+}
+
+
+otexstream & operator<<(otexstream & ots, odocstream_manip pf)
+{
+ ots.os() << pf;
+ if (pf == static_cast<odocstream_manip>(endl)) {
+ ots.lastChar('\n');
+ ots.texrow().newline();
+ }
+ return ots;
+}
+
+
+otexstream & operator<<(otexstream & ots, docstring const & s)
+{
+ size_t const len = s.length();
+
+ // Check whether there's something to output
+ if (len == 0)
+ return ots;
+
+ if (ots.protectSpace()) {
+ if (!ots.canBreakLine() && s[0] == ' ')
+ ots.os() << "{}";
+ ots.protectSpace(false);
+ }
+
+ if (contains(s, 0xF0000)) {
+ // Some encoding changes for the underlying stream are embedded
+ // in the docstring. The encoding names to be used are enclosed
+ // between the code points 0xF0000 and 0xF0001, the first two
+ // characters of plane 15, which is a Private Use Area whose
+ // codepoints don't have any associated glyph.
+ docstring s1;
+ docstring s2 = split(s, s1, 0xF0000);
+ while (true) {
+ if (!s1.empty())
+ ots.os() << s1;
+ if (s2.empty())
+ break;
+ docstring enc;
+ docstring const s3 = split(s2, enc, 0xF0001);
+ if (!contains(s2, 0xF0001))
+ s2 = split(enc, s1, 0xF0000);
+ else {
+ ots.os() << setEncoding(to_ascii(enc));
+ s2 = split(s3, s1, 0xF0000);
+ }
+ }
+ } else
+ ots.os() << s;
+
+ ots.lastChar(s[len - 1]);
+ ots.texrow().newlines(count(s.begin(), s.end(), '\n'));
+ ots.canBreakLine(s[len - 1] != '\n');
+ return ots;
+}
+
+
+otexstream & operator<<(otexstream & ots, string const & s)
+{
+ ots << from_utf8(s);
+ return ots;
+}
+
+
+otexstream & operator<<(otexstream & ots, char const * s)
+{
+ ots << from_utf8(s);
+ return ots;
+}
+
+
+otexstream & operator<<(otexstream & ots, char c)
+{
+ if (ots.protectSpace()) {
+ if (!ots.canBreakLine() && c == ' ')
+ ots.os() << "{}";
+ ots.protectSpace(false);
+ }
+ ots.os() << c;
+ ots.lastChar(c);
+ if (c == '\n')
+ ots.texrow().newline();
+ ots.canBreakLine(c != '\n');
+ return ots;
+}
+
+
+template <typename Type>
+otexstream & operator<<(otexstream & ots, Type value)
+{
+ ots.os() << value;
+ ots.lastChar(0);
+ ots.canBreakLine(true);
+ ots.protectSpace(false);
+ return ots;
+}
+
+template otexstream & operator<< <SetEnc>(otexstream & os, SetEnc);
+template otexstream & operator<< <double>(otexstream &, double);
+template otexstream & operator<< <int>(otexstream &, int);
+template otexstream & operator<< <unsigned int>(otexstream &, unsigned int);
+template otexstream & operator<< <unsigned long>(otexstream &, unsigned long);
+
}
#if ! defined(USE_WCHAR_T) && defined(__GNUC__)
projects / lyx.git / blobdiff