fix #10199 Non-encodable characters with XeTeX and utf8-plain

XeTeX with TeX fonts is only safe with ASCII input encoding (see #9740)
and we therefore force "ascii" when exporting with XeTeX and 8-bit TeX-fonts.

However, "utf8-plain" is a "power-user" option, which allows to switch off LyX's
encoding of the LaTeX file:
keep this also for "XeTeX with TeX fonts".

The user is responsible to ensure all characters can be processed and are
correctly shown in the output. The provided test sample shows the problems
with this encoding without special measures (like loading fontspec in the
user-preamble or a document class).

diff --git a/autotests/export/utf8-plain-with-tex-fonts.lyx b/autotests/export/utf8-plain-with-tex-fonts.lyx
new file mode 100644 (file)
index 0000000..da31b31
--- /dev/null
+++ b/autotests/export/utf8-plain-with-tex-fonts.lyx
@@ -0,0 +1,87 @@
+#LyX 2.3 created this file. For more info see http://www.lyx.org/
+\lyxformat 509
+\begin_document
+\begin_header
+\save_transient_properties true
+\origin unavailable
+\textclass article
+\use_default_options true
+\begin_modules
+logicalmkup
+\end_modules
+\maintain_unincluded_children false
+\language english
+\language_package default
+\inputencoding utf8-plain
+\fontencoding global
+\font_roman "default" "default"
+\font_sans "default" "default"
+\font_typewriter "default" "default"
+\font_math "auto" "auto"
+\font_default_family default
+\use_non_tex_fonts false
+\font_sc false
+\font_osf false
+\font_sf_scale 100 100
+\font_tt_scale 100 100
+\use_microtype false
+\graphics default
+\default_output_format default
+\output_sync 0
+\bibtex_command default
+\index_command default
+\paperfontsize default
+\spacing single
+\use_hyperref false
+\papersize default
+\use_geometry false
+\use_package amsmath 1
+\use_package amssymb 1
+\use_package cancel 1
+\use_package esint 1
+\use_package mathdots 1
+\use_package mathtools 1
+\use_package mhchem 1
+\use_package stackrel 1
+\use_package stmaryrd 1
+\use_package undertilde 1
+\cite_engine basic
+\cite_engine_type default
+\biblio_style plain
+\use_bibtopic false
+\use_indices false
+\paperorientation portrait
+\suppress_date false
+\justification true
+\use_refstyle 1
+\index Index
+\shortcut idx
+\color #008000
+\end_index
+\secnumdepth 3
+\tocdepth 3
+\paragraph_separation indent
+\paragraph_indentation default
+\quotes_language english
+\papercolumns 1
+\papersides 1
+\paperpagestyle default
+\tracking_changes false
+\output_changes false
+\html_math_output 0
+\html_css_as_file 0
+\html_be_strict false
+\end_header
+
+\begin_body
+
+\begin_layout Standard
+¹²³¼½¬¸@ł€¶←
+\end_layout
+
+\begin_layout Standard
+ätsch Grüße, em- and en-dash: — ―
+\end_layout
+
+\end_body
+\end_document

diff --git a/development/autotests/suspiciousTests b/development/autotests/suspiciousTests
index 3d0d579d30fec0c3d6e2a413a16f202952588c3b..f293a7166db7a2e801af844d47a3acc705734db0 100644 (file)
--- a/development/autotests/suspiciousTests
+++ b/development/autotests/suspiciousTests
@@ -120,6 +120,11 @@ export/doc/(de|es|fr)/Additional_pdf4_texF
 # \ding{58} and \ding{171} not supported with XeTeX and system fonts:
 export/examples/fr/ListesPuces_pdf4_systemF
 
+# inputencoding="utf8-plain" with Xe/LuaTeX: characters with
+# Unicode point > 256 lead to errors with 8-bit fonts
+# (We keep this option for power users setting Unicode fonts in the preamble
+# or document class file).
+export/export/utf8-plain-with-tex-fonts_.*_texF
 
 Sublabel: texissues
 #

diff --git a/development/autotests/unreliableTests b/development/autotests/unreliableTests
index ee18fe12f054187f45986bcfbcbca6fdf2e36595..8d6bbeb16f3ee272730fcc8f68b6228556a4e7a3 100644 (file)
--- a/development/autotests/unreliableTests
+++ b/development/autotests/unreliableTests
@@ -92,3 +92,7 @@ export/doc/es/.*_(pdf[45]|dvi3)_texF
 # Non-ASCII char in verbatim environment is dropped (with a warning when
 # doing this interactively) with ASCII (and hence also with XeTeX).
 export/doc/(|de/|es/|fr/)EmbeddedObjects_pdf4_texF
+
+# inputencoding="utf8-plain" with 8-bit TeX: non-ASCII characters are passed
+#  as two 8-bit characters leading to garbage in the output
+export/export/utf8-plain-with-tex-fonts_(dvi|pdf.?)

diff --git a/src/Buffer.cpp b/src/Buffer.cpp
index 9b92d8af63a966d03afe3140672d29b9699c7673..f4d238fb293299c15f813ba440a052defdfa5b02 100644 (file)
--- a/src/Buffer.cpp
+++ b/src/Buffer.cpp
@@ -1659,9 +1659,11 @@ bool Buffer::makeLaTeXFile(FileName const & fname,
 {
        OutputParams runparams = runparams_in;
 
-       // XeTeX with TeX fonts is only safe with ASCII encoding,
-       // but the "flavor" is not known in BufferParams::encoding().
-       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX))
+       // XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740),
+       // Check here, because the "flavor" is not known in BufferParams::encoding()
+       // (power users can override this safety measure selecting "utf8-plain").
+       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
+           && (runparams.encoding->name() != "utf8-plain"))
                runparams.encoding = encodings.fromLyXName("ascii");
 
        string const encoding = runparams.encoding->iconvName();
@@ -1747,8 +1749,10 @@ void Buffer::writeLaTeXSource(otexstream & os,
        OutputParams runparams = runparams_in;
 
        // XeTeX with TeX fonts is only safe with ASCII encoding,
-       // but the "flavor" is not known in BufferParams::encoding().
-       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX))
+       // Check here, because the "flavor" is not known in BufferParams::encoding()
+       // (power users can override this safety measure selecting "utf8-plain").
+       if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX)
+           && (runparams.encoding->name() != "utf8-plain"))
                runparams.encoding = encodings.fromLyXName("ascii");
        // FIXME: when only the current paragraph is shown, this is ignored
        //        (or not reached) and characters encodable in the current

diff --git a/src/BufferParams.cpp b/src/BufferParams.cpp
index 2d113bad4579bbc7851700c03ce4e59048a92c7e..373eee5dd0d0d98359ab47944e18cb75d341a46d 100644 (file)
--- a/src/BufferParams.cpp
+++ b/src/BufferParams.cpp
@@ -2973,8 +2973,8 @@ void BufferParams::writeEncodingPreamble(otexstream & os,
        // XeTeX/LuaTeX: (see also #9740)
        // With Unicode fonts we use utf8-plain without encoding package.
        // With TeX fonts, we cannot use utf8-plain, but "inputenc" fails.
-       // XeTeX must use ASCII encoding, for LuaTeX, we load
-       // "luainputenc" (see below).
+       // XeTeX must use ASCII encoding (see Buffer.cpp),
+       //  for LuaTeX, we load "luainputenc" (see below).
        if (useNonTeXFonts || features.runparams().flavor == OutputParams::XETEX)
                return;