From 6def81f30ff4f73a9c222651f602221c67535503 Mon Sep 17 00:00:00 2001 From: =?utf8?q?G=C3=BCnter=20Milde?= Date: Sat, 16 Mar 2019 12:43:50 +0100 Subject: [PATCH] Fix #10600: XeTeX with 8-bit TeX fonts requires input encodings "utf8" or "ascii". * New: support also utf8 (working around false positive test in "inputenc.sty"). * Do not force the change of input encoding to "ascii". Deny compilation with XeTeX if a document uses TeX fonts and a non-supported input encoding. --- development/autotests/ignoredTests | 9 +++- development/autotests/invertedTests | 27 ++++-------- development/autotests/unreliableTests | 2 +- lib/examples/ko/Welcome.lyx | 9 ++-- lib/examples/ru/Welcome.lyx | 33 +++++++++++---- lib/examples/uk/Welcome.lyx | 17 ++++---- src/Buffer.cpp | 26 +++--------- src/BufferParams.cpp | 59 ++++++++++++++++----------- src/output_latex.cpp | 23 ++++++----- 9 files changed, 115 insertions(+), 90 deletions(-) diff --git a/development/autotests/ignoredTests b/development/autotests/ignoredTests index dd8973baba..eaab1dba3d 100644 --- a/development/autotests/ignoredTests +++ b/development/autotests/ignoredTests @@ -26,12 +26,19 @@ export/examples/Modules/Noweb2LyX_lyx16 export/doc/(ja/|)MergedManuals_(xhtml|lyx2[2-9]) - # =========================================== Sublabel: special # Documents that do not work with all exports # =========================================== +# XeTeX with TeX-fonts works only for input encodings "utf8" and "ascii". +# special test cases for XeTeX with 8-bit fonts (ignore matches in this file): +!export/.*(utf8|ascii)_pdf4_texF +!export/examples/(|ar|ca|cs|da|de|el|es|eu|fa|fr|gl|he|hu|id|it|ko|nb|nl|pl|pt|pt_BR|pt_PT|ro|ru|sk|sl|sr|sv|uk|zh_CN)/Welcome_pdf4_texF +# Documentation and templates use "auto" 8-bit encodings. +.*_pdf4_texF +export/(doc|examples|templates)/(|ar|ca|cs|da|de|el|es|eu|fa|fr|gl|he|hu|id|it|ja|ko|nb|nl|pl|pt|pt_BR|pt_PT|ro|ru|sk|sl|sr|sv|uk|zh_CN)/.*pdf4_texF + # Inside these two files, we state that they should only be expected # to work with pdflatex and possibly lualatex. export/examples/Modules/PDF_Form_(dvi.*|pdf[^25]*) diff --git a/development/autotests/invertedTests b/development/autotests/invertedTests index dd69061518..533a28b94f 100644 --- a/development/autotests/invertedTests +++ b/development/autotests/invertedTests @@ -141,14 +141,6 @@ export/export/latex/LongestLabelWithUnderscore.* #11455 "Unable to process argument!" with multiple 1.6.x roundtrips export/templates/Articles/IEEE_Transactions_Journal_lyx16 -#10600 forced use of "ascii" inputenc with XeTeX -# Non-ASCII in ERT, fails with inputenc==ASCII -# Could not find LaTeX command for character 'í' (code point 0xed) -export/doc/(de|es)/Additional_pdf4_texF -export/export/latex/xetex-inputenc-utf8_pdf4_texF -# no unicodesymbols replacements for Arab charaters -export/export/latex/supported-languages_pdf4_texF - #11520 "listerrors" fails with pdflatex, lualatex, and xelatex # Example for lib/scripts/listerrors # Requires noweb. Seems to be restricted to plain "latex". @@ -218,18 +210,18 @@ export/doc/(|de/|es/|fr/)UserGuide_pdf4_texF # Babel-Russian uses UTF-8 for auto-strings if it detects Xe/LuaTeX. # This fails unless the inputencoding is set to utf-8, too. export/doc/ru/(Intro|Tutorial)_.*_texF -export/examples/ru/.*(LyXified|Welcome).*_texF +export/examples/ru/.*LyXified.*_texF # Babel-Ukrainian uses UTF-8 if it detects Xe/LuaTeX (for "\date"). # With 8-bit TeX fonts this results in: # * garbled output with PDF (LuaTeX) # * missing glyphs with XeTeX and DVI (LuaTeX) # New in TL18 -> cf. unreliableTests.varying_versions. -# Workaround: suppressing auto-date or using date inset (new in LyX 2.4) -# in the source documents. +# Workaround: suppressing auto-date, setting inputenc to "utf8", or using +# date inset (new in LyX 2.4) in the source documents. export/export/latex/uk-babel-date-missing-chars-with-ascii_(pdf4|dvi3)_texF # the "dvi3" tests for these pass, but have wrong output. -export/(doc|examples)/uk/(Intro|Welcome)_(pdf4)_texF +export/doc/uk/Intro_pdf4_texF # Fails with XeTeX/LuaTeX # Reported upstream (https://www.mail-archive.com/lyx-devel@lists.lyx.org/msg184435.html) @@ -292,7 +284,9 @@ export/templates/es/Presentations/Beamer_pdf4_texF # # see discussion at https://www.mail-archive.com/search?l=mid&q=20161114035123.s5lt4ib4x4obtptp%40steph # -# Babel-Arabic loads "inputenc" which fails with Xe/LuaTeX: +# Babel-Arabic loads "inputenc" which fails with LuaTeX +# Wit XeTeX, it works if "utf8" input encoding is set (as in ar/Welcome.lyx) +!export/examples/ar/Welcome_pdf4_texF export/.*/ar/.*_texF # Polyglossia bug with LuaTeX and LTR languages (also Hebrew and Farsi): # Undefined control sequences "\luatexpardir" and "\luatextextdir". @@ -306,8 +300,6 @@ export/export/latex/supported-languages_(dvi3|pdf5)_texF export/examples/fa/Welcome_(dvi3|pdf5)_systemF # missing glyphs (em space and "English" quotes) with XeTeX + system fonts. export/examples/fa/Welcome_pdf4_systemF -# missing ASCII replacements (required by XeTeX + TeX fonts) -export/examples/fa/Welcome_pdf4_texF # "invalid utf-8 sequence" with Babel-Farsi, LuaTeX, and TeX fonts. export/examples/fa/Welcome_(dvi3|pdf5)_texF @@ -315,9 +307,8 @@ export/examples/fa/Welcome_(dvi3|pdf5)_texF # does not currently work with LuaTeX (undefined command) (cf. Arabic) # terminal message: "Language hebrew not found in language.dat.lua" export/.*/he/.*(dvi3|pdf5)_systemF -# Babel-Arabic loads "inputenc" which fails with Xe/LuaTeX -# additionally: "! Right-to-Left Support Error: use TeX--XeT or e-TeX engine." -# and (with XeTeX) missing ASCII replacements. +# Babel-Arabic loads "inputenc" which fails with LuaTeX +# Xe/LuaTeX + TeX fonts: "! Right-to-Left Support Error: use TeX--XeT or e-TeX engine." export/.*/he/.*_texF # See comment in unreliableTests diff --git a/development/autotests/unreliableTests b/development/autotests/unreliableTests index 1e0aa688dc..e812c27e15 100644 --- a/development/autotests/unreliableTests +++ b/development/autotests/unreliableTests @@ -119,7 +119,7 @@ export/examples/(|fr/)seminar_(dvi|pdf).* # * garbled output with LuaTeX + TeX fonts (since TL18) # * missing chars with XeTeX + TeX fonts (see invertedTests) export/export/latex/uk-babel-date-missing-chars-with-ascii_.*_texF -export/(doc|examples)/uk/(Intro|Welcome)_.*_texF +export/doc/uk/Intro_.*_texF # new problem with TL18 (see invertedTests) export/export/latex/luainputenc-utf8_pdf5_texF diff --git a/lib/examples/ko/Welcome.lyx b/lib/examples/ko/Welcome.lyx index 79cb89eb26..c1e5f2ed0b 100644 --- a/lib/examples/ko/Welcome.lyx +++ b/lib/examples/ko/Welcome.lyx @@ -1,5 +1,5 @@ #LyX 2.4 created this file. For more info see https://www.lyx.org/ -\lyxformat 544 +\lyxformat 571 \begin_document \begin_header \save_transient_properties true @@ -12,8 +12,8 @@ \maintain_unincluded_children false \language korean \language_package default -\inputencoding utf8-cjk -\fontencoding global +\inputencoding utf8 +\fontencoding auto \font_roman "lmodern" "UnJamoBatang" \font_sans "default" "UnJamoNovel" \font_typewriter "default" "UnTaza" @@ -32,6 +32,8 @@ \output_sync 0 \bibtex_command default \index_command default +\float_placement class +\float_alignment class \paperfontsize default \spacing single \use_hyperref false @@ -72,6 +74,7 @@ \papercolumns 1 \papersides 1 \paperpagestyle default +\tablestyle default \tracking_changes false \output_changes false \html_math_output 0 diff --git a/lib/examples/ru/Welcome.lyx b/lib/examples/ru/Welcome.lyx index 5f237307eb..010583bcf9 100644 --- a/lib/examples/ru/Welcome.lyx +++ b/lib/examples/ru/Welcome.lyx @@ -1,5 +1,5 @@ -#LyX 2.3 created this file. For more info see http://www.lyx.org/ -\lyxformat 544 +#LyX 2.4 created this file. For more info see https://www.lyx.org/ +\lyxformat 571 \begin_document \begin_header \save_transient_properties true @@ -10,10 +10,10 @@ \language russian \language_package default \inputencoding utf8 -\fontencoding global -\font_roman "default" "DejaVu Serif" -\font_sans "default" "DejaVu Sans" -\font_typewriter "default" "DejaVu Sans Mono" +\fontencoding auto +\font_roman "DejaVuSerif" "DejaVu Serif" +\font_sans "DejaVuSans" "DejaVu Sans" +\font_typewriter "DejaVuSansMono" "DejaVu Sans Mono" \font_math "auto" "auto" \font_default_family default \use_non_tex_fonts false @@ -28,6 +28,8 @@ \output_sync 0 \bibtex_command default \index_command default +\float_placement class +\float_alignment class \paperfontsize default \spacing single \use_hyperref false @@ -68,6 +70,7 @@ \papercolumns 1 \papersides 1 \paperpagestyle default +\tablestyle default \tracking_changes false \output_changes false \html_math_output 0 @@ -129,7 +132,11 @@ status collapsed \begin_layout Enumerate Вывод программы \SpecialChar LyX выглядит потрясающе! Чтобы в этом убедиться, выберите пункт - меню + меню +\begin_inset Newline newline +\end_inset + + \family sans Документ\SpecialChar menuseparator Просмотреть @@ -186,6 +193,18 @@ arg "buffer-view" \end_layout \begin_layout Enumerate +\begin_inset ERT +status open + +\begin_layout Plain Layout + + +\backslash +sloppy +\end_layout + +\end_inset + Домашняя страница \SpecialChar LyX находится по адресу \begin_inset Flex URL diff --git a/lib/examples/uk/Welcome.lyx b/lib/examples/uk/Welcome.lyx index c953fd16f3..0ce74eb8b1 100644 --- a/lib/examples/uk/Welcome.lyx +++ b/lib/examples/uk/Welcome.lyx @@ -1,5 +1,5 @@ -#LyX 2.3 created this file. For more info see http://www.lyx.org/ -\lyxformat 544 +#LyX 2.4 created this file. For more info see https://www.lyx.org/ +\lyxformat 571 \begin_document \begin_header \save_transient_properties true @@ -10,16 +10,16 @@ \language ukrainian \language_package default \inputencoding utf8 -\fontencoding global -\font_roman "default" "DejaVu Serif" -\font_sans "default" "DejaVu Sans" -\font_typewriter "courier" "DejaVu Sans Mono" +\fontencoding auto +\font_roman "DejaVuSerif" "DejaVu Serif" +\font_sans "DejaVuSans" "DejaVu Sans" +\font_typewriter "DejaVuSansMono" "DejaVu Sans Mono" \font_math "auto" "auto" \font_default_family default \use_non_tex_fonts false \font_sc false \font_osf false -\font_sf_scale 95 100 +\font_sf_scale 95 95 \font_tt_scale 100 100 \use_microtype false \use_dash_ligatures false @@ -28,6 +28,8 @@ \output_sync 0 \bibtex_command default \index_command default +\float_placement class +\float_alignment class \paperfontsize default \spacing single \use_hyperref false @@ -68,6 +70,7 @@ \papercolumns 1 \papersides 1 \paperpagestyle default +\tablestyle default \tracking_changes false \output_changes false \html_math_output 0 diff --git a/src/Buffer.cpp b/src/Buffer.cpp index 854dd736b8..e71c83122d 100644 --- a/src/Buffer.cpp +++ b/src/Buffer.cpp @@ -1719,13 +1719,6 @@ Buffer::ExportStatus Buffer::makeLaTeXFile(FileName const & fname, { OutputParams runparams = runparams_in; - // XeTeX with TeX fonts is only safe with ASCII encoding (see also #9740), - // Check here, because the "flavor" is not known in BufferParams::encoding() - // (power users can override this safety measure selecting "utf8-plain"). - if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX) - && (runparams.encoding->name() != "utf8-plain")) - runparams.encoding = encodings.fromLyXName("ascii"); - string const encoding = runparams.encoding->iconvName(); LYXERR(Debug::LATEX, "makeLaTeXFile encoding: " << encoding << ", fname=" << fname.realPath()); @@ -1823,16 +1816,6 @@ Buffer::ExportStatus Buffer::writeLaTeXSource(otexstream & os, OutputParams runparams = runparams_in; - // XeTeX with TeX fonts is only safe with ASCII encoding, - // Check here, because the "flavor" is not known in BufferParams::encoding() - // (power users can override this safety measure selecting "utf8-plain"). - if (!params().useNonTeXFonts && (runparams.flavor == OutputParams::XETEX) - && (runparams.encoding->name() != "utf8-plain")) - runparams.encoding = encodings.fromLyXName("ascii"); - // FIXME: when only the current paragraph is shown, this is ignored - // (or not reached) and characters encodable in the current - // encoding are not converted to ASCII-representation. - // Some macros rely on font encoding runparams.main_fontenc = params().main_font_encoding(); @@ -4473,9 +4456,12 @@ Buffer::ExportStatus Buffer::doExport(string const & target, bool put_in_tempdir if (!put_in_tempdir) { // Only show this alert if this is an export to a non-temporary // file (not for previewing). - Alert::error(_("Couldn't export file"), bformat( - _("No information for exporting the format %1$s."), - theFormats().prettyName(format))); + docstring s = _("No information for exporting the format %1$s."); + if (format == "pdf4") + s += _("\nHint: use non-TeX fonts or set input encoding to " + "'utf8' or 'ascii'"); + Alert::error(_("Couldn't export file"), + bformat(s, theFormats().prettyName(format))); } return ExportNoPathToFormat; } diff --git a/src/BufferParams.cpp b/src/BufferParams.cpp index 2234ca4b56..5704940ecd 100644 --- a/src/BufferParams.cpp +++ b/src/BufferParams.cpp @@ -102,9 +102,20 @@ static char const * const tex_graphics[] = { }; - namespace lyx { +// XeTeX with TeX fonts: +// run in 8-bit emulation mode and trick `inputenc` into working with XeTeX +static docstring const xetex_pre_inputenc = from_ascii( + "\\XeTeXinputencoding \"bytes\" % current file\n" + "\\XeTeXdefaultencoding \"bytes\" % included files\n" + "\\makeatletter\n" + "\\let\\origUmathchar\\Umathchar\n" + "\\let\\Umathchar\\@undefined\n"); +static docstring const xetex_post_inputenc = from_ascii( + "\\let\\Umathchar\\origUmathchar\n" + "\\makeatother\n"); + // Local translators namespace { @@ -1769,7 +1780,7 @@ bool BufferParams::writeLaTeX(otexstream & os, LaTeXFeatures & features, << from_ascii(fonts_default_family) << "}\n"; // set font encoding - // XeTeX and LuaTeX (with OS fonts) do not need fontenc + // non-TeX fonts use font encoding TU (set by fontspec) if (!useNonTeXFonts && !features.isProvided("fontenc") && main_font_encoding() != "default") { // get main font encodings @@ -2314,8 +2325,8 @@ bool BufferParams::writeLaTeX(otexstream & os, LaTeXFeatures & features, // esint and the other packages that provide special glyphs if (features.mustProvide("tipa") && useNonTeXFonts && !features.isProvided("xunicode")) { - // The package officially only supports XeTeX, but also works - // with LuaTeX. Thus we work around its XeTeX test. + // The `xunicode` package officially only supports XeTeX, + // but also works with LuaTeX. We work around its XeTeX test. if (features.runparams().flavor != OutputParams::XETEX) { os << "% Pretend to xunicode that we are XeTeX\n" << "\\def\\XeTeXpicfile{}\n"; @@ -2631,13 +2642,17 @@ FormatList const & BufferParams::exportableFormats(bool only_viewable) const if (useNonTeXFonts) { excludes.insert("latex"); excludes.insert("pdflatex"); - } - FormatList result = - theConverters().getReachable(backs[0], only_viewable, true, excludes); + } else if (inputenc != "ascii" && inputenc != "utf8" + && inputenc != "utf8x" && inputenc != "utf8-plain") + // XeTeX with TeX fonts requires input encoding ascii or utf8 + // (https://www.tug.org/pipermail/xetex/2010-April/016452.html). + excludes.insert("xetex"); + FormatList result = theConverters().getReachable(backs[0], only_viewable, + true, excludes); for (vector::const_iterator it = backs.begin() + 1; it != backs.end(); ++it) { FormatList r = theConverters().getReachable(*it, only_viewable, - false, excludes); + false, excludes); result.insert(result.end(), r.begin(), r.end()); } sort(result.begin(), result.end(), Format::formatSorter); @@ -2661,7 +2676,9 @@ vector BufferParams::backends() const v.push_back("pdflatex"); v.push_back("latex"); } - v.push_back("xetex"); + if (useNonTeXFonts || inputenc == "ascii" || inputenc == "utf8" + || inputenc == "utf8x" || inputenc == "utf8-plain") + v.push_back("xetex"); v.push_back("luatex"); v.push_back("dviluatex"); } @@ -3224,12 +3241,8 @@ docstring BufferParams::getGraphicsDriver(string const & package) const void BufferParams::writeEncodingPreamble(otexstream & os, LaTeXFeatures & features) const { - // XeTeX/LuaTeX: (see also #9740) - // With Unicode fonts we use utf8-plain without encoding package. - // With TeX fonts, we cannot use utf8-plain, but "inputenc" fails. - // XeTeX must use ASCII encoding (see Buffer.cpp), - // for LuaTeX, we load "luainputenc" (see below). - if (useNonTeXFonts || features.runparams().flavor == OutputParams::XETEX) + // With no-TeX fonts we use utf8-plain without encoding package. + if (useNonTeXFonts) return; if (inputenc == "auto") { @@ -3241,7 +3254,8 @@ void BufferParams::writeEncodingPreamble(otexstream & os, // Create list of inputenc options: set encoding_set; // luainputenc fails with more than one encoding - if (!features.runparams().isFullUnicode()) // if we reach this point, this means LuaTeX with TeX fonts + if (features.runparams().flavor != OutputParams::LUATEX + && features.runparams().flavor != OutputParams::DVILUATEX) // list all input encodings used in the document encoding_set = features.getEncodingSet(doc_encoding); @@ -3276,7 +3290,8 @@ void BufferParams::writeEncodingPreamble(otexstream & os, case Encoding::none: case Encoding::CJK: case Encoding::japanese: - if (encoding().iconvName() != "UTF-8") + if (encoding().iconvName() != "UTF-8" + && !features.runparams().isFullUnicode()) // don't default to [utf8]{inputenc} with TeXLive >= 18 os << "\\ifdefined\\UseRawInputEncoding\n" << " \\UseRawInputEncoding\\fi\n"; @@ -3287,12 +3302,16 @@ void BufferParams::writeEncodingPreamble(otexstream & os, if (features.isRequired("japanese") || features.isProvided("inputenc")) break; + if (features.runparams().flavor == OutputParams::XETEX) + os << xetex_pre_inputenc; os << "\\usepackage[" << from_ascii(encoding().latexName()); if (features.runparams().flavor == OutputParams::LUATEX || features.runparams().flavor == OutputParams::DVILUATEX) os << "]{luainputenc}\n"; else os << "]{inputenc}\n"; + if (features.runparams().flavor == OutputParams::XETEX) + os << xetex_post_inputenc; break; } } @@ -3412,12 +3431,6 @@ string const BufferParams::loadFonts(LaTeXFeatures & features) const Encoding const & BufferParams::encoding() const { // Main encoding for LaTeX output. - // - // Exception: XeTeX with 8-bit TeX fonts requires ASCII (see #9740). - // As the "flavor" is only known once export started, this - // cannot be handled here. Instead, runparams.encoding is set - // to ASCII in Buffer::makeLaTeXFile (for export) - // and Buffer::writeLaTeXSource (for preview). if (useNonTeXFonts) return *(encodings.fromLyXName("utf8-plain")); if (inputenc == "auto" || inputenc == "default") diff --git a/src/output_latex.cpp b/src/output_latex.cpp index 3489a11bb6..784a9e58bb 100644 --- a/src/output_latex.cpp +++ b/src/output_latex.cpp @@ -1336,14 +1336,16 @@ void TeXOnePar(Buffer const & buf, } } + // Information about local language is stored as a font feature. // If this is the last paragraph, and a local_font was set upon entering - // the inset, and we're using "auto" or "default" encoding, and not - // compiling with XeTeX or LuaTeX, the encoding - // should be set back to that local_font's encoding. + // the inset and we're using "auto" or "default" encoding (and not + // compiling with LuaTeX), ensure the encoding is set back to the default + // encoding of the local language. if (runparams.isLastPar && runparams_in.local_font != 0 && runparams_in.encoding != runparams_in.local_font->language()->encoding() && (bparams.inputenc == "auto" || bparams.inputenc == "default") - && !runparams.isFullUnicode() + && runparams.flavor != OutputParams::LUATEX + && runparams.flavor != OutputParams::DVILUATEX ) { runparams_in.encoding = runparams_in.local_font->language()->encoding(); os << setEncoding(runparams_in.encoding->iconvName()); @@ -1632,12 +1634,13 @@ pair switchEncoding(odocstream & os, BufferParams const & bparams, OutputParams const & runparams, Encoding const & newEnc, bool force, bool noswitchmacro) { - // XeTeX/LuaTeX use only one encoding per document: - // * with useNonTeXFonts: "utf8plain", - // * with XeTeX and TeX fonts: "ascii" (inputenc fails), - // * with LuaTeX and TeX fonts: only one encoding accepted by luainputenc. - if (runparams.isFullUnicode() || newEnc.name() == "inherit") - return make_pair(false, 0); + // Never switch encoding with non-TeX fonts (always "utf8plain") or + // with LuaTeX and TeX fonts (only one encoding accepted by luainputenc). + if (bparams.useNonTeXFonts + || runparams.flavor == OutputParams::LUATEX + || runparams.flavor == OutputParams::DVILUATEX + || newEnc.name() == "inherit") + return make_pair(false, 0); Encoding const & oldEnc = *runparams.encoding; bool moving_arg = runparams.moving_arg; -- 2.39.2