From 72a488d7e6b56432263c80dd92cd6acc565e03a7 Mon Sep 17 00:00:00 2001 From: Enrico Forestieri Date: Sun, 19 Mar 2017 20:50:34 +0100 Subject: [PATCH] Fix output of en- and em-dashes with TeX fonts This commit fixes the regression introduced in 2.2 about the output of en- and em-dashes. In 2.2 en- and em-dashes are output as the \textendash and \textemdash macros when using TeX fonts, causing changed output in old documents and also bugs (for example, #10490). Now documents produced with older versions work again as intended, while documents produced with 2.2 can be made to produce the exact same output by simply checking "Don't use ligatures for en-and em-dashes" in Document->Settings->Fonts. When exporting documents using TeX fonts to earlier versions, in order to avoid changed output, a zero-width space character is inserted after each en/em-dash if dash ligatures are allowed. These characters are removed when reloading documents with 2.3, so that they don't accumulate. --- development/FORMAT | 7 ++ lib/RELEASE-NOTES | 19 ++++ lib/lyx2lyx/lyx_2_3.py | 95 ++++++++++++++++++- src/BufferParams.cpp | 4 + src/BufferParams.h | 2 + src/Paragraph.cpp | 15 +++ src/frontends/qt4/GuiDocument.cpp | 4 + src/frontends/qt4/ui/FontUi.ui | 30 ++++-- src/tex2lyx/test/CJK.lyx.lyx | 2 +- src/tex2lyx/test/CJKutf8.lyx.lyx | 2 +- src/tex2lyx/test/DummyDocument.lyx.lyx | 2 +- src/tex2lyx/test/Dummy~Document.lyx.lyx | 2 +- src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx | 2 +- src/tex2lyx/test/algo2e.lyx.lyx | 2 +- .../test/box-color-size-space-align.lyx.lyx | 2 +- src/tex2lyx/test/test-insets-basic.lyx.lyx | 2 +- src/tex2lyx/test/test-insets.lyx.lyx | 2 +- src/tex2lyx/test/test-memoir.lyx.lyx | 2 +- src/tex2lyx/test/test-modules.lyx.lyx | 2 +- .../test/test-refstyle-theorems.lyx.lyx | 2 +- src/tex2lyx/test/test-scr.lyx.lyx | 2 +- src/tex2lyx/test/test-structure.lyx.lyx | 2 +- src/tex2lyx/test/test.lyx.lyx | 2 +- src/tex2lyx/test/verbatim.lyx.lyx | 2 +- src/version.h | 4 +- 25 files changed, 183 insertions(+), 29 deletions(-) diff --git a/development/FORMAT b/development/FORMAT index 38c6ec1d21..0bb7de09d0 100644 --- a/development/FORMAT +++ b/development/FORMAT @@ -7,6 +7,13 @@ changes happened in particular if possible. A good example would be ----------------------- +2017-03-19 Enrico Forestieri + * Format incremented to 535: support for en/em-dash as ligatures. + The en- and em-dashes (U+2013 and U+2014) are now exported as + the font ligatures -- and --- when they would have been exported + as the macros \textendash and \textemdash, unless instructed + otherwise by a document preference. + 2017-02-04 Jürgen Spitzmüller * Format incremented to 534: Support for chapterbib - New buffer param value \multibib child diff --git a/lib/RELEASE-NOTES b/lib/RELEASE-NOTES index 8debb19b16..55e2f02137 100644 --- a/lib/RELEASE-NOTES +++ b/lib/RELEASE-NOTES @@ -13,6 +13,12 @@ be safely dissolved, as it will be automatically inserted at export time if needed, as usual. +* LyX now outputs en- and em-dashes as -- and --- ligatures when exporting to + latex using TeX fonts, as done in version 2.1 and earlier. In version 2.2 + they were instead output as the macros \textendash and \textemdash, causing + changed output with old documents and bugs. The 2.2 behavior can be restored + by don't allowing using dash ligatures in Document→Settings→Fonts. + !!!The following pref variables were added in 2.3: @@ -82,3 +88,16 @@ !!Caveats when upgrading from earlier versions to 2.3.x +* When loading documents created with LyX 2.2, you might need to check + "Don't use ligatures for en- and em-dashes" in Document→Settings→Fonts + to avoid changed output if they contain en- or em-dashes and use TeX fonts. + You don't need to do this for documents created with earlier versions. + +* If the "Use non-TeX fonts" and "Don't use ligatures for en- and em-dashes" + document preferences are not checked, when exporting documents containing + en- and em-dashes to the format of LyX 2.0 or earlier, the following line + has to be manually added to the unicodesymbols file of that LyX version:
+ 0x200b "\\hspace{0pt}" "" "" "" "" # ZERO WIDTH SPACE
+ This avoids "uncodable character" issues if the document is actually + loaded by that LyX version. LyX 2.1 and later versions already have the + necessary definition in their unicodesymbols file. diff --git a/lib/lyx2lyx/lyx_2_3.py b/lib/lyx2lyx/lyx_2_3.py index 9fbe12d07a..53eff206ad 100644 --- a/lib/lyx2lyx/lyx_2_3.py +++ b/lib/lyx2lyx/lyx_2_3.py @@ -1840,6 +1840,97 @@ def revert_chapterbib(document): # 7. Chapterbib proper add_to_preamble(document, ["\\usepackage{chapterbib}"]) + + +def convert_dashligatures(document): + " Remove a zero-length space (U+200B) after en- and em-dashes. " + + i = 0 + while i < len(document.body): + words = document.body[i].split() + # Skip some document parts where dashes are not converted + if len(words) > 1 and words[0] == "\\begin_inset" and \ + words[1] in ["CommandInset", "ERT", "External", "Formula", \ + "FormulaMacro", "Graphics", "IPA", "listings"]: + j = find_end_of_inset(document.body, i) + if j == -1: + document.warning("Malformed LyX document: Can't find end of " \ + + words[1] + " inset at line " + str(i)) + i += 1 + else: + i = j + continue + if len(words) > 0 and words[0] in ["\\leftindent", \ + "\\paragraph_spacing", "\\align", "\\labelwidthstring"]: + i += 1 + continue + + start = 0 + while True: + j = document.body[i].find(u"\u2013", start) # en-dash + k = document.body[i].find(u"\u2014", start) # em-dash + if j == -1 and k == -1: + break + if j == -1 or (k != -1 and k < j): + j = k + after = document.body[i][j+1:] + if after.startswith(u"\u200B"): + document.body[i] = document.body[i][:j+1] + after[1:] + else: + if len(after) == 0 and document.body[i+1].startswith(u"\u200B"): + document.body[i+1] = document.body[i+1][1:] + break + start = j+1 + i += 1 + + +def revert_dashligatures(document): + " Remove font ligature settings for en- and em-dashes. " + i = find_token(document.header, "\\use_dash_ligatures", 0) + if i == -1: + return + use_dash_ligatures = get_bool_value(document.header, "\\use_dash_ligatures", i) + del document.header[i] + use_non_tex_fonts = False + i = find_token(document.header, "\\use_non_tex_fonts", 0) + if i != -1: + use_non_tex_fonts = get_bool_value(document.header, "\\use_non_tex_fonts", i) + if not use_dash_ligatures or use_non_tex_fonts: + return + + # Add a zero-length space (U+200B) after en- and em-dashes + i = 0 + while i < len(document.body): + words = document.body[i].split() + # Skip some document parts where dashes are not converted + if len(words) > 1 and words[0] == "\\begin_inset" and \ + words[1] in ["CommandInset", "ERT", "External", "Formula", \ + "FormulaMacro", "Graphics", "IPA", "listings"]: + j = find_end_of_inset(document.body, i) + if j == -1: + document.warning("Malformed LyX document: Can't find end of " \ + + words[1] + " inset at line " + str(i)) + i += 1 + else: + i = j + continue + if len(words) > 0 and words[0] in ["\\leftindent", \ + "\\paragraph_spacing", "\\align", "\\labelwidthstring"]: + i += 1 + continue + + start = 0 + while True: + j = document.body[i].find(u"\u2013", start) # en-dash + k = document.body[i].find(u"\u2014", start) # em-dash + if j == -1 and k == -1: + break + if j == -1 or (k != -1 and k < j): + j = k + after = document.body[i][j+1:] + document.body[i] = document.body[i][:j+1] + u"\u200B" + after + start = j+1 + i += 1 ## @@ -1873,10 +1964,12 @@ convert = [ [531, []], [532, [convert_literalparam]], [533, []], - [534, []] + [534, []], + [535, [convert_dashligatures]] ] revert = [ + [534, [revert_dashligatures]], [533, [revert_chapterbib]], [532, [revert_multibib]], [531, [revert_literalparam]], diff --git a/src/BufferParams.cpp b/src/BufferParams.cpp index f20078bdff..03b1bd5e23 100644 --- a/src/BufferParams.cpp +++ b/src/BufferParams.cpp @@ -415,6 +415,7 @@ BufferParams::BufferParams() fonts_default_family = "default"; useNonTeXFonts = false; use_microtype = false; + use_dash_ligatures = true; fonts_expert_sc = false; fonts_old_figures = false; fonts_sans_scale[0] = 100; @@ -812,6 +813,8 @@ string BufferParams::readToken(Lexer & lex, string const & token, lex >> fonts_cjk; } else if (token == "\\use_microtype") { lex >> use_microtype; + } else if (token == "\\use_dash_ligatures") { + lex >> use_dash_ligatures; } else if (token == "\\paragraph_separation") { string parsep; lex >> parsep; @@ -1196,6 +1199,7 @@ void BufferParams::writeFile(ostream & os, Buffer const * buf) const os << "\\font_cjk " << fonts_cjk << '\n'; } os << "\\use_microtype " << convert(use_microtype) << '\n'; + os << "\\use_dash_ligatures " << convert(use_dash_ligatures) << '\n'; os << "\\graphics " << graphics_driver << '\n'; os << "\\default_output_format " << default_output_format << '\n'; os << "\\output_sync " << output_sync << '\n'; diff --git a/src/BufferParams.h b/src/BufferParams.h index 200b6d4396..30a157eb6f 100644 --- a/src/BufferParams.h +++ b/src/BufferParams.h @@ -280,6 +280,8 @@ public: std::string fonts_cjk; /// use LaTeX microtype package bool use_microtype; + /// use font ligatures for en- and em-dashes + bool use_dash_ligatures; /// Spacing & spacing(); Spacing const & spacing() const; diff --git a/src/Paragraph.cpp b/src/Paragraph.cpp index eb5b1114b4..7b0b3552dd 100644 --- a/src/Paragraph.cpp +++ b/src/Paragraph.cpp @@ -1274,6 +1274,21 @@ void Paragraph::Private::latexSpecialChar(otexstream & os, // written. (Asger) break; + case 0x2013: + case 0x2014: + if (bparams.use_dash_ligatures && !bparams.useNonTeXFonts) { + if (c == 0x2013) { + // en-dash + os << "--"; + column +=2; + } else { + // em-dash + os << "---"; + column +=3; + } + break; + } + // fall through default: if (c == '\0') return; diff --git a/src/frontends/qt4/GuiDocument.cpp b/src/frontends/qt4/GuiDocument.cpp index 2293ce3acf..01a5eddae2 100644 --- a/src/frontends/qt4/GuiDocument.cpp +++ b/src/frontends/qt4/GuiDocument.cpp @@ -838,6 +838,8 @@ GuiDocument::GuiDocument(GuiView & lv) this, SLOT(change_adaptor())); connect(fontModule->microtypeCB, SIGNAL(clicked()), this, SLOT(change_adaptor())); + connect(fontModule->dashesCB, SIGNAL(clicked()), + this, SLOT(change_adaptor())); connect(fontModule->scaleSansSB, SIGNAL(valueChanged(int)), this, SLOT(change_adaptor())); connect(fontModule->scaleTypewriterSB, SIGNAL(valueChanged(int)), @@ -3046,6 +3048,7 @@ void GuiDocument::applyView() fromqstr(fontModule->cjkFontLE->text()); bp_.use_microtype = fontModule->microtypeCB->isChecked(); + bp_.use_dash_ligatures = !fontModule->dashesCB->isChecked(); bp_.fonts_sans_scale[nontexfonts] = fontModule->scaleSansSB->value(); bp_.fonts_sans_scale[!nontexfonts] = fontModule->font_sf_scale; @@ -3550,6 +3553,7 @@ void GuiDocument::paramsToDialog() fontModule->cjkFontLE->setText(QString()); fontModule->microtypeCB->setChecked(bp_.use_microtype); + fontModule->dashesCB->setChecked(!bp_.use_dash_ligatures); fontModule->fontScCB->setChecked(bp_.fonts_expert_sc); fontModule->fontOsfCB->setChecked(bp_.fonts_old_figures); diff --git a/src/frontends/qt4/ui/FontUi.ui b/src/frontends/qt4/ui/FontUi.ui index 416c3fbc65..3877b37627 100644 --- a/src/frontends/qt4/ui/FontUi.ui +++ b/src/frontends/qt4/ui/FontUi.ui @@ -242,7 +242,27 @@ + + + + Activate extensions such as character protrusion and font expansion via the microtype package + + + Enable micr&o-typographic extensions + + + + + + Use \textendash and \textemdash instead of -- and --- for en- and em-dashes + + + Don't use ligatures for en- and &em-dashes + + + + Qt::Vertical @@ -255,16 +275,6 @@ - - - - Activate extensions such as character protrusion and font expansion via the microtype package - - - Enable micr&o-typographic extensions - - - diff --git a/src/tex2lyx/test/CJK.lyx.lyx b/src/tex2lyx/test/CJK.lyx.lyx index 656d858e9a..d6cceb748d 100644 --- a/src/tex2lyx/test/CJK.lyx.lyx +++ b/src/tex2lyx/test/CJK.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/CJKutf8.lyx.lyx b/src/tex2lyx/test/CJKutf8.lyx.lyx index d48441343e..34f1b7800e 100644 --- a/src/tex2lyx/test/CJKutf8.lyx.lyx +++ b/src/tex2lyx/test/CJKutf8.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/DummyDocument.lyx.lyx b/src/tex2lyx/test/DummyDocument.lyx.lyx index 9e618ed770..fe5320c388 100644 --- a/src/tex2lyx/test/DummyDocument.lyx.lyx +++ b/src/tex2lyx/test/DummyDocument.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/Dummy~Document.lyx.lyx b/src/tex2lyx/test/Dummy~Document.lyx.lyx index aed81be0d8..79283501a1 100644 --- a/src/tex2lyx/test/Dummy~Document.lyx.lyx +++ b/src/tex2lyx/test/Dummy~Document.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx b/src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx index 38a5c6b381..c83051683f 100644 --- a/src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx +++ b/src/tex2lyx/test/XeTeX-polyglossia.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/algo2e.lyx.lyx b/src/tex2lyx/test/algo2e.lyx.lyx index 61a1423eca..01771a793b 100644 --- a/src/tex2lyx/test/algo2e.lyx.lyx +++ b/src/tex2lyx/test/algo2e.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/box-color-size-space-align.lyx.lyx b/src/tex2lyx/test/box-color-size-space-align.lyx.lyx index 2ec71846bc..b6ce2e4758 100644 --- a/src/tex2lyx/test/box-color-size-space-align.lyx.lyx +++ b/src/tex2lyx/test/box-color-size-space-align.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-insets-basic.lyx.lyx b/src/tex2lyx/test/test-insets-basic.lyx.lyx index 9dcb18171d..d111e967c7 100644 --- a/src/tex2lyx/test/test-insets-basic.lyx.lyx +++ b/src/tex2lyx/test/test-insets-basic.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-insets.lyx.lyx b/src/tex2lyx/test/test-insets.lyx.lyx index f1f8dcec68..1d6404a5fe 100644 --- a/src/tex2lyx/test/test-insets.lyx.lyx +++ b/src/tex2lyx/test/test-insets.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-memoir.lyx.lyx b/src/tex2lyx/test/test-memoir.lyx.lyx index e3a7be318f..069485019e 100644 --- a/src/tex2lyx/test/test-memoir.lyx.lyx +++ b/src/tex2lyx/test/test-memoir.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-modules.lyx.lyx b/src/tex2lyx/test/test-modules.lyx.lyx index 615d9b1e71..c2ff8f4ff4 100644 --- a/src/tex2lyx/test/test-modules.lyx.lyx +++ b/src/tex2lyx/test/test-modules.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-refstyle-theorems.lyx.lyx b/src/tex2lyx/test/test-refstyle-theorems.lyx.lyx index 129fa7e113..969c25756e 100644 --- a/src/tex2lyx/test/test-refstyle-theorems.lyx.lyx +++ b/src/tex2lyx/test/test-refstyle-theorems.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-scr.lyx.lyx b/src/tex2lyx/test/test-scr.lyx.lyx index 048311333e..45979611e7 100644 --- a/src/tex2lyx/test/test-scr.lyx.lyx +++ b/src/tex2lyx/test/test-scr.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test-structure.lyx.lyx b/src/tex2lyx/test/test-structure.lyx.lyx index 8d502c19b8..e2eb17cca4 100644 --- a/src/tex2lyx/test/test-structure.lyx.lyx +++ b/src/tex2lyx/test/test-structure.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/test.lyx.lyx b/src/tex2lyx/test/test.lyx.lyx index 3c2a857d63..f0c45fa0ec 100644 --- a/src/tex2lyx/test/test.lyx.lyx +++ b/src/tex2lyx/test/test.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/tex2lyx/test/verbatim.lyx.lyx b/src/tex2lyx/test/verbatim.lyx.lyx index ea5ed082cf..5ab7608131 100644 --- a/src/tex2lyx/test/verbatim.lyx.lyx +++ b/src/tex2lyx/test/verbatim.lyx.lyx @@ -1,5 +1,5 @@ #LyX file created by tex2lyx 2.3 -\lyxformat 534 +\lyxformat 535 \begin_document \begin_header \save_transient_properties true diff --git a/src/version.h b/src/version.h index 43a9b356d3..5a03e0d312 100644 --- a/src/version.h +++ b/src/version.h @@ -32,8 +32,8 @@ extern char const * const lyx_version_info; // Do not remove the comment below, so we get merge conflict in // independent branches. Instead add your own. -#define LYX_FORMAT_LYX 534 // spitz: chapterbib support -#define LYX_FORMAT_TEX2LYX 534 +#define LYX_FORMAT_LYX 535 // ef: support for en/em-dash as ligatures +#define LYX_FORMAT_TEX2LYX 535 #if LYX_FORMAT_TEX2LYX != LYX_FORMAT_LYX #ifndef _MSC_VER -- 2.39.2