X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=development%2Fautotests%2FunreliableTests;h=8d6bbeb16f3ee272730fcc8f68b6228556a4e7a3;hb=0c7bd9a57f2a308bb9659200eda3b7e45f8d5d3c;hp=b11f6f97c9a057bf76a6f2109ea28aa62e717656;hpb=1d66b6432a06ec649dd68ad89d13860a00f9de78;p=lyx.git

diff --git a/development/autotests/unreliableTests b/development/autotests/unreliableTests
index b11f6f97c9..8d6bbeb16f 100644
--- a/development/autotests/unreliableTests
+++ b/development/autotests/unreliableTests
@@ -1,36 +1,98 @@
-# Tests using some non-standard tex class
-Sublabel: nonstandard
-export/templates/IUCr-article_(dvi|pdf).*
-export/templates/ja_beamer-conference-ornate-20min_(pdf2|pdf[45]_(texF|systemF))
-export/templates/es_beamer-conference-ornate-20min_(pdf2|pdf[45]_(texF|systemF))
-export/templates/kluwer_pdf[45]_systemF
-export/examples/modernCV_pdf4_(tex|system)F
-export/templates/ectaart_(dvi3|pdf5)_(tex|system)F
-export/examples/fa/splash_(dvi|pdf|pdf[23]|(dvi3|pdf4|pdf5)_(texF|systemF))
+# Regular expressions for tests that should not be executed with
+# ctest -L export or ctest -L inverted because they do not work as expected
+# (either unreliable or invalid).
 #
+# Patterns will be automatically enclosed with '^' and '$'.
+# Matching tests get the label "unreliable" and do not get the label "export".
 #
+# TODO: Name problem: *invalid* tests (wrong output) are not *unreliable*.
+#   Use "unfit" or "unapplicable" as better label and name of pattern file?
+
+
+Sublabel: nonstandard
+# ===================
+# Documents with additional requirements,
+# e.g. a class or package file not in TeXLive.
+# TODO: rename to "extra"?
+
+export/templates/IUCr-article_(dvi|pdf).*
+# does not work with pdflatex, lualatex (PDF) or XeTeX (postscript specials?).
+export/templates/(ja|es)_beamer-conference-ornate-20min_(pdf2|pdf[45]).*
+export/templates/kluwer_.*_systemF
+export/templates/ectaart_(dvi3|pdf5).*
+
 # 1.) missing farsi package with lfeenc.def
 # 2.) LuaTeX does not support Farsi yet. See:
 #     https://github.com/reutenauer/polyglossia/commit/ccb0e9e2c6411170ad779b05ff5076f1193cc323
-# Tests depending on local configuration, OS, TeX distribution,
-# package versions, or the phase of the moon.
+export/examples/fa/splash_(dvi|pdf|pdf[23]|(dvi3|pdf4|pdf5)_(texF|systemF))
+
+# requires knitr, requires rjournal.sty (not in TeXLive)
+# incompatible with fontspec?
+export/templates/RJournal_(dvi3|pdf[45])_systemF
+
+# Requires aa.cls (not in TeXLive).
+# Fails with XeTeX/LuaTeX
+# Reported upstream (https://www.mail-archive.com/lyx-devel@lists.lyx.org/msg184435.html)
+# Is there a new version fixing the problem?
+export/examples/aa_sample_(dvi3|pdf[45])_.*
+export/templates/aa_(dvi3|pdf[45])_.*
+
+# Requires iucr.cls (not in TeXLive)
+# Is there a new version fixing the problem?
+# Does LuaTeX produce sensible results?
+export/templates/IUCr-article_pdf4_systemF
+
+# Requires acmsiggraph.cls (not in TeXLive)
+# see http://wiki.lyx.org/Examples/AcmSiggraph
+export/templates/ACM-siggraph_(dvi|pdf).*
+
 Sublabel: erratic
+# Tests depending on local configuration, or the phase of the moon.
+
 # for unknonwn reasons, seminar fails at some places:
 export/examples/(|fr/)seminar_.*
+
+# Export fails the first time but works the second time. (Why?)
+export/doc/es/Customization_pdf4_texF
+
+
+Sublabel: varying_versions
+# ========================
+# Test depending on TeX distribution, package versions or OS
+
+# document class aastex.cls replaced by aastex6.cls (v.1 2016/02/16)
+# in CTAN and TeXLive in march 2016
+# Template and sample use the new version via aastex6.layout (cf. #10027)
+# -> fail if aastex6.cls is missing (TeXLive15 original)
+export/templates/aastex_(dvi|pdf).*
+export/examples/aas_sample_(dvi|pdf).*
+
+
+Sublabel: wrong_output
+# ====================
+# Export does not fail but the resulting document has errors.
 #
-Sublabel: 1wrong 2output
-#
-# seminar.sty uses Postscript specials 
+# These tests are actually not "unreliable" (different result for different
+# runs) but *invalid* (not measuring what they should measure).
+
+# seminar.sty uses Postscript specials
 # -> wrong output with pdflatex/LuaTeX and DVI (missing landscape slides).
 export/examples/(|fr/)seminar_pdf[25].*
 export/examples/(|fr/)seminar_dvi.*
-#
-Sublabel: todo 1wrong 2output
-#
+
 # Babel-Spanish uses Babel's "strings" feature to define
 # separate auto-strings using UTF-8 literals.
 # Babel uses the "unicode" strings if it detects XeTeX or LuaTeX.
 # This is wrong for Xe/Lua with 8-bit TeX-fonts.
-# set inputenc to utf8? In the document or in lib/languages?
+# set inputenc to utf8?
+# (Changing the default in lib/languages requires more tests for utf8 first.)
 export/examples/es/linguistics_pdf4_texF
 export/doc/es/.*_(pdf[45]|dvi3)_texF
+
+# Non-ASCII char in verbatim environment is dropped (with a warning when
+# doing this interactively) with ASCII (and hence also with XeTeX).
+export/doc/(|de/|es/|fr/)EmbeddedObjects_pdf4_texF
+
+# inputencoding="utf8-plain" with 8-bit TeX: non-ASCII characters are passed
+#  as two 8-bit characters leading to garbage in the output
+export/export/utf8-plain-with-tex-fonts_(dvi|pdf.?)