ctests: rename pattern file "suspiciousTests" to "invertedTests".

diff --git a/development/autotests/ExportTests.cmake b/development/autotests/ExportTests.cmake
index d1d6bfabc8ac8065e77918b891e28720b808eece..7dd61683de21c13d93ddeb4fb0deb8a91970ca4d 100644 (file)
--- a/development/autotests/ExportTests.cmake
+++ b/development/autotests/ExportTests.cmake
@@ -277,7 +277,7 @@ assignLabelDepth(1 "unreliable" "inverted")
 assignLabelDepth(2 "suspended")
 assignLabelDepth(-1 "examples" "manuals" "mathmacros" "templates" "autotests")
 
-loadTestList(suspiciousTests suspiciousTests 7)
+loadTestList(invertedTests invertedTests 7)
 loadTestList(ignoredTests ignoredTests 0)
 loadTestList(suspendedTests suspendedTests 6)
 loadTestList(unreliableTests unreliableTests 5)
@@ -318,7 +318,7 @@ foreach(libsubfolderx autotests/export lib/doc lib/examples lib/templates autote
     foreach(_lyx_format_num 16 21 22)
       set(TestName "export/${libsubfolder}/${f}_lyx${_lyx_format_num}")
       set(mytestlabel ${testlabel} "lyx2lyx" "load")
-      maketestname(TestName inverted suspiciousTests ignoredTests unreliableTests mytestlabel)
+      maketestname(TestName inverted invertedTests ignoredTests unreliableTests mytestlabel)
       if(TestName)
         add_test(NAME ${TestName}
           WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@@ -343,7 +343,7 @@ foreach(libsubfolderx autotests/export lib/doc lib/examples lib/templates autote
       # For use of lyx2lyx we need the python executable
       set(mytestlabel ${lyx2lyxtestlabel})
       set(TestName "lyx2lyx/${libsubfolder}/${f}")
-      maketestname(TestName inverted suspiciousTests ignoredTests unreliableTests mytestlabel)
+      maketestname(TestName inverted invertedTests ignoredTests unreliableTests mytestlabel)
       if(TestName)
         add_test(NAME ${TestName}
           WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@@ -359,7 +359,7 @@ foreach(libsubfolderx autotests/export lib/doc lib/examples lib/templates autote
     set(loadtestlabel "load")
     set(mytestlabel ${loadtestlabel})
     set(TestName "check_load/${libsubfolder}/${f}")
-    maketestname(TestName inverted suspiciousTests ignoredTests unreliableTests mytestlabel)
+    maketestname(TestName inverted invertedTests ignoredTests unreliableTests mytestlabel)
     if(TestName)
       add_test(NAME ${TestName}
         WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@@ -387,7 +387,7 @@ foreach(libsubfolderx autotests/export lib/doc lib/examples lib/templates autote
           set(TestName "export/${libsubfolder}/${f}_${format}_${fonttype}")
         endif()
         set(mytestlabel ${testlabel})
-        maketestname(TestName inverted suspiciousTests ignoredTests unreliableTests mytestlabel)
+        maketestname(TestName inverted invertedTests ignoredTests unreliableTests mytestlabel)
         if(TestName)
           add_test(NAME ${TestName}
             WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"

diff --git a/development/autotests/invertedTests b/development/autotests/invertedTests
index 25558a19a265a1b62aa26c82ce011a17204becf1..4e3602e80eb8a05c54c7b7816f2a1e143d3abb73 100644 (file)
--- a/development/autotests/invertedTests
+++ b/development/autotests/invertedTests
@@ -3,9 +3,9 @@
 # Patterns will be automatically enclosed with '^' and '$'.
 # Matching tests get the label "inverted"
 # or (if also matching a pattern in "suspendedTests") "suspended".
-# They get also the test-feature 'inverted', i.e.
+# They get also the test-property 'WILL_FAIL', i.e.
 # they are reported as failing if the export works without error.
-
+# https://cmake.org/cmake/help/v3.0/prop_test/WILL_FAIL.html
 
 Sublabel: todo
 #

diff --git a/development/autotests/unreliableTests b/development/autotests/unreliableTests
index ef68ca913b73cb6067e9575be321eca3ba8cbc8e..5aff59d403d252f9b1ce07f7f7e4f6e8b8689ac1 100644 (file)
--- a/development/autotests/unreliableTests
+++ b/development/autotests/unreliableTests
@@ -10,10 +10,10 @@
 
 
 Sublabel: nonstandard
-# ===================
+#####################
 # Documents with additional requirements,
 # e.g. a class or package file not in TeXLive.
-# TODO: rename to "extra"?
+# TODO: rename to "extra" or "exotic"?
 
 export/templates/IUCr-article_(dvi|pdf).*
 # does not work with pdflatex, lualatex (PDF) or XeTeX (postscript specials?).
@@ -47,6 +47,7 @@ export/templates/IUCr-article_pdf4_systemF
 export/templates/ACM-siggraph_(dvi|pdf).*
 
 Sublabel: erratic
+#################
 # Tests depending on local configuration, or the phase of the moon.
 
 # for unknonwn reasons, seminar fails at some places:
@@ -57,8 +58,8 @@ export/doc/es/Customization_pdf4_texF
 
 
 Sublabel: varying_versions
-# ========================
-# Test depending on TeX distribution, package versions or OS
+##########################
+# Test depending on TeX distribution, package versions or OS.
 
 # document class aastex.cls replaced by aastex6.cls (v.1 2016/02/16)
 # in CTAN and TeXLive in march 2016
@@ -69,7 +70,7 @@ export/examples/aas_sample_(dvi|pdf).*
 
 
 Sublabel: wrong_output
-# ====================
+######################
 # Export does not fail but the resulting document has errors.
 #
 # These tests are actually not "unreliable" (different result for different
@@ -99,7 +100,7 @@ export/export/utf8-plain-with-tex-fonts_(dvi|pdf.?)
 
 # The following document mixes inTitle layouts and leads to
 # likely unintended output. See #10347
-export/mixing_inTitle_layouts.lyx
+export/export/mixing_inTitle_layouts(pdf|dvi).*
 
 # Characters with accents are not output
 # Reported to LuaTeX mailing list here:

diff --git a/lib/doc/Development.lyx b/lib/doc/Development.lyx
index 6eb2f60eeb4b3fda96f60c5390494f6ad3c3b5fb..92628dfb1a6b6d4955289caac26245705e6e5518 100644 (file)
--- a/lib/doc/Development.lyx
+++ b/lib/doc/Development.lyx
@@ -1,5 +1,5 @@
-#LyX 2.2 created this file. For more info see http://www.lyx.org/
-\lyxformat 508
+#LyX 2.3 created this file. For more info see http://www.lyx.org/
+\lyxformat 512
 \begin_document
 \begin_header
 \save_transient_properties true
@@ -25,6 +25,7 @@ logicalmkup
 \font_osf false
 \font_sf_scale 100 100
 \font_tt_scale 100 100
+\use_microtype false
 \graphics default
 \default_output_format pdf2
 \output_sync 1
@@ -3193,7 +3194,7 @@ status collapsed
 status collapsed
 
 \begin_layout Plain Layout
-development/autotests/suspiciousTests
+development/autotests/invertedTests
 \end_layout
 
 \end_inset
@@ -3333,15 +3334,15 @@ inverted
 \emph on
 uninvert
 \emph default
- the test by removing the labeling pattern from 
+ the test by removing the pattern from the 
 \begin_inset Quotes eld
 \end_inset
 
-suspiciousTests
+invertedTests
 \begin_inset Quotes erd
 \end_inset
 
-) (see 
+ file) (see 
 \begin_inset CommandInset ref
 LatexCommand ref
 reference "par:Inverted-tests"
@@ -3353,7 +3354,11 @@ reference "par:Inverted-tests"
 
 \begin_layout Itemize
 If the export did not fail previously but led to wrong output (PDF, say),
- it is in fact an improvement when the test now fails, label it as 
+\begin_inset Foot
+status collapsed
+
+\begin_layout Plain Layout
+Non-failing test with wrong output should be labeledas 
 \begin_inset Quotes eld
 \end_inset
 
@@ -3371,9 +3376,25 @@ reference "par:Unreliable-tests"
 ).
 \end_layout
 
+\end_inset
+
+ it is in fact an improvement when the test now fails.
+ 
+\emph on
+Invert
+\emph default
+ the failing test case (see 
+\begin_inset CommandInset ref
+LatexCommand ref
+reference "par:Inverted-tests"
+
+\end_inset
+
+).
+\end_layout
+
 \begin_layout Itemize
-In case of tests failing due to missing requirements (when only a subset
- of TeXLive is installed or a test labeled 
+In case of tests failing due to missing requirements (tests labeled 
 \begin_inset Quotes eld
 \end_inset
 
@@ -3381,8 +3402,9 @@ unreliable:nonstandard
 \begin_inset Quotes erd
 \end_inset
 
- fails), ignore the failure, ask for someone else to run the test, or install
- the missing ressources and try again.
+ or testing on a system withonly a subset of TeXLive installed), ignore
+ the failure, ask for someone else to run the test, or install the missing
+ ressources and try again.
 \end_layout
 
 \begin_layout Paragraph
@@ -3401,7 +3423,7 @@ Test cases whose name matches a pattern in the file
 status collapsed
 
 \begin_layout Plain Layout
-development/autotests/suspiciousTests
+development/autotests/invertedTests
 \end_layout
 
 \end_inset
@@ -3456,7 +3478,7 @@ The following sublabels are currently present in
 status collapsed
 
 \begin_layout Plain Layout
-suspiciousTests
+invertedTests
 \end_layout
 
 \end_inset
@@ -3522,12 +3544,20 @@ otherwise, add a pattern here.
 
 \end_deeper
 \begin_layout Description
-attic Documents in the attic.
- (Kept for reference and format conversion test.)
+attic Documents in the attic (kept for reference and format conversion test).
+ Usually 
+\begin_inset Quotes eld
+\end_inset
+
+Wontfix
+\begin_inset Quotes erd
+\end_inset
+
+.
 \end_layout
 
 \begin_layout Subparagraph
-suspended 
+suspended tests
 \end_layout
 
 \begin_layout Standard
@@ -3702,43 +3732,45 @@ status collapsed
 
 \end_layout
 
-\begin_layout Description
-nonstandard Documents with additional requirements, e.g.
- a class or package file not in TeXLive.
- 
-\begin_inset Note Note
-status open
+\begin_layout Standard
+The following sublabels are currently present in 
+\begin_inset Flex Code
+status collapsed
 
 \begin_layout Plain Layout
-TODO: rename to "extra"?
+unreliableTests
 \end_layout
 
 \end_inset
 
-
-\end_layout
-
-\begin_deeper
-\begin_layout Standard
-These tests are labeled as 
-\family typewriter
-'nonstandard'.
+:
 \end_layout
 
-\end_deeper
 \begin_layout Description
-erratic Tests depending on local configuration, OS, TeX distribution, package
- versions, or the phase of the moon.
+nonstandard Documents with additional requirements, e.g.
+ a class or package file not in TeXLive.
  
 \begin_inset Note Note
 status open
 
 \begin_layout Plain Layout
-TODO: use 
-\emph on
-erratic
-\emph default
- only for the phase-of-moon dependency?
+TODO: rename to 
+\begin_inset Quotes eld
+\end_inset
+
+extra
+\begin_inset Quotes erd
+\end_inset
+
+ or 
+\begin_inset Quotes eld
+\end_inset
+
+exotic
+\begin_inset Quotes erd
+\end_inset
+
+?
 \end_layout
 
 \end_inset
@@ -3746,20 +3778,23 @@ erratic
 
 \end_layout
 
-\begin_deeper
-\begin_layout Standard
-These tests are labeled as 
-\family typewriter
-'erratic'.
+\begin_layout Description
+erratic Tests depending on local configuration or the phase of the moon.
+ 
+\end_layout
+
+\begin_layout Description
+varying_versions Test depending on TeX distribution, package versions or
+ OS.
 \end_layout
 
-\end_deeper
 \begin_layout Description
 wrong
 \begin_inset space ~
 \end_inset
 
-output Export does not fail but the resulting document has errors.
+output Export does not fail but the resulting document has (undetected)
+ errors.
 \end_layout
 
 \begin_deeper
@@ -3770,11 +3805,7 @@ status open
 
 \begin_layout Plain Layout
 \paragraph_spacing single
-These tests are actually not 
-\emph on
-unreliable
-\emph default
- but 
+These tests are in a strict sense not unreliable but 
 \emph on
 invalid
 \emph default
@@ -3831,8 +3862,8 @@ Output Stop if tests not selected here
 
 \end_deeper
 \begin_layout Description
-unreliableTests: Tests selected either pass or fail, but that is dependent
- on the system where the test is run.
+unreliableTests: Tests selected pass or fail dependent on the system where
+ the test is run.
  Selected tests gain the label 'unreliable'.
 \end_layout
 
@@ -3849,7 +3880,7 @@ Output Stop if test selected, gain label 'unreliable'.
 
 \end_deeper
 \begin_layout Description
-suspiciousTests 
+invertedTests 
 \begin_inset space \space{}
 \end_inset
 
@@ -3864,12 +3895,13 @@ Input Each test which passed 'unreliableTests'
 
 \begin_layout Labeling
 \labelwidthstring 00.00.0000
-Output Stop if not selected.
+Output Stop if not selected, gain test-property 'WILL_FAIL' (i.e.
+ tests are reported as failing if the export works without error.) If no
+ subselection applies, gain labels 'export' and 'inverted'.
 \end_layout
 
 \begin_layout Standard
-The following file is meant as subselections of 'suspiciousTests'.
- If neither subselection applies, test gains labels 'export' and 'inverted'
+The following filter perfoms a subselection of 'invertedTests':
 \end_layout
 
 \begin_layout Description
@@ -3881,7 +3913,7 @@ suspendedTests Tests selected here gain the label 'suspended' but _not_
 \begin_deeper
 \begin_layout Labeling
 \labelwidthstring 00.00.0000
-Input Each test selected by 'suspiciousTests' 
+Input Each test selected by 'invertedTests' 
 \end_layout
 
 \begin_layout Labeling
@@ -3898,7 +3930,7 @@ The following table may clarify label assignement
 
 \begin_layout Standard
 \begin_inset Tabular
-<lyxtabular version="3" rows="7" columns="12">
+<lyxtabular version="3" rows="7" columns="9">
 <features tabularvalignment="middle">
 <column alignment="left" valignment="top" width="0pt">
 <column alignment="left" valignment="top" width="0pt">
@@ -3909,15 +3941,12 @@ The following table may clarify label assignement
 <column alignment="center" valignment="top">
 <column alignment="center" valignment="top">
 <column alignment="center" valignment="top">
-<column alignment="center" valignment="top">
-<column alignment="center" valignment="top">
-<column alignment="center" valignment="top">
 <row>
 <cell multicolumn="1" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Test found in file:
+Test matching pattern in file:
 \end_layout
 
 \end_inset
@@ -3949,16 +3978,16 @@ Test found in file:
 
 \end_inset
 </cell>
-<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Marked in ctest, Assigned label
+Assigned label
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" usebox="none">
+<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -3976,7 +4005,7 @@ Marked in ctest, Assigned label
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell multicolumn="2" alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -3985,90 +4014,90 @@ Marked in ctest, Assigned label
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+test property
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+</row>
+<row>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+ignoredTests
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+unreliableTests
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+invertedTests
 \end_layout
 
 \end_inset
 </cell>
-</row>
-<row>
 <cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Ignored
+suspendedTests
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Unreliable
+export
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Suspicious
+unreliable
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Suspended
+inverted
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-export
+suspended
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4077,34 +4106,54 @@ export
 
 \end_inset
 </cell>
-<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+</row>
+<row>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-unreliable
+Yes
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
+-
+\end_layout
 
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+-
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-inverted
+-
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+-
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4113,16 +4162,25 @@ inverted
 
 \end_inset
 </cell>
-<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-suspended
+-
 \end_layout
 
 \end_inset
 </cell>
-<cell multicolumn="2" alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+-
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4133,11 +4191,11 @@ suspended
 </cell>
 </row>
 <row>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multirow="3" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Yes
+No
 \end_layout
 
 \end_inset
@@ -4146,7 +4204,7 @@ Yes
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+Yes
 \end_layout
 
 \end_inset
@@ -4181,13 +4239,22 @@ Yes
 <cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
+\begin_layout Plain Layout
++
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" usebox="none">
+\begin_inset Text
+
 \begin_layout Plain Layout
 -
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4196,11 +4263,49 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell multirow="4" alignment="left" valignment="middle" topline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+</cell>
+<cell multirow="3" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
+No
+\end_layout
+
+\end_inset
+</cell>
+<cell multirow="3" alignment="left" valignment="middle" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
 
+\begin_layout Plain Layout
+Yes
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+Yes
 \end_layout
 
 \end_inset
@@ -4223,7 +4328,7 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4236,36 +4341,45 @@ Yes
 \begin_inset Text
 
 \begin_layout Plain Layout
--
++
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+WILL_FAIL
 \end_layout
 
 \end_inset
 </cell>
 </row>
 <row>
-<cell multirow="3" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multirow="4" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-No
+
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multirow="4" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Yes
+
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
 \end_layout
 
 \end_inset
@@ -4274,7 +4388,7 @@ Yes
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+No
 \end_layout
 
 \end_inset
@@ -4282,6 +4396,15 @@ Yes
 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
+\begin_layout Plain Layout
++
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" usebox="none">
+\begin_inset Text
+
 \begin_layout Plain Layout
 -
 \end_layout
@@ -4291,13 +4414,60 @@ Yes
 <cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
+\begin_layout Plain Layout
++
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+\begin_inset Text
+
 \begin_layout Plain Layout
 -
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+WILL_FAIL
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+</cell>
+<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+No
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4306,7 +4476,7 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4315,7 +4485,7 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4324,7 +4494,7 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4333,7 +4503,7 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4342,103 +4512,139 @@ Yes
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
 \end_layout
 
 \end_inset
 </cell>
 </row>
+</lyxtabular>
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+\begin_inset Note Note
+status open
+
+\begin_layout Plain Layout
+Without the 
+\begin_inset Quotes eld
+\end_inset
+
+suspendedTests
+\begin_inset Quotes erd
+\end_inset
+
+ filter, this would be far less complicated:
+\end_layout
+
+\begin_layout Plain Layout
+\begin_inset Tabular
+<lyxtabular version="3" rows="6" columns="7">
+<features tabularvalignment="middle">
+<column alignment="left" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="0pt">
+<column alignment="left" valignment="top" width="0pt">
+<column alignment="center" valignment="top">
+<column alignment="center" valignment="top">
+<column alignment="center" valignment="top">
+<column alignment="center" valignment="top">
 <row>
-<cell multirow="4" alignment="left" valignment="middle" topline="true" leftline="true" usebox="none">
+<cell multicolumn="1" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+Test matching pattern in file:
 \end_layout
 
 \end_inset
 </cell>
-<cell multirow="3" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multicolumn="2" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-No
+
 \end_layout
 
 \end_inset
 </cell>
-<cell multirow="3" alignment="left" valignment="middle" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multicolumn="2" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Yes
+
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multicolumn="1" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-Yes
+Label
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+<cell multicolumn="2" alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+test property
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+</row>
+<row>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+ignoredTests
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-+
+unreliableTests
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+invertedTests
 \end_layout
 
 \end_inset
@@ -4447,32 +4653,30 @@ Yes
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+export
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-+
+unreliable
 \end_layout
 
 \end_inset
 </cell>
-</row>
-<row>
-<cell multirow="4" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+inverted
 \end_layout
 
 \end_inset
 </cell>
-<cell multirow="4" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4481,11 +4685,22 @@ Yes
 
 \end_inset
 </cell>
-<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+</row>
+<row>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
+Yes
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
 
+\begin_layout Plain Layout
+-
 \end_layout
 
 \end_inset
@@ -4494,7 +4709,7 @@ Yes
 \begin_inset Text
 
 \begin_layout Plain Layout
-No
+-
 \end_layout
 
 \end_inset
@@ -4512,12 +4727,12 @@ No
 \begin_inset Text
 
 \begin_layout Plain Layout
-+
+
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4526,7 +4741,36 @@ No
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+</cell>
+</row>
+<row>
+<cell multirow="3" alignment="left" valignment="middle" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+No
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+Yes
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4539,7 +4783,7 @@ No
 \begin_inset Text
 
 \begin_layout Plain Layout
-+
+-
 \end_layout
 
 \end_inset
@@ -4553,7 +4797,7 @@ No
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4566,14 +4810,14 @@ No
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
 \end_layout
 
 \end_inset
 </cell>
 </row>
 <row>
-<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multirow="4" alignment="left" valignment="middle" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4582,34 +4826,34 @@ No
 
 \end_inset
 </cell>
-<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell multirow="3" alignment="left" valignment="middle" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-
+No
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
-No
+Yes
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
++
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4618,7 +4862,7 @@ No
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" bottomline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4627,20 +4871,40 @@ No
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+WILL_FAIL
 \end_layout
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" bottomline="true" usebox="none">
+</row>
+<row>
+<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
+\end_layout
+
+\end_inset
+</cell>
+<cell multirow="4" alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+
+\end_layout
+
+\end_inset
+</cell>
+<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+\begin_inset Text
+
+\begin_layout Plain Layout
+No
 \end_layout
 
 \end_inset
@@ -4649,7 +4913,7 @@ No
 \begin_inset Text
 
 \begin_layout Plain Layout
--
++
 \end_layout
 
 \end_inset
@@ -4663,7 +4927,7 @@ No
 
 \end_inset
 </cell>
-<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
+<cell alignment="center" valignment="top" topline="true" bottomline="true" rightline="true" usebox="none">
 \begin_inset Text
 
 \begin_layout Plain Layout
@@ -4676,7 +4940,7 @@ No
 \begin_inset Text
 
 \begin_layout Plain Layout
--
+
 \end_layout
 
 \end_inset
@@ -4687,6 +4951,11 @@ No
 \end_inset
 
 
+\end_layout
+
+\end_inset
+
+
 \end_layout
 
 \begin_layout Subsubsection