Cmake export tests: Renamed nonstandardTests to unreliableTests

and introduce sublabels

Sublabels section in *.Tests starts with 'Sublabel: name'
and is valid until start of a new sublabel.
'name' contains only ascii characters [a-z]+

diff --git a/development/autotests/ExportTests.cmake b/development/autotests/ExportTests.cmake
index bb05af9273183b23c399417e769250069c4665e6..44d410dd5f8d326dfe227ef681a49ccb13597910 100644 (file)
--- a/development/autotests/ExportTests.cmake
+++ b/development/autotests/ExportTests.cmake
@@ -63,41 +63,60 @@ macro(getoutputformats filepath varname)
   set(${varname} ${out_formats})
 endmacro()
 
-macro(findexpr found testname listname)
+macro(findexpr found testname listname rsublabel)
   set(_found 0)
+  set(sublabel "")
   foreach(_itrx ${${listname}})
-    set(_itr "^${_itrx}$")
-    if(${testname} MATCHES ${_itr})
-      set(_found 1)
-      break()
+    if ("${_itrx}" MATCHES "^Sublabel:[ \t]*\([a-z]+\)[ \t]*$")
+      set(sublabel "${CMAKE_MATCH_1}")
+    else()
+      set(_itr "^${_itrx}$")
+      if (${testname} MATCHES "${_itr}")
+        set(_found 1)
+        break()
+      endif()
     endif()
   endforeach()
+  if (${_found})
+    if (NOT "${sublabel}" STREQUAL "")
+      set(${rsublabel} ${sublabel})
+    endif()
+  endif()
   set(${found} ${_found})
 endmacro()
 
-macro(maketestname testname reverted listreverted listignored listnonstandard)
+macro(maketestname testname reverted listreverted listignored listunreliable listlabels)
+  set(sublabel "")
   string(REGEX MATCH "\\/[a-z][a-z](_[A-Z][A-Z])?\\/" _v ${${testname}})
   if(_v)
     string(REGEX REPLACE "\\/" "" _v ${_v})
     set(listrevertedx ${listreverted}_${_v})
     set(listignoredx ${listignored}_${_v})
-    set(listnonstandardx ${listnonstandard}_${_v})
+    set(listunreliablex ${listunreliable}_${_v})
   else()
     set(listrevertedx ${listreverted})
     set(listignoredx ${listignored})
-    set(listnonstandardx ${listnonstandard})
+    set(listunreliablex ${listunreliable})
   endif()
-  findexpr(mfound ${testname} ${listignoredx})
+  findexpr(mfound ${testname} ${listignoredx} sublabel)
   if (NOT mfound)
-    # check if they are non-standard
-    findexpr(nsfound ${testname} ${listnonstandardx})
+    # check if they are unreliable
+    set(sublabel "")
+    findexpr(nsfound ${testname} ${listunreliablex} sublabel)
     if (nsfound)
-      set(${testname} "NON_STANDARD.${${testname}}")
+      if (NOT "${sublabel}" STREQUAL "")
+        list(APPEND ${listlabels} ${sublabel})
+        string(TOUPPER "${sublabel}_" tmpprefix)
+      else()
+        set(tmpprefix "")
+      endif()
+      set(${testname} "UNRELIABLE.${tmpprefix}${${testname}}")
     else ()
       string(REGEX MATCH "_(systemF|texF|pdf3|pdf2|pdf|dvi|lyx16|xhtml)$" _v ${${testname}})
+      set(sublabel "")
       # check if test _may_ be in listreverted
       if (_v)
-        findexpr(mfound ${testname} ${listrevertedx})
+        findexpr(mfound ${testname} ${listrevertedx} sublabel)
       else()
         set(mfound OFF)
       endif()
@@ -106,6 +125,9 @@ macro(maketestname testname reverted listreverted listignored listnonstandard)
       else()
         set(${reverted} 1)
         set(${testname} "INVERTED_SEE-README.ctest_${${testname}}")
+        if (NOT sublabel STREQUAL "")
+          list(APPEND ${listlabels} ${sublabel})
+        endif()
       endif()
     endif()
   else()
@@ -115,18 +137,26 @@ endmacro()
 
 macro(loadTestList filename resList)
   # Create list of strings from a file without comments
-  file(STRINGS ${filename} tempList)
+  file(STRINGS ${filename} tempList ENCODING "UTF-8")
   set(${resList})
+  set(sublabel)
   foreach(_l ${tempList})
-    string(REGEX REPLACE "[ \t]*#.*" "" _newl "${_l}")
+    set(_newl "${_l}")
+    string(REGEX REPLACE "[ \t]*#.*$" "" _newl "${_l}")
     if(_newl)
-      #message(STATUS "Testing ${_newl}")
       string(REGEX REPLACE "(\\/|\\||\\(|\\))" "  " _vxx ${_newl})
       string(REGEX MATCHALL " ([a-z][a-z](_[A-Z][A-Z])?) " _vx ${_vxx})
+      if (_newl MATCHES "^Sublabel:")
+        set(sublabel ${_newl})
+      endif()
       if(_vx)
         foreach(_v ${_vx})
           string(REGEX REPLACE " " "" _v ${_v})
           #message(STATUS " ==> ${resList}_${_v}")
+          if (NOT ${sublabel} STREQUAL "${sublabel}_${_v}")
+            list(APPEND ${resList}_${_v} "${sublabel}")
+            set(${sublabel}_${_v} "${sublabel}")
+          endif()
           list(APPEND ${resList}_${_v} "${_newl}")
         endforeach()
         if(_newl MATCHES "\\(\\|")
@@ -144,21 +174,26 @@ endmacro()
 loadTestList(revertedTests revertedTests)
 loadTestList(ignoredTests ignoredTests)
 loadTestList(suspendedTests suspendedTests)
-loadTestList(nonstandardTests nonstandardTests)
+loadTestList(unreliableTests unreliableTests)
 
 macro(handlesuspended TestName reverted testlabel)
   set(mylabel ${${testlabel}})
   set(myreverted ${reverted})
+  set(sublabel)
 
-  # check for nonstandard
-  if ("${TestName}" MATCHES "^NON_STANDARD")
-    set(mylabel "nonstandard")
+  # check for unreliable
+  if ("${TestName}" MATCHES "^UNRELIABLE")
+    list(REMOVE_ITEM mylabel "export" "reverted" "templates" "mathmacros" "manuals")
+    list(APPEND mylabel "unreliable")
     set(myreverted 0)
   elseif (${reverted})
     # check suspension only for reverted tests
-    findexpr(tfound TestName suspendedTests)
+    findexpr(tfound TestName suspendedTests sublabel)
     if (tfound)
       set(mylabel "suspended")
+      if (sublabel)
+        list(APPEND mylabel ${sublabel})
+      endif()
       set(myreverted 0) # if test is to be suspended, remove the 'reverted' flag
     endif()
   endif()
@@ -200,7 +235,8 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
     # Strip extension
     string(REGEX REPLACE "\\.lyx$" "" f ${f})
     set(TestName "export/${libsubfolder}/${f}_lyx16")
-    maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
+    set(mytestlabel ${testlabel})
+    maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
     if(TestName)
       add_test(NAME ${TestName}
         WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@@ -216,13 +252,14 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
         -DTOP_SRC_DIR=${TOP_SRC_DIR}
         -DPERL_EXECUTABLE=${PERL_EXECUTABLE}
         -P "${TOP_SRC_DIR}/development/autotests/export.cmake")
-      handlesuspended(${TestName} ${reverted} testlabel) # checking for suspended lyx16 exports
+      handlesuspended(${TestName} ${reverted} mytestlabel) # checking for suspended lyx16 exports
     endif()
     if(LYX_PYTHON_EXECUTABLE)
       set(lyx2lyxtestlabel "lyx2lyx")
       # For use of lyx2lyx we need the python executable
+      set(mytestlabel ${lyx2lyxtestlabel})
       set(TestName "lyx2lyx/${libsubfolder}/${f}")
-      maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
+      maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
       if(TestName)
         add_test(NAME ${TestName}
           WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@@ -232,13 +269,14 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
           "-DLYX_TESTS_USERDIR=${LYX_TESTS_USERDIR}"
           "-DLYXFILE=${LIBSUB_SRC_DIR}/${f}.lyx"
           -P "${TOP_SRC_DIR}/development/autotests/lyx2lyxtest.cmake")
-        handlesuspended(${TestName} ${reverted} lyx2lyxtestlabel)
+        handlesuspended(${TestName} ${reverted} mytestlabel)
       endif()
     endif()
+    set(loadtestlabel "load")
+    set(mytestlabel ${loadtestlabel})
     set(TestName "check_load/${libsubfolder}/${f}")
-    maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
+    maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
     if(TestName)
-      set(loadtestlabel "load")
       add_test(NAME ${TestName}
         WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
         COMMAND ${CMAKE_COMMAND} -DLYXFILE=${LIBSUB_SRC_DIR}/${f}.lyx
@@ -247,7 +285,7 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
         -DPARAMS_DIR=${TOP_SRC_DIR}/development/autotests
         -DWORKDIR=${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}
         -P "${TOP_SRC_DIR}/development/autotests/check_load.cmake")
-      handlesuspended(${TestName} ${reverted} loadtestlabel)
+      handlesuspended(${TestName} ${reverted} mytestlabel)
       set_tests_properties(${TestName} PROPERTIES RUN_SERIAL ON)
     endif()
     getoutputformats("${LIBSUB_SRC_DIR}/${f}.lyx" formatlist)
@@ -263,7 +301,8 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
         else()
           set(TestName "export/${libsubfolder}/${f}_${format}_${fonttype}")
         endif()
-        maketestname(TestName reverted revertedTests ignoredTests nonstandardTests)
+        set(mytestlabel ${testlabel})
+        maketestname(TestName reverted revertedTests ignoredTests unreliableTests mytestlabel)
         if(TestName)
           add_test(NAME ${TestName}
             WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${LYX_HOME}"
@@ -279,7 +318,7 @@ foreach(libsubfolderx lib/doc lib/examples lib/templates development/mathmacros
             -DTOP_SRC_DIR=${TOP_SRC_DIR}
             -DPERL_EXECUTABLE=${PERL_EXECUTABLE}
             -P "${TOP_SRC_DIR}/development/autotests/export.cmake")
-          handlesuspended(${TestName} ${reverted} testlabel) # check for suspended pdf/dvi exports
+          handlesuspended(${TestName} ${reverted} mytestlabel) # check for suspended pdf/dvi exports
         endif()
       endforeach()
     endforeach()

diff --git a/development/autotests/unreliableTests b/development/autotests/unreliableTests
new file mode 100644 (file)
index 0000000..b295b67
--- /dev/null
+++ b/development/autotests/unreliableTests
@@ -0,0 +1,18 @@
+# Tests using some non-standard tex class
+Sublabel: nonstandard
+export/templates/IUCr-article_(dvi|pdf).*
+export/templates/es_beamer-conference-ornate-20min_pdf4_texF
+export/templates/kluwer_pdf[45]_systemF
+export/examples/modernCV_pdf4_(tex|system)F
+export/templates/ectaart_(dvi3|pdf5)_(tex|system)F
+export/examples/fa/splash_(dvi|pdf|pdf[23]|(dvi3|pdf4|pdf5)_(texF|systemF))
+#
+#
+# 1.) missing farsi package with lfeenc.def
+# 2.) LuaTeX does not support Farsi yet. See:
+#     https://github.com/reutenauer/polyglossia/commit/ccb0e9e2c6411170ad779b05ff5076f1193cc323
+# Tests depending on local configuration, OS, TeX distribution,
+# package versions, or the phase of the moon.
+Sublabel: erratic
+export/examples/(|fr/)seminar_(dvi|pdf|pdf[23]|pdf4_texF|pdf5_systemF)
+#