X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=lib%2Fdoc%2FDevelopment.lyx;h=eb2917a6362ff6fa7ba0e8473c95b60d9a1564a5;hb=693ca643fc1e6e9d797d8a074285136987b82add;hp=aeff56185d5c173b18d222fd32af4fa2be62f7d3;hpb=88b3fa32256062775d7e165a6f8c858290a5bb00;p=lyx.git diff --git a/lib/doc/Development.lyx b/lib/doc/Development.lyx index aeff56185d..eb2917a636 100644 --- a/lib/doc/Development.lyx +++ b/lib/doc/Development.lyx @@ -1,7 +1,8 @@ #LyX 2.2 created this file. For more info see http://www.lyx.org/ -\lyxformat 503 +\lyxformat 504 \begin_document \begin_header +\save_transient_properties true \origin /systemlyxdir/doc/ \textclass scrartcl \options BCOR8mm,captions=tableheading @@ -26,7 +27,7 @@ logicalmkup \font_tt_scale 100 100 \graphics default \default_output_format pdf2 -\output_sync 0 +\output_sync 1 \bibtex_command default \index_command default \paperfontsize 12 @@ -81,9 +82,9 @@ logicalmkup \papersides 2 \paperpagestyle headings \tracking_changes false +\output_changes false \html_math_output 0 \html_css_as_file 0 -\output_changes false \html_be_strict true \end_header @@ -331,6 +332,12 @@ Implement and test the new feature, including the reading and writing of \end_layout \begin_layout Enumerate +\begin_inset CommandInset label +LatexCommand label +name "enu:Describe_format" + +\end_inset + Describe the new format in \begin_inset Flex Code status collapsed @@ -384,6 +391,12 @@ lib/lyx2lyx/LyX.py \end_layout \begin_layout Enumerate +\begin_inset CommandInset label +LatexCommand label +name "enu:Add-an-entry" + +\end_inset + Add an entry to both format lists (for conversion and reversion) in \begin_inset Newline newline \end_inset @@ -551,49 +564,39 @@ python development/tools/updatedocs.py \end_inset . - Then, revert the change to + Look at the resulting changes using the command \begin_inset Flex Code status collapsed \begin_layout Plain Layout -LFUNs.lyx +git diff \end_layout \end_inset - because that file is meant to be generated separately: +. + If anything looks surprising, please investigate. + Keep in mind that the case of \begin_inset Flex Code status collapsed \begin_layout Plain Layout -git checkout lib/doc/LFUNs.lyx -\end_layout - -\end_inset - - -\begin_inset Note Note -status collapsed - -\begin_layout Plain Layout -TODO: this step should be done within updatedocs.py +LFUNs.lyx \end_layout \end_inset -. - Look at the resulting changes using the command + is special, because it is first generated with \begin_inset Flex Code status collapsed \begin_layout Plain Layout -git diff +gen_lfuns.py \end_layout \end_inset -. - If anything looks surprising, please investigate. + before being converted to the latest format. Finally, commit using \begin_inset Flex Code status collapsed @@ -683,6 +686,96 @@ Updating the file format number of bind/ui files \end_layout \begin_layout Standard +A change to the functionality of existing LFUNs can require a conversion + of +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.bind +\end_layout + +\end_inset + + and +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.ui +\end_layout + +\end_inset + + files, and therefore an increment of the LFUN format, as well as a conversion + of Info insets in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files for manuals. + The latter cannot be done automatically and requires also a \SpecialChar LyX + format increase + (think of e.g. + someone who might have made a set of \SpecialChar LyX + teaching manuals for use in their + own group) +\begin_inset Foot +status open + +\begin_layout Plain Layout +\begin_inset Flex URL +status open + +\begin_layout Plain Layout + +http://www.lyx.org/trac/ticket/9794 +\end_layout + +\end_inset + + +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Enumerate +Increment the LFUN file format number in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +src/LyXAction.h +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Enumerate +Implement the LFUN conversion in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +lib/scripts/prefs2prefs_lfuns.py +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Enumerate See step \begin_inset CommandInset ref LatexCommand ref @@ -720,6 +813,82 @@ bash development/tools/updatelfuns.sh . \end_layout +\begin_layout Enumerate +Update Info insets in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files. + To do so, increment the \SpecialChar LyX + format and proceed as in +\begin_inset CommandInset ref +LatexCommand ref +reference "subsec:update_lyx_files" + +\end_inset + +, steps +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:Describe_format" + +\end_inset + +- +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:updatefiles" + +\end_inset + +. + In the lyx2lyx implementation ( +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:Add-an-entry" + +\end_inset + +th step), implement a conversion similar to the one in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +prefs2prefs_lfuns.py +\end_layout + +\end_inset + + above, as well as a corresponding reversion; for this one can use +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +convert_info_insets +\end_layout + +\end_inset + + from +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +lib/lyx2lyx/lyx2lyx_tools.py +\end_layout + +\end_inset + +. + +\end_layout + \begin_layout Subsection Backporting new styles to the stable version \end_layout @@ -1237,7 +1406,7 @@ Commit the changes to the repository, or send a patch to the development \end_layout \begin_layout Subsection -ctest automatic tests (cmake only) +ctest automatic tests \end_layout \begin_layout Standard @@ -1246,7 +1415,7 @@ Some tests are located in the status collapsed \begin_layout Plain Layout -development/autotests +development/autotests/ \end_layout \end_inset @@ -1254,24 +1423,28 @@ development/autotests subfolder of the \SpecialChar LyX source code distribution. -\begin_inset Flex Code -status collapsed +\begin_inset Foot +status open \begin_layout Plain Layout -cmake +The README document in this folder only describes the +\begin_inset Quotes eld +\end_inset + +keytests +\begin_inset Quotes erd +\end_inset + + subset of autotests! \end_layout \end_inset - is required to run the automatic \SpecialChar LyX - tests, running them is not implemented - for autotools. \end_layout \begin_layout Standard -The \SpecialChar LyX - tests can be run by the commands +These tests can be run by the commands \begin_inset Flex Code status collapsed @@ -1281,7 +1454,12 @@ ctest \end_inset - (all platforms) or + in the +\emph on + build directory +\emph default + (all platforms) or (when using a make based build system and not MSVC) + \begin_inset Flex Code status collapsed @@ -1291,17 +1469,37 @@ make test \end_inset - (when using a make based build system and not MSVC) in the + in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +autotests/ +\end_layout + +\end_inset + + subfolder of the +\emph on + build directory +\emph default +. + The test logs are written to the \begin_inset Flex Code status collapsed \begin_layout Plain Layout -autotests +Testing/Temporary/ \end_layout \end_inset - subfolder of the build directory. + subfolder of the +\emph on + +\emph default +build directory. + \end_layout \begin_layout Subsubsection @@ -1324,9 +1522,8 @@ reuse \begin_inset Quotes erd \end_inset - documentation, template, and example files trying to export them to all - supported output formats. - In addition, there are a number of dedicated sample documents under + documentation, template, and example documents. + In addition, there are a number of dedicated sample documents in the \begin_inset Flex Code status collapsed @@ -1336,7 +1533,22 @@ autotests/export/ \end_inset -. + subfolder of the \SpecialChar LyX + source code distribution. + All samples are (after copying and eventual processing by scripts) exported + to various output formats via the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +—export-to +\end_layout + +\end_inset + + command line option. + The tests checks for errors reported by LyX. + (However, error-free export is no guarantee for an error-free output document.) \end_layout \begin_layout Paragraph @@ -1357,6 +1569,67 @@ When making a non-trivial change to a .layout file, run the export and layout \begin_layout Itemize When making non-trivial changes to a .lyx file, run the export tests correspondin g to that .lyx file. + +\begin_inset Foot +status collapsed + +\begin_layout Plain Layout +This rule is due to revision. + +\end_layout + +\begin_layout Plain Layout +There is an objection from the documentation maintainer that working on + the documentation must not be complicated by having to consider non-standard + exports. +\end_layout + +\begin_layout Itemize +successful compiling/testing an edited documentation file with pdflatex + suffices to ensure it can be commited, not tests with other exports are + required. +\end_layout + +\begin_layout Plain Layout +If sudden failures with other exports due to “half-tested” documentation + updates are a problem for the test maintainer, the test suite should use + copies that are +\end_layout + +\begin_layout Itemize +copied to a cache dir (autotests/samples/doc/, say) but not changed, +\end_layout + +\begin_layout Itemize +updated regularely (but on a time chosen by the test suite maintainer) from + the originals in lib/doc/ +\end_layout + +\begin_layout Plain Layout +This way, +\end_layout + +\begin_layout Itemize +no test will fail due to ongoing work on documentation, +\end_layout + +\begin_layout Itemize +the documentation is still tested in full (with some delay), +\end_layout + +\begin_layout Itemize +failures with non-default export can be examined and handled accordingly + in one run with the cache update, +\end_layout + +\begin_layout Itemize +“interesting failures” (like the nested-language+polyglossia problem in + es/Customization can be separated and moved into dedicated test samples. +\end_layout + +\end_inset + + \end_layout \begin_layout Itemize @@ -1412,18 +1685,231 @@ reference "subsec:Interpreting-export-tests" \begin_layout Paragraph \begin_inset CommandInset label LatexCommand label -name "par:Configuring-ctests" +name "par:export-test-output-formats" \end_inset -Configuring the tests +Output formats \end_layout \begin_layout Standard -To enable the export autotests, add the -\begin_inset Flex Code -status collapsed - +The following output formats are currently tested for each sample document + (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Export-test-filtering" + +\end_inset + + for exceptions): +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +lyx16 LyX 1.6 file format (lyx2lyx) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +lyx21 LyX 2.1 file format (lyx2lyx) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +xhtml LyXHTML (native LyX HTML export) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +LaTeX: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi DVI (8-bit latex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi3_texF DVI (LuaTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi3_systemF DVI (LuaTeX with Unicode fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf2 PDF (pdflatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf4_texF PDF (XeTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf4_systemF PDF (XeTeX with Unicode fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf5_texF PDF (LuaTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf5_systemF PDF (LuaTeX with Unicode fonts) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +LaTeX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +postprocessing: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf DVI -> PS (dvips) -> PDF (ps2pdf) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf3 DVI -> PDF (dvipdfm) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +not +\begin_inset space ~ +\end_inset + +tested: (or only if set as default output format in the document source) +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +latex LaTeX (plain) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +luatex LaTeX (LuaTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dviluatex LaTeX (dviluatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdflatex LaTeX (pdflatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +platex LaTeX (pLaTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +xetex LaTeX (XeTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +eps3 EPS (encapsulated Postscript) (cropped) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +ps DVI -> Postscript (dvips) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +odf +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +text (nor text2, ..., text4) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +textparagraph +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +word +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +word2 +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +wordhtml +\end_layout + +\end_deeper +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Configuring-ctests" + +\end_inset + +Configuring the tests +\end_layout + +\begin_layout Standard +To enable the export autotests, add the +\begin_inset Flex Code +status collapsed + \begin_layout Plain Layout -DLYX_ENABLE_EXPORT_TESTS=ON \end_layout @@ -1461,6 +1947,12 @@ reference "subsec:Interpreting-export-tests" \end_layout \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:ctest-options" + +\end_inset + Running the tests \end_layout @@ -1475,12 +1967,23 @@ ctest \end_inset +. + A full, up-to-date TeXLive installation is recommended to run the tests. + Otherwise, some tests will fail. + Tests with additional requirements are labeled +\begin_inset Quotes eld +\end_inset + +unreliable:nonstandard +\begin_inset Quotes erd +\end_inset + . \end_layout \begin_layout Standard -To run only some of the tests, use command line options: +To run only some of the tests, use command line options (see examples below): \end_layout \begin_layout Labeling @@ -1513,48 +2016,43 @@ status collapsed \end_layout -\begin_deeper -\begin_layout Standard -For example, to run only the export tests, you can use +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest -L export +-E \end_layout \end_inset -. + Exclude the tests whose names match the given regular expression. \end_layout -\end_deeper \begin_layout Labeling \labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout --E +-LE \end_layout \end_inset - Exclude the tests whose names match the given regular expression. -\end_layout - -\begin_layout Labeling -\labelwidthstring -R + Exclude the tests whose labels match the given regular expression. + Cannot be combined with \begin_inset Flex Code status collapsed \begin_layout Plain Layout --LE +-L \end_layout \end_inset - Exclude the tests whose labels match the given regular expression. +. \end_layout \begin_layout Standard @@ -1612,7 +2110,7 @@ print-labels \end_layout \begin_layout Standard -Other relevant options are: +Other useful options are: \end_layout \begin_layout Labeling @@ -1636,7 +2134,6 @@ We are still working on getting the tests to run in parallel. pass when run sequentially. A reasonable approach is to first run the tests in parallel and then run the failed tests sequentially. - \end_layout \begin_layout Standard @@ -1671,6 +2168,35 @@ rerun-failed \end_layout +\begin_layout Standard +When specifying a subset of the tests (e.g. + using +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\SpecialChar nobreakdash +R +\end_layout + +\end_inset + +), the same subset must be specified when using the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\SpecialChar nobreakdash +\SpecialChar nobreakdash +rerun-failed +\end_layout + +\end_inset + + option because it is the test numbers that are used to index which tests + failed on the previous run. +\end_layout + \begin_layout Standard \noindent Note that some tests cannot be run in parallel. @@ -1751,6 +2277,68 @@ man ctest ) the full list of command line options. \end_layout +\begin_layout Paragraph +Examples +\end_layout + +\begin_layout Itemize +run only the export tests: +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -L export +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +run inverted tests: +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -L "inverted|suspended" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +list all export tests which match any of the labelling patterns: +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -N -R " +\backslash +..*_export/" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +exclude rarely used output formats and post-processing tests +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -L export -E "_(texF|dvi3|pdf3?)" +\end_layout + +\end_inset + + +\end_layout + \begin_layout Paragraph \begin_inset CommandInset label LatexCommand label @@ -1766,8 +2354,7 @@ A test can fail for several reasons, not all of them bad. \end_layout \begin_layout Enumerate -The .lyx file could have been added recently and some formats are not expected - to work well. +A new or edited sample document may be incompatible with some output formats. \end_layout \begin_layout Enumerate @@ -1822,123 +2409,323 @@ Because the .lyx files are exported in several formats, it is not surprising \begin_inset Quotes eld \end_inset -inverting +inverting +\begin_inset Quotes erd +\end_inset + + the tests, that is, by marking the test as +\begin_inset Quotes eld +\end_inset + +passing +\begin_inset Quotes erd +\end_inset + + if the export exits with error and as +\begin_inset Quotes eld +\end_inset + +failing +\begin_inset Quotes erd +\end_inset + + if the export succeeds +\emph on +. + +\emph default + It follows that these expected failures will not show up as failed tests + in the test results and thus will not pollute the +\begin_inset Quotes eld +\end_inset + +good +\begin_inset Quotes erd +\end_inset + + tests. + If the export actually succeeds, then the test will fail. + The purpose of this failure is to get your attention—something has changed, + possibly for the better. +\end_layout + +\begin_layout Standard +We try to document why a test is inverted or ignored. + See the comment (prefixed with +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +# +\end_layout + +\end_inset + +) above the block in which the test is listed as inverted or ignored in + the files +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/suspiciousTests +\end_layout + +\end_inset + +, +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/unreliableTests +\end_layout + +\end_inset + + and +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/ignoredTests +\end_layout + +\end_inset + +. + +\end_layout + +\begin_layout Standard +A good question is why do we enable the tests for non-default formats? The + answer is that if a non-default route is broken it is often because a bug + was introduced in LyX and not because a document-specific change was made + that is not supported by the route. + In other words, there is a high signal/noise ratio in the export tests + for some non-default formats. + +\end_layout + +\begin_layout Standard +When a test or several tests fail, consider checking the files in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +Testing/Temporary/ +\end_layout + +\end_inset + + subdirectory of your build directory. + In this subdirectory are three files: the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +LastTestsFailed.log +\end_layout + +\end_inset + + simply lists the tests that failed on your last +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest +\end_layout + +\end_inset + + command; the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +LastTest.log +\end_layout + +\end_inset + + file contains the output from the tests (and often has details explaining + why a test failed); and the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +CTestCostData.txt +\end_layout + +\end_inset + + file lists the times that it took to run the tests. +\end_layout + +\begin_layout Paragraph +What action should you take if a test fails? +\end_layout + +\begin_layout Standard +\paragraph_spacing single +It is always good to check manually why something fails and if it passes + if the PDF output is good. +\end_layout + +\begin_layout Itemize +Generally, if a change breaks compilation for the target format (for the + manuals pdf2) without solving some important other issue, +\emph on +fix or revert the commit +\emph default + that led to failure. +\end_layout + +\begin_layout Itemize +If it is not possible to (immediately) fix the failure but there are reasons + not to revert the commit (e.g. + it fixes another more important issue), +\emph on +invert +\emph default + the failing test case (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Inverted-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +If an +\emph on +inverted +\emph default + test case fails because the export now works, +\emph on +uninvert +\emph default + the test by removing the labeling pattern from +\begin_inset Quotes eld +\end_inset + +suspiciousTests +\begin_inset Quotes erd +\end_inset + +) (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Inverted-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +If the export did not fail previously but led to wrong output (PDF, say), + it is in fact an improvement when the test now fails, label it as +\begin_inset Quotes eld +\end_inset + +unreliable:wrong:output \begin_inset Quotes erd \end_inset - the tests, that is, by marking the test as -\begin_inset Quotes eld -\end_inset + ( +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Unreliable-tests" -passing -\begin_inset Quotes erd \end_inset - if the export exits with error and as +). +\end_layout + +\begin_layout Itemize +In case of tests failing due to missing requirements (when only a subset + of TeXLive is installed or a test labeled \begin_inset Quotes eld \end_inset -failing +unreliable:nonstandard \begin_inset Quotes erd \end_inset - if the export succeeds -\emph on -. + fails), ignore the failure, ask for someone else to run the test, or install + the missing ressources and try again. +\end_layout -\emph default - It follows that these expected failures will not show up as failed tests - in the test results and thus will not pollute the -\begin_inset Quotes eld -\end_inset +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Inverted-tests" -good -\begin_inset Quotes erd \end_inset - tests. - If the export actually succeeds, then the test will fail. - The purpose of this failure is to get your attention—something has changed, - possibly for the better. +Inverted tests \end_layout \begin_layout Standard -We try to document why a test is inverted or ignored. - See the comment (prefixed with +Test cases whose name matches a pattern in the file \begin_inset Flex Code status collapsed \begin_layout Plain Layout -# +development/autotests/suspiciousTests \end_layout \end_inset -) above the block in which the test is listed as inverted or ignored in - the files + get the label +\emph on +inverted +\emph default +. + They get also the test property \begin_inset Flex Code status collapsed \begin_layout Plain Layout -development/autotests/suspiciousTests +WILL_FAIL \end_layout \end_inset - and -\begin_inset Flex Code +, i.e. + they are reported as failing if the export works without error +\begin_inset Flex URL status collapsed \begin_layout Plain Layout -development/autotests/ignoredTests + +https://cmake.org/cmake/help/v3.0/command/set_tests_properties.html \end_layout \end_inset . - It is possible that an export goes from succeeding to failing just because - the document was changed and the document was never expected to work with - a certain export format in the first case. - Once this is confirmed to be the situation, the test can be inverted. \end_layout \begin_layout Standard -A good question is why do we enable the tests for non-default formats? The - answer is that if a non-default route is broken it is often because a bug - was introduced in LyX and not because a document-specific change was made - that is not supported by the route. - In other words, there is a high signal/noise ratio in the export tests - for some non-default formats. - -\end_layout - -\begin_layout Standard -What action should you take if a test fails? This depends: -\end_layout +Add failing cases to this file, if they cannot be solved +\begin_inset Quotes eld +\end_inset -\begin_layout Standard -Generally, if a change breaks compilation for the target format (for the - manuals pdf2) without solving some important other issue, fix or revert - the commit that led to failure. - If a change breaks compilation for some non-target format (for the manuals - everything except pdf2), invert the test. - -\end_layout +immediately +\begin_inset Quotes erd +\end_inset -\begin_layout Standard -A special case is given, if the export was succeeding before but showing - garbled text in the PDF output. - Try to establish, that when the compilation succeeded before the resulting - PDF was good. - Otherwise, it is in fact an improvement when a test fails. - Now it might fail with a clear message of "language xyz not supported". - It is always good to check manually why something fails and if it passes - if the PDF output is good. + but it is expected that the export will work in a foreseeable future, e.g. + low priority issues like failures to export to a non-target format (for + the manuals everything except pdf2). \end_layout \begin_layout Standard -Sometimes a test is fixed as side-effect of some change. - We should uninvert a test (remove it from the +The following sublabels are currently present in \begin_inset Flex Code status collapsed @@ -1948,87 +2735,100 @@ suspiciousTests \end_inset - file) in order to preserve the fix. +: \end_layout -\begin_layout Standard -When a test or several tests fail, consider checking the files in the -\begin_inset Flex Code -status collapsed +\begin_layout Description +todo test failures that require attention: +\end_layout -\begin_layout Plain Layout -Testing/Temporary/ +\begin_deeper +\begin_layout Itemize +minor issues to explore and properly sort later, \end_layout -\end_inset +\begin_layout Itemize +easyfix issues, +\end_layout - subdirectory of your build directory. - In this subdirectory are three files: the file -\begin_inset Flex Code -status collapsed +\begin_layout Itemize +LyX bugs to report at trac (move pattern to section "lyxbugs" once done). +\end_layout -\begin_layout Plain Layout -LastTestsFailed.log +\end_deeper +\begin_layout Description +lyxbugs LyX bugs with a Trac number. \end_layout -\end_inset +\begin_layout Description +ert Export failures due to "raw" LaTeX use in ERT or preamble code. +\end_layout - simply lists the tests that failed on your last -\begin_inset Flex Code -status collapsed +\begin_deeper +\begin_layout Standard +"Wontfix" if demonstrating correct use and OK in the default output format. +\end_layout -\begin_layout Plain Layout -ctest +\end_deeper +\begin_layout Description +texissues Export fails due to LaTeX limitations like non-ASCII characters + in verbatim or listings, incompatible packages, ... \end_layout -\end_inset +\begin_deeper +\begin_layout Standard +"Wontfix" if documents demonstrate correct use in the default output format: +\end_layout - command; the -\begin_inset Flex Code -status collapsed +\begin_layout Itemize +If the source can be made more robust without becoming "hackish", fix the + source, +\end_layout -\begin_layout Plain Layout -LastTest.log +\begin_layout Itemize +if LyX could be enhanced to care for a permanent TeX limitation, file a + ticket at trac and add a pattern under lyxbugs, \end_layout -\end_inset +\begin_layout Itemize +otherwise, add a pattern here. +\end_layout - file contains the output from the tests (and often has details explaining - why a test failed); and the +\end_deeper +\begin_layout Description +attic Documents in the attic. + (Kept for reference and format conversion test.) +\end_layout + +\begin_layout Subparagraph +suspended +\end_layout + +\begin_layout Standard +Test cases whose name additionally matches a pattern in the file \begin_inset Flex Code status collapsed \begin_layout Plain Layout -CTestCostData.txt +development/autotests/suspendedTests \end_layout \end_inset - file lists the times that it took to run the tests. -\end_layout - -\begin_layout Paragraph -Inverted tests -\end_layout - -\begin_layout Standard -These tests fail if the export does + get the label \emph on -not +suspended \emph default - return an error. - -\end_layout - -\begin_layout Description -inverted Export fails, but the test cases are subject to be examined. - It is expected that the export will work in a foreseeable future. - They are labeled 'inverted'. -\end_layout - -\begin_layout Description -suspended Some inverted tests are labeled 'suspended'. - This means, they are not executed using +(instead of +\emph on +export +\emph default +and +\emph on + inverted +\emph default +). + This means they are not executed using \begin_inset Flex Code status collapsed @@ -2049,6 +2849,18 @@ ctest -L inverted \end_inset . + However, they also get the test property +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +WILL_FAIL +\end_layout + +\end_inset + +, i.e. + they are reported as failing if the export works without error. From time to time they still have to be checked using \begin_inset Flex Code status collapsed @@ -2062,7 +2874,6 @@ ctest -L suspended . \end_layout -\begin_deeper \begin_layout Standard These tests are suspended, because the export fails for known reasons which cannot ATM be resolved. @@ -2083,15 +2894,45 @@ status collapsed \end_inset parameter nothing changes. - Suspended or not, tests will be executed depending only on the regexes - parameters given to the ctest command. + Suspended or not, tests will be executed depending only on the selecting + regular expression given to the ctest command (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:ctest-options" + +\end_inset + +). \end_layout -\end_deeper \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Unreliable-tests" + +\end_inset + Unreliable tests \end_layout +\begin_layout Standard +Test cases whose name matches a pattern in the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/unreliableTests +\end_layout + +\end_inset + + get the label +\emph on +unreliable +\emph default +. +\end_layout + \begin_layout Standard These tests are not executed using \begin_inset Flex Code @@ -2120,11 +2961,24 @@ ctest -L inverted \begin_layout Standard They pass or fail for various reasons not related to LyX (nonstandard, erratic) or pass but should rather fail (wrong output). + +\begin_inset Note Note +status collapsed + +\begin_layout Plain Layout +*invalid* tests (wrong output) are not *unreliable*. + # Use "unfit" or "unapplicable" as better label and name of pattern file? + +\end_layout + +\end_inset + + \end_layout \begin_layout Description nonstandard Documents with additional requirements, e.g. - a class or package file not on CTAN. + a class or package file not in TeXLive. \begin_inset Note Note status open @@ -2158,7 +3012,7 @@ TODO: use \emph on erratic \emph default - only for the phase-of-moon dependency. + only for the phase-of-moon dependency? \end_layout \end_inset @@ -2185,6 +3039,11 @@ output Export does not fail but the resulting document has errors. \begin_deeper \begin_layout Standard \paragraph_spacing single +\begin_inset Note Note +status open + +\begin_layout Plain Layout +\paragraph_spacing single These tests are actually not \emph on unreliable @@ -2196,8 +3055,19 @@ invalid (not measuring what they should measure). \end_layout +\end_inset + + +\end_layout + \end_deeper \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Export-test-filtering" + +\end_inset + Export test filtering \end_layout @@ -3131,6 +4001,27 @@ development/autotests/filterCheckWarnings Under cmake, the tests are labeled as 'load'. \end_layout +\begin_layout Subsubsection +Keytests +\end_layout + +\begin_layout Standard +Automated tests based on the "MonKey Testing" keytest program. + They are documented in the README document in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests +\end_layout + +\end_inset + + subfolder of the \SpecialChar LyX + source code distribution. + T +\end_layout + \begin_layout Subsubsection lyx21 tests \end_layout @@ -3222,40 +4113,6 @@ Associated test results can be examined in ctest-log directory in files of the form 'LastFailed.*URLS.log' \end_layout -\begin_layout Subsubsection -Test labels -\end_layout - -\begin_layout Standard -ctest label commands: -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -\SpecialChar nobreakdash -print-labels shows all assigned labels -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -L -\begin_inset space ~ -\end_inset - -labelname executes all tests to which this label is assigned to. - A test may have more that one label. -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -j -\begin_inset space ~ -\end_inset - -number executes tests in parallel using 'number' simultaneously processes. - Some tests are marked as 'sequencial', for them this parameter has no effect. -\end_layout - \begin_layout Section Development policies \end_layout