X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=lib%2Fdoc%2FDevelopment.lyx;h=eb2917a6362ff6fa7ba0e8473c95b60d9a1564a5;hb=693ca643fc1e6e9d797d8a074285136987b82add;hp=0b0f941494884d44c0025c86c46899cdcaf47f51;hpb=b5c693eb0f99a7e5dec4f0da909803aa355c64f7;p=lyx.git diff --git a/lib/doc/Development.lyx b/lib/doc/Development.lyx index 0b0f941494..eb2917a636 100644 --- a/lib/doc/Development.lyx +++ b/lib/doc/Development.lyx @@ -1,8 +1,9 @@ #LyX 2.2 created this file. For more info see http://www.lyx.org/ -\lyxformat 503 +\lyxformat 504 \begin_document \begin_header -\origin /systemlyxdir/doc +\save_transient_properties true +\origin /systemlyxdir/doc/ \textclass scrartcl \options BCOR8mm,captions=tableheading \use_default_options false @@ -26,7 +27,7 @@ logicalmkup \font_tt_scale 100 100 \graphics default \default_output_format pdf2 -\output_sync 0 +\output_sync 1 \bibtex_command default \index_command default \paperfontsize 12 @@ -81,9 +82,9 @@ logicalmkup \papersides 2 \paperpagestyle headings \tracking_changes false +\output_changes false \html_math_output 0 \html_css_as_file 0 -\output_changes false \html_be_strict true \end_header @@ -331,6 +332,12 @@ Implement and test the new feature, including the reading and writing of \end_layout \begin_layout Enumerate +\begin_inset CommandInset label +LatexCommand label +name "enu:Describe_format" + +\end_inset + Describe the new format in \begin_inset Flex Code status collapsed @@ -384,6 +391,12 @@ lib/lyx2lyx/LyX.py \end_layout \begin_layout Enumerate +\begin_inset CommandInset label +LatexCommand label +name "enu:Add-an-entry" + +\end_inset + Add an entry to both format lists (for conversion and reversion) in \begin_inset Newline newline \end_inset @@ -551,49 +564,39 @@ python development/tools/updatedocs.py \end_inset . - Then, revert the change to + Look at the resulting changes using the command \begin_inset Flex Code status collapsed \begin_layout Plain Layout -LFUNs.lyx +git diff \end_layout \end_inset - because that file is meant to be generated separately: +. + If anything looks surprising, please investigate. + Keep in mind that the case of \begin_inset Flex Code status collapsed \begin_layout Plain Layout -git checkout lib/doc/LFUNs.lyx -\end_layout - -\end_inset - - -\begin_inset Note Note -status collapsed - -\begin_layout Plain Layout -TODO: this step should be done within updatedocs.py +LFUNs.lyx \end_layout \end_inset -. - Look at the resulting changes using the command + is special, because it is first generated with \begin_inset Flex Code status collapsed \begin_layout Plain Layout -git diff +gen_lfuns.py \end_layout \end_inset -. - If anything looks surprising, please investigate. + before being converted to the latest format. Finally, commit using \begin_inset Flex Code status collapsed @@ -683,6 +686,96 @@ Updating the file format number of bind/ui files \end_layout \begin_layout Standard +A change to the functionality of existing LFUNs can require a conversion + of +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.bind +\end_layout + +\end_inset + + and +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.ui +\end_layout + +\end_inset + + files, and therefore an increment of the LFUN format, as well as a conversion + of Info insets in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files for manuals. + The latter cannot be done automatically and requires also a \SpecialChar LyX + format increase + (think of e.g. + someone who might have made a set of \SpecialChar LyX + teaching manuals for use in their + own group) +\begin_inset Foot +status open + +\begin_layout Plain Layout +\begin_inset Flex URL +status open + +\begin_layout Plain Layout + +http://www.lyx.org/trac/ticket/9794 +\end_layout + +\end_inset + + +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Enumerate +Increment the LFUN file format number in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +src/LyXAction.h +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Enumerate +Implement the LFUN conversion in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +lib/scripts/prefs2prefs_lfuns.py +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Enumerate See step \begin_inset CommandInset ref LatexCommand ref @@ -720,6 +813,82 @@ bash development/tools/updatelfuns.sh . \end_layout +\begin_layout Enumerate +Update Info insets in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files. + To do so, increment the \SpecialChar LyX + format and proceed as in +\begin_inset CommandInset ref +LatexCommand ref +reference "subsec:update_lyx_files" + +\end_inset + +, steps +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:Describe_format" + +\end_inset + +- +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:updatefiles" + +\end_inset + +. + In the lyx2lyx implementation ( +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:Add-an-entry" + +\end_inset + +th step), implement a conversion similar to the one in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +prefs2prefs_lfuns.py +\end_layout + +\end_inset + + above, as well as a corresponding reversion; for this one can use +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +convert_info_insets +\end_layout + +\end_inset + + from +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +lib/lyx2lyx/lyx2lyx_tools.py +\end_layout + +\end_inset + +. + +\end_layout + \begin_layout Subsection Backporting new styles to the stable version \end_layout @@ -1237,7 +1406,7 @@ Commit the changes to the repository, or send a patch to the development \end_layout \begin_layout Subsection -ctest automatic tests (cmake only) +ctest automatic tests \end_layout \begin_layout Standard @@ -1246,7 +1415,7 @@ Some tests are located in the status collapsed \begin_layout Plain Layout -development/autotests +development/autotests/ \end_layout \end_inset @@ -1254,24 +1423,28 @@ development/autotests subfolder of the \SpecialChar LyX source code distribution. -\begin_inset Flex Code -status collapsed +\begin_inset Foot +status open \begin_layout Plain Layout -cmake +The README document in this folder only describes the +\begin_inset Quotes eld +\end_inset + +keytests +\begin_inset Quotes erd +\end_inset + + subset of autotests! \end_layout \end_inset - is required to run the automatic \SpecialChar LyX - tests, running them is not implemented - for autotools. \end_layout \begin_layout Standard -The \SpecialChar LyX - tests can be run by the commands +These tests can be run by the commands \begin_inset Flex Code status collapsed @@ -1281,7 +1454,12 @@ ctest \end_inset - (all platforms) or + in the +\emph on + build directory +\emph default + (all platforms) or (when using a make based build system and not MSVC) + \begin_inset Flex Code status collapsed @@ -1291,17 +1469,37 @@ make test \end_inset - (when using a make based build system and not MSVC) in the + in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +autotests/ +\end_layout + +\end_inset + + subfolder of the +\emph on + build directory +\emph default +. + The test logs are written to the \begin_inset Flex Code status collapsed \begin_layout Plain Layout -autotests +Testing/Temporary/ \end_layout \end_inset - subfolder of the build directory. + subfolder of the +\emph on + +\emph default +build directory. + \end_layout \begin_layout Subsubsection @@ -1324,8 +1522,33 @@ reuse \begin_inset Quotes erd \end_inset - documentation, template, and example files trying to export them to all - supported output formats. + documentation, template, and example documents. + In addition, there are a number of dedicated sample documents in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +autotests/export/ +\end_layout + +\end_inset + + subfolder of the \SpecialChar LyX + source code distribution. + All samples are (after copying and eventual processing by scripts) exported + to various output formats via the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +—export-to +\end_layout + +\end_inset + + command line option. + The tests checks for errors reported by LyX. + (However, error-free export is no guarantee for an error-free output document.) \end_layout \begin_layout Paragraph @@ -1346,48 +1569,109 @@ When making a non-trivial change to a .layout file, run the export and layout \begin_layout Itemize When making non-trivial changes to a .lyx file, run the export tests correspondin g to that .lyx file. + +\begin_inset Foot +status collapsed + +\begin_layout Plain Layout +This rule is due to revision. + +\end_layout + +\begin_layout Plain Layout +There is an objection from the documentation maintainer that working on + the documentation must not be complicated by having to consider non-standard + exports. \end_layout \begin_layout Itemize -When making non-trivial changes to LyX's \SpecialChar LaTeX - export code (e.g. - touching the encoding code or package handling code that you expect will - change the exported \SpecialChar LaTeX - in some way): +successful compiling/testing an edited documentation file with pdflatex + suffices to ensure it can be commited, not tests with other exports are + required. \end_layout -\begin_deeper -\begin_layout Standard -\paragraph_spacing single -Consider running all of the export tests before and after your change. - If there are differences, please reconcile these (i.e. - fix the bug or fix the tests) -\emph on -before -\emph default - committing. - Ask for help if you're not sure what to. +\begin_layout Plain Layout +If sudden failures with other exports due to “half-tested” documentation + updates are a problem for the test maintainer, the test suite should use + copies that are \end_layout -\begin_layout Standard -If you do not want to run the tests, +\begin_layout Itemize +copied to a cache dir (autotests/samples/doc/, say) but not changed, \end_layout \begin_layout Itemize -post the patch on the list and others will run the tests and eventually - ask for fixes, or +updated regularely (but on a time chosen by the test suite maintainer) from + the originals in lib/doc/ +\end_layout + +\begin_layout Plain Layout +This way, \end_layout \begin_layout Itemize -commit, but be prepared to fix eventually arising problems or to revert - the commit if there is no easy fix. +no test will fail due to ongoing work on documentation, \end_layout -\end_deeper \begin_layout Itemize -Understand how to interpret test failures. - If your commit is found to have broken a test, you should be able to interpret - the test results when made aware of them. +the documentation is still tested in full (with some delay), +\end_layout + +\begin_layout Itemize +failures with non-default export can be examined and handled accordingly + in one run with the cache update, +\end_layout + +\begin_layout Itemize +“interesting failures” (like the nested-language+polyglossia problem in + es/Customization can be separated and moved into dedicated test samples. +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +When making non-trivial changes to LyX's \SpecialChar LaTeX + export code (e.g. + touching the encoding code or package handling code that you expect will + change the exported \SpecialChar LaTeX + in some way): +\end_layout + +\begin_deeper +\begin_layout Standard +\paragraph_spacing single +Consider running all of the export tests before and after your change. + If there are differences, please reconcile these (i.e. + fix the bug or fix the tests) +\emph on +before +\emph default + committing. + Ask for help if you're not sure what to. +\end_layout + +\begin_layout Standard +If you do not want to run the tests, +\end_layout + +\begin_layout Itemize +post the patch on the list and others will run the tests and eventually + ask for fixes, or +\end_layout + +\begin_layout Itemize +commit, but be prepared to fix eventually arising problems or to revert + the commit if there is no easy fix. +\end_layout + +\end_deeper +\begin_layout Itemize +Understand how to interpret test failures. + If your commit is found to have broken a test, you should be able to interpret + the test results when made aware of them. See Section \begin_inset CommandInset ref LatexCommand ref @@ -1399,7 +1683,226 @@ reference "subsec:Interpreting-export-tests" \end_layout \begin_layout Paragraph -Configuring the tests +\begin_inset CommandInset label +LatexCommand label +name "par:export-test-output-formats" + +\end_inset + +Output formats +\end_layout + +\begin_layout Standard +The following output formats are currently tested for each sample document + (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Export-test-filtering" + +\end_inset + + for exceptions): +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +lyx16 LyX 1.6 file format (lyx2lyx) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +lyx21 LyX 2.1 file format (lyx2lyx) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +xhtml LyXHTML (native LyX HTML export) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +LaTeX: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi DVI (8-bit latex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi3_texF DVI (LuaTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi3_systemF DVI (LuaTeX with Unicode fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf2 PDF (pdflatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf4_texF PDF (XeTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf4_systemF PDF (XeTeX with Unicode fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf5_texF PDF (LuaTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf5_systemF PDF (LuaTeX with Unicode fonts) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +LaTeX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +postprocessing: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf DVI -> PS (dvips) -> PDF (ps2pdf) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf3 DVI -> PDF (dvipdfm) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +not +\begin_inset space ~ +\end_inset + +tested: (or only if set as default output format in the document source) +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +latex LaTeX (plain) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +luatex LaTeX (LuaTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dviluatex LaTeX (dviluatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdflatex LaTeX (pdflatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +platex LaTeX (pLaTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +xetex LaTeX (XeTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +eps3 EPS (encapsulated Postscript) (cropped) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +ps DVI -> Postscript (dvips) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +odf +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +text (nor text2, ..., text4) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +textparagraph +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +word +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +word2 +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +wordhtml +\end_layout + +\end_deeper +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Configuring-ctests" + +\end_inset + +Configuring the tests \end_layout \begin_layout Standard @@ -1444,6 +1947,12 @@ reference "subsec:Interpreting-export-tests" \end_layout \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:ctest-options" + +\end_inset + Running the tests \end_layout @@ -1458,12 +1967,23 @@ ctest \end_inset +. + A full, up-to-date TeXLive installation is recommended to run the tests. + Otherwise, some tests will fail. + Tests with additional requirements are labeled +\begin_inset Quotes eld +\end_inset + +unreliable:nonstandard +\begin_inset Quotes erd +\end_inset + . \end_layout \begin_layout Standard -To run only some of the tests, use command line options: +To run only some of the tests, use command line options (see examples below): \end_layout \begin_layout Labeling @@ -1496,49 +2016,43 @@ status collapsed \end_layout -\begin_deeper -\begin_layout Standard -For example, to run only the export tests, you can use +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest -L export +-E \end_layout \end_inset -. - + Exclude the tests whose names match the given regular expression. \end_layout -\end_deeper \begin_layout Labeling \labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout --E +-LE \end_layout \end_inset - Exclude the tests whose names match the given regular expression. -\end_layout - -\begin_layout Labeling -\labelwidthstring -R + Exclude the tests whose labels match the given regular expression. + Cannot be combined with \begin_inset Flex Code status collapsed \begin_layout Plain Layout --LE +-L \end_layout \end_inset - Exclude the tests whose labels match the given regular expression. +. \end_layout \begin_layout Standard @@ -1596,7 +2110,7 @@ print-labels \end_layout \begin_layout Standard -Other relevant options are: +Other useful options are: \end_layout \begin_layout Labeling @@ -1620,7 +2134,6 @@ We are still working on getting the tests to run in parallel. pass when run sequentially. A reasonable approach is to first run the tests in parallel and then run the failed tests sequentially. - \end_layout \begin_layout Standard @@ -1656,37 +2169,66 @@ rerun-failed \end_layout \begin_layout Standard -\noindent -Note that some tests cannot be run in parallel. - These tests are marked in the code with the +When specifying a subset of the tests (e.g. + using \begin_inset Flex Code status collapsed \begin_layout Plain Layout -\noindent -RUN_SERIAL ON +\SpecialChar nobreakdash +R \end_layout \end_inset - CMake property. -\end_layout - -\end_deeper -\begin_layout Labeling -\labelwidthstring -R +), the same subset must be specified when using the \begin_inset Flex Code status collapsed \begin_layout Plain Layout \SpecialChar nobreakdash \SpecialChar nobreakdash -timeout +rerun-failed \end_layout \end_inset - Set a global timeout on all tests that do not already have a timeout set + option because it is the test numbers that are used to index which tests + failed on the previous run. +\end_layout + +\begin_layout Standard +\noindent +Note that some tests cannot be run in parallel. + These tests are marked in the code with the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\noindent +RUN_SERIAL ON +\end_layout + +\end_inset + + CMake property. +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring -R +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\SpecialChar nobreakdash +\SpecialChar nobreakdash +timeout +\end_layout + +\end_inset + + Set a global timeout on all tests that do not already have a timeout set on them. \end_layout @@ -1735,6 +2277,68 @@ man ctest ) the full list of command line options. \end_layout +\begin_layout Paragraph +Examples +\end_layout + +\begin_layout Itemize +run only the export tests: +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -L export +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +run inverted tests: +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -L "inverted|suspended" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +list all export tests which match any of the labelling patterns: +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -N -R " +\backslash +..*_export/" +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +exclude rarely used output formats and post-processing tests +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest -L export -E "_(texF|dvi3|pdf3?)" +\end_layout + +\end_inset + + +\end_layout + \begin_layout Paragraph \begin_inset CommandInset label LatexCommand label @@ -1750,8 +2354,7 @@ A test can fail for several reasons, not all of them bad. \end_layout \begin_layout Enumerate -The .lyx file could have been added recently and some formats are not expected - to work well. +A new or edited sample document may be incompatible with some output formats. \end_layout \begin_layout Enumerate @@ -1818,111 +2421,311 @@ passing \begin_inset Quotes erd \end_inset - if the export exits with error and as + if the export exits with error and as +\begin_inset Quotes eld +\end_inset + +failing +\begin_inset Quotes erd +\end_inset + + if the export succeeds +\emph on +. + +\emph default + It follows that these expected failures will not show up as failed tests + in the test results and thus will not pollute the +\begin_inset Quotes eld +\end_inset + +good +\begin_inset Quotes erd +\end_inset + + tests. + If the export actually succeeds, then the test will fail. + The purpose of this failure is to get your attention—something has changed, + possibly for the better. +\end_layout + +\begin_layout Standard +We try to document why a test is inverted or ignored. + See the comment (prefixed with +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +# +\end_layout + +\end_inset + +) above the block in which the test is listed as inverted or ignored in + the files +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/suspiciousTests +\end_layout + +\end_inset + +, +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/unreliableTests +\end_layout + +\end_inset + + and +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/ignoredTests +\end_layout + +\end_inset + +. + +\end_layout + +\begin_layout Standard +A good question is why do we enable the tests for non-default formats? The + answer is that if a non-default route is broken it is often because a bug + was introduced in LyX and not because a document-specific change was made + that is not supported by the route. + In other words, there is a high signal/noise ratio in the export tests + for some non-default formats. + +\end_layout + +\begin_layout Standard +When a test or several tests fail, consider checking the files in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +Testing/Temporary/ +\end_layout + +\end_inset + + subdirectory of your build directory. + In this subdirectory are three files: the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +LastTestsFailed.log +\end_layout + +\end_inset + + simply lists the tests that failed on your last +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest +\end_layout + +\end_inset + + command; the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +LastTest.log +\end_layout + +\end_inset + + file contains the output from the tests (and often has details explaining + why a test failed); and the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +CTestCostData.txt +\end_layout + +\end_inset + + file lists the times that it took to run the tests. +\end_layout + +\begin_layout Paragraph +What action should you take if a test fails? +\end_layout + +\begin_layout Standard +\paragraph_spacing single +It is always good to check manually why something fails and if it passes + if the PDF output is good. +\end_layout + +\begin_layout Itemize +Generally, if a change breaks compilation for the target format (for the + manuals pdf2) without solving some important other issue, +\emph on +fix or revert the commit +\emph default + that led to failure. +\end_layout + +\begin_layout Itemize +If it is not possible to (immediately) fix the failure but there are reasons + not to revert the commit (e.g. + it fixes another more important issue), +\emph on +invert +\emph default + the failing test case (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Inverted-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +If an +\emph on +inverted +\emph default + test case fails because the export now works, +\emph on +uninvert +\emph default + the test by removing the labeling pattern from +\begin_inset Quotes eld +\end_inset + +suspiciousTests +\begin_inset Quotes erd +\end_inset + +) (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Inverted-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +If the export did not fail previously but led to wrong output (PDF, say), + it is in fact an improvement when the test now fails, label it as +\begin_inset Quotes eld +\end_inset + +unreliable:wrong:output +\begin_inset Quotes erd +\end_inset + + ( +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Unreliable-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +In case of tests failing due to missing requirements (when only a subset + of TeXLive is installed or a test labeled \begin_inset Quotes eld \end_inset -failing +unreliable:nonstandard \begin_inset Quotes erd \end_inset - if the export succeeds -\emph on -. + fails), ignore the failure, ask for someone else to run the test, or install + the missing ressources and try again. +\end_layout -\emph default - It follows that these expected failures will not show up as failed tests - in the test results and thus will not pollute the -\begin_inset Quotes eld -\end_inset +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Inverted-tests" -good -\begin_inset Quotes erd \end_inset - tests. - If the export actually succeeds, then the test will fail. - The purpose of this failure is to get your attention—something has changed, - possibly for the better. +Inverted tests \end_layout \begin_layout Standard -We try to document why a test is inverted or ignored. - See the comment (prefixed with +Test cases whose name matches a pattern in the file \begin_inset Flex Code status collapsed \begin_layout Plain Layout -# +development/autotests/suspiciousTests \end_layout \end_inset -) above the block in which the test is listed as inverted or ignored in - the files + get the label +\emph on +inverted +\emph default +. + They get also the test property \begin_inset Flex Code status collapsed \begin_layout Plain Layout -development/autotests/suspiciousTests +WILL_FAIL \end_layout \end_inset - and -\begin_inset Flex Code +, i.e. + they are reported as failing if the export works without error +\begin_inset Flex URL status collapsed \begin_layout Plain Layout -development/autotests/ignoredTests + +https://cmake.org/cmake/help/v3.0/command/set_tests_properties.html \end_layout \end_inset . - It is possible that an export goes from succeeding to failing just because - the document was changed and the document was never expected to work with - a certain export format in the first case. - Once this is confirmed to be the situation, the test can be inverted. -\end_layout - -\begin_layout Standard -A good question is why do we enable the tests for non-default formats? The - answer is that if a non-default route is broken it is often because a bug - was introduced in LyX and not because a document-specific change was made - that is not supported by the route. - In other words, there is a high signal/noise ratio in the export tests - for some non-default formats. - \end_layout \begin_layout Standard -What action should you take if a test fails? This depends: -\end_layout +Add failing cases to this file, if they cannot be solved +\begin_inset Quotes eld +\end_inset -\begin_layout Standard -Generally, if a change breaks compilation for the target format (for the - manuals pdf2) without solving some important other issue, fix or revert - the commit that led to failure. - If a change breaks compilation for some non-target format (for the manuals - everything except pdf2), invert the test. - -\end_layout +immediately +\begin_inset Quotes erd +\end_inset -\begin_layout Standard -A special case is given, if the export was succeeding before but showing - garbled text in the PDF output. - Try to establish, that when the compilation succeeded before the resulting - PDF was good. - Otherwise, it is in fact an improvement when a test fails. - Now it might fail with a clear message of "language xyz not supported". - It is always good to check manually why something fails and if it passes - if the PDF output is good. + but it is expected that the export will work in a foreseeable future, e.g. + low priority issues like failures to export to a non-target format (for + the manuals everything except pdf2). \end_layout \begin_layout Standard -Sometimes a test is fixed as side-effect of some change. - We should uninvert a test (remove it from the +The following sublabels are currently present in \begin_inset Flex Code status collapsed @@ -1932,82 +2735,100 @@ suspiciousTests \end_inset - file) in order to preserve the fix. +: \end_layout -\begin_layout Standard -When a test or several tests fail, consider checking the files in the -\begin_inset Flex Code -status collapsed - -\begin_layout Plain Layout -Testing/Temporary/ +\begin_layout Description +todo test failures that require attention: \end_layout -\end_inset - - subdirectory of your build directory. - In this subdirectory are three files: the file -\begin_inset Flex Code -status collapsed - -\begin_layout Plain Layout -LastTestsFailed.log +\begin_deeper +\begin_layout Itemize +minor issues to explore and properly sort later, \end_layout -\end_inset +\begin_layout Itemize +easyfix issues, +\end_layout - simply lists the tests that failed on your last -\begin_inset Flex Code -status collapsed +\begin_layout Itemize +LyX bugs to report at trac (move pattern to section "lyxbugs" once done). +\end_layout -\begin_layout Plain Layout -ctest +\end_deeper +\begin_layout Description +lyxbugs LyX bugs with a Trac number. \end_layout -\end_inset +\begin_layout Description +ert Export failures due to "raw" LaTeX use in ERT or preamble code. +\end_layout - command; the -\begin_inset Flex Code -status collapsed +\begin_deeper +\begin_layout Standard +"Wontfix" if demonstrating correct use and OK in the default output format. +\end_layout -\begin_layout Plain Layout -LastTest.log +\end_deeper +\begin_layout Description +texissues Export fails due to LaTeX limitations like non-ASCII characters + in verbatim or listings, incompatible packages, ... \end_layout -\end_inset +\begin_deeper +\begin_layout Standard +"Wontfix" if documents demonstrate correct use in the default output format: +\end_layout - file contains the output from the tests (and often has details explaining - why a test failed); and the -\begin_inset Flex Code -status collapsed +\begin_layout Itemize +If the source can be made more robust without becoming "hackish", fix the + source, +\end_layout -\begin_layout Plain Layout -CTestCostData.txt +\begin_layout Itemize +if LyX could be enhanced to care for a permanent TeX limitation, file a + ticket at trac and add a pattern under lyxbugs, \end_layout -\end_inset +\begin_layout Itemize +otherwise, add a pattern here. +\end_layout - file lists the times that it took to run the tests. +\end_deeper +\begin_layout Description +attic Documents in the attic. + (Kept for reference and format conversion test.) \end_layout -\begin_layout Paragraph -Inverted tests +\begin_layout Subparagraph +suspended \end_layout \begin_layout Standard -These tests are expected to always fail. -\end_layout +Test cases whose name additionally matches a pattern in the file +\begin_inset Flex Code +status collapsed -\begin_layout Description -inverted These tests are expected to fail, but are subject to be examined. - It is expected that they will pass in a foreseeable future. - They are labeled 'inverted'. +\begin_layout Plain Layout +development/autotests/suspendedTests \end_layout -\begin_layout Description -suspended Some inverted tests are labeled 'suspended'. - This means, they are not executed using +\end_inset + + get the label +\emph on +suspended +\emph default +(instead of +\emph on +export +\emph default +and +\emph on + inverted +\emph default +). + This means they are not executed using \begin_inset Flex Code status collapsed @@ -2028,6 +2849,18 @@ ctest -L inverted \end_inset . + However, they also get the test property +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +WILL_FAIL +\end_layout + +\end_inset + +, i.e. + they are reported as failing if the export works without error. From time to time they still have to be checked using \begin_inset Flex Code status collapsed @@ -2041,10 +2874,9 @@ ctest -L suspended . \end_layout -\begin_deeper \begin_layout Standard -These tests are suspended, because they fail for known reasons which cannot - ATM be resolved. +These tests are suspended, because the export fails for known reasons which + cannot ATM be resolved. But it is expected the reason might disappear in the future. Be it new TL or better handling in \SpecialChar LyX . @@ -2062,15 +2894,45 @@ status collapsed \end_inset parameter nothing changes. - Suspended or not, tests will be executed depending only on the regexes - parameters given to the ctest command. + Suspended or not, tests will be executed depending only on the selecting + regular expression given to the ctest command (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:ctest-options" + +\end_inset + +). \end_layout -\end_deeper \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Unreliable-tests" + +\end_inset + Unreliable tests \end_layout +\begin_layout Standard +Test cases whose name matches a pattern in the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/unreliableTests +\end_layout + +\end_inset + + get the label +\emph on +unreliable +\emph default +. +\end_layout + \begin_layout Standard These tests are not executed using \begin_inset Flex Code @@ -2099,11 +2961,24 @@ ctest -L inverted \begin_layout Standard They pass or fail for various reasons not related to LyX (nonstandard, erratic) or pass but should rather fail (wrong output). + +\begin_inset Note Note +status collapsed + +\begin_layout Plain Layout +*invalid* tests (wrong output) are not *unreliable*. + # Use "unfit" or "unapplicable" as better label and name of pattern file? + +\end_layout + +\end_inset + + \end_layout \begin_layout Description nonstandard Documents with additional requirements, e.g. - a class or package file not on CTAN. + a class or package file not in TeXLive. \begin_inset Note Note status open @@ -2137,7 +3012,7 @@ TODO: use \emph on erratic \emph default - only for the phase-of-moon dependency. + only for the phase-of-moon dependency? \end_layout \end_inset @@ -2164,6 +3039,11 @@ output Export does not fail but the resulting document has errors. \begin_deeper \begin_layout Standard \paragraph_spacing single +\begin_inset Note Note +status open + +\begin_layout Plain Layout +\paragraph_spacing single These tests are actually not \emph on unreliable @@ -2175,13 +3055,25 @@ invalid (not measuring what they should measure). \end_layout +\end_inset + + +\end_layout + \end_deeper \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Export-test-filtering" + +\end_inset + Export test filtering \end_layout \begin_layout Standard -There are files which control the assignment of a label to a test. +The assignment of a label to a test is controlled by a set of files with + regular expressions that are matched against the test names. \end_layout \begin_layout Description @@ -2189,7 +3081,15 @@ ignoredTests (small file) \begin_inset Newline newline \end_inset -Tests selected here are withdrawn +Tests selected here are withdrawn in the configuration step (cf. + +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Configuring-ctests" + +\end_inset + +). \end_layout \begin_deeper @@ -3101,6 +4001,36 @@ development/autotests/filterCheckWarnings Under cmake, the tests are labeled as 'load'. \end_layout +\begin_layout Subsubsection +Keytests +\end_layout + +\begin_layout Standard +Automated tests based on the "MonKey Testing" keytest program. + They are documented in the README document in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests +\end_layout + +\end_inset + + subfolder of the \SpecialChar LyX + source code distribution. + T +\end_layout + +\begin_layout Subsubsection +lyx21 tests +\end_layout + +\begin_layout Standard +These tests combine lyx2lyx tests with check_load tests. + They fail if either fails. +\end_layout + \begin_layout Subsubsection URL tests \end_layout @@ -3183,40 +4113,6 @@ Associated test results can be examined in ctest-log directory in files of the form 'LastFailed.*URLS.log' \end_layout -\begin_layout Subsubsection -Test labels -\end_layout - -\begin_layout Standard -ctest label commands: -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -\SpecialChar nobreakdash -print-labels shows all assigned labels -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -L -\begin_inset space ~ -\end_inset - -labelname executes all tests to which this label is assigned to. - A test may have more that one label. -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -j -\begin_inset space ~ -\end_inset - -number executes tests in parallel using 'number' simultaneously processes. - Some tests are marked as 'sequencial', for them this parameter has no effect. -\end_layout - \begin_layout Section Development policies \end_layout