X-Git-Url: https://git.lyx.org/gitweb/?a=blobdiff_plain;f=lib%2Fdoc%2FDevelopment.lyx;h=eb2917a6362ff6fa7ba0e8473c95b60d9a1564a5;hb=693ca643fc1e6e9d797d8a074285136987b82add;hp=ca67d8d60faa7793dc259d91de53d4df012375d7;hpb=d515544bb44546eb709f4ccc64b010f41814d6df;p=lyx.git diff --git a/lib/doc/Development.lyx b/lib/doc/Development.lyx index ca67d8d60f..eb2917a636 100644 --- a/lib/doc/Development.lyx +++ b/lib/doc/Development.lyx @@ -1,7 +1,8 @@ #LyX 2.2 created this file. For more info see http://www.lyx.org/ -\lyxformat 503 +\lyxformat 504 \begin_document \begin_header +\save_transient_properties true \origin /systemlyxdir/doc/ \textclass scrartcl \options BCOR8mm,captions=tableheading @@ -25,8 +26,8 @@ logicalmkup \font_sf_scale 100 100 \font_tt_scale 100 100 \graphics default -\default_output_format default -\output_sync 0 +\default_output_format pdf2 +\output_sync 1 \bibtex_command default \index_command default \paperfontsize 12 @@ -331,6 +332,12 @@ Implement and test the new feature, including the reading and writing of \end_layout \begin_layout Enumerate +\begin_inset CommandInset label +LatexCommand label +name "enu:Describe_format" + +\end_inset + Describe the new format in \begin_inset Flex Code status collapsed @@ -384,6 +391,12 @@ lib/lyx2lyx/LyX.py \end_layout \begin_layout Enumerate +\begin_inset CommandInset label +LatexCommand label +name "enu:Add-an-entry" + +\end_inset + Add an entry to both format lists (for conversion and reversion) in \begin_inset Newline newline \end_inset @@ -551,49 +564,39 @@ python development/tools/updatedocs.py \end_inset . - Then, revert the change to + Look at the resulting changes using the command \begin_inset Flex Code status collapsed \begin_layout Plain Layout -LFUNs.lyx +git diff \end_layout \end_inset - because that file is meant to be generated separately: +. + If anything looks surprising, please investigate. + Keep in mind that the case of \begin_inset Flex Code status collapsed \begin_layout Plain Layout -git checkout lib/doc/LFUNs.lyx -\end_layout - -\end_inset - - -\begin_inset Note Note -status collapsed - -\begin_layout Plain Layout -TODO: this step should be done within updatedocs.py +LFUNs.lyx \end_layout \end_inset -. - Look at the resulting changes using the command + is special, because it is first generated with \begin_inset Flex Code status collapsed \begin_layout Plain Layout -git diff +gen_lfuns.py \end_layout \end_inset -. - If anything looks surprising, please investigate. + before being converted to the latest format. Finally, commit using \begin_inset Flex Code status collapsed @@ -649,11 +652,130 @@ python development/tools/updatelayouts.py . \end_layout +\begin_layout Standard +Note that we do not update the local layout of our +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files because users would then not be able to export to older formats. + For example, if a 2.2.0 user exported a template to 2.1.x format and tried + to open the file in LyX 2.1.x, there would be an error because the file would + contain a local layout whose format is too new. + The root reason for this is that we do not support converting layouts to + older layout formats, as we do for the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + file format. +\end_layout + \begin_layout Subsection Updating the file format number of bind/ui files \end_layout \begin_layout Standard +A change to the functionality of existing LFUNs can require a conversion + of +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.bind +\end_layout + +\end_inset + + and +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.ui +\end_layout + +\end_inset + + files, and therefore an increment of the LFUN format, as well as a conversion + of Info insets in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files for manuals. + The latter cannot be done automatically and requires also a \SpecialChar LyX + format increase + (think of e.g. + someone who might have made a set of \SpecialChar LyX + teaching manuals for use in their + own group) +\begin_inset Foot +status open + +\begin_layout Plain Layout +\begin_inset Flex URL +status open + +\begin_layout Plain Layout + +http://www.lyx.org/trac/ticket/9794 +\end_layout + +\end_inset + + +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Enumerate +Increment the LFUN file format number in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +src/LyXAction.h +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Enumerate +Implement the LFUN conversion in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +lib/scripts/prefs2prefs_lfuns.py +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Enumerate See step \begin_inset CommandInset ref LatexCommand ref @@ -691,6 +813,82 @@ bash development/tools/updatelfuns.sh . \end_layout +\begin_layout Enumerate +Update Info insets in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +.lyx +\end_layout + +\end_inset + + files. + To do so, increment the \SpecialChar LyX + format and proceed as in +\begin_inset CommandInset ref +LatexCommand ref +reference "subsec:update_lyx_files" + +\end_inset + +, steps +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:Describe_format" + +\end_inset + +- +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:updatefiles" + +\end_inset + +. + In the lyx2lyx implementation ( +\begin_inset CommandInset ref +LatexCommand ref +reference "enu:Add-an-entry" + +\end_inset + +th step), implement a conversion similar to the one in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +prefs2prefs_lfuns.py +\end_layout + +\end_inset + + above, as well as a corresponding reversion; for this one can use +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +convert_info_insets +\end_layout + +\end_inset + + from +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +lib/lyx2lyx/lyx2lyx_tools.py +\end_layout + +\end_inset + +. + +\end_layout + \begin_layout Subsection Backporting new styles to the stable version \end_layout @@ -1208,7 +1406,7 @@ Commit the changes to the repository, or send a patch to the development \end_layout \begin_layout Subsection -automatic LyX tests (cmake only) +ctest automatic tests \end_layout \begin_layout Standard @@ -1217,7 +1415,7 @@ Some tests are located in the status collapsed \begin_layout Plain Layout -development/autotests +development/autotests/ \end_layout \end_inset @@ -1225,24 +1423,28 @@ development/autotests subfolder of the \SpecialChar LyX source code distribution. -\begin_inset Flex Code -status collapsed +\begin_inset Foot +status open \begin_layout Plain Layout -cmake +The README document in this folder only describes the +\begin_inset Quotes eld +\end_inset + +keytests +\begin_inset Quotes erd +\end_inset + + subset of autotests! \end_layout \end_inset - is required to run the automatic \SpecialChar LyX - tests, running them is not implemented - for autotools. \end_layout \begin_layout Standard -The \SpecialChar LyX - tests can be run by the commands +These tests can be run by the commands \begin_inset Flex Code status collapsed @@ -1252,7 +1454,12 @@ ctest \end_inset - (all platforms) or + in the +\emph on + build directory +\emph default + (all platforms) or (when using a make based build system and not MSVC) + \begin_inset Flex Code status collapsed @@ -1262,17 +1469,37 @@ make test \end_inset - (when using a make based build system and not MSVC) in the + in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +autotests/ +\end_layout + +\end_inset + + subfolder of the +\emph on + build directory +\emph default +. + The test logs are written to the \begin_inset Flex Code status collapsed \begin_layout Plain Layout -autotests +Testing/Temporary/ \end_layout \end_inset - subfolder of the build directory. + subfolder of the +\emph on + +\emph default +build directory. + \end_layout \begin_layout Subsubsection @@ -1287,27 +1514,41 @@ e it is important to keep them up-to-date and understand how they work. \end_layout \begin_layout Standard -The export tests require the +The export tests +\begin_inset Quotes eld +\end_inset + +reuse +\begin_inset Quotes erd +\end_inset + + documentation, template, and example documents. + In addition, there are a number of dedicated sample documents in the \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest +autotests/export/ \end_layout \end_inset - command that comes with the + subfolder of the \SpecialChar LyX + source code distribution. + All samples are (after copying and eventual processing by scripts) exported + to various output formats via the \begin_inset Flex Code status collapsed \begin_layout Plain Layout -cmake +—export-to \end_layout \end_inset - build system + command line option. + The tests checks for errors reported by LyX. + (However, error-free export is no guarantee for an error-free output document.) \end_layout \begin_layout Paragraph @@ -1328,45 +1569,344 @@ When making a non-trivial change to a .layout file, run the export and layout \begin_layout Itemize When making non-trivial changes to a .lyx file, run the export tests correspondin g to that .lyx file. + +\begin_inset Foot +status collapsed + +\begin_layout Plain Layout +This rule is due to revision. + \end_layout -\begin_layout Itemize -When making non-trivial changes to LyX's \SpecialChar LaTeX - export code (e.g. - touching the encoding code or package handling code that you expect will - change the exported \SpecialChar LaTeX - in some way), consider running all of the export tests - before and after your change. - If there are differences, please reconcile these (i.e. - fix the bug or fix the tests) -\emph on -before -\emph default - committing. - Ask for help if you're not sure what to do or if you do not want to run - the tests, post the patch on the list and others will run the tests. +\begin_layout Plain Layout +There is an objection from the documentation maintainer that working on + the documentation must not be complicated by having to consider non-standard + exports. \end_layout \begin_layout Itemize -Understand how to interpret test failures. - If your commit is found to have broken a test, you should be able to interpret - the test results when made aware of them. - See Section -\begin_inset CommandInset ref -LatexCommand ref -reference "subsec:Interpreting-export-tests" +successful compiling/testing an edited documentation file with pdflatex + suffices to ensure it can be commited, not tests with other exports are + required. +\end_layout -\end_inset +\begin_layout Plain Layout +If sudden failures with other exports due to “half-tested” documentation + updates are a problem for the test maintainer, the test suite should use + copies that are +\end_layout -. +\begin_layout Itemize +copied to a cache dir (autotests/samples/doc/, say) but not changed, \end_layout -\begin_layout Paragraph -Configuring the tests +\begin_layout Itemize +updated regularely (but on a time chosen by the test suite maintainer) from + the originals in lib/doc/ \end_layout -\begin_layout Standard -To enable these tests, add the +\begin_layout Plain Layout +This way, +\end_layout + +\begin_layout Itemize +no test will fail due to ongoing work on documentation, +\end_layout + +\begin_layout Itemize +the documentation is still tested in full (with some delay), +\end_layout + +\begin_layout Itemize +failures with non-default export can be examined and handled accordingly + in one run with the cache update, +\end_layout + +\begin_layout Itemize +“interesting failures” (like the nested-language+polyglossia problem in + es/Customization can be separated and moved into dedicated test samples. +\end_layout + +\end_inset + + +\end_layout + +\begin_layout Itemize +When making non-trivial changes to LyX's \SpecialChar LaTeX + export code (e.g. + touching the encoding code or package handling code that you expect will + change the exported \SpecialChar LaTeX + in some way): +\end_layout + +\begin_deeper +\begin_layout Standard +\paragraph_spacing single +Consider running all of the export tests before and after your change. + If there are differences, please reconcile these (i.e. + fix the bug or fix the tests) +\emph on +before +\emph default + committing. + Ask for help if you're not sure what to. +\end_layout + +\begin_layout Standard +If you do not want to run the tests, +\end_layout + +\begin_layout Itemize +post the patch on the list and others will run the tests and eventually + ask for fixes, or +\end_layout + +\begin_layout Itemize +commit, but be prepared to fix eventually arising problems or to revert + the commit if there is no easy fix. +\end_layout + +\end_deeper +\begin_layout Itemize +Understand how to interpret test failures. + If your commit is found to have broken a test, you should be able to interpret + the test results when made aware of them. + See Section +\begin_inset CommandInset ref +LatexCommand ref +reference "subsec:Interpreting-export-tests" + +\end_inset + +. +\end_layout + +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:export-test-output-formats" + +\end_inset + +Output formats +\end_layout + +\begin_layout Standard +The following output formats are currently tested for each sample document + (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Export-test-filtering" + +\end_inset + + for exceptions): +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +lyx16 LyX 1.6 file format (lyx2lyx) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +lyx21 LyX 2.1 file format (lyx2lyx) +\end_layout + +\begin_layout Labeling +\labelwidthstring 00.00.0000 +xhtml LyXHTML (native LyX HTML export) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +LaTeX: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi DVI (8-bit latex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi3_texF DVI (LuaTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dvi3_systemF DVI (LuaTeX with Unicode fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf2 PDF (pdflatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf4_texF PDF (XeTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf4_systemF PDF (XeTeX with Unicode fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf5_texF PDF (LuaTeX with 8-bit TeX fonts) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf5_systemF PDF (LuaTeX with Unicode fonts) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +LyX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +LaTeX +\begin_inset space ~ +\end_inset + ++ +\begin_inset space ~ +\end_inset + +postprocessing: +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf DVI -> PS (dvips) -> PDF (ps2pdf) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdf3 DVI -> PDF (dvipdfm) +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring 00.00.0000 +not +\begin_inset space ~ +\end_inset + +tested: (or only if set as default output format in the document source) +\end_layout + +\begin_deeper +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +latex LaTeX (plain) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +luatex LaTeX (LuaTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +dviluatex LaTeX (dviluatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +pdflatex LaTeX (pdflatex) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +platex LaTeX (pLaTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +xetex LaTeX (XeTeX) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +eps3 EPS (encapsulated Postscript) (cropped) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +ps DVI -> Postscript (dvips) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +odf +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +text (nor text2, ..., text4) +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +textparagraph +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +word +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +word2 +\end_layout + +\begin_layout Labeling +\labelwidthstring pdf5msystemFM +wordhtml +\end_layout + +\end_deeper +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Configuring-ctests" + +\end_inset + +Configuring the tests +\end_layout + +\begin_layout Standard +To enable the export autotests, add the \begin_inset Flex Code status collapsed @@ -1407,6 +1947,12 @@ reference "subsec:Interpreting-export-tests" \end_layout \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:ctest-options" + +\end_inset + Running the tests \end_layout @@ -1422,133 +1968,176 @@ ctest \end_inset . - To run only some of the tests, use the command -\begin_inset Flex Code -status collapsed + A full, up-to-date TeXLive installation is recommended to run the tests. + Otherwise, some tests will fail. + Tests with additional requirements are labeled +\begin_inset Quotes eld +\end_inset -\begin_layout Plain Layout -ctest -R +unreliable:nonstandard +\begin_inset Quotes erd +\end_inset + +. + \end_layout -\end_inset +\begin_layout Standard +To run only some of the tests, use command line options (see examples below): +\end_layout -, where +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout - +-R \end_layout \end_inset - is a regular expression that matches test names. - To run only the export tests, you can use + Run only the tests whose names match the given regular expression. +\end_layout + +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest -L export +-L \end_layout \end_inset -. - For the list of test categories available in addition to -\begin_inset Quotes eld -\end_inset - -export -\begin_inset Quotes erd -\end_inset + Run only the tests whose labels match the given regular expression. + A test may have more that one label. + +\end_layout -, run +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest \SpecialChar nobreakdash -\SpecialChar nobreakdash -print-labels +-E \end_layout \end_inset -. - It is often useful to list the tests without running them (e.g. - if you want to know how many tests there are or whether your + Exclude the tests whose names match the given regular expression. +\end_layout + +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout - +-LE \end_layout \end_inset - regular expression did what you expected). - This can be done with the + Exclude the tests whose labels match the given regular expression. + Cannot be combined with \begin_inset Flex Code status collapsed \begin_layout Plain Layout --N +-L \end_layout \end_inset - or +. +\end_layout + +\begin_layout Standard +The following options help to find good selection patterns: +\end_layout + +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout -\SpecialChar nobreakdash -\SpecialChar nobreakdash -show-only +-N \end_layout \end_inset - argument. - We are still working on getting the tests to run in parallel which is supported - by the + List the tests that would be run but not actually run them. + +\end_layout + +\begin_deeper +\begin_layout Standard +Useful in conjunction with the -R, -L, -E and -LE options, e.g., if you want + to know how many tests there are or whether your \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest + \end_layout \end_inset - command with the + regular expression did what you expected. +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout --j +\SpecialChar nobreakdash +\SpecialChar nobreakdash +print-labels \end_layout \end_inset - or + print the list of all labels associated with the test set. + Can also be combined with -R, -L, -E, ... + +\end_layout + +\begin_layout Standard +Other useful options are: +\end_layout + +\begin_layout Labeling +\labelwidthstring -R \begin_inset Flex Code status collapsed \begin_layout Plain Layout -\SpecialChar nobreakdash -\SpecialChar nobreakdash -parallel +-j \end_layout \end_inset - argument. + Run the tests in parallel using the given number of jobs. +\end_layout + +\begin_deeper +\begin_layout Standard +We are still working on getting the tests to run in parallel. However, when running the tests in parallel, sometimes tests fail that pass when run sequentially. A reasonable approach is to first run the tests in parallel and then run the failed tests sequentially. - For example, to run 8 jobs at a time: +\end_layout + +\begin_layout Standard +For example, to run 8 jobs at a time: \end_layout \begin_layout Standard @@ -1579,6 +2168,35 @@ rerun-failed \end_layout +\begin_layout Standard +When specifying a subset of the tests (e.g. + using +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\SpecialChar nobreakdash +R +\end_layout + +\end_inset + +), the same subset must be specified when using the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\SpecialChar nobreakdash +\SpecialChar nobreakdash +rerun-failed +\end_layout + +\end_inset + + option because it is the test numbers that are used to index which tests + failed on the previous run. +\end_layout + \begin_layout Standard \noindent Note that some tests cannot be run in parallel. @@ -1593,63 +2211,132 @@ RUN_SERIAL ON \end_inset - CMake property. + CMake property. +\end_layout + +\end_deeper +\begin_layout Labeling +\labelwidthstring -R +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +\SpecialChar nobreakdash +\SpecialChar nobreakdash +timeout +\end_layout + +\end_inset + + Set a global timeout on all tests that do not already have a timeout set + on them. +\end_layout + +\begin_deeper +\begin_layout Standard +There have been bugs in LyX and in \SpecialChar LaTeX + which cause compilation to hang, and + without a timeout a test might never stop (in one case there was even a + memory leak). + If a test times out, the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest +\end_layout + +\end_inset + + command exits with error, but you can distinguish between a timed out test + and a failed test in the output reported at the end of the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest +\end_layout + +\end_inset + + command. +\end_layout + +\end_deeper +\begin_layout Standard +See the manual ( +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +man ctest +\end_layout + +\end_inset + +) the full list of command line options. \end_layout -\begin_layout Standard -In some situations the option +\begin_layout Paragraph +Examples +\end_layout + +\begin_layout Itemize +run only the export tests: \begin_inset Flex Code status collapsed \begin_layout Plain Layout -\SpecialChar nobreakdash -\SpecialChar nobreakdash -timeout +ctest -L export \end_layout \end_inset - is useful. - There have been bugs in LyX and in \SpecialChar LaTeX - which cause compilation to hang, and - without a timeout a test might never stop (in one case there was even a - memory leak). - If a test times out, the + +\end_layout + +\begin_layout Itemize +run inverted tests: \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest +ctest -L "inverted|suspended" \end_layout \end_inset - command exits with error, but you can distinguish between a timed out test - and a failed test in the output reported at the end of the + +\end_layout + +\begin_layout Itemize +list all export tests which match any of the labelling patterns: \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest +ctest -N -R " +\backslash +..*_export/" \end_layout \end_inset - command. + \end_layout -\begin_layout Standard -See the manual ( +\begin_layout Itemize +exclude rarely used output formats and post-processing tests \begin_inset Flex Code status collapsed \begin_layout Plain Layout -man ctest +ctest -L export -E "_(texF|dvi3|pdf3?)" \end_layout \end_inset -) the full list of command line options. + \end_layout \begin_layout Paragraph @@ -1667,8 +2354,7 @@ A test can fail for several reasons, not all of them bad. \end_layout \begin_layout Enumerate -The .lyx file could have been added recently and some formats are not expected - to work well. +A new or edited sample document may be incompatible with some output formats. \end_layout \begin_layout Enumerate @@ -1677,7 +2363,7 @@ A dependency is not met (e.g. class file). One hint that this is the case is that the corresponding \begin_inset Flex Code -status open +status collapsed \begin_layout Plain Layout check_load @@ -1781,137 +2467,474 @@ status collapsed status collapsed \begin_layout Plain Layout -development/autotests/suspiciousTests +development/autotests/suspiciousTests +\end_layout + +\end_inset + +, +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/unreliableTests +\end_layout + +\end_inset + + and +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/ignoredTests +\end_layout + +\end_inset + +. + +\end_layout + +\begin_layout Standard +A good question is why do we enable the tests for non-default formats? The + answer is that if a non-default route is broken it is often because a bug + was introduced in LyX and not because a document-specific change was made + that is not supported by the route. + In other words, there is a high signal/noise ratio in the export tests + for some non-default formats. + +\end_layout + +\begin_layout Standard +When a test or several tests fail, consider checking the files in the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +Testing/Temporary/ +\end_layout + +\end_inset + + subdirectory of your build directory. + In this subdirectory are three files: the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +LastTestsFailed.log +\end_layout + +\end_inset + + simply lists the tests that failed on your last +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +ctest +\end_layout + +\end_inset + + command; the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +LastTest.log +\end_layout + +\end_inset + + file contains the output from the tests (and often has details explaining + why a test failed); and the +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +CTestCostData.txt +\end_layout + +\end_inset + + file lists the times that it took to run the tests. +\end_layout + +\begin_layout Paragraph +What action should you take if a test fails? +\end_layout + +\begin_layout Standard +\paragraph_spacing single +It is always good to check manually why something fails and if it passes + if the PDF output is good. +\end_layout + +\begin_layout Itemize +Generally, if a change breaks compilation for the target format (for the + manuals pdf2) without solving some important other issue, +\emph on +fix or revert the commit +\emph default + that led to failure. +\end_layout + +\begin_layout Itemize +If it is not possible to (immediately) fix the failure but there are reasons + not to revert the commit (e.g. + it fixes another more important issue), +\emph on +invert +\emph default + the failing test case (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Inverted-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +If an +\emph on +inverted +\emph default + test case fails because the export now works, +\emph on +uninvert +\emph default + the test by removing the labeling pattern from +\begin_inset Quotes eld +\end_inset + +suspiciousTests +\begin_inset Quotes erd +\end_inset + +) (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Inverted-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +If the export did not fail previously but led to wrong output (PDF, say), + it is in fact an improvement when the test now fails, label it as +\begin_inset Quotes eld +\end_inset + +unreliable:wrong:output +\begin_inset Quotes erd +\end_inset + + ( +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Unreliable-tests" + +\end_inset + +). +\end_layout + +\begin_layout Itemize +In case of tests failing due to missing requirements (when only a subset + of TeXLive is installed or a test labeled +\begin_inset Quotes eld +\end_inset + +unreliable:nonstandard +\begin_inset Quotes erd +\end_inset + + fails), ignore the failure, ask for someone else to run the test, or install + the missing ressources and try again. +\end_layout + +\begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Inverted-tests" + +\end_inset + +Inverted tests +\end_layout + +\begin_layout Standard +Test cases whose name matches a pattern in the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/suspiciousTests +\end_layout + +\end_inset + + get the label +\emph on +inverted +\emph default +. + They get also the test property +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +WILL_FAIL +\end_layout + +\end_inset + +, i.e. + they are reported as failing if the export works without error +\begin_inset Flex URL +status collapsed + +\begin_layout Plain Layout + +https://cmake.org/cmake/help/v3.0/command/set_tests_properties.html +\end_layout + +\end_inset + +. +\end_layout + +\begin_layout Standard +Add failing cases to this file, if they cannot be solved +\begin_inset Quotes eld +\end_inset + +immediately +\begin_inset Quotes erd +\end_inset + + but it is expected that the export will work in a foreseeable future, e.g. + low priority issues like failures to export to a non-target format (for + the manuals everything except pdf2). +\end_layout + +\begin_layout Standard +The following sublabels are currently present in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +suspiciousTests \end_layout \end_inset - and -\begin_inset Flex Code -status collapsed +: +\end_layout -\begin_layout Plain Layout -development/autotests/ignoredTests +\begin_layout Description +todo test failures that require attention: \end_layout -\end_inset +\begin_deeper +\begin_layout Itemize +minor issues to explore and properly sort later, +\end_layout -. - It is possible that an export goes from succeeding to failing just because - the document was changed and the document was never expected to work with - a certain export format in the first case. - Once this is confirmed to be the situation, the test can be inverted. +\begin_layout Itemize +easyfix issues, \end_layout +\begin_layout Itemize +LyX bugs to report at trac (move pattern to section "lyxbugs" once done). +\end_layout + +\end_deeper +\begin_layout Description +lyxbugs LyX bugs with a Trac number. +\end_layout + +\begin_layout Description +ert Export failures due to "raw" LaTeX use in ERT or preamble code. +\end_layout + +\begin_deeper \begin_layout Standard -A good question is why do we enable the tests for non-default formats? The - answer is that if a non-default route is broken it is often because a bug - was introduced in LyX and not because a document-specific change was made - that is not supported by the route. - In other words, there is a high signal/noise ratio in the export tests - for some non-default formats. - +"Wontfix" if demonstrating correct use and OK in the default output format. +\end_layout + +\end_deeper +\begin_layout Description +texissues Export fails due to LaTeX limitations like non-ASCII characters + in verbatim or listings, incompatible packages, ... \end_layout +\begin_deeper \begin_layout Standard -What action should you take if a test fails? First, check manually that - when the compilation succeeded before the resulting PDF was good. - In fact, sometimes it is an improvement when a test fails. - If you check manually, it might be the case that the export was succeeding - before but showing garbled text in a PDF output. - Now it might fail with a clear message of "language xyz not supported". - It is always good to check manually why something fails and if it passes - if the PDF output is good. +"Wontfix" if documents demonstrate correct use in the default output format: +\end_layout + +\begin_layout Itemize +If the source can be made more robust without becoming "hackish", fix the + source, +\end_layout + +\begin_layout Itemize +if LyX could be enhanced to care for a permanent TeX limitation, file a + ticket at trac and add a pattern under lyxbugs, +\end_layout + +\begin_layout Itemize +otherwise, add a pattern here. +\end_layout + +\end_deeper +\begin_layout Description +attic Documents in the attic. + (Kept for reference and format conversion test.) +\end_layout + +\begin_layout Subparagraph +suspended \end_layout \begin_layout Standard -Sometimes a test is fixed as side-effect of some change. - We should uninvert a test (remove it from the +Test cases whose name additionally matches a pattern in the file \begin_inset Flex Code status collapsed \begin_layout Plain Layout -suspiciousTests +development/autotests/suspendedTests \end_layout \end_inset - file) in order to preserve the fix. -\end_layout - -\begin_layout Standard -When a test or several tests fail, consider checking the files in the + get the label +\emph on +suspended +\emph default +(instead of +\emph on +export +\emph default +and +\emph on + inverted +\emph default +). + This means they are not executed using \begin_inset Flex Code status collapsed \begin_layout Plain Layout -Testing/Temporary/ +ctest -L export \end_layout \end_inset - subdirectory of your build directory. - In this subdirectory are three files: the file + or \begin_inset Flex Code status collapsed \begin_layout Plain Layout -LastTestsFailed.log +ctest -L inverted \end_layout \end_inset - simply lists the tests that failed on your last +. + However, they also get the test property \begin_inset Flex Code status collapsed \begin_layout Plain Layout -ctest +WILL_FAIL \end_layout \end_inset - command; the +, i.e. + they are reported as failing if the export works without error. + From time to time they still have to be checked using \begin_inset Flex Code status collapsed \begin_layout Plain Layout -LastTest.log +ctest -L suspended \end_layout \end_inset - file contains the output from the tests (and often has details explaining - why a test failed); and the +. +\end_layout + +\begin_layout Standard +These tests are suspended, because the export fails for known reasons which + cannot ATM be resolved. + But it is expected the reason might disappear in the future. + Be it new TL or better handling in \SpecialChar LyX +. +\end_layout + +\begin_layout Standard +For ctest commands without the \begin_inset Flex Code status collapsed \begin_layout Plain Layout -CTestCostData.txt +-L \end_layout \end_inset - file lists the times that it took to run the tests. + parameter nothing changes. + Suspended or not, tests will be executed depending only on the selecting + regular expression given to the ctest command (see +\begin_inset CommandInset ref +LatexCommand ref +reference "par:ctest-options" + +\end_inset + +). \end_layout \begin_layout Paragraph -Inverted tests +\begin_inset CommandInset label +LatexCommand label +name "par:Unreliable-tests" + +\end_inset + +Unreliable tests \end_layout \begin_layout Standard -These tests are expected to always fail. +Test cases whose name matches a pattern in the file +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests/unreliableTests \end_layout -\begin_layout Description -inverted These tests are expected to fail, but are subject to be examined. - It is expected that they will pass in a foreseeable future. - They are labeled 'inverted'. +\end_inset + + get the label +\emph on +unreliable +\emph default +. \end_layout -\begin_layout Description -suspended Some inverted tests are labeled 'suspended'. - This means, they are not executed using +\begin_layout Standard +These tests are not executed using \begin_inset Flex Code status collapsed @@ -1932,53 +2955,41 @@ ctest -L inverted \end_inset . - From time to time they still have to be checked using -\begin_inset Flex Code + +\end_layout + +\begin_layout Standard +They pass or fail for various reasons not related to LyX (nonstandard, erratic) + or pass but should rather fail (wrong output). + +\begin_inset Note Note status collapsed \begin_layout Plain Layout -ctest -L suspended +*invalid* tests (wrong output) are not *unreliable*. + # Use "unfit" or "unapplicable" as better label and name of pattern file? + \end_layout \end_inset -. -\end_layout -\begin_deeper -\begin_layout Standard -These tests are suspended, because they fail for known reasons which cannot - ATM be resolved. - But it is expected the reason might disappear in the future. - Be it new TL or better handling in \SpecialChar LyX -. \end_layout -\begin_layout Standard -For ctest commands without the -\begin_inset Flex Code -status collapsed +\begin_layout Description +nonstandard Documents with additional requirements, e.g. + a class or package file not in TeXLive. + +\begin_inset Note Note +status open \begin_layout Plain Layout --L +TODO: rename to "extra"? \end_layout \end_inset - parameter nothing changes. - Suspended or not, tests will be executed depending only on the regexes - parameters given to the ctest command. -\end_layout - -\end_deeper -\begin_layout Paragraph -Unreliable tests -\end_layout -\begin_layout Description -nonstandard Requires non-standard ressources (\SpecialChar LaTeX - packages and document classes, - fonts, ...) that are not a requirement for running this test suite. \end_layout \begin_deeper @@ -1990,17 +3001,23 @@ These tests are labeled as \end_deeper \begin_layout Description -erratic Tests with -\begin_inset Quotes eld -\end_inset +erratic Tests depending on local configuration, OS, TeX distribution, package + versions, or the phase of the moon. + +\begin_inset Note Note +status open + +\begin_layout Plain Layout +TODO: use +\emph on +erratic +\emph default + only for the phase-of-moon dependency? +\end_layout -arbitrary -\begin_inset Quotes erd \end_inset - result, depending on local configuration, OS, \SpecialChar TeX - distribution, package versions, - or the phase of the moon. + \end_layout \begin_deeper @@ -2010,21 +3027,69 @@ These tests are labeled as 'erratic'. \end_layout +\end_deeper +\begin_layout Description +wrong +\begin_inset space ~ +\end_inset + +output Export does not fail but the resulting document has errors. +\end_layout + +\begin_deeper +\begin_layout Standard +\paragraph_spacing single +\begin_inset Note Note +status open + +\begin_layout Plain Layout +\paragraph_spacing single +These tests are actually not +\emph on +unreliable +\emph default + but +\emph on +invalid +\emph default + (not measuring what they should measure). +\end_layout + +\end_inset + + +\end_layout + \end_deeper \begin_layout Paragraph +\begin_inset CommandInset label +LatexCommand label +name "par:Export-test-filtering" + +\end_inset + Export test filtering \end_layout \begin_layout Standard -There are files which control the assignment of a label to a test. +The assignment of a label to a test is controlled by a set of files with + regular expressions that are matched against the test names. \end_layout \begin_layout Description -ignoredTests (smal file) +ignoredTests (small file) \begin_inset Newline newline \end_inset -Tests selected here are withdrawn +Tests selected here are withdrawn in the configuration step (cf. + +\begin_inset CommandInset ref +LatexCommand ref +reference "par:Configuring-ctests" + +\end_inset + +). \end_layout \begin_deeper @@ -2936,6 +4001,36 @@ development/autotests/filterCheckWarnings Under cmake, the tests are labeled as 'load'. \end_layout +\begin_layout Subsubsection +Keytests +\end_layout + +\begin_layout Standard +Automated tests based on the "MonKey Testing" keytest program. + They are documented in the README document in +\begin_inset Flex Code +status collapsed + +\begin_layout Plain Layout +development/autotests +\end_layout + +\end_inset + + subfolder of the \SpecialChar LyX + source code distribution. + T +\end_layout + +\begin_layout Subsubsection +lyx21 tests +\end_layout + +\begin_layout Standard +These tests combine lyx2lyx tests with check_load tests. + They fail if either fails. +\end_layout + \begin_layout Subsubsection URL tests \end_layout @@ -2996,7 +4091,7 @@ ctest -L url \begin_inset Newline newline \end_inset -runns all tests with label +runs all tests with label \family typewriter 'url' \end_layout @@ -3010,7 +4105,7 @@ ctest -R 'check_.*urls' \begin_inset Newline newline \end_inset -runns the tests 'check_accessible_urls' +runs the tests 'check_accessible_urls' \end_layout \begin_layout Standard @@ -3018,40 +4113,6 @@ Associated test results can be examined in ctest-log directory in files of the form 'LastFailed.*URLS.log' \end_layout -\begin_layout Subsubsection -Test labels -\end_layout - -\begin_layout Standard -ctest label commands: -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -\SpecialChar nobreakdash -print-labels shows all assigned labels -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -L -\begin_inset space ~ -\end_inset - -labelname executes all tests to which this label is assigned to. - A test may have more that one label. -\end_layout - -\begin_layout Description -\SpecialChar nobreakdash -j -\begin_inset space ~ -\end_inset - -number executes tests in parallel using 'number' simultaneously processes. - Some tests are marked as 'sequencial', for them this parameter has no effect. -\end_layout - \begin_layout Section Development policies \end_layout