From 5f50ae8f08c98dd18cfd85c69af286f9e3bf1033 Mon Sep 17 00:00:00 2001 From: Tommaso Cucinotta Date: Sat, 29 Jan 2011 14:42:53 +0000 Subject: [PATCH] Added basic automated testing capability, based on the MonKey Test (keytest.py) by McCabe-Dansted. As of now, I plan to add a few regression tests for the Advanced Find & Replace using this engine. It can be used for automated testing of other LyX subsystems as well. Just write your own *-in.txt scripts and add them into the autotests/ folder. TODOs: - perhaps simplify keytest.py - check how this behaves under various types of failure of the test (e.g., LyX crashing, hanging, ...) just to be sure it goes on with the other tests anyway. git-svn-id: svn://svn.lyx.org/lyx/lyx-devel/trunk@37364 a592a061-630c-0410-9148-cb99ea01b6c8 --- development/autotests/README | 13 + development/autotests/findadv-01-in.txt | 12 + development/autotests/findadv-02-in.txt | 12 + development/autotests/hello-world-in.txt | 42 +++ development/autotests/keytest.py | 350 +++++++++++++++++++++++ development/autotests/run-tests.sh | 42 +++ development/autotests/single-test.sh | 23 ++ 7 files changed, 494 insertions(+) create mode 100644 development/autotests/README create mode 100644 development/autotests/findadv-01-in.txt create mode 100644 development/autotests/findadv-02-in.txt create mode 100644 development/autotests/hello-world-in.txt create mode 100755 development/autotests/keytest.py create mode 100755 development/autotests/run-tests.sh create mode 100755 development/autotests/single-test.sh diff --git a/development/autotests/README b/development/autotests/README new file mode 100644 index 0000000000..0cdefbd358 --- /dev/null +++ b/development/autotests/README @@ -0,0 +1,13 @@ +Automated tests based on the "MonKey Testing" keytest program. + +In order to launch all the tests available in this folder, just run + + ./run-tests.sh + +If you wish to launch one or more specific tests, then provide them +as argument to the script, e.g.: + + ./run-tests.sh *-in.txt + +For failed tests you get a folder with the logs that help you identify +the problem. diff --git a/development/autotests/findadv-01-in.txt b/development/autotests/findadv-01-in.txt new file mode 100644 index 0000000000..c731269044 --- /dev/null +++ b/development/autotests/findadv-01-in.txt @@ -0,0 +1,12 @@ +# After replacing with something that matches the search pattern, the next +# match should not consider the just replaced text (#4388) +# +TestBegin test.lyx -dbg find > lyx-log.txt 2>&1 +KK: aaa\[Home] +KK: \CF +KK: a\[Tab] +KK: aba\[Return]\[Return]\[Return] +KK: \Afs +KK: \Ax +KK: buffer-export latex\[Return] +TestEnd grep "^abaabaa$" test.tex diff --git a/development/autotests/findadv-02-in.txt b/development/autotests/findadv-02-in.txt new file mode 100644 index 0000000000..91cd265a1d --- /dev/null +++ b/development/autotests/findadv-02-in.txt @@ -0,0 +1,12 @@ +# Basic test on refactoring of mathematics notation +# +TestBegin test.lyx -dbg find > lyx-log.txt 2>&1 +KK: \Cm +KK: x^(a) +\\frac 1+x^(a) \[Down]1-x^(a) \C\[Home] +KK: \CF +KK: \Cmx^(a) \[Tab] +KK: \Cmx_a \[Return]\[Return]\[Return] +KK: \Afs +KK: \Ax +KK: buffer-export latex\[Return] +TestEnd grep '^\$x_{a}+\\frac{1+x_{a}}{1-x\^{(a)}}\$' test.tex diff --git a/development/autotests/hello-world-in.txt b/development/autotests/hello-world-in.txt new file mode 100644 index 0000000000..1b5295e961 --- /dev/null +++ b/development/autotests/hello-world-in.txt @@ -0,0 +1,42 @@ +# Sample test script for regression tests +# +# Adapted from the syntax of the original testkeys.py by McCabe-Dansted. +# Run this script by providing it as single argument to run-test.sh +# +# WARNING: Syntax is poorly parsed: no extra spaces anywhere +# +# Lines starting with a '#' on the 1st column are comments +# +# Exec +# Causes the execution of in a os.system() +# +# TestBegin +# Starts a test case under deterministic conditions: +# Any currently running instance of LyX is killed +# A new LyX instance is spawned, with the command-line args +# +# TestEnd +# Ends a test case, killing the LyX instance under test, and +# executing the commands into an os.system(). The exit status +# of the commands sequence determines success or failure of the test +# +# KK: +# Feeds the running LyX instance with the specified key characters. +# Special sequences: +# \A ...... sends the sequence [Alt+] +# \[] ...... sends a cursor movement, = Right, Left, Up, Down +# \[Tab] ........ sends a Tab character +# \[Return] ..... sends a Return +# +# +# This example instructs LyX to type "Hello World", then exports it as +# LaTeX, and verifies that the typed characters are indeed present in the +# exported document. +# +Exec rm -f /tmp/test.lyx /tmp/test.tex +TestBegin /tmp/test.lyx 2> /tmp/log 2>&1 +KK: Hello World +KK: \Afs +KK: \Ax +KK: buffer-export latex\[Return] +TestEnd grep "Hello World" /tmp/test.tex diff --git a/development/autotests/keytest.py b/development/autotests/keytest.py new file mode 100755 index 0000000000..618c8b9f17 --- /dev/null +++ b/development/autotests/keytest.py @@ -0,0 +1,350 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# This script generates hundreds of random keypresses per second, +# and sends them to the lyx window +# It requires xvkbd and wmctrl +# It generates a log of the KEYCODES it sends as development/keystest/out/KEYCODES +# +# Adapted by Tommaso Cucinotta from the original MonKey Test by +# John McCabe-Dansted. + +import random +import os +import re +import sys +import time +#from subprocess import call +import subprocess + +print 'Beginning keytest.py' + +FNULL = open('/dev/null', 'w') + +DELAY = '59' + +class CommandSource: + + def __init__(self): + keycode = [ + "\[Left]", + '\[Right]', + '\[Down]', + '\[Up]', + '\[BackSpace]', + '\[Delete]', + '\[Escape]', + ] + keycode[:0] = keycode + keycode[:0] = keycode + + keycode[:0] = ['\\'] + + for k in range(97, 123): + keycode[:0] = chr(k) + + for k in range(97, 123): + keycode[:0] = ["\A" + chr(k)] + + for k in range(97, 123): + keycode[:0] = ["\A" + chr(k)] + + for k in range(97, 123): + keycode[:0] = ["\C" + chr(k)] + + self.keycode = keycode + self.count = 0 + self.count_max = 1999 + + def getCommand(self): + self.count = self.count + 1 + if self.count % 200 == 0: + return 'RaiseLyx' + elif self.count > self.count_max: + os._exit(0) + else: + keystr = '' + for k in range(1, 2): + keystr = keystr + self.keycode[random.randint(1, + len(self.keycode)) - 1] + return 'KK: ' + keystr + + +class CommandSourceFromFile(CommandSource): + + def __init__(self, filename, p): + + self.infile = open(filename, 'r') + self.lines = self.infile.readlines() + self.infile.close() + linesbak = self.lines + self.p = p + print p, self.p, 'self.p' + self.i = 0 + self.count = 0 + self.loops = 0 + + # Now we start randomly dropping lines, which we hope are redundant + # p is the probability that any given line will be removed + + if p > 0.001: + if random.uniform(0, 1) < 0.5: + print 'randomdrop_independant\n' + self.randomdrop_independant() + else: + print 'randomdrop_slice\n' + self.randomdrop_slice() + if screenshot_out is None: + count_atleast = 100 + else: + count_atleast = 1 + self.max_count = max(len(self.lines) + 20, count_atleast) + if len(self.lines) < 1: + self.lines = linesbak + + def randomdrop_independant(self): + p = self.p + + # The next couple of lines are to ensure that at least one line is dropped + + drop = random.randint(0, len(self.lines) - 1) + del self.lines[drop] + #p = p - 1 / len(self.lines) + origlines = self.lines + self.lines = [] + for l in origlines: + if random.uniform(0, 1) < self.p: + print 'Randomly dropping line ' + l + '\n' + else: + self.lines.append(l) + print 'LINES\n' + print self.lines + sys.stdout.flush() + + def randomdrop_slice(self): + lines = self.lines + if random.uniform(0, 1) < 0.4: + lines.append(lines[0]) + del lines[0] + num_lines = len(lines) + max_drop = max(5, num_lines / 5) + num_drop = random.randint(1, 5) + drop_mid = random.randint(0, num_lines) + drop_start = max(drop_mid - num_drop / 2, 0) + drop_end = min(drop_start + num_drop, num_lines) + print drop_start, drop_mid, drop_end + print lines + del lines[drop_start:drop_end] + print lines + self.lines = lines + + def getCommand(self): + if self.count >= self.max_count: + os._exit(0) + if self.i >= len(self.lines): + self.loops = self.loops + 1 + if self.loops >= max_loops: + os._exit(0) + self.i = 0 + return 'Loop' + line = self.lines[self.i] + self.count = self.count + 1 + self.i = self.i + 1 + #print 'Line read: <<' + line + '>>\n' + sys.stdout.write('r') + return line.rstrip('\n').rstrip() + + +def lyx_sleeping(): + if lyx_pid is None: + return True + fname = '/proc/' + lyx_pid + '/status' + if not os.path.exists(fname): + return False + f = open(fname, 'r') + lines = f.readlines() + sleeping = lines[1].find('(sleeping)') > 0 + + # print 'LYX_STATE', lines[1] , 'SLEEPING=', sleeping + + return sleeping + + +def sendKeystring(keystr, LYX_PID): + + # print "sending keystring "+keystr+"\n" + + if not re.match(".*\w.*", keystr): + print 'print .' + keystr + '.\n' + keystr = 'a' + before_secs = time.time() + while not lyx_sleeping(): + time.sleep(0.02) + sys.stdout.write('.') + sys.stdout.flush() + if time.time() - before_secs > 180: + print 'Killing due to freeze (KILL_FREEZE)' + + # Do profiling, but sysprof has no command line interface? + # os.system("killall -KILL lyx") + + os._exit(1) + if not screenshot_out is None: + while not lyx_sleeping(): + time.sleep(0.02) + sys.stdout.write('.') + sys.stdout.flush() + print 'Making Screenshot: ' + screenshot_out + ' OF ' + infilename + time.sleep(0.2) + os.system('import -window root '+screenshot_out+str(x.count)+".png") + time.sleep(0.1) + sys.stdout.flush() + if (subprocess.call( + ["xvkbd", "-xsendevent", "-delay", DELAY, "-text", keystr], + stdout=FNULL,stderr=FNULL + ) == 0): + sys.stdout.write('*') + else: + sys.stdout.write('X') + +def system_retry(num_retry, cmd): + i = 0 + rtn = os.system(cmd) + while ( ( i < num_retry ) and ( rtn != 0) ): + i = i + 1 + rtn=os.system(cmd) + time.sleep(1) + if ( rtn != 0 ): + print "Command Failed: "+cmd + print " EXITING!\n" + os._exit(1) + +def RaiseWindow(): + #os.system("echo x-session-manager PID: $X_PID.") + #os.system("echo x-session-manager open files: `lsof -p $X_PID | grep ICE-unix | wc -l`") + ####os.system("wmctrl -l | ( grep '"+lyx_window_name+"' || ( killall lyx ; sleep 1 ; killall -9 lyx ))") + #os.system("wmctrl -R '"+lyx_window_name+"' ;sleep 0.1") + system_retry(30, "wmctrl -R '"+lyx_window_name+"'") + + +lyx_pid = os.environ.get('LYX_PID') +print 'lyx_pid: ' + str(lyx_pid) + '\n' +infilename = os.environ.get('KEYTEST_INFILE') +outfilename = os.environ.get('KEYTEST_OUTFILE') +max_drop = os.environ.get('MAX_DROP') +lyx_window_name = os.environ.get('LYX_WINDOW_NAME') +screenshot_out = os.environ.get('SCREENSHOT_OUT') + +max_loops = os.environ.get('MAX_LOOPS') +if max_loops is None: + max_loops = 3 + +lyx_exe = os.environ.get('LYX_EXE') +if lyx_exe is None: + lyx_exe = "lyx" + +file_new_command = os.environ.get('FILE_NEW_COMMAND') +if file_new_command is None: + file_new_command = "\Afn" + +ResetCommand = os.environ.get('RESET_COMMAND') +if ResetCommand is None: + ResetCommand = "\[Escape]\[Escape]\[Escape]\[Escape]" + file_new_command + #ResetCommand="\[Escape]\[Escape]\[Escape]\[Escape]\Cw\Cw\Cw\Cw\Cw\Afn" + +if lyx_window_name is None: + lyx_window_name = 'LyX' + +print 'outfilename: ' + outfilename + '\n' +print 'max_drop: ' + max_drop + '\n' + +if infilename is None: + print 'infilename is None\n' + x = CommandSource() + print 'Using x=CommandSource\n' +else: + print 'infilename: ' + infilename + '\n' + probability_we_drop_a_command = random.uniform(0, float(max_drop)) + print 'probability_we_drop_a_command: ' + print '%s' % probability_we_drop_a_command + print '\n' + x = CommandSourceFromFile(infilename, probability_we_drop_a_command) + print 'Using x=CommandSourceFromFile\n' + +outfile = open(outfilename, 'w') + +if lyx_pid != "": + RaiseWindow() + sendKeystring("\Afn", lyx_pid) + +write_commands = True + +while True: + #os.system('echo -n LOADAVG:; cat /proc/loadavg') + c = x.getCommand() + outfile.writelines(c + '\n') + outfile.flush() + if c[0] == '#': + print "\nIgnoring comment line: " + c + elif c[0:9] == 'TestBegin': + print "\n" + lyx_pid=os.popen("pidof lyx").read() + if lyx_pid != "": + print "Found running instance(s) of LyX: " + lyx_pid + ": killing them all\n" + os.system("killall lyx") + time.sleep(0.2) + print "Starting LyX . . ." + os.system(lyx_exe + c[9:] + "&") + while True: + lyx_pid=os.popen("pidof lyx").read().rstrip() + lyx_window_name=os.popen("wmctrl -l | grep 'lyx$\\|LyX:' | sed -e 's/.*\\([Ll]y[Xx].*\\)$/\\1/'").read().rstrip() + if lyx_window_name != "": + break + print 'lyx_win: ' + lyx_window_name + '\n' + print "Waiting for LyX to show up . . ." + time.sleep(1) + print 'lyx_pid: ' + lyx_pid + '\n' + print 'lyx_win: ' + lyx_window_name + '\n' + #RaiseWindow() + #sendKeystring("\Afn", lyx_pid) + elif c[0:5] == 'Sleep': + print "\nSleeping for " + c[6:] + "\n" + time.sleep(float(c[6:])) + elif c[0:4] == 'Exec': + cmd = c[5:].rstrip() + print "\nExecuting " + cmd + "\n" + os.system(cmd) + elif c == 'Loop': + outfile.close() + outfile = open(outfilename + '+', 'w') + print 'Now Looping' + elif c == 'RaiseLyx': + print 'Raising Lyx' + RaiseWindow() + elif c[0:4] == 'KK: ': + if os.path.exists('/proc/' + lyx_pid + '/status'): + sendKeystring(c[4:], lyx_pid) + else: + ##os.system('killall lyx; sleep 2 ; killall -9 lyx') + print 'No path /proc/' + lyx_pid + '/status, exiting' + os._exit(1) + elif c[0:4] == 'KD: ': + DELAY = c[4:].rstrip('\n') + print 'Setting DELAY to ' + DELAY + '.' + elif c == 'Loop': + RaiseWindow() + sendKeystring(ResetCommand, lyx_pid) + elif c[0:7] == 'TestEnd': + time.sleep(1) + print "\nTerminating lyx instance: " + str(lyx_pid) + "\n" + os.system("kill -9 " + str(lyx_pid) + "\n"); + cmd = c[7:].rstrip() + result = os.system(cmd) + print "Test case terminated: " + if result == 0: + print "Ok\n" + os._exit(0) + else: + print "FAIL\n" + os._exit(1) + else: + print "Unrecognised Command '" + c + "'\n" diff --git a/development/autotests/run-tests.sh b/development/autotests/run-tests.sh new file mode 100755 index 0000000000..ba4067d1f7 --- /dev/null +++ b/development/autotests/run-tests.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# This script runs all the test scripts it finds in the current folder +# Tests are identified as having a file name of *-in.txt +# For failed tests, the collected output is kept in the corresponding folder + +export LYX_EXE=../../../src/lyx +export KEYTEST=../keytest.py + +if [ "$#" -eq 0 ]; then + TESTS=$(ls *-in.txt); +else + TESTS=$* +fi + +echo +echo "Running test cases . . ." +failed=0 +for t in $(echo "$TESTS" | sed -e 's/-in.txt//g'); do + rm -rf "out-$t" + mkdir "out-$t" + cd "out-$t" + printf "%40s: " $t + if ../single-test.sh "../$t-in.txt" > keytest-log.txt 2>&1; then + echo Ok + cd .. + rm -rf "out-$t"; + else + echo FAILED + cd .. + failed=$[$failed+1]; + fi; +done + +echo +if [ $failed -eq 0 ]; then + echo "All tests SUCCESSFUL" +else + echo "There were $failed FAILED tests"; +fi + +echo diff --git a/development/autotests/single-test.sh b/development/autotests/single-test.sh new file mode 100755 index 0000000000..6f1a7b0131 --- /dev/null +++ b/development/autotests/single-test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script invokes the keytest.py script with the simple set-up needed +# to re-run deterministic regression tests that one would like to have. +# +# Usage: run-test.sh +# +# See the hello-world-in.txt for an example syntax and description + +KEYTEST=${KEYTEST:-./keytest.py} + +export KEYTEST_OUTFILE=out.txt +export KEYTEST_INFILE=in-sample.txt +if [ "$1" != "" ]; then + KEYTEST_INFILE="$1"; +fi + +export MAX_DROP=0 +export LYX_WINDOW_NAME=$(wmctrl -l | grep LyX | sed -e 's/.*\(LyX: .*\)$/\1/') +export MAX_LOOPS=1 +export LYX_EXE=${LYX_EXE:-../../src/lyx} + +$KEYTEST -- 2.39.2