Small fixes for testing in VM
[ci.git] / test-in-vm.sh
blobc962272dec8dfc94d6794b5573554c2c9e13d1d7
1 #!/bin/bash
4 # Copyright (c) 2016 Vojtech Horky
5 # All rights reserved.
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
11 # - Redistributions of source code must retain the above copyright
12 # notice, this list of conditions and the following disclaimer.
13 # - Redistributions in binary form must reproduce the above copyright
14 # notice, this list of conditions and the following disclaimer in the
15 # documentation and/or other materials provided with the distribution.
16 # - The name of the author may not be used to endorse or promote products
17 # derived from this software without specific prior written permission.
19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 # This code was originally inspired by "virtual testing lab" scripts created
33 # by Jan Buchar for testing his firewall implementation in HelenOS:
34 # https://code.launchpad.net/~teyras/hpf-virtlab/trunk
37 xx_echo() {
38 echo ">>" "$@"
41 xx_echo2() {
42 echo " -" "$@"
45 xx_debug() {
46 $XX_DEBUG_ECHO " :" "$@"
49 xx_run_debug() {
50 xx_debug "$@"
51 "$@"
54 xx_activity() {
55 $XX_ACTIVITY_ECHO " +" "$@"
58 xx_fatal() {
59 echo "!!" "$@"
60 xx_shutdown
63 ## Prepares pipe to QEMU monitor.
64 # Feed QEMU commands to stdin of this function.
66 # Do not forget to check for return code as checking it inside pipe
67 # is not very useful (we cannot terminate the script from that).
69 # @param 1 Machine id.
70 xx_do_qemu_command_pipe() {
71 socat STDIN "UNIX-CONNECT:$XX_TEMP/$1.monitor"
74 ## Sends command to QEMU monitor.
75 # This function terminates the scenario on failure of the monitoring pipe.
77 # @param 1 Machine id.
78 # @param 2 Command to send.
79 xx_do_qemu_command() {
80 xx_debug "echo '$2' | socat STDIN 'UNIX-CONNECT:$XX_TEMP/$1.monitor'"
81 echo "$2" | socat STDIN "UNIX-CONNECT:$XX_TEMP/$1.monitor"
82 res=$?
83 if [ $res -ne 0 ]; then
84 xx_shutdown
88 ## Types the text to a specific machine.
90 # @param 1 Machine id.
91 # @param 2 Text to type.
92 xx_do_type() {
93 ( echo "$2" | fold -w 1 | sed \
94 -e 's/ /spc/' \
95 -e 's/\./dot/' \
96 -e 's/-/minus/' \
97 -e 's/+/shift-equal/' \
98 -e 's/_/shift-minus/' \
99 -e 's/@/shift-2/' \
100 -e 's/:/shift-semicolon/' \
101 -e 's/\//slash/' \
102 -e 's/=/equal/' \
103 -e 's/|/shift-backslash/' \
104 -e 's/\([[:upper:]]\)/shift-\L\1/' \
105 -e 's/\\/backslash/' | \
106 while read code; do
107 xx_do_qemu_command "$1" "sendkey $code"
108 done
110 res=$?
111 if [ $res -ne 0 ]; then
112 xx_shutdown
116 xx_get_var() {
117 local varname="$1"
118 local default="$2"
119 shift 2
120 local i
121 for i in "$@"; do
122 case $i in
123 $varname=*)
124 echo "$i" | cut '-d=' -f 2-
125 return
129 esac
130 done
131 echo "$default"
134 xx_get_def_var() {
135 local i
136 for i in "$@"; do
137 case $i in
138 *=*)
141 echo "$i"
143 esac
144 done
145 echo
148 xx_get_boolean_var() {
149 value=`xx_get_var "$@" | tr 'A-Z' 'a-z'`
150 if [ "$value" = "yes" -o "$value" = "true" ]; then
151 echo "true"
152 elif [ "$value" = "no" -o "$value" = "false" ]; then
153 echo "false"
154 else
155 echo
160 xx_shutdown() {
161 local i
162 for i in $XX_KNOWN_MACHINES; do
163 xx_stop_machine name="$i"
164 done
165 exit 1
168 xx_assert_var() {
169 if [ -z "$2" ]; then
170 xx_echo "Option $1 not specified or invalid, terminating."
171 xx_shutdown
175 xx_do_check_var() {
176 if [ -z "$2" ]; then
177 xx_fatal "Option $1 not specified with no suitable default."
179 if [ -n "$3" ]; then
180 if ! echo "$2" | grep -q "$3"; then
181 xx_fatal "Option $1 does not match '$3' (got '$2')."
186 xx_do_compute_timeout() {
187 echo $(( `date +%s` + 10 * $1 ))
191 xx_start_machine() {
192 local cdrom=`xx_get_var cdrom "$XX_CDROM_FILE" "$@"`
193 local name=`xx_get_var name default "$@"`
194 local wait_for_vterm=`xx_get_boolean_var vterm true "$@"`
196 xx_do_check_var "xx_start_machine/name" "$name" '^[a-zA-Z][0-9a-zA-Z]*$'
197 xx_do_check_var "xx_start_machine/cdrom" "$cdrom"
199 local extra_opts=""
200 if $XX_HEADLESS; then
201 extra_opts="-display none"
203 if $XX_USE_KVM; then
204 extra_opts="$extra_opts -enable-kvm"
207 xx_echo "Starting machine $name from $cdrom."
209 local qemu_command=""
210 if [ $XX_ARCH == "ia32" ]; then
211 qemu_command=qemu-system-i386
212 elif [ $XX_ARCH == "amd64" ]; then
213 qemu_command=qemu-system-x86_64
215 if [ -z "$qemu_command" ]; then
216 xx_fatal "Unable to find proper emulator."
219 xx_run_debug $qemu_command \
220 $extra_opts \
221 -device e1000,vlan=0 -net user \
222 -redir udp:8080::8080 -redir udp:8081::8081 \
223 -redir tcp:8080::8080 -redir tcp:8081::8081 \
224 -redir tcp:2223::2223 \
225 -usb \
226 -m 256 \
227 -daemonize -pidfile "$XX_TEMP/$name.pid" \
228 -monitor "unix:$XX_TEMP/$name.monitor,server,nowait" \
229 -boot d \
230 -cdrom "$cdrom"
231 sleep 1
232 xx_do_qemu_command "$name" "sendkey ret"
234 XX_LAST_MACHINE=$name
236 XX_KNOWN_MACHINES="$XX_KNOWN_MACHINES $name"
238 if $wait_for_vterm; then
239 xx_echo2 "Waiting for OS to boot into GUI..."
240 if ! xx_do_wait_for_text "$name" `xx_do_compute_timeout 15` "to see a few survival tips"; then
241 xx_fatal "OS have not booted into a known state."
249 xx_stop_machine() {
250 local name=`xx_get_var name $XX_LAST_MACHINE "$@"`
252 xx_echo "Forcefully killing machine $name."
253 xx_do_qemu_command "$name" "quit"
254 sleep 1
256 if [ "$name" = "$XX_LAST_MACHINE" ]; then
257 XX_LAST_MACHINE=""
262 ## Wait for text to appear on machine console.
264 # @param 1 Machine id.
265 # @param 2 UNIX end-time (date %s + TIME_OUT).
266 # @param 3 Text to match.
267 xx_do_wait_for_text() {
268 while true; do
269 xx_activity "Taking screenshot, looking for '$3'."
271 xx_do_screenshot "$1" "$XX_TEMP/$1-full.ppm" \
272 "$XX_TEMP/$1-term.png" "$XX_TEMP/$1-term.txt"
274 if grep -q "$3" <"$XX_TEMP/$1-term.txt"; then
275 return 0
278 if [ `date +%s` -gt $2 ]; then
279 return 1
282 sleep 1
283 done
286 xx_assert() {
287 local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
288 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
289 local error_msg=`xx_get_var error "" "$@"`
290 local text=`xx_get_def_var "$@"`
292 xx_echo "Checking that '$text' will appear on $machine within ${timeout}s."
294 if ! xx_do_wait_for_text "$machine" `xx_do_compute_timeout $timeout` "$text"; then
295 xx_fatal "Failed to recognize '$text' on $machine."
299 xx_die_on() {
300 local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
301 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
302 local error_msg=`xx_get_var message "" "$@"`
303 local text=`xx_get_def_var "$@"`
305 xx_echo "Checking that '$text' will not appear on $machine within ${timeout}s."
307 if xx_do_wait_for_text "$machine" `xx_do_compute_timeout $timeout` "$text"; then
308 xx_fatal "Prohibited text '$text' spotted on $machine."
312 xx_sleep() {
313 local amount=`xx_get_def_var "$@"`
315 xx_echo "Waiting for ${amount}s."
316 sleep $amount
320 xx_cls() {
321 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
323 xx_echo "Clearing the screen on $machine."
324 for i in `seq 1 35`; do
325 xx_do_qemu_command "$machine" "sendkey ret"
326 done
327 sleep 1
330 xx_do_ocr_prepare() {
331 convert "$1" -crop "8x16" +repage +adjoin txt:- \
332 | sed -e 's|[0-9]*,[0-9]*: ([^)]*)[ ]*#\([0-9A-Fa-f]\{6\}\).*|\1|' -e 's:^#.*:@:' -e 's#000000#0#g' -e 's#FFFFFF#F#' \
333 | sed -e ':a' -e 'N;s#\n##;s#^@##;/@$/{s#@$##p;d}' -e 't a'
337 xx_do_ocr() {
338 xx_do_ocr_prepare "$1" | sed -f "$XX_MY_HOME/ocr.sed"
341 xx_do_screenshot() {
342 xx_do_qemu_command "$1" "screendump $2"
343 if [ -n "$3" ]; then
344 convert "$2" -crop 640x480+4+24 +repage -colors 2 -monochrome "$3"
345 if [ -n "$4" ]; then
346 xx_do_ocr "$3" | paste -sd '' | fold -w 80 >"$4"
351 ## Wait for text to appear on machine console.
353 # @param 1 Machine id.
354 # @param 2 UNIX end-time (date %s + TIME_OUT).
355 # @param 3 Text that shall not be matched.
356 # @param 4 Text that shall be matched before timeout.
357 # @param 5 Consider prompt as a success.
358 # @param 6 Check for standard Bdsh error messages.
359 # @retval 0 Expected text was matched.
360 # @retval 1 Prompt text was matched.
361 # @retval 2 Time-out, nothing matched.
362 # @retval 3 Unexpected text was matched.
363 # @retval 4 Standard Bdsh error message detected.
364 xx_do_complex_text_waiting() {
365 xx_debug "waiting: $1/$2 match='$4', no-match='$3'"
366 xx_debug "waiting: $1/$2 prompt_is_success=$5 check_for_bdsh_error=$6"
368 while true; do
369 xx_activity "Taking screenshot, checking for specific output."
371 xx_do_screenshot "$1" "$XX_TEMP/$1-full.ppm" \
372 "$XX_TEMP/$1-term.png" "$XX_TEMP/$1-term.txt"
374 if [ -n "$3" ] && grep -q "$3" <"$XX_TEMP/$1-term.txt"; then
375 return 3
378 if [ -n "$4" ] && grep -q "$4" <"$XX_TEMP/$1-term.txt" ; then
379 return 0
382 if $5 && grep -q '^.*/ # _[ ]*$' <"$XX_TEMP/$1-term.txt" ; then
383 return 0
386 if $6 && grep -q -e 'Cannot spawn' -e 'Command failed' <"$XX_TEMP/$1-term.txt"; then
387 return 4
390 if [ `date +%s` -gt $2 ]; then
391 return 2
394 sleep 1
395 done
398 xx_cmd() {
399 local cmd=`xx_get_def_var "$@"`
400 local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
401 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
402 local error_msg=`xx_get_var error "" "$@"`
403 local text=`xx_get_var assert "" "$@"`
404 local negtext=`xx_get_var die_on "" "$@"`
405 local text_is_empty=`if [ -n "$text" ]; then echo false; else echo true; fi`
407 xx_echo "Sending '$cmd' to $machine."
408 xx_do_type "$machine" "$cmd"
409 xx_do_qemu_command "$machine" "sendkey ret"
412 xx_do_complex_text_waiting "$machine" `xx_do_compute_timeout $timeout` \
413 "$negtext" "$text" $text_is_empty true
414 local res=$?
416 xx_debug "xx_do_complex_text_waiting = $res"
418 case $res in
419 0|1)
420 return 0
423 if $text_is_empty; then
424 xx_fatal "Command timed-out."
425 else
426 xx_fatal "Failed to match '$text'."
429 3|4)
430 if [ -n "$error_msg" ]; then
431 xx_fatal "$error_msg"
432 else
433 xx_fatal "Command failed."
437 xx_fatal "Internal error, we shall never reach this line."
439 esac
444 xx_do_print_help() {
445 echo "Usage: $1 [options] scenario-file [scenarios-file ...]"
446 cat <<'EOF_USAGE'
447 where [options] are:
449 --help Print this help and exit.
450 --headless Hide the screen of the virtual machine.
451 --root=DIR Find HelenOS image in the source directory.
452 --image=FILE File with main HelenOS image (specify --arch).
453 --arch=ARCH Architecture of the image file (see --image).
454 --no-kvm Do not try to run QEMU with KVM enabled.
455 --fail-fast Exit with first error.
456 --debug Print (a lot of) debugging messages.
458 EOF_USAGE
462 XX_MY_HOME=`which -- "$0" 2>/dev/null`
463 # Maybe, we are running Bash
464 [ -z "$XX_MY_HOME" ] && XX_MY_HOME=`which -- "$BASH_SOURCE" 2>/dev/null`
465 XX_MY_HOME=`dirname -- "$XX_MY_HOME"`
467 XX_DEBUG_ECHO=:
468 XX_ACTIVITY_ECHO=:
469 XX_TRAIN_OCR=false
470 XX_HEADLESS=false
471 XX_USE_KVM=true
472 XX_TEMP="$PWD/tmp-vm/"
473 XX_HELENOS_ROOT="."
474 XX_AUTODETECT_HELENOS=false
475 XX_ARCH=""
476 XX_CDROM_FILE=""
477 XX_FAIL_FAST=false
479 XX_KNOWN_MACHINES=""
480 XX_DEFAULT_TIMEOUT=5
481 XX_LAST_MACHINE=default
484 # Replace with getopt eventually.
485 while [ $# -gt 0 ]; do
486 case "$1" in
487 --headless)
488 XX_HEADLESS=true
490 --train-ocr)
491 XX_TRAIN_OCR=true
493 --train-ocr=*)
494 XX_TRAIN_OCR=true
495 XX_TRAIN_OCR_FILE=`echo "$1" | cut '-d=' -f 2-`
497 --debug)
498 XX_DEBUG_ECHO=echo
500 --activity)
501 XX_ACTIVITY_ECHO=echo
503 --root=*)
504 XX_HELENOS_ROOT=`echo "$1" | cut '-d=' -f 2-`
505 XX_AUTODETECT_HELENOS=true
507 --image=*)
508 XX_CDROM_FILE=`echo "$1" | cut '-d=' -f 2-`
509 XX_AUTODETECT_HELENOS=false
511 --arch=*)
512 XX_ARCH=`echo "$1" | cut '-d=' -f 2-`
513 XX_AUTODETECT_HELENOS=false
515 --temp=*)
516 XX_TEMP=`echo "$1" | cut '-d=' -f 2-`
518 --no-kvm)
519 XX_USE_KVM=false
521 --fail-fast)
522 XX_FAIL_FAST=true
524 --help|-h|-?)
525 xx_do_print_help "$0"
526 exit 0
528 --*)
529 xx_fatal "Unknown option $1."
532 break
534 esac
535 shift
536 done
538 mkdir -p "$XX_TEMP"
540 if $XX_AUTODETECT_HELENOS; then
541 if ! [ -r "$XX_HELENOS_ROOT/Makefile.config" ]; then
542 xx_fatal "Cannot open $XX_HELENOS_ROOT/Makefile.config."
544 XX_ARCH=`grep '^PLATFORM = ' "$XX_HELENOS_ROOT/Makefile.config" | cut '-d=' -f 2- | tr -d ' '`
545 XX_CDROM_FILE=$XX_HELENOS_ROOT/`cat "$XX_HELENOS_ROOT/defaults/$XX_ARCH/output"`
549 if $XX_TRAIN_OCR; then
550 if [ -z "$XX_TRAIN_OCR_FILE" ]; then
551 echo "Usage notes for --train-ocr command-line switch."
552 echo
553 echo "Prepare HelenOS image.iso with train-ocr.txt in its root."
554 echo
555 echo "Run this script with the ISO and with --train-ocr=out.sed"
556 echo "where out.sed is output file with OCR commands."
557 echo
558 echo "If the script finishes with no error you can replace the"
559 echo "old OCR scheme (ocr.sed) with the new one in out.sed."
560 echo
561 exit
564 # Set to false to debug the script below without running QEMU
565 # all the time
566 if true; then
567 xx_start_machine name=ocr vterm=false
568 xx_echo "Will display the training set on the VM screen"
569 xx_echo "(this will take a while)."
570 sleep 20
571 xx_do_type ocr "cat train-ocr.txt"
572 sleep 1
573 xx_do_qemu_command ocr "sendkey ret"
574 sleep 5
575 xx_do_screenshot ocr "$XX_TEMP/ocr-full.ppm" "$XX_TEMP/ocr-term.png"
576 xx_stop_machine
579 xx_echo "Doing OCR..."
580 xx_do_ocr_prepare "$XX_TEMP/ocr-term.png" \
582 prev1_line=""
583 prev2_line=""
584 prev3_line=""
585 counter=0
586 while read current_line; do
587 if [ $counter -gt 5 ]; then
588 while read current_line; do
589 if ! [ "$current_line" = "$prev1_line" -o "$current_line" = "$prev2_line" ]; then
590 break
592 done
593 alphabet=`sed -n 's#^\(.\)\(.\)\(\1\2\)\{4,\}##p' train-ocr.txt`
594 alphabet_size=`echo "$alphabet" | wc -c`
595 for i in `seq 1 $(( $alphabet_size - 1))`; do
596 symbol="`echo \"$alphabet\" | fold -w 1 | sed -n \"${i}p\" | sed 's#[*\.\&\\:\/\[]\|\]#\\\\&#g'`"
597 echo "s:$current_line:$symbol:"
598 read current_line
599 done
600 echo "$current_line" | tr 'F' '0' | sed 's#.*#s:&:_:#'
601 echo "$current_line" | tr '0F' '.' | sed 's#.*#s:&:?:#'
602 break
604 if [ "$current_line" = "$prev2_line" -a "$prev1_line" = "$prev3_line" -a "$prev1_line" != "$prev2_line" ]; then
605 counter=$(( $counter + 1 ))
606 else
607 counter=0
609 prev3_line="$prev2_line"
610 prev2_line="$prev1_line"
611 prev1_line="$current_line"
612 done >"$XX_TRAIN_OCR_FILE"
614 xx_echo "New OCR scheme is in $XX_TRAIN_OCR_FILE."
615 exit
619 XX_RESULT_SUMMARY="$XX_TEMP/summary.$$.txt"
621 date '+# Execution started on %Y-%m-%d %H:%M.' >"$XX_RESULT_SUMMARY"
622 echo '# =======================================' >>"$XX_RESULT_SUMMARY"
625 # Run it
626 for i in "$@"; do
627 echo "# Starting scenario $i..."
629 . $i
631 if [ $? -eq 0 ]; then
632 res="OK"
633 else
634 res="FAIL"
636 echo "# Scenario $i terminated, $res."
637 printf '%-35s %4s\n' "$i" "$res" >>"$XX_RESULT_SUMMARY"
638 if $FAIL_FAST && [ "$res" = "FAIL" ]; then
639 exit 1
641 done
644 date '+# Execution finished on %Y-%m-%d %H:%M.' >>"$XX_RESULT_SUMMARY"
646 # Display the results.
647 echo
649 cat "$XX_RESULT_SUMMARY"