HTML report: link to summary matrix
[ci.git] / test-in-vm.sh
blob4a70dd22c1b960667b694fd7740165d92e69f1b7
1 #!/bin/bash
4 # Copyright (c) 2016 Vojtech Horky
5 # All rights reserved.
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
11 # - Redistributions of source code must retain the above copyright
12 # notice, this list of conditions and the following disclaimer.
13 # - Redistributions in binary form must reproduce the above copyright
14 # notice, this list of conditions and the following disclaimer in the
15 # documentation and/or other materials provided with the distribution.
16 # - The name of the author may not be used to endorse or promote products
17 # derived from this software without specific prior written permission.
19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 # This code was originally inspired by "virtual testing lab" scripts created
33 # by Jan Buchar for testing his firewall implementation in HelenOS:
34 # https://code.launchpad.net/~teyras/hpf-virtlab/trunk
37 xx_echo() {
38 echo ">>" "$@"
41 xx_echo2() {
42 echo " -" "$@"
45 xx_debug() {
46 $XX_DEBUG_ECHO " :" "$@"
49 xx_debug_filter() {
50 sed 's#.*# : &#'
54 xx_run_debug() {
55 xx_debug "$@"
56 "$@"
59 xx_activity() {
60 $XX_ACTIVITY_ECHO " +" "$@"
63 xx_fatal() {
64 echo "!!" "$@"
65 xx_shutdown
68 ## Prepares pipe to QEMU monitor.
69 # Feed QEMU commands to stdin of this function.
71 # Do not forget to check for return code as checking it inside pipe
72 # is not very useful (we cannot terminate the script from that).
74 # @param 1 Machine id.
75 xx_do_qemu_command_pipe() {
76 socat STDIN "UNIX-CONNECT:$XX_TEMP/$1.monitor"
79 ## Sends command to QEMU monitor.
80 # This function terminates the scenario on failure of the monitoring pipe.
82 # @param 1 Machine id.
83 # @param 2 Command to send.
84 xx_do_qemu_command() {
85 xx_debug "echo '$2' | socat STDIN 'UNIX-CONNECT:$XX_TEMP/$1.monitor'"
86 echo "$2" | socat STDIN "UNIX-CONNECT:$XX_TEMP/$1.monitor"
87 res=$?
88 if [ $res -ne 0 ]; then
89 xx_shutdown
93 ## Types the text to a specific machine.
95 # @param 1 Machine id.
96 # @param 2 Text to type.
97 xx_do_type() {
98 ( echo "$2" | fold -w 1 | sed \
99 -e 's/ /spc/' \
100 -e 's/\./dot/' \
101 -e 's/-/minus/' \
102 -e 's/+/shift-equal/' \
103 -e 's/_/shift-minus/' \
104 -e 's/@/shift-2/' \
105 -e 's/:/shift-semicolon/' \
106 -e 's/\//slash/' \
107 -e 's/=/equal/' \
108 -e 's/|/shift-backslash/' \
109 -e 's/\([[:upper:]]\)/shift-\L\1/' \
110 -e 's/\\/backslash/' | \
111 while read code; do
112 xx_do_qemu_command "$1" "sendkey $code"
113 done
115 res=$?
116 if [ $res -ne 0 ]; then
117 xx_shutdown
121 xx_get_var() {
122 local varname="$1"
123 local default="$2"
124 shift 2
125 local i
126 for i in "$@"; do
127 case $i in
128 $varname=*)
129 echo "$i" | cut '-d=' -f 2-
130 return
134 esac
135 done
136 echo "$default"
139 xx_get_def_var() {
140 local i
141 for i in "$@"; do
142 echo "$i" | grep -v '^[a-zA-Z][a-zA-Z0-9_]*='
143 done
144 echo
147 xx_get_boolean_var() {
148 value=`xx_get_var "$@" | tr 'A-Z' 'a-z'`
149 if [ "$value" = "yes" -o "$value" = "true" ]; then
150 echo "true"
151 elif [ "$value" = "no" -o "$value" = "false" ]; then
152 echo "false"
153 else
154 echo
159 xx_shutdown() {
160 local i
161 for i in $XX_KNOWN_MACHINES; do
162 xx_stop_machine name="$i"
163 done
164 exit 1
167 xx_assert_var() {
168 if [ -z "$2" ]; then
169 xx_echo "Option $1 not specified or invalid, terminating."
170 xx_shutdown
174 xx_do_check_var() {
175 if [ -z "$2" ]; then
176 xx_fatal "Option $1 not specified with no suitable default."
178 if [ -n "$3" ]; then
179 if ! echo "$2" | grep -q "$3"; then
180 xx_fatal "Option $1 does not match '$3' (got '$2')."
185 xx_do_compute_timeout() {
186 echo $(( `date +%s` + 10 * $1 ))
190 xx_start_machine() {
191 local cdrom=`xx_get_var cdrom "$XX_CDROM_FILE" "$@"`
192 local name=`xx_get_var name default "$@"`
193 local wait_for_vterm=`xx_get_boolean_var vterm true "$@"`
195 xx_do_check_var "xx_start_machine/name" "$name" '^[a-zA-Z][0-9a-zA-Z]*$'
196 xx_do_check_var "xx_start_machine/cdrom" "$cdrom"
198 local extra_opts=""
199 if $XX_HEADLESS; then
200 extra_opts="-display none"
202 if $XX_USE_KVM; then
203 extra_opts="$extra_opts -enable-kvm"
206 xx_echo "Starting machine $name from $cdrom."
208 local qemu_command=""
209 if [ "$XX_ARCH" == "ia32" ]; then
210 qemu_command=qemu-system-i386
211 elif [ "$XX_ARCH" == "amd64" ]; then
212 qemu_command=qemu-system-x86_64
214 if [ -z "$qemu_command" ]; then
215 xx_fatal "Unable to find proper emulator."
218 xx_run_debug $qemu_command \
219 $extra_opts \
220 -device e1000,vlan=0 -net user \
221 -redir udp:8080::8080 -redir udp:8081::8081 \
222 -redir tcp:8080::8080 -redir tcp:8081::8081 \
223 -redir tcp:2223::2223 \
224 -usb \
225 -m "${XX_MEMORY_MB}" \
226 -daemonize -pidfile "$XX_TEMP/$name.pid" \
227 -monitor "unix:$XX_TEMP/$name.monitor,server,nowait" \
228 -boot d \
229 -cdrom "$cdrom"
230 sleep 1
231 xx_do_qemu_command "$name" "sendkey ret"
233 XX_LAST_MACHINE=$name
235 XX_KNOWN_MACHINES="$XX_KNOWN_MACHINES $name"
237 if $wait_for_vterm; then
238 xx_echo2 "Waiting for OS to boot into GUI..."
239 if ! xx_do_wait_for_text "$name" `xx_do_compute_timeout 60` "to see a few survival tips"; then
240 xx_fatal "OS have not booted into a known state."
248 xx_stop_machine() {
249 local name=`xx_get_var name $XX_LAST_MACHINE "$@"`
251 xx_echo "Forcefully killing machine $name."
252 xx_do_qemu_command "$name" "quit"
253 sleep 1
255 if [ "$name" = "$XX_LAST_MACHINE" ]; then
256 XX_LAST_MACHINE=""
261 ## Wait for text to appear on machine console.
263 # @param 1 Machine id.
264 # @param 2 UNIX end-time (date %s + TIME_OUT).
265 # @param 3 Text to match.
266 xx_do_wait_for_text() {
267 while true; do
268 xx_activity "Taking screenshot, looking for '$3'."
270 xx_do_screenshot "$1" "$XX_TEMP/$1-full.ppm" \
271 "$XX_TEMP/$1-term.png" "$XX_TEMP/$1-term.txt"
273 if grep -q "$3" <"$XX_TEMP/$1-term.txt"; then
274 return 0
277 if [ `date +%s` -gt $2 ]; then
278 return 1
281 sleep 1
282 done
285 xx_assert() {
286 local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
287 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
288 local error_msg=`xx_get_var error "" "$@"`
289 local text=`xx_get_def_var "$@"`
291 xx_echo "Checking that '$text' will appear on $machine within ${timeout}s."
293 if ! xx_do_wait_for_text "$machine" `xx_do_compute_timeout $timeout` "$text"; then
294 xx_fatal "Failed to recognize '$text' on $machine."
298 xx_die_on() {
299 local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
300 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
301 local error_msg=`xx_get_var message "" "$@"`
302 local text=`xx_get_def_var "$@"`
304 xx_echo "Checking that '$text' will not appear on $machine within ${timeout}s."
306 if xx_do_wait_for_text "$machine" `xx_do_compute_timeout $timeout` "$text"; then
307 xx_fatal "Prohibited text '$text' spotted on $machine."
311 xx_sleep() {
312 local amount=`xx_get_def_var "$@"`
314 xx_echo "Waiting for ${amount}s."
315 sleep $amount
319 xx_cls() {
320 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
322 xx_echo "Clearing the screen on $machine."
323 for i in `seq 1 35`; do
324 xx_do_qemu_command "$machine" "sendkey ret"
325 done
326 sleep 1
329 xx_do_ocr_prepare() {
330 convert "$1" -crop "8x16" +repage +adjoin txt:- \
331 | sed -e 's|[0-9]*,[0-9]*: ([^)]*)[ ]*#\([0-9A-Fa-f]\{6\}\).*|\1|' -e 's:^#.*:@:' -e 's#000000#0#g' -e 's#FFFFFF#F#' \
332 | sed -e ':a' -e 'N;s#\n##;s#^@##;/@$/{s#@$##p;d}' -e 't a'
336 xx_do_ocr() {
337 xx_do_ocr_prepare "$1" | sed -f "$XX_MY_HOME/ocr.sed"
340 xx_do_screenshot() {
341 xx_do_qemu_command "$1" "screendump $2"
342 if [ -n "$3" ]; then
343 # Looping as the screenshot might take some time to save
344 local retries=3
345 while true; do
346 if convert "$2" -crop 640x480+4+24 +repage -colors 2 -monochrome "$3"; then
347 break
349 retries=$(( $retries - 1 ))
350 if [ $retries -le 0 ]; then
351 xx_fatal "Failed to convert screen to monochrome."
353 sleep 1
354 done
356 if [ -n "$4" ]; then
357 xx_do_ocr "$3" | paste -sd '' | fold -w 80 >"$4"
358 if $XX_DUMP_TERMINAL; then
359 local print_it=false
360 if [ -e "$4.previous" ]; then
361 if ! cmp --quiet "$4.previous" "$4"; then
362 print_it=true
364 else
365 print_it=true
368 if $print_it; then
369 xx_debug "Terminal content:"
370 sed 's#.*# | &#' <"$4" | xx_debug_filter
372 cat "$4" >"$4.previous"
378 ## Wait for text to appear on machine console.
380 # @param 1 Machine id.
381 # @param 2 UNIX end-time (date %s + TIME_OUT).
382 # @param 3 Text that shall not be matched.
383 # @param 4 Text that shall be matched before timeout.
384 # @param 5 Consider prompt as a success.
385 # @param 6 Check for standard Bdsh error messages.
386 # @retval 0 Expected text was matched.
387 # @retval 1 Prompt text was matched.
388 # @retval 2 Time-out, nothing matched.
389 # @retval 3 Unexpected text was matched.
390 # @retval 4 Standard Bdsh error message detected.
391 xx_do_complex_text_waiting() {
392 xx_debug "waiting: $1/$2 match='$4', no-match='$3'"
393 xx_debug "waiting: $1/$2 prompt_is_success=$5 check_for_bdsh_error=$6"
395 while true; do
396 xx_activity "Taking screenshot, checking for specific output."
398 xx_do_screenshot "$1" "$XX_TEMP/$1-full.ppm" \
399 "$XX_TEMP/$1-term.png" "$XX_TEMP/$1-term.txt"
401 if [ -n "$3" ] && grep -q "$3" <"$XX_TEMP/$1-term.txt"; then
402 return 3
405 if [ -n "$4" ] && grep -q "$4" <"$XX_TEMP/$1-term.txt" ; then
406 return 0
409 if $6 && grep -q -e 'Cannot spawn' -e 'Command failed' <"$XX_TEMP/$1-term.txt"; then
410 return 4
413 if $5 && grep -q '^/[-a-zA-Z0-9_/.]* # _[ ]*$' <"$XX_TEMP/$1-term.txt" ; then
414 return 0
417 if [ `date +%s` -gt $2 ]; then
418 return 2
421 sleep 1
422 done
425 xx_cmd() {
426 local cmd=`xx_get_def_var "$@"`
427 local timeout=`xx_get_var timeout $XX_DEFAULT_TIMEOUT "$@"`
428 local machine=`xx_get_var machine $XX_LAST_MACHINE "$@"`
429 local error_msg=`xx_get_var error "" "$@"`
430 local text=`xx_get_var assert "" "$@"`
431 local negtext=`xx_get_var die_on "" "$@"`
432 local text_is_empty=`if [ -n "$text" ]; then echo false; else echo true; fi`
434 xx_echo "Sending '$cmd' to $machine."
435 xx_do_type "$machine" "$cmd"
436 xx_do_qemu_command "$machine" "sendkey ret"
439 xx_do_complex_text_waiting "$machine" `xx_do_compute_timeout $timeout` \
440 "$negtext" "$text" $text_is_empty true
441 local res=$?
443 xx_debug "xx_do_complex_text_waiting = $res"
445 case $res in
446 0|1)
447 return 0
450 if $text_is_empty; then
451 xx_fatal "Command timed-out."
452 else
453 xx_fatal "Failed to match '$text'."
456 3|4)
457 if [ -n "$error_msg" ]; then
458 xx_fatal "$error_msg"
459 else
460 xx_fatal "Command failed."
464 xx_fatal "Internal error, we shall never reach this line."
466 esac
471 xx_do_print_help() {
472 echo "Usage: $1 [options] scenario-file [scenarios-file ...]"
473 cat <<'EOF_USAGE'
474 where [options] are:
476 --help Print this help and exit.
477 --headless Hide the screen of the virtual machine.
478 --root=DIR Find HelenOS image in the source directory.
479 --image=FILE File with main HelenOS image (specify --arch).
480 --arch=ARCH Architecture of the image file (see --image).
481 --no-kvm Do not try to run QEMU with KVM enabled.
482 --memory=INT Size of VM RAM in INT MB.
483 --fail-fast Exit with first error.
484 --debug Print (a lot of) debugging messages.
485 --activity Print messages about background activity.
486 --dump-terminal Print contents of the vterm (implies --debug).
487 --train-ocr For experts: train OCR for vterm contents.
489 EOF_USAGE
493 XX_MY_HOME=`which -- "$0" 2>/dev/null`
494 # Maybe, we are running Bash
495 [ -z "$XX_MY_HOME" ] && XX_MY_HOME=`which -- "$BASH_SOURCE" 2>/dev/null`
496 XX_MY_HOME=`dirname -- "$XX_MY_HOME"`
498 XX_DEBUG_ECHO=:
499 XX_ACTIVITY_ECHO=:
500 XX_DUMP_TERMINAL=false
501 XX_TRAIN_OCR=false
502 XX_HEADLESS=false
503 XX_USE_KVM=true
504 XX_TEMP="$PWD/tmp-vm/"
505 XX_HELENOS_ROOT="."
506 XX_AUTODETECT_HELENOS=false
507 XX_ARCH=""
508 XX_CDROM_FILE=""
509 XX_MEMORY_MB=""
510 XX_FAIL_FAST=false
512 XX_KNOWN_MACHINES=""
513 XX_DEFAULT_TIMEOUT=5
514 XX_LAST_MACHINE=default
517 # Replace with getopt eventually.
518 while [ $# -gt 0 ]; do
519 case "$1" in
520 --headless)
521 XX_HEADLESS=true
523 --train-ocr)
524 XX_TRAIN_OCR=true
526 --train-ocr=*)
527 XX_TRAIN_OCR=true
528 XX_TRAIN_OCR_FILE=`echo "$1" | cut '-d=' -f 2-`
530 --debug)
531 XX_DEBUG_ECHO=echo
533 --activity)
534 XX_ACTIVITY_ECHO=echo
536 --dump-terminal)
537 XX_DUMP_TERMINAL=true
538 XX_DEBUG_ECHO=echo
540 --root=*)
541 XX_HELENOS_ROOT=`echo "$1" | cut '-d=' -f 2-`
542 XX_AUTODETECT_HELENOS=true
544 --image=*)
545 XX_CDROM_FILE=`echo "$1" | cut '-d=' -f 2-`
546 XX_AUTODETECT_HELENOS=false
548 --arch=*)
549 XX_ARCH=`echo "$1" | cut '-d=' -f 2-`
550 XX_AUTODETECT_HELENOS=false
552 --memory=*)
553 XX_MEMORY_MB=`echo "$1" | cut '-d=' -f 2-`
554 if ! echo "$XX_MEMORY_MB" | grep -q '^[1-9][0-9]*$'; then
555 xx_fatal "--memory has to be provided as an integer."
558 --temp=*)
559 XX_TEMP=`echo "$1" | cut '-d=' -f 2-`
561 --no-kvm)
562 XX_USE_KVM=false
564 --fail-fast)
565 XX_FAIL_FAST=true
567 --help|-h|-?)
568 xx_do_print_help "$0"
569 exit 0
571 --*)
572 xx_fatal "Unknown option $1."
575 break
577 esac
578 shift
579 done
581 [ -z "$XX_MEMORY_MB" ] && XX_MEMORY_MB=256
583 mkdir -p "$XX_TEMP"
585 if $XX_AUTODETECT_HELENOS; then
586 if ! [ -r "$XX_HELENOS_ROOT/Makefile.config" ]; then
587 xx_fatal "Cannot open $XX_HELENOS_ROOT/Makefile.config."
589 XX_ARCH=`grep '^PLATFORM = ' "$XX_HELENOS_ROOT/Makefile.config" | cut '-d=' -f 2- | tr -d ' '`
590 XX_CDROM_FILE=$XX_HELENOS_ROOT/`cat "$XX_HELENOS_ROOT/defaults/$XX_ARCH/output"`
594 if $XX_TRAIN_OCR; then
595 if [ -z "$XX_TRAIN_OCR_FILE" ]; then
596 echo "Usage notes for --train-ocr command-line switch."
597 echo
598 echo "Prepare HelenOS image.iso with train-ocr.txt in its root."
599 echo
600 echo "Run this script with the ISO and with --train-ocr=out.sed"
601 echo "where out.sed is output file with OCR commands."
602 echo
603 echo "If the script finishes with no error you can replace the"
604 echo "old OCR scheme (ocr.sed) with the new one in out.sed."
605 echo
606 exit
609 # Set to false to debug the script below without running QEMU
610 # all the time
611 if true; then
612 xx_start_machine name=ocr vterm=false
613 xx_echo "Will display the training set on the VM screen"
614 xx_echo "(this will take a while)."
615 sleep 20
616 xx_do_type ocr "cat train-ocr.txt"
617 sleep 1
618 xx_do_qemu_command ocr "sendkey ret"
619 sleep 5
620 xx_do_screenshot ocr "$XX_TEMP/ocr-full.ppm" "$XX_TEMP/ocr-term.png"
621 xx_stop_machine
624 xx_echo "Doing OCR..."
625 xx_do_ocr_prepare "$XX_TEMP/ocr-term.png" \
627 prev1_line=""
628 prev2_line=""
629 prev3_line=""
630 counter=0
631 while read current_line; do
632 if [ $counter -gt 5 ]; then
633 while read current_line; do
634 if ! [ "$current_line" = "$prev1_line" -o "$current_line" = "$prev2_line" ]; then
635 break
637 done
638 alphabet=`sed -n 's#^\(.\)\(.\)\(\1\2\)\{4,\}##p' train-ocr.txt`
639 alphabet_size=`echo "$alphabet" | wc -c`
640 for i in `seq 1 $(( $alphabet_size - 1))`; do
641 symbol="`echo \"$alphabet\" | fold -w 1 | sed -n \"${i}p\" | sed 's#[*\.\&\\:\/\[]\|\]#\\\\&#g'`"
642 echo "s:$current_line:$symbol:"
643 read current_line
644 done
645 echo "$current_line" | tr 'F' '0' | sed 's#.*#s:&:_:#'
646 echo "$current_line" | tr '0F' '.' | sed 's#.*#s:&:?:#'
647 break
649 if [ "$current_line" = "$prev2_line" -a "$prev1_line" = "$prev3_line" -a "$prev1_line" != "$prev2_line" ]; then
650 counter=$(( $counter + 1 ))
651 else
652 counter=0
654 prev3_line="$prev2_line"
655 prev2_line="$prev1_line"
656 prev1_line="$current_line"
657 done >"$XX_TRAIN_OCR_FILE"
659 xx_echo "New OCR scheme is in $XX_TRAIN_OCR_FILE."
660 exit
664 XX_RESULT_SUMMARY="$XX_TEMP/summary.$$.txt"
666 date '+# Execution started on %Y-%m-%d %H:%M.' >"$XX_RESULT_SUMMARY"
667 echo '# =======================================' >>"$XX_RESULT_SUMMARY"
670 # Run it
671 for i in "$@"; do
672 echo "# Starting scenario $i..."
674 . $i
676 if [ $? -eq 0 ]; then
677 res="OK"
678 else
679 res="FAIL"
681 echo "# Scenario $i terminated, $res."
682 printf '%-35s %4s\n' "$i" "$res" >>"$XX_RESULT_SUMMARY"
683 if $XX_FAIL_FAST && [ "$res" = "FAIL" ]; then
684 exit 1
686 done
689 date '+# Execution finished on %Y-%m-%d %H:%M.' >>"$XX_RESULT_SUMMARY"
691 # Display the results.
692 echo
694 cat "$XX_RESULT_SUMMARY"