tests/image-fuzzer/runner.py

   1 #!/usr/bin/env python
   2
   3 # Tool for running fuzz tests
   4 #
   5 # Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20
  21 import sys
  22 import os
  23 import signal
  24 import subprocess
  25 import random
  26 import shutil
  27 from itertools import count
  28 import time
  29 import getopt
  30 import StringIO
  31 import resource
  32
  33 try:
  34     import json
  35 except ImportError:
  36     try:
  37         import simplejson as json
  38     except ImportError:
  39         print >>sys.stderr, \
  40             "Warning: Module for JSON processing is not found.\n" \
  41             "'--config' and '--command' options are not supported."
  42
  43 # Backing file sizes in MB
  44 MAX_BACKING_FILE_SIZE = 10
  45 MIN_BACKING_FILE_SIZE = 1
  46
  47
  48 def multilog(msg, *output):
  49     """ Write an object to all of specified file descriptors."""
  50     for fd in output:
  51         fd.write(msg)
  52         fd.flush()
  53
  54
  55 def str_signal(sig):
  56     """ Convert a numeric value of a system signal to the string one
  57     defined by the current operational system.
  58     """
  59     for k, v in signal.__dict__.items():
  60         if v == sig:
  61             return k
  62
  63
  64 def run_app(fd, q_args):
  65     """Start an application with specified arguments and return its exit code
  66     or kill signal depending on the result of execution.
  67     """
  68
  69     class Alarm(Exception):
  70         """Exception for signal.alarm events."""
  71         pass
  72
  73     def handler(*arg):
  74         """Notify that an alarm event occurred."""
  75         raise Alarm
  76
  77     signal.signal(signal.SIGALRM, handler)
  78     signal.alarm(600)
  79     term_signal = signal.SIGKILL
  80     devnull = open('/dev/null', 'r+')
  81     process = subprocess.Popen(q_args, stdin=devnull,
  82                                stdout=subprocess.PIPE,
  83                                stderr=subprocess.PIPE)
  84     try:
  85         out, err = process.communicate()
  86         signal.alarm(0)
  87         fd.write(out)
  88         fd.write(err)
  89         fd.flush()
  90         return process.returncode
  91
  92     except Alarm:
  93         os.kill(process.pid, term_signal)
  94         fd.write('The command was terminated by timeout.\n')
  95         fd.flush()
  96         return -term_signal
  97
  98
  99 class TestException(Exception):
 100     """Exception for errors risen by TestEnv objects."""
 101     pass
 102
 103
 104 class TestEnv(object):
 105
 106     """Test object.
 107
 108     The class sets up test environment, generates backing and test images
 109     and executes application under tests with specified arguments and a test
 110     image provided.
 111
 112     All logs are collected.
 113
 114     The summary log will contain short descriptions and statuses of tests in
 115     a run.
 116
 117     The test log will include application (e.g. 'qemu-img') logs besides info
 118     sent to the summary log.
 119     """
 120
 121     def __init__(self, test_id, seed, work_dir, run_log,
 122                  cleanup=True, log_all=False):
 123         """Set test environment in a specified work directory.
 124
 125         Path to qemu-img and qemu-io will be retrieved from 'QEMU_IMG' and
 126         'QEMU_IO' environment variables.
 127         """
 128         if seed is not None:
 129             self.seed = seed
 130         else:
 131             self.seed = str(random.randint(0, sys.maxint))
 132         random.seed(self.seed)
 133
 134         self.init_path = os.getcwd()
 135         self.work_dir = work_dir
 136         self.current_dir = os.path.join(work_dir, 'test-' + test_id)
 137         self.qemu_img = os.environ.get('QEMU_IMG', 'qemu-img')\
 138                                   .strip().split(' ')
 139         self.qemu_io = os.environ.get('QEMU_IO', 'qemu-io').strip().split(' ')
 140         self.commands = [['qemu-img', 'check', '-f', 'qcow2', '$test_img'],
 141                          ['qemu-img', 'info', '-f', 'qcow2', '$test_img'],
 142                          ['qemu-io', '$test_img', '-c', 'read $off $len'],
 143                          ['qemu-io', '$test_img', '-c', 'write $off $len'],
 144                          ['qemu-io', '$test_img', '-c',
 145                           'aio_read $off $len'],
 146                          ['qemu-io', '$test_img', '-c',
 147                           'aio_write $off $len'],
 148                          ['qemu-io', '$test_img', '-c', 'flush'],
 149                          ['qemu-io', '$test_img', '-c',
 150                           'discard $off $len'],
 151                          ['qemu-io', '$test_img', '-c',
 152                           'truncate $off']]
 153         for fmt in ['raw', 'vmdk', 'vdi', 'cow', 'qcow2', 'file',
 154                     'qed', 'vpc']:
 155             self.commands.append(
 156                 ['qemu-img', 'convert', '-f', 'qcow2', '-O', fmt,
 157                  '$test_img', 'converted_image.' + fmt])
 158
 159         try:
 160             os.makedirs(self.current_dir)
 161         except OSError, e:
 162             print >>sys.stderr, \
 163                 "Error: The working directory '%s' cannot be used. Reason: %s"\
 164                 % (self.work_dir, e[1])
 165             raise TestException
 166         self.log = open(os.path.join(self.current_dir, "test.log"), "w")
 167         self.parent_log = open(run_log, "a")
 168         self.failed = False
 169         self.cleanup = cleanup
 170         self.log_all = log_all
 171
 172     def _create_backing_file(self):
 173         """Create a backing file in the current directory.
 174
 175         Return a tuple of a backing file name and format.
 176
 177         Format of a backing file is randomly chosen from all formats supported
 178         by 'qemu-img create'.
 179         """
 180         # All formats supported by the 'qemu-img create' command.
 181         backing_file_fmt = random.choice(['raw', 'vmdk', 'vdi', 'cow', 'qcow2',
 182                                           'file', 'qed', 'vpc'])
 183         backing_file_name = 'backing_img.' + backing_file_fmt
 184         backing_file_size = random.randint(MIN_BACKING_FILE_SIZE,
 185                                            MAX_BACKING_FILE_SIZE) * (1 << 20)
 186         cmd = self.qemu_img + ['create', '-f', backing_file_fmt,
 187                                backing_file_name, str(backing_file_size)]
 188         temp_log = StringIO.StringIO()
 189         retcode = run_app(temp_log, cmd)
 190         if retcode == 0:
 191             temp_log.close()
 192             return (backing_file_name, backing_file_fmt)
 193         else:
 194             multilog("Warning: The %s backing file was not created.\n\n"
 195                      % backing_file_fmt, sys.stderr, self.log, self.parent_log)
 196             self.log.write("Log for the failure:\n" + temp_log.getvalue() +
 197                            '\n\n')
 198             temp_log.close()
 199             return (None, None)
 200
 201     def execute(self, input_commands=None, fuzz_config=None):
 202         """ Execute a test.
 203
 204         The method creates backing and test images, runs test app and analyzes
 205         its exit status. If the application was killed by a signal, the test
 206         is marked as failed.
 207         """
 208         if input_commands is None:
 209             commands = self.commands
 210         else:
 211             commands = input_commands
 212
 213         os.chdir(self.current_dir)
 214         backing_file_name, backing_file_fmt = self._create_backing_file()
 215         img_size = image_generator.create_image('test.img',
 216                                                 backing_file_name,
 217                                                 backing_file_fmt,
 218                                                 fuzz_config)
 219         for item in commands:
 220             shutil.copy('test.img', 'copy.img')
 221             # 'off' and 'len' are multiple of the sector size
 222             sector_size = 512
 223             start = random.randrange(0, img_size + 1, sector_size)
 224             end = random.randrange(start, img_size + 1, sector_size)
 225
 226             if item[0] == 'qemu-img':
 227                 current_cmd = list(self.qemu_img)
 228             elif item[0] == 'qemu-io':
 229                 current_cmd = list(self.qemu_io)
 230             else:
 231                 multilog("Warning: test command '%s' is not defined.\n" \
 232                          % item[0], sys.stderr, self.log, self.parent_log)
 233                 continue
 234             # Replace all placeholders with their real values
 235             for v in item[1:]:
 236                 c = (v
 237                      .replace('$test_img', 'copy.img')
 238                      .replace('$off', str(start))
 239                      .replace('$len', str(end - start)))
 240                 current_cmd.append(c)
 241
 242             # Log string with the test header
 243             test_summary = "Seed: %s\nCommand: %s\nTest directory: %s\n" \
 244                            "Backing file: %s\n" \
 245                            % (self.seed, " ".join(current_cmd),
 246                               self.current_dir, backing_file_name)
 247
 248             temp_log = StringIO.StringIO()
 249             try:
 250                 retcode = run_app(temp_log, current_cmd)
 251             except OSError, e:
 252                 multilog(test_summary + "Error: Start of '%s' failed. " \
 253                          "Reason: %s\n\n" % (os.path.basename(
 254                              current_cmd[0]), e[1]),
 255                          sys.stderr, self.log, self.parent_log)
 256                 raise TestException
 257
 258             if retcode < 0:
 259                 self.log.write(temp_log.getvalue())
 260                 multilog(test_summary + "FAIL: Test terminated by signal " +
 261                          "%s\n\n" % str_signal(-retcode), sys.stderr, self.log,
 262                          self.parent_log)
 263                 self.failed = True
 264             else:
 265                 if self.log_all:
 266                     self.log.write(temp_log.getvalue())
 267                     multilog(test_summary + "PASS: Application exited with" +
 268                              " the code '%d'\n\n" % retcode, sys.stdout,
 269                              self.log, self.parent_log)
 270             temp_log.close()
 271             os.remove('copy.img')
 272
 273     def finish(self):
 274         """Restore the test environment after a test execution."""
 275         self.log.close()
 276         self.parent_log.close()
 277         os.chdir(self.init_path)
 278         if self.cleanup and not self.failed:
 279             shutil.rmtree(self.current_dir)
 280
 281 if __name__ == '__main__':
 282
 283     def usage():
 284         print """
 285         Usage: runner.py [OPTION...] TEST_DIR IMG_GENERATOR
 286
 287         Set up test environment in TEST_DIR and run a test in it. A module for
 288         test image generation should be specified via IMG_GENERATOR.
 289         Example:
 290         runner.py -c '[["qemu-img", "info", "$test_img"]]' /tmp/test qcow2
 291
 292         Optional arguments:
 293           -h, --help                    display this help and exit
 294           -d, --duration=NUMBER         finish tests after NUMBER of seconds
 295           -c, --command=JSON            run tests for all commands specified in
 296                                         the JSON array
 297           -s, --seed=STRING             seed for a test image generation,
 298                                         by default will be generated randomly
 299           --config=JSON                 take fuzzer configuration from the JSON
 300                                         array
 301           -k, --keep_passed             don't remove folders of passed tests
 302           -v, --verbose                 log information about passed tests
 303
 304         JSON:
 305
 306         '--command' accepts a JSON array of commands. Each command presents
 307         an application under test with all its paramaters as a list of strings,
 308         e.g.
 309           ["qemu-io", "$test_img", "-c", "write $off $len"]
 310
 311         Supported application aliases: 'qemu-img' and 'qemu-io'.
 312         Supported argument aliases: $test_img for the fuzzed image, $off
 313         for an offset, $len for length.
 314
 315         Values for $off and $len will be generated based on the virtual disk
 316         size of the fuzzed image
 317         Paths to 'qemu-img' and 'qemu-io' are retrevied from 'QEMU_IMG' and
 318         'QEMU_IO' environment variables
 319
 320         '--config' accepts a JSON array of fields to be fuzzed, e.g.
 321           '[["header"], ["header", "version"]]'
 322         Each of the list elements can consist of a complex image element only
 323         as ["header"] or ["feature_name_table"] or an exact field as
 324         ["header", "version"]. In the first case random portion of the element
 325         fields will be fuzzed, in the second one the specified field will be
 326         fuzzed always.
 327
 328         If '--config' argument is specified, fields not listed in
 329         the configuration array will not be fuzzed.
 330         """
 331
 332     def run_test(test_id, seed, work_dir, run_log, cleanup, log_all,
 333                  command, fuzz_config):
 334         """Setup environment for one test and execute this test."""
 335         try:
 336             test = TestEnv(test_id, seed, work_dir, run_log, cleanup,
 337                            log_all)
 338         except TestException:
 339             sys.exit(1)
 340
 341         # Python 2.4 doesn't support 'finally' and 'except' in the same 'try'
 342         # block
 343         try:
 344             try:
 345                 test.execute(command, fuzz_config)
 346             except TestException:
 347                 sys.exit(1)
 348         finally:
 349             test.finish()
 350
 351     def should_continue(duration, start_time):
 352         """Return True if a new test can be started and False otherwise."""
 353         current_time = int(time.time())
 354         return (duration is None) or (current_time - start_time < duration)
 355
 356     try:
 357         opts, args = getopt.gnu_getopt(sys.argv[1:], 'c:hs:kvd:',
 358                                        ['command=', 'help', 'seed=', 'config=',
 359                                         'keep_passed', 'verbose', 'duration='])
 360     except getopt.error, e:
 361         print >>sys.stderr, \
 362             "Error: %s\n\nTry 'runner.py --help' for more information" % e
 363         sys.exit(1)
 364
 365     command = None
 366     cleanup = True
 367     log_all = False
 368     seed = None
 369     config = None
 370     duration = None
 371
 372     for opt, arg in opts:
 373         if opt in ('-h', '--help'):
 374             usage()
 375             sys.exit()
 376         elif opt in ('-c', '--command'):
 377             try:
 378                 command = json.loads(arg)
 379             except (TypeError, ValueError, NameError), e:
 380                 print >>sys.stderr, \
 381                     "Error: JSON array of test commands cannot be loaded.\n" \
 382                     "Reason: %s" % e
 383                 sys.exit(1)
 384         elif opt in ('-k', '--keep_passed'):
 385             cleanup = False
 386         elif opt in ('-v', '--verbose'):
 387             log_all = True
 388         elif opt in ('-s', '--seed'):
 389             seed = arg
 390         elif opt in ('-d', '--duration'):
 391             duration = int(arg)
 392         elif opt == '--config':
 393             try:
 394                 config = json.loads(arg)
 395             except (TypeError, ValueError, NameError), e:
 396                 print >>sys.stderr, \
 397                     "Error: JSON array with the fuzzer configuration cannot" \
 398                     " be loaded\nReason: %s" % e
 399                 sys.exit(1)
 400
 401     if not len(args) == 2:
 402         print >>sys.stderr, \
 403             "Expected two parameters\nTry 'runner.py --help'" \
 404             " for more information."
 405         sys.exit(1)
 406
 407     work_dir = os.path.realpath(args[0])
 408     # run_log is created in 'main', because multiple tests are expected to
 409     # log in it
 410     run_log = os.path.join(work_dir, 'run.log')
 411
 412     # Add the path to the image generator module to sys.path
 413     sys.path.append(os.path.realpath(os.path.dirname(args[1])))
 414     # Remove a script extension from image generator module if any
 415     generator_name = os.path.splitext(os.path.basename(args[1]))[0]
 416
 417     try:
 418         image_generator = __import__(generator_name)
 419     except ImportError, e:
 420         print >>sys.stderr, \
 421             "Error: The image generator '%s' cannot be imported.\n" \
 422             "Reason: %s" % (generator_name, e)
 423         sys.exit(1)
 424
 425     # Enable core dumps
 426     resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
 427     # If a seed is specified, only one test will be executed.
 428     # Otherwise runner will terminate after a keyboard interruption
 429     start_time = int(time.time())
 430     test_id = count(1)
 431     while should_continue(duration, start_time):
 432         try:
 433             run_test(str(test_id.next()), seed, work_dir, run_log, cleanup,
 434                      log_all, command, config)
 435         except (KeyboardInterrupt, SystemExit):
 436             sys.exit(1)
 437
 438         if seed is not None:
 439             break