utils/test_importer.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2008 Google Inc. All Rights Reserved.
   4 """
   5 This utility allows for easy updating, removing and importing
   6 of tests into the autotest_web afe_autotests table.
   7
   8 Example of updating client side tests:
   9 ./test_importer.py -t /usr/local/autotest/client/tests
  10
  11 If, for example, not all of your control files adhere to the standard outlined
  12 at http://autotest.kernel.org/wiki/ControlRequirements, you can force options:
  13
  14 ./test_importer.py --test-type server -t /usr/local/autotest/server/tests
  15
  16 You would need to pass --add-noncompliant to include such control files,
  17 however.  An easy way to check for compliance is to run in dry mode:
  18
  19 ./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest
  20
  21 Or to check a single control file, you can use check_control_file_vars.py.
  22
  23 Running with no options is equivalent to --add-all --db-clear-tests.
  24
  25 Most options should be fairly self explanatory, use --help to display them.
  26 """
  27
  28
  29 import common
  30 import logging, re, os, sys, optparse, compiler
  31 from autotest_lib.frontend import setup_django_environment
  32 from autotest_lib.frontend.afe import models
  33 from autotest_lib.client.common_lib import control_data, utils
  34 from autotest_lib.client.common_lib import logging_config, logging_manager
  35
  36
  37 class TestImporterLoggingConfig(logging_config.LoggingConfig):
  38     def configure_logging(self, results_dir=None, verbose=False):
  39         super(TestImporterLoggingConfig, self).configure_logging(
  40                                                                use_console=True,
  41                                                                verbose=verbose)
  42
  43
  44 # Global
  45 DRY_RUN = False
  46 DEPENDENCIES_NOT_FOUND = set()
  47
  48
  49 def update_all(autotest_dir, add_noncompliant, add_experimental):
  50     """
  51     Function to scan through all tests and add them to the database.
  52
  53     This function invoked when no parameters supplied to the command line.
  54     It 'synchronizes' the test database with the current contents of the
  55     client and server test directories.  When test code is discovered
  56     in the file system new tests may be added to the db.  Likewise,
  57     if test code is not found in the filesystem, tests may be removed
  58     from the db.  The base test directories are hard-coded to client/tests,
  59     client/site_tests, server/tests and server/site_tests.
  60
  61     @param autotest_dir: prepended to path strings (/usr/local/autotest).
  62     @param add_noncompliant: attempt adding test with invalid control files.
  63     @param add_experimental: add tests with experimental attribute set.
  64     """
  65     for path in [ 'server/tests', 'server/site_tests', 'client/tests',
  66                   'client/site_tests']:
  67         test_path = os.path.join(autotest_dir, path)
  68         if not os.path.exists(test_path):
  69             continue
  70         logging.info("Scanning %s", test_path)
  71         tests = []
  72         tests = get_tests_from_fs(test_path, "^control.*",
  73                                  add_noncompliant=add_noncompliant)
  74         update_tests_in_db(tests, add_experimental=add_experimental,
  75                            add_noncompliant=add_noncompliant,
  76                            autotest_dir=autotest_dir)
  77     test_suite_path = os.path.join(autotest_dir, 'test_suites')
  78     if os.path.exists(test_suite_path):
  79         logging.info("Scanning %s", test_suite_path)
  80         tests = get_tests_from_fs(test_suite_path, '.*',
  81                                  add_noncompliant=add_noncompliant)
  82         update_tests_in_db(tests, add_experimental=add_experimental,
  83                            add_noncompliant=add_noncompliant,
  84                            autotest_dir=autotest_dir)
  85
  86     profilers_path = os.path.join(autotest_dir, "client/profilers")
  87     if os.path.exists(profilers_path):
  88         logging.info("Scanning %s", profilers_path)
  89         profilers = get_tests_from_fs(profilers_path, '.*py$')
  90         update_profilers_in_db(profilers, add_noncompliant=add_noncompliant,
  91                                description='NA')
  92     # Clean bad db entries
  93     db_clean_broken(autotest_dir)
  94
  95
  96 def update_samples(autotest_dir, add_noncompliant, add_experimental):
  97     """
  98     Add only sample tests to the database from the filesystem.
  99
 100     This function invoked when -S supplied on command line.
 101     Only adds tests to the database - does not delete any.
 102     Samples tests are formatted slightly differently than other tests.
 103
 104     @param autotest_dir: prepended to path strings (/usr/local/autotest).
 105     @param add_noncompliant: attempt adding test with invalid control files.
 106     @param add_experimental: add tests with experimental attribute set.
 107     """
 108     sample_path = os.path.join(autotest_dir, 'server/samples')
 109     if os.path.exists(sample_path):
 110         logging.info("Scanning %s", sample_path)
 111         tests = get_tests_from_fs(sample_path, '.*srv$',
 112                                   add_noncompliant=add_noncompliant)
 113         update_tests_in_db(tests, add_experimental=add_experimental,
 114                            add_noncompliant=add_noncompliant,
 115                            autotest_dir=autotest_dir)
 116
 117
 118 def db_clean_broken(autotest_dir):
 119     """
 120     Remove tests from autotest_web that do not have valid control files
 121
 122     This function invoked when -c supplied on the command line and when
 123     running update_all().  Removes tests from database which are not
 124     found in the filesystem.  Also removes profilers which are just
 125     a special case of tests.
 126
 127     @param autotest_dir: prepended to path strings (/usr/local/autotest).
 128     """
 129     for test in models.Test.objects.all():
 130         full_path = os.path.join(autotest_dir, test.path)
 131         if not os.path.isfile(full_path):
 132             logging.info("Removing %s", test.path)
 133             _log_or_execute(repr(test), test.delete)
 134
 135     # Find profilers that are no longer present
 136     for profiler in models.Profiler.objects.all():
 137         full_path = os.path.join(autotest_dir, "client", "profilers",
 138                                  profiler.name)
 139         if not os.path.exists(full_path):
 140             logging.info("Removing %s", profiler.name)
 141             _log_or_execute(repr(profiler), profiler.delete)
 142
 143
 144 def db_clean_all(autotest_dir):
 145     """
 146     Remove all tests from autotest_web - very destructive
 147
 148     This function invoked when -C supplied on the command line.
 149     Removes ALL tests from the database.
 150
 151     @param autotest_dir: prepended to path strings (/usr/local/autotest).
 152     """
 153     for test in models.Test.objects.all():
 154         full_path = os.path.join(autotest_dir, test.path)
 155         logging.info("Removing %s", test.path)
 156         _log_or_execute(repr(test), test.delete)
 157
 158     # Find profilers that are no longer present
 159     for profiler in models.Profiler.objects.all():
 160         full_path = os.path.join(autotest_dir, "client", "profilers",
 161                                  profiler.name)
 162         logging.info("Removing %s", profiler.name)
 163         _log_or_execute(repr(profiler), profiler.delete)
 164
 165
 166 def update_profilers_in_db(profilers, description='NA',
 167                            add_noncompliant=False):
 168     """
 169     Add only profilers to the database from the filesystem.
 170
 171     This function invoked when -p supplied on command line.
 172     Only adds profilers to the database - does not delete any.
 173     Profilers are formatted slightly differently than tests.
 174
 175     @param profilers: list of profilers found in the file system.
 176     @param description: simple text to satisfy docstring.
 177     @param add_noncompliant: attempt adding test with invalid control files.
 178     """
 179     for profiler in profilers:
 180         name = os.path.basename(profiler)
 181         if name.endswith('.py'):
 182             name = name[:-3]
 183         if not profilers[profiler]:
 184             if add_noncompliant:
 185                 doc = description
 186             else:
 187                 logging.warn("Skipping %s, missing docstring", profiler)
 188                 continue
 189         else:
 190             doc = profilers[profiler]
 191
 192         model = models.Profiler.objects.get_or_create(name=name)[0]
 193         model.description = doc
 194         _log_or_execute(repr(model), model.save)
 195
 196
 197 def update_tests_in_db(tests, dry_run=False, add_experimental=False,
 198                        add_noncompliant=False, autotest_dir=None):
 199     """
 200     Scans through all tests and add them to the database.
 201
 202     This function invoked when -t supplied and for update_all.
 203     When test code is discovered in the file system new tests may be added
 204
 205     @param tests: list of tests found in the filesystem.
 206     @param dry_run: not used at this time.
 207     @param add_experimental: add tests with experimental attribute set.
 208     @param add_noncompliant: attempt adding test with invalid control files.
 209     @param autotest_dir: prepended to path strings (/usr/local/autotest).
 210     """
 211     site_set_attributes_module = utils.import_site_module(
 212         __file__, 'autotest_lib.utils.site_test_importer_attributes')
 213
 214     for test in tests:
 215         new_test = models.Test.objects.get_or_create(
 216                 path=test.replace(autotest_dir, '').lstrip('/'))[0]
 217         logging.info("Processing %s", new_test.path)
 218
 219         # Set the test's attributes
 220         data = tests[test]
 221         _set_attributes_clean(new_test, data)
 222
 223         # Custom Attribute Update
 224         if site_set_attributes_module:
 225             site_set_attributes_module._set_attributes_custom(new_test, data)
 226
 227         # This only takes place if --add-noncompliant is provided on the CLI
 228         if not new_test.name:
 229             test_new_test = test.split('/')
 230             if test_new_test[-1] == 'control':
 231                 new_test.name = test_new_test[-2]
 232             else:
 233                 control_name = "%s:%s"
 234                 control_name %= (test_new_test[-2],
 235                                  test_new_test[-1])
 236                 new_test.name = control_name.replace('control.', '')
 237
 238         # Experimental Check
 239         if not add_experimental and new_test.experimental:
 240             continue
 241
 242         _log_or_execute(repr(new_test), new_test.save)
 243         add_label_dependencies(new_test)
 244
 245         # save TestParameter
 246         for para_name in data.test_parameters:
 247             test_parameter = models.TestParameter.objects.get_or_create(
 248                 test=new_test, name=para_name)[0]
 249             test_parameter.save()
 250
 251
 252 def _set_attributes_clean(test, data):
 253     """
 254     First pass sets the attributes of the Test object from file system.
 255
 256     @param test: a test object to be populated for the database.
 257     @param data: object with test data from the file system.
 258     """
 259     test_type = { 'client' : 1,
 260                   'server' : 2, }
 261     test_time = { 'short' : 1,
 262                   'medium' : 2,
 263                   'long' : 3, }
 264
 265
 266     string_attributes = ('name', 'author', 'test_class', 'test_category',
 267                          'test_category', 'sync_count')
 268     for attribute in string_attributes:
 269         setattr(test, attribute, getattr(data, attribute))
 270
 271     test.description = data.doc
 272     test.dependencies = ", ".join(data.dependencies)
 273
 274     int_attributes = ('experimental', 'run_verify')
 275     for attribute in int_attributes:
 276         setattr(test, attribute, int(getattr(data, attribute)))
 277
 278     try:
 279         test.test_type = int(data.test_type)
 280         if test.test_type != 1 and test.test_type != 2:
 281             raise Exception('Incorrect number %d for test_type' %
 282                             test.test_type)
 283     except ValueError:
 284         pass
 285     try:
 286         test.test_time = int(data.time)
 287         if test.test_time < 1 or test.time > 3:
 288             raise Exception('Incorrect number %d for time' % test.time)
 289     except ValueError:
 290         pass
 291
 292     if not test.test_time and str == type(data.time):
 293         test.test_time = test_time[data.time.lower()]
 294     if not test.test_type and str == type(data.test_type):
 295         test.test_type = test_type[data.test_type.lower()]
 296
 297
 298 def add_label_dependencies(test):
 299     """
 300     Add proper many-to-many relationships from DEPENDENCIES field.
 301
 302     @param test: test object for the database.
 303     """
 304
 305     # clear out old relationships
 306     _log_or_execute(repr(test), test.dependency_labels.clear,
 307                     subject='clear dependencies from')
 308
 309     for label_name in test.dependencies.split(','):
 310         label_name = label_name.strip().lower()
 311         if not label_name:
 312             continue
 313
 314         try:
 315             label = models.Label.objects.get(name=label_name)
 316         except models.Label.DoesNotExist:
 317             log_dependency_not_found(label_name)
 318             continue
 319
 320         _log_or_execute(repr(label), test.dependency_labels.add, label,
 321                         subject='add dependency to %s' % test.name)
 322
 323
 324 def log_dependency_not_found(label_name):
 325     """
 326     Exception processing when label not found in database.
 327
 328     @param label_name: from test dependencies.
 329     """
 330     if label_name in DEPENDENCIES_NOT_FOUND:
 331         return
 332     logging.info("Dependency %s not found", label_name)
 333     DEPENDENCIES_NOT_FOUND.add(label_name)
 334
 335
 336 def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False):
 337     """
 338     Find control files in file system and load a list with their info.
 339
 340     @param parent_dir: directory to search recursively.
 341     @param control_pattern: name format of control file.
 342     @param add_noncompliant: ignore control file parse errors.
 343
 344     @return dictionary of the form: tests[file_path] = parsed_object
 345     """
 346     tests = {}
 347     profilers = False
 348     if 'client/profilers' in parent_dir:
 349         profilers = True
 350     for dir in [ parent_dir ]:
 351         files = recursive_walk(dir, control_pattern)
 352         for file in files:
 353             if '__init__.py' in file or '.svn' in file:
 354                 continue
 355             if not profilers:
 356                 if not add_noncompliant:
 357                     try:
 358                         found_test = control_data.parse_control(file,
 359                                                             raise_warnings=True)
 360                         tests[file] = found_test
 361                     except control_data.ControlVariableException, e:
 362                         logging.warn("Skipping %s\n%s", file, e)
 363                     except Exception, e:
 364                         logging.error("Bad %s\n%s", file, e)
 365                 else:
 366                     found_test = control_data.parse_control(file)
 367                     tests[file] = found_test
 368             else:
 369                 tests[file] = compiler.parseFile(file).doc
 370     return tests
 371
 372
 373 def recursive_walk(path, wildcard):
 374     """
 375     Recursively go through a directory.
 376
 377     This function invoked by get_tests_from_fs().
 378
 379     @param path: base directory to start search.
 380     @param wildcard: name format to match.
 381
 382     @return A list of files that match wildcard
 383     """
 384     files = []
 385     directories = [ path ]
 386     while len(directories)>0:
 387         directory = directories.pop()
 388         for name in os.listdir(directory):
 389             fullpath = os.path.join(directory, name)
 390             if os.path.isfile(fullpath):
 391                 # if we are a control file
 392                 if re.search(wildcard, name):
 393                     files.append(fullpath)
 394             elif os.path.isdir(fullpath):
 395                 directories.append(fullpath)
 396     return files
 397
 398
 399 def _log_or_execute(content, func, *args, **kwargs):
 400     """
 401     Log a message if dry_run is enabled, or execute the given function.
 402
 403     Relies on the DRY_RUN global variable.
 404
 405     @param content: the actual log message.
 406     @param func: function to execute if dry_run is not enabled.
 407     @param subject: (Optional) The type of log being written. Defaults to
 408                      the name of the provided function.
 409     """
 410     subject = kwargs.get('subject', func.__name__)
 411
 412     if DRY_RUN:
 413         logging.info("Would %s: %s",  subject, content)
 414     else:
 415         func(*args)
 416
 417
 418 def _create_whitelist_set(whitelist_path):
 419     """
 420     Create a set with contents from a whitelist file for membership testing.
 421
 422     @param whitelist_path: full path to the whitelist file.
 423
 424     @return set with files listed one/line - newlines included.
 425     """
 426     f = open(whitelist_path, 'r')
 427     whitelist_set = set([line.strip() for line in f])
 428     f.close()
 429     return whitelist_set
 430
 431
 432 def update_from_whitelist(whitelist_set, add_experimental, add_noncompliant,
 433                           autotest_dir):
 434     """
 435     Scans through all tests in the whitelist and add them to the database.
 436
 437     This function invoked when -w supplied.
 438
 439     @param whitelist_set: set of tests in full-path form from a whitelist.
 440     @param add_experimental: add tests with experimental attribute set.
 441     @param add_noncompliant: attempt adding test with invalid control files.
 442     @param autotest_dir: prepended to path strings (/usr/local/autotest).
 443     """
 444     tests = {}
 445     profilers = {}
 446     for file_path in whitelist_set:
 447         if file_path.find('client/profilers') == -1:
 448             try:
 449                 found_test = control_data.parse_control(file_path,
 450                                                         raise_warnings=True)
 451                 tests[file_path] = found_test
 452             except control_data.ControlVariableException, e:
 453                 logging.warn("Skipping %s\n%s", file, e)
 454         else:
 455             profilers[file_path] = compiler.parseFile(file_path).doc
 456
 457     if len(tests) > 0:
 458         update_tests_in_db(tests, add_experimental=add_experimental,
 459                            add_noncompliant=add_noncompliant,
 460                            autotest_dir=autotest_dir)
 461     if len(profilers) > 0:
 462         update_profilers_in_db(profilers, add_noncompliant=add_noncompliant,
 463                                description='NA')
 464
 465
 466 def main(argv):
 467     """Main function"""
 468
 469     global DRY_RUN
 470     parser = optparse.OptionParser()
 471     parser.add_option('-c', '--db-clean-tests',
 472                       dest='clean_tests', action='store_true',
 473                       default=False,
 474                 help='Clean client and server tests with invalid control files')
 475     parser.add_option('-C', '--db-clear-all-tests',
 476                       dest='clear_all_tests', action='store_true',
 477                       default=False,
 478                 help='Clear ALL client and server tests')
 479     parser.add_option('-d', '--dry-run',
 480                       dest='dry_run', action='store_true', default=False,
 481                       help='Dry run for operation')
 482     parser.add_option('-A', '--add-all',
 483                       dest='add_all', action='store_true',
 484                       default=False,
 485                       help='Add site_tests, tests, and test_suites')
 486     parser.add_option('-S', '--add-samples',
 487                       dest='add_samples', action='store_true',
 488                       default=False,
 489                       help='Add samples.')
 490     parser.add_option('-E', '--add-experimental',
 491                       dest='add_experimental', action='store_true',
 492                       default=True,
 493                       help='Add experimental tests to frontend')
 494     parser.add_option('-N', '--add-noncompliant',
 495                       dest='add_noncompliant', action='store_true',
 496                       default=False,
 497                       help='Add non-compliant tests (i.e. tests that do not '
 498                            'define all required control variables)')
 499     parser.add_option('-p', '--profile-dir', dest='profile_dir',
 500                       help='Directory to recursively check for profiles')
 501     parser.add_option('-t', '--tests-dir', dest='tests_dir',
 502                       help='Directory to recursively check for control.*')
 503     parser.add_option('-r', '--control-pattern', dest='control_pattern',
 504                       default='^control.*',
 505                help='The pattern to look for in directories for control files')
 506     parser.add_option('-v', '--verbose',
 507                       dest='verbose', action='store_true', default=False,
 508                       help='Run in verbose mode')
 509     parser.add_option('-w', '--whitelist-file', dest='whitelist_file',
 510                       help='Filename for list of test names that must match')
 511     parser.add_option('-z', '--autotest-dir', dest='autotest_dir',
 512                       default=os.path.join(os.path.dirname(__file__), '..'),
 513                       help='Autotest directory root')
 514     options, args = parser.parse_args()
 515
 516     logging_manager.configure_logging(TestImporterLoggingConfig(),
 517                                       verbose=options.verbose)
 518
 519     DRY_RUN = options.dry_run
 520     if DRY_RUN:
 521         logging.getLogger().setLevel(logging.WARN)
 522
 523     # Make sure autotest_dir is the absolute path
 524     options.autotest_dir = os.path.abspath(options.autotest_dir)
 525
 526     if len(args) > 0:
 527         logging.error("Invalid option(s) provided: %s", args)
 528         parser.print_help()
 529         return 1
 530
 531     if options.verbose:
 532         logging.getLogger().setLevel(logging.DEBUG)
 533
 534     if len(argv) == 1 or (len(argv) == 2 and options.verbose):
 535         update_all(options.autotest_dir, options.add_noncompliant,
 536                    options.add_experimental)
 537         db_clean_broken(options.autotest_dir)
 538         return 0
 539
 540     if options.clear_all_tests:
 541         if (options.clean_tests or options.add_all or options.add_samples or
 542             options.add_noncompliant):
 543             logging.error(
 544                 "Can only pass --autotest-dir, --dry-run and --verbose with "
 545                 "--db-clear-all-tests")
 546             return 1
 547         db_clean_all(options.autotest_dir)
 548
 549     whitelist_set = None
 550     if options.whitelist_file:
 551         if options.add_all:
 552             logging.error("Cannot pass both --add-all and --whitelist-file")
 553             return 1
 554         whitelist_path = os.path.abspath(options.whitelist_file)
 555         if not os.path.isfile(whitelist_path):
 556             logging.error("--whitelist-file (%s) not found", whitelist_path)
 557             return 1
 558         logging.info("Using whitelist file %s", whitelist_path)
 559         whitelist_set =  _create_whitelist_set(whitelist_path)
 560         update_from_whitelist(whitelist_set,
 561                               add_experimental=options.add_experimental,
 562                               add_noncompliant=options.add_noncompliant,
 563                               autotest_dir=options.autotest_dir)
 564     if options.add_all:
 565         update_all(options.autotest_dir, options.add_noncompliant,
 566                    options.add_experimental)
 567     if options.add_samples:
 568         update_samples(options.autotest_dir, options.add_noncompliant,
 569                        options.add_experimental)
 570     if options.tests_dir:
 571         options.tests_dir = os.path.abspath(options.tests_dir)
 572         tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
 573                                   add_noncompliant=options.add_noncompliant)
 574         update_tests_in_db(tests, add_experimental=options.add_experimental,
 575                            add_noncompliant=options.add_noncompliant,
 576                            autotest_dir=options.autotest_dir)
 577     if options.profile_dir:
 578         profilers = get_tests_from_fs(options.profile_dir, '.*py$')
 579         update_profilers_in_db(profilers,
 580                                add_noncompliant=options.add_noncompliant,
 581                                description='NA')
 582     if options.clean_tests:
 583         db_clean_broken(options.autotest_dir)
 584
 585
 586 if __name__ == "__main__":
 587     main(sys.argv)