utils/tko_publish.py

   1 #!/usr/bin/python
   2 """
   3 This script will scan an autotest server results directory for job result
   4 directories that have completed and that have not yet been published on
   5 a remote dashboard server matching given filtering options and for those it
   6 finds it will rsync them to the tko server and mark them as published (it uses
   7 a <jobdir>/.tko_published flag file to determine if a jobdir results directory
   8 has been published yet).
   9 """
  10
  11 import sys, os, re, optparse
  12
  13 import common
  14 from autotest_lib.client.common_lib import utils
  15 from autotest_lib.server import frontend
  16
  17 options = optparse.Values()
  18
  19 USAGE="""tko-publish [options] <resultsdir> <rsync-destination-path>
  20
  21 Where:
  22 <resultsdir>              A path to the directory having the job results
  23                           directories to publish.
  24
  25 <rsync-destination-path>  A valid rsync destination path where to upload the
  26                           job result directories.
  27                           Example: user@machine.org:/home/autotest/results"""
  28 PUBLISH_FLAGFILE = '.tko_published'
  29 RSYNC_COMMAND = 'rsync -aqz "%s" "%s"'
  30
  31
  32 def get_job_dirs(path):
  33     regex = re.compile('[1-9][0-9]*-')
  34     jobdirs = []
  35
  36     for dir in os.listdir(path):
  37         # skip directories not matching the job result dir pattern
  38         if not regex.match(dir):
  39             continue
  40
  41         dir = os.path.join(options.resultsdir, dir)
  42         if (os.path.isdir(dir)
  43                 and not os.path.exists(os.path.join(dir, PUBLISH_FLAGFILE))):
  44             jobdirs.append(dir)
  45
  46     return jobdirs
  47
  48
  49 def publish_job(jobdir):
  50     cmd = RSYNC_COMMAND % (jobdir, options.dest)
  51     utils.system(cmd)
  52
  53     # mark the jobdir as published
  54     fd = open(os.path.join(jobdir, PUBLISH_FLAGFILE), 'w')
  55     fd.close()
  56     print 'Published', jobdir
  57
  58
  59 def main():
  60     jobdirs = get_job_dirs(options.resultsdir)
  61
  62     afe = frontend.AFE()
  63     # the way AFE API is right now is to give a whole list of jobs and can't
  64     # get specific jobs so minimize the queries caching the result
  65     finished_jobs = afe.get_jobs(finished=True)
  66
  67     if options.jobname_pattern:
  68         jobname_pattern = re.compile(options.jobname_pattern)
  69     else:
  70         jobname_pattern = None
  71
  72     # for each unpublished possible jobdir find it in the database and see
  73     # if it is completed
  74     for jobdir in jobdirs:
  75         job_id = int(os.path.basename(jobdir).split('-')[0])
  76         job = [job for job in finished_jobs if job.id == job_id]
  77
  78         if len(job) != 1:
  79             continue
  80
  81         if jobname_pattern:
  82             # does it match the jobname pattern?
  83             if not jobname_pattern.match(job[0].name):
  84                 continue
  85
  86         # does it match the wanted job owner
  87         if options.job_owner and options.job_owner != job[0].owner:
  88             continue
  89
  90         publish_job(jobdir)
  91
  92
  93 if __name__ == '__main__':
  94     parser = optparse.OptionParser(usage=USAGE)
  95     parser.add_option('--jobname-pattern', dest='jobname_pattern',
  96                       help='Regexp pattern to match against job names, by '
  97                       "default there won't be any matching done",
  98                       default=None)
  99     parser.add_option('--job-owner', dest='job_owner', default=None,
 100                       help='Job owner username to match against for the '
 101                       'published jobs, by default no matching is done.')
 102     options, args = parser.parse_args()
 103
 104     if len(args) < 2:
 105         print USAGE
 106         sys.exit(-1)
 107
 108     options.resultsdir = args[0]
 109     options.dest = args[1]
 110     main()