testing/talos/talos_from_code.py

   1 #! /usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 #
   7 # Script name:   talos_from_code.py
   8 # Purpose:       Read from a talos.json file the different files to download for a talos job
   9 # Author(s):     Zambrano Gasparnian, Armen <armenzg@mozilla.com>
  10 # Target:        Python 2.5
  11 #
  12 from optparse import OptionParser
  13 import json
  14 import re
  15 import urllib2
  16 import urlparse
  17 import sys
  18 import os
  19
  20 def main():
  21     '''
  22     This script downloads a talos.json file which indicates which files to download
  23     for a talos job.
  24     See a talos.json file for a better understand:
  25     http://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
  26     '''
  27     parser = OptionParser()
  28     parser.add_option("--talos-json-url", dest="talos_json_url", type="string",
  29                       help="It indicates from where to download the talos.json file.")
  30     (options, args) = parser.parse_args()
  31
  32     # 1) check that the url was passed
  33     if options.talos_json_url == None:
  34         print "You need to specify --talos-json-url."
  35         sys.exit(1)
  36
  37     # 2) try to download the talos.json file
  38     try:
  39         jsonFilename = download_file(options.talos_json_url)
  40     except Exception, e:
  41         print "ERROR: We tried to download the talos.json file but something failed."
  42         print "ERROR: %s" % str(e)
  43         sys.exit(1)
  44
  45     # 3) download the necessary files
  46     print "INFO: talos.json URL: %s" % options.talos_json_url
  47     try:
  48         key = 'talos.zip'
  49         entity = get_value(jsonFilename, key)
  50         if passesRestrictions(options.talos_json_url, entity["url"]):
  51             # the key is at the same time the filename e.g. talos.zip
  52             print "INFO: Downloading %s as %s" % (entity["url"], os.path.join(entity["path"], key))
  53             download_file(entity["url"], entity["path"], key)
  54         else:
  55             print "ERROR: You have tried to download a file " + \
  56                   "from: %s " % fileUrl + \
  57                   "which is a location different than http://talos-bundles.pvt.build.mozilla.org/"
  58             print "ERROR: This is only allowed for the certain branches."
  59             sys.exit(1)
  60     except Exception, e:
  61         print "ERROR: %s" % str(e)
  62         sys.exit(1)
  63
  64 def passesRestrictions(talosJsonUrl, fileUrl):
  65     '''
  66     Only certain branches are exempted from having to host their downloadable files
  67     in talos-bundles.pvt.build.mozilla.org
  68     '''
  69     if talosJsonUrl.startswith("http://hg.mozilla.org/try/") == True or \
  70        talosJsonUrl.startswith("https://hg.mozilla.org/try/") == True or \
  71        talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") == True or \
  72        talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/") == True:
  73         return True
  74     else:
  75         p = re.compile('^http://talos-bundles.pvt.build.mozilla.org/')
  76         m = p.match(fileUrl)
  77         if m == None:
  78             return False
  79         return True
  80
  81 def get_filename_from_url(url):
  82     '''
  83     This returns the filename of the file we're trying to download
  84     '''
  85     parsed = urlparse.urlsplit(url.rstrip('/'))
  86     if parsed.path != '':
  87         return parsed.path.rsplit('/', 1)[-1]
  88     else:
  89         print "ERROR: We were trying to download a file from %s " + \
  90               "but the URL seems to be incorrect."
  91         sys.exit(1)
  92
  93 def download_file(url, path="", saveAs=None):
  94     '''
  95     It downloads a file from URL to the indicated path
  96     '''
  97     req = urllib2.Request(url)
  98     f = urllib2.urlopen(req)
  99     if path != "" and not os.path.isdir(path):
 100         try:
 101             os.makedirs(path)
 102             print "INFO: directory %s created" % path
 103         except Exception, e:
 104             print "ERROR: %s" % str(e)
 105             sys.exit(1)
 106     filename = saveAs if saveAs else get_filename_from_url(url)
 107     local_file = open(os.path.join(path, filename), 'wb')
 108     local_file.write(f.read())
 109     local_file.close()
 110     return filename
 111
 112 def get_value(json_filename, key):
 113     '''
 114     It loads up a JSON file and returns the value for the given string
 115     '''
 116     f = open(json_filename, 'r')
 117     return json.load(f)[key]
 118
 119 if __name__ == '__main__':
 120     main()