testing/talos/talos_from_code.py

   1 #! /usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 #
   7 # Script name:   talos_from_code.py
   8 # Purpose:       Read from a talos.json file the different files to download for a talos job
   9 # Author(s):     Zambrano Gasparnian, Armen <armenzg@mozilla.com>
  10 # Target:        Python 2.5
  11 #
  12 import json
  13 import os
  14 import re
  15 import sys
  16 from optparse import OptionParser
  17
  18 import six
  19
  20
  21 def main():
  22     """
  23     This script downloads a talos.json file which indicates which files to download
  24     for a talos job.
  25     See a talos.json file for a better understand:
  26     https://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
  27     """
  28     parser = OptionParser()
  29     parser.add_option(
  30         "--talos-json-url",
  31         dest="talos_json_url",
  32         type="string",
  33         help="It indicates from where to download the talos.json file.",
  34     )
  35     (options, args) = parser.parse_args()
  36
  37     # 1) check that the url was passed
  38     if options.talos_json_url is None:
  39         print("You need to specify --talos-json-url.")
  40         sys.exit(1)
  41
  42     # 2) try to download the talos.json file
  43     try:
  44         jsonFilename = download_file(options.talos_json_url)
  45     except Exception as e:
  46         print("ERROR: We tried to download the talos.json file but something failed.")
  47         print("ERROR: %s" % str(e))
  48         sys.exit(1)
  49
  50     # 3) download the necessary files
  51     print("INFO: talos.json URL: %s" % options.talos_json_url)
  52     try:
  53         key = "talos.zip"
  54         entity = get_value(jsonFilename, key)
  55         if passesRestrictions(options.talos_json_url, entity["url"]):
  56             # the key is at the same time the filename e.g. talos.zip
  57             print(
  58                 "INFO: Downloading %s as %s"
  59                 % (entity["url"], os.path.join(entity["path"], key))
  60             )
  61             download_file(entity["url"], entity["path"], key)
  62         else:
  63             print(
  64                 "ERROR: You have tried to download a file "
  65                 + "from: %s " % entity["url"]
  66                 + "which is a location different than http://talos-bundles.pvt.build.mozilla.org/"
  67             )
  68             print("ERROR: This is only allowed for the certain branches.")
  69             sys.exit(1)
  70     except Exception as e:
  71         print("ERROR: %s" % str(e))
  72         sys.exit(1)
  73
  74
  75 def passesRestrictions(talosJsonUrl, fileUrl):
  76     """
  77     Only certain branches are exempted from having to host their downloadable files
  78     in talos-bundles.pvt.build.mozilla.org
  79     """
  80     if (
  81         talosJsonUrl.startswith("http://hg.mozilla.org/try/")
  82         or talosJsonUrl.startswith("https://hg.mozilla.org/try/")
  83         or talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/")
  84         or talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/")
  85         or talosJsonUrl.startswith("http://hg.mozilla.org/projects/ash/")
  86         or talosJsonUrl.startswith("https://hg.mozilla.org/projects/ash/")
  87     ):
  88         return True
  89     else:
  90         p = re.compile("^http://talos-bundles.pvt.build.mozilla.org/")
  91         m = p.match(fileUrl)
  92         if m is None:
  93             return False
  94         return True
  95
  96
  97 def get_filename_from_url(url):
  98     """
  99     This returns the filename of the file we're trying to download
 100     """
 101     parsed = six.moves.urllib.parse.urlsplit(url.rstrip("/"))
 102     if parsed.path != "":
 103         return parsed.path.rsplit("/", 1)[-1]
 104     else:
 105         print(
 106             "ERROR: We were trying to download a file from %s "
 107             + "but the URL seems to be incorrect."
 108         )
 109         sys.exit(1)
 110
 111
 112 def download_file(url, path="", saveAs=None):
 113     """
 114     It downloads a file from URL to the indicated path
 115     """
 116     req = six.moves.urllib.request.Request(url)
 117     f = six.moves.urllib.request.urlopen(req)
 118     if path != "" and not os.path.isdir(path):
 119         try:
 120             os.makedirs(path)
 121             print("INFO: directory %s created" % path)
 122         except Exception as e:
 123             print("ERROR: %s" % str(e))
 124             sys.exit(1)
 125     filename = saveAs if saveAs else get_filename_from_url(url)
 126     local_file = open(os.path.join(path, filename), "wb")
 127     local_file.write(f.read())
 128     local_file.close()
 129     return filename
 130
 131
 132 def get_value(json_filename, key):
 133     """
 134     It loads up a JSON file and returns the value for the given string
 135     """
 136     f = open(json_filename, "r")
 137     return json.load(f)[key]
 138
 139
 140 if __name__ == "__main__":
 141     main()