netcore/dl-test-assets.py

   1 #!/usr/bin/env python3
   2
   3 import sys
   4 import os
   5 import subprocess
   6 import xml.etree.ElementTree as ET
   7 import zipfile
   8 import urllib.request, urllib.parse, urllib.error
   9 import multiprocessing
  10
  11 if len(sys.argv) < 4:
  12     print("Usage: dl-test-assets.py <path to assets.xml> <base url> <output dir>")
  13     sys.exit(1)
  14
  15 infile_name = sys.argv [1]
  16 base_url = sys.argv [2]
  17 outdir = sys.argv [3]
  18 tree = ET.parse(infile_name)
  19 root = tree.getroot()
  20
  21 if not os.path.exists(outdir):
  22     os.makedirs(outdir)
  23
  24 def downloadAsset(elem):
  25     if elem.tag != "Blob":
  26         return
  27     id = elem.attrib ["Id"]
  28     filename = os.path.basename(id)
  29     print("Downloading " + filename)
  30     try:
  31         name, hdrs = urllib.request.urlretrieve(base_url + "/" + id, outdir + "/" + filename)
  32     except IOError as e:
  33         print("Download failed for " + id)
  34         sys.exit (1)
  35     print("Extracting " + filename)
  36     with zipfile.ZipFile(outdir + "/" + filename) as zf:
  37         zf.extractall(outdir + "/extracted/" + filename[:-4])
  38
  39 pool = multiprocessing.Pool(multiprocessing.cpu_count())
  40 results = pool.map(downloadAsset, root)
  41 pool.close()
  42 pool.join()