5 # The contents of this file are subject to the terms of the
6 # Common Development and Distribution License (the "License").
7 # You may not use this file except in compliance with the License.
9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 # or http://www.opensolaris.org/os/licensing.
11 # See the License for the specific language governing permissions
12 # and limitations under the License.
14 # When distributing Covered Code, include this CDDL HEADER in each
15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 # If applicable, add the following below this CDDL HEADER, with the
17 # fields enclosed by brackets "[]" replaced with your own identifying
18 # information: Portions Copyright [yyyy] [name of copyright owner]
22 # Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
25 # fetch.py - a file download utility
27 # A simple program similiar to wget(1), but handles local file copy, ignores
28 # directories, and verifies file hashes.
34 from urllib import splittype
35 from urllib2 import urlopen
36 from urllib2 import Request
39 def printIOError(e, txt):
40 """ Function to decode and print IOError type exception """
41 print "I/O Error: " + txt + ": "
44 print str(message) + " (" + str(code) + ")"
48 def validate(file, hash):
49 algorithm, hashvalue = hash.split(':')
51 # force migration away from sha1
52 if algorithm == "sha1":
55 m = hashlib.new(algorithm)
70 return "%s:%s" % (algorithm, m.hexdigest())
72 def validate_container(filename, hash):
74 file = open(filename, 'r')
76 printIOError(e, "Can't open file " + filename)
78 return validate(file, hash)
81 def validate_payload(filename, hash):
86 expr_bz = re.compile('.+\.bz2$', re.IGNORECASE)
87 expr_gz = re.compile('.+\.gz$', re.IGNORECASE)
88 expr_tgz = re.compile('.+\.tgz$', re.IGNORECASE)
91 if expr_bz.match(filename):
92 file = bz2.BZ2File(filename, 'r')
93 elif expr_gz.match(filename):
94 file = gzip.GzipFile(filename, 'r')
95 elif expr_tgz.match(filename):
96 file = gzip.GzipFile(filename, 'r')
100 printIOError(e, "Can't open archive " + filename)
102 return validate(file, hash)
105 def download(url, filename = None, user_agent_arg = None):
110 if user_agent_arg != None:
111 req.add_header("User-Agent", user_agent_arg)
114 printIOError(e, "Can't open url " + url)
117 # 3xx, 4xx and 5xx (f|ht)tp codes designate unsuccessfull action
118 if src.getcode() and (3 <= int(src.getcode()/100) <= 5):
119 print "Error code: " + str(src.getcode())
123 filename = src.geturl().split('/')[-1]
126 dst = open(filename, 'wb');
128 printIOError(e, "Can't open file " + filename + " for writing")
141 # return the name of the file that we downloaded the data to.
144 def download_paths(search, filename, url):
148 tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
150 search += tmp.split(' ')
152 file = os.path.basename(filename)
154 urls = [ base + '/' + file for base in search ]
156 # filename should always be first
158 urls.remove(filename)
159 urls.insert(0, filename)
161 # command line url is a fallback, so it's last
162 if url != None and url not in urls:
168 print "Usage: %s [-a|--user-agent (user-agent)] [-f|--file (file)] " \
169 "[-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] " \
170 "-u|--url (url)" % (sys.argv[0].split('/')[-1])
177 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
179 user_agent_arg = None
187 opts, args = getopt.getopt(sys.argv[1:], "a:f:h:ls:u:",
188 ["file=", "link", "hash=", "search=", "url=",
190 except getopt.GetoptError, err:
194 for opt, arg in opts:
195 if opt in [ "-a", "--user-agent" ]:
197 elif opt in [ "-f", "--file" ]:
199 elif opt in [ "-l", "--link" ]:
201 elif opt in [ "-h", "--hash" ]:
203 elif opt in [ "-s", "--search" ]:
204 search_list.append(arg)
205 elif opt in [ "-u", "--url" ]:
208 assert False, "unknown option"
213 for url in download_paths(search_list, file_arg, url_arg):
214 print "Source %s..." % url,
216 scheme, path = splittype(url)
219 if scheme in [ None, 'file' ]:
220 if os.path.exists(path) == False:
221 print "not found, skipping file copy"
224 if link_arg == False:
225 print "\n copying..."
226 shutil.copy2(path, name)
228 print "\n linking..."
229 os.symlink(path, name)
232 elif scheme in [ 'http', 'https', 'ftp' ]:
233 print "\n downloading...",
234 name = download(url, file_arg, user_agent_arg)
239 print "\n validating...",
241 print "skipping (no hash)"
244 realhash = validate_container(name, hash_arg)
245 if realhash == hash_arg:
249 payloadhash = validate_payload(name, hash_arg)
250 if payloadhash == hash_arg:
253 print "corruption detected"
254 print " expected: %s" % hash_arg
255 print " actual: %s" % realhash
256 print " payload: %s" % payloadhash
265 if __name__ == "__main__":