Lib/urllib.py

   1 """Open an arbitrary URL.
   2
   3 See the following document for more info on URLs:
   4 "Names and Addresses, URIs, URLs, URNs, URCs", at
   5 http://www.w3.org/pub/WWW/Addressing/Overview.html
   6
   7 See also the HTTP spec (from which the error codes are derived):
   8 "HTTP - Hypertext Transfer Protocol", at
   9 http://www.w3.org/pub/WWW/Protocols/
  10
  11 Related standards and specs:
  12 - RFC1808: the "relative URL" spec. (authoritative status)
  13 - RFC1738 - the "URL standard". (authoritative status)
  14 - RFC1630 - the "URI spec". (informational status)
  15
  16 The object returned by URLopener().open(file) will differ per
  17 protocol.  All you know is that is has methods read(), readline(),
  18 readlines(), fileno(), close() and info().  The read*(), fileno()
  19 and close() methods work like those of open files.
  20 The info() method returns a mimetools.Message object which can be
  21 used to query various info about the object, if available.
  22 (mimetools.Message objects are queried with the getheader() method.)
  23 """
  24
  25 import string
  26 import socket
  27 import os
  28 import time
  29 import sys
  30 from urlparse import urljoin as basejoin
  31
  32 __all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
  33            "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
  34            "urlencode", "url2pathname", "pathname2url", "splittag",
  35            "localhost", "thishost", "ftperrors", "basejoin", "unwrap",
  36            "splittype", "splithost", "splituser", "splitpasswd", "splitport",
  37            "splitnport", "splitquery", "splitattr", "splitvalue",
  38            "getproxies"]
  39
  40 __version__ = '1.17'    # XXX This version is not always updated :-(
  41
  42 MAXFTPCACHE = 10        # Trim the ftp cache beyond this size
  43
  44 # Helper for non-unix systems
  45 if os.name == 'nt':
  46     from nturl2path import url2pathname, pathname2url
  47 elif os.name == 'riscos':
  48     from rourl2path import url2pathname, pathname2url
  49 else:
  50     def url2pathname(pathname):
  51         """OS-specific conversion from a relative URL of the 'file' scheme
  52         to a file system path; not recommended for general use."""
  53         return unquote(pathname)
  54
  55     def pathname2url(pathname):
  56         """OS-specific conversion from a file system path to a relative URL
  57         of the 'file' scheme; not recommended for general use."""
  58         return quote(pathname)
  59
  60 # This really consists of two pieces:
  61 # (1) a class which handles opening of all sorts of URLs
  62 #     (plus assorted utilities etc.)
  63 # (2) a set of functions for parsing URLs
  64 # XXX Should these be separated out into different modules?
  65
  66
  67 # Shortcut for basic usage
  68 _urlopener = None
  69 def urlopen(url, data=None, proxies=None):
  70     """Create a file-like object for the specified URL to read from."""
  71     from warnings import warnpy3k
  72     warnpy3k("urllib.urlopen() has been removed in Python 3.0 in "
  73              "favor of urllib2.urlopen()", stacklevel=2)
  74
  75     global _urlopener
  76     if proxies is not None:
  77         opener = FancyURLopener(proxies=proxies)
  78     elif not _urlopener:
  79         opener = FancyURLopener()
  80         _urlopener = opener
  81     else:
  82         opener = _urlopener
  83     if data is None:
  84         return opener.open(url)
  85     else:
  86         return opener.open(url, data)
  87 def urlretrieve(url, filename=None, reporthook=None, data=None):
  88     global _urlopener
  89     if not _urlopener:
  90         _urlopener = FancyURLopener()
  91     return _urlopener.retrieve(url, filename, reporthook, data)
  92 def urlcleanup():
  93     if _urlopener:
  94         _urlopener.cleanup()
  95     _safe_quoters.clear()
  96     ftpcache.clear()
  97
  98 # check for SSL
  99 try:
 100     import ssl
 101 except:
 102     _have_ssl = False
 103 else:
 104     _have_ssl = True
 105
 106 # exception raised when downloaded size does not match content-length
 107 class ContentTooShortError(IOError):
 108     def __init__(self, message, content):
 109         IOError.__init__(self, message)
 110         self.content = content
 111
 112 ftpcache = {}
 113 class URLopener:
 114     """Class to open URLs.
 115     This is a class rather than just a subroutine because we may need
 116     more than one set of global protocol-specific options.
 117     Note -- this is a base class for those who don't want the
 118     automatic handling of errors type 302 (relocated) and 401
 119     (authorization needed)."""
 120
 121     __tempfiles = None
 122
 123     version = "Python-urllib/%s" % __version__
 124
 125     # Constructor
 126     def __init__(self, proxies=None, **x509):
 127         if proxies is None:
 128             proxies = getproxies()
 129         assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
 130         self.proxies = proxies
 131         self.key_file = x509.get('key_file')
 132         self.cert_file = x509.get('cert_file')
 133         self.addheaders = [('User-Agent', self.version)]
 134         self.__tempfiles = []
 135         self.__unlink = os.unlink # See cleanup()
 136         self.tempcache = None
 137         # Undocumented feature: if you assign {} to tempcache,
 138         # it is used to cache files retrieved with
 139         # self.retrieve().  This is not enabled by default
 140         # since it does not work for changing documents (and I
 141         # haven't got the logic to check expiration headers
 142         # yet).
 143         self.ftpcache = ftpcache
 144         # Undocumented feature: you can use a different
 145         # ftp cache by assigning to the .ftpcache member;
 146         # in case you want logically independent URL openers
 147         # XXX This is not threadsafe.  Bah.
 148
 149     def __del__(self):
 150         self.close()
 151
 152     def close(self):
 153         self.cleanup()
 154
 155     def cleanup(self):
 156         # This code sometimes runs when the rest of this module
 157         # has already been deleted, so it can't use any globals
 158         # or import anything.
 159         if self.__tempfiles:
 160             for file in self.__tempfiles:
 161                 try:
 162                     self.__unlink(file)
 163                 except OSError:
 164                     pass
 165             del self.__tempfiles[:]
 166         if self.tempcache:
 167             self.tempcache.clear()
 168
 169     def addheader(self, *args):
 170         """Add a header to be used by the HTTP interface only
 171         e.g. u.addheader('Accept', 'sound/basic')"""
 172         self.addheaders.append(args)
 173
 174     # External interface
 175     def open(self, fullurl, data=None):
 176         """Use URLopener().open(file) instead of open(file, 'r')."""
 177         fullurl = unwrap(toBytes(fullurl))
 178         # percent encode url, fixing lame server errors for e.g, like space
 179         # within url paths.
 180         fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
 181         if self.tempcache and fullurl in self.tempcache:
 182             filename, headers = self.tempcache[fullurl]
 183             fp = open(filename, 'rb')
 184             return addinfourl(fp, headers, fullurl)
 185         urltype, url = splittype(fullurl)
 186         if not urltype:
 187             urltype = 'file'
 188         if urltype in self.proxies:
 189             proxy = self.proxies[urltype]
 190             urltype, proxyhost = splittype(proxy)
 191             host, selector = splithost(proxyhost)
 192             url = (host, fullurl) # Signal special case to open_*()
 193         else:
 194             proxy = None
 195         name = 'open_' + urltype
 196         self.type = urltype
 197         name = name.replace('-', '_')
 198         if not hasattr(self, name):
 199             if proxy:
 200                 return self.open_unknown_proxy(proxy, fullurl, data)
 201             else:
 202                 return self.open_unknown(fullurl, data)
 203         try:
 204             if data is None:
 205                 return getattr(self, name)(url)
 206             else:
 207                 return getattr(self, name)(url, data)
 208         except socket.error, msg:
 209             raise IOError, ('socket error', msg), sys.exc_info()[2]
 210
 211     def open_unknown(self, fullurl, data=None):
 212         """Overridable interface to open unknown URL type."""
 213         type, url = splittype(fullurl)
 214         raise IOError, ('url error', 'unknown url type', type)
 215
 216     def open_unknown_proxy(self, proxy, fullurl, data=None):
 217         """Overridable interface to open unknown URL type."""
 218         type, url = splittype(fullurl)
 219         raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
 220
 221     # External interface
 222     def retrieve(self, url, filename=None, reporthook=None, data=None):
 223         """retrieve(url) returns (filename, headers) for a local object
 224         or (tempfilename, headers) for a remote object."""
 225         url = unwrap(toBytes(url))
 226         if self.tempcache and url in self.tempcache:
 227             return self.tempcache[url]
 228         type, url1 = splittype(url)
 229         if filename is None and (not type or type == 'file'):
 230             try:
 231                 fp = self.open_local_file(url1)
 232                 hdrs = fp.info()
 233                 fp.close()
 234                 return url2pathname(splithost(url1)[1]), hdrs
 235             except IOError:
 236                 pass
 237         fp = self.open(url, data)
 238         try:
 239             headers = fp.info()
 240             if filename:
 241                 tfp = open(filename, 'wb')
 242             else:
 243                 import tempfile
 244                 garbage, path = splittype(url)
 245                 garbage, path = splithost(path or "")
 246                 path, garbage = splitquery(path or "")
 247                 path, garbage = splitattr(path or "")
 248                 suffix = os.path.splitext(path)[1]
 249                 (fd, filename) = tempfile.mkstemp(suffix)
 250                 self.__tempfiles.append(filename)
 251                 tfp = os.fdopen(fd, 'wb')
 252             try:
 253                 result = filename, headers
 254                 if self.tempcache is not None:
 255                     self.tempcache[url] = result
 256                 bs = 1024*8
 257                 size = -1
 258                 read = 0
 259                 blocknum = 0
 260                 if reporthook:
 261                     if "content-length" in headers:
 262                         size = int(headers["Content-Length"])
 263                     reporthook(blocknum, bs, size)
 264                 while 1:
 265                     block = fp.read(bs)
 266                     if block == "":
 267                         break
 268                     read += len(block)
 269                     tfp.write(block)
 270                     blocknum += 1
 271                     if reporthook:
 272                         reporthook(blocknum, bs, size)
 273             finally:
 274                 tfp.close()
 275         finally:
 276             fp.close()
 277
 278         # raise exception if actual size does not match content-length header
 279         if size >= 0 and read < size:
 280             raise ContentTooShortError("retrieval incomplete: got only %i out "
 281                                        "of %i bytes" % (read, size), result)
 282
 283         return result
 284
 285     # Each method named open_<type> knows how to open that type of URL
 286
 287     def open_http(self, url, data=None):
 288         """Use HTTP protocol."""
 289         import httplib
 290         user_passwd = None
 291         proxy_passwd= None
 292         if isinstance(url, str):
 293             host, selector = splithost(url)
 294             if host:
 295                 user_passwd, host = splituser(host)
 296                 host = unquote(host)
 297             realhost = host
 298         else:
 299             host, selector = url
 300             # check whether the proxy contains authorization information
 301             proxy_passwd, host = splituser(host)
 302             # now we proceed with the url we want to obtain
 303             urltype, rest = splittype(selector)
 304             url = rest
 305             user_passwd = None
 306             if urltype.lower() != 'http':
 307                 realhost = None
 308             else:
 309                 realhost, rest = splithost(rest)
 310                 if realhost:
 311                     user_passwd, realhost = splituser(realhost)
 312                 if user_passwd:
 313                     selector = "%s://%s%s" % (urltype, realhost, rest)
 314                 if proxy_bypass(realhost):
 315                     host = realhost
 316
 317             #print "proxy via http:", host, selector
 318         if not host: raise IOError, ('http error', 'no host given')
 319
 320         if proxy_passwd:
 321             import base64
 322             proxy_auth = base64.b64encode(proxy_passwd).strip()
 323         else:
 324             proxy_auth = None
 325
 326         if user_passwd:
 327             import base64
 328             auth = base64.b64encode(user_passwd).strip()
 329         else:
 330             auth = None
 331         h = httplib.HTTP(host)
 332         if data is not None:
 333             h.putrequest('POST', selector)
 334             h.putheader('Content-Type', 'application/x-www-form-urlencoded')
 335             h.putheader('Content-Length', '%d' % len(data))
 336         else:
 337             h.putrequest('GET', selector)
 338         if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
 339         if auth: h.putheader('Authorization', 'Basic %s' % auth)
 340         if realhost: h.putheader('Host', realhost)
 341         for args in self.addheaders: h.putheader(*args)
 342         h.endheaders(data)
 343         errcode, errmsg, headers = h.getreply()
 344         fp = h.getfile()
 345         if errcode == -1:
 346             if fp: fp.close()
 347             # something went wrong with the HTTP status line
 348             raise IOError, ('http protocol error', 0,
 349                             'got a bad status line', None)
 350         # According to RFC 2616, "2xx" code indicates that the client's
 351         # request was successfully received, understood, and accepted.
 352         if (200 <= errcode < 300):
 353             return addinfourl(fp, headers, "http:" + url, errcode)
 354         else:
 355             if data is None:
 356                 return self.http_error(url, fp, errcode, errmsg, headers)
 357             else:
 358                 return self.http_error(url, fp, errcode, errmsg, headers, data)
 359
 360     def http_error(self, url, fp, errcode, errmsg, headers, data=None):
 361         """Handle http errors.
 362         Derived class can override this, or provide specific handlers
 363         named http_error_DDD where DDD is the 3-digit error code."""
 364         # First check if there's a specific handler for this error
 365         name = 'http_error_%d' % errcode
 366         if hasattr(self, name):
 367             method = getattr(self, name)
 368             if data is None:
 369                 result = method(url, fp, errcode, errmsg, headers)
 370             else:
 371                 result = method(url, fp, errcode, errmsg, headers, data)
 372             if result: return result
 373         return self.http_error_default(url, fp, errcode, errmsg, headers)
 374
 375     def http_error_default(self, url, fp, errcode, errmsg, headers):
 376         """Default error handler: close the connection and raise IOError."""
 377         void = fp.read()
 378         fp.close()
 379         raise IOError, ('http error', errcode, errmsg, headers)
 380
 381     if _have_ssl:
 382         def open_https(self, url, data=None):
 383             """Use HTTPS protocol."""
 384
 385             import httplib
 386             user_passwd = None
 387             proxy_passwd = None
 388             if isinstance(url, str):
 389                 host, selector = splithost(url)
 390                 if host:
 391                     user_passwd, host = splituser(host)
 392                     host = unquote(host)
 393                 realhost = host
 394             else:
 395                 host, selector = url
 396                 # here, we determine, whether the proxy contains authorization information
 397                 proxy_passwd, host = splituser(host)
 398                 urltype, rest = splittype(selector)
 399                 url = rest
 400                 user_passwd = None
 401                 if urltype.lower() != 'https':
 402                     realhost = None
 403                 else:
 404                     realhost, rest = splithost(rest)
 405                     if realhost:
 406                         user_passwd, realhost = splituser(realhost)
 407                     if user_passwd:
 408                         selector = "%s://%s%s" % (urltype, realhost, rest)
 409                 #print "proxy via https:", host, selector
 410             if not host: raise IOError, ('https error', 'no host given')
 411             if proxy_passwd:
 412                 import base64
 413                 proxy_auth = base64.b64encode(proxy_passwd).strip()
 414             else:
 415                 proxy_auth = None
 416             if user_passwd:
 417                 import base64
 418                 auth = base64.b64encode(user_passwd).strip()
 419             else:
 420                 auth = None
 421             h = httplib.HTTPS(host, 0,
 422                               key_file=self.key_file,
 423                               cert_file=self.cert_file)
 424             if data is not None:
 425                 h.putrequest('POST', selector)
 426                 h.putheader('Content-Type',
 427                             'application/x-www-form-urlencoded')
 428                 h.putheader('Content-Length', '%d' % len(data))
 429             else:
 430                 h.putrequest('GET', selector)
 431             if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
 432             if auth: h.putheader('Authorization', 'Basic %s' % auth)
 433             if realhost: h.putheader('Host', realhost)
 434             for args in self.addheaders: h.putheader(*args)
 435             h.endheaders(data)
 436             errcode, errmsg, headers = h.getreply()
 437             fp = h.getfile()
 438             if errcode == -1:
 439                 if fp: fp.close()
 440                 # something went wrong with the HTTP status line
 441                 raise IOError, ('http protocol error', 0,
 442                                 'got a bad status line', None)
 443             # According to RFC 2616, "2xx" code indicates that the client's
 444             # request was successfully received, understood, and accepted.
 445             if (200 <= errcode < 300):
 446                 return addinfourl(fp, headers, "https:" + url, errcode)
 447             else:
 448                 if data is None:
 449                     return self.http_error(url, fp, errcode, errmsg, headers)
 450                 else:
 451                     return self.http_error(url, fp, errcode, errmsg, headers,
 452                                            data)
 453
 454     def open_file(self, url):
 455         """Use local file or FTP depending on form of URL."""
 456         if not isinstance(url, str):
 457             raise IOError, ('file error', 'proxy support for file protocol currently not implemented')
 458         if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
 459             return self.open_ftp(url)
 460         else:
 461             return self.open_local_file(url)
 462
 463     def open_local_file(self, url):
 464         """Use local file."""
 465         import mimetypes, mimetools, email.utils
 466         try:
 467             from cStringIO import StringIO
 468         except ImportError:
 469             from StringIO import StringIO
 470         host, file = splithost(url)
 471         localname = url2pathname(file)
 472         try:
 473             stats = os.stat(localname)
 474         except OSError, e:
 475             raise IOError(e.errno, e.strerror, e.filename)
 476         size = stats.st_size
 477         modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
 478         mtype = mimetypes.guess_type(url)[0]
 479         headers = mimetools.Message(StringIO(
 480             'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
 481             (mtype or 'text/plain', size, modified)))
 482         if not host:
 483             urlfile = file
 484             if file[:1] == '/':
 485                 urlfile = 'file://' + file
 486             return addinfourl(open(localname, 'rb'),
 487                               headers, urlfile)
 488         host, port = splitport(host)
 489         if not port \
 490            and socket.gethostbyname(host) in (localhost(), thishost()):
 491             urlfile = file
 492             if file[:1] == '/':
 493                 urlfile = 'file://' + file
 494             return addinfourl(open(localname, 'rb'),
 495                               headers, urlfile)
 496         raise IOError, ('local file error', 'not on local host')
 497
 498     def open_ftp(self, url):
 499         """Use FTP protocol."""
 500         if not isinstance(url, str):
 501             raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')
 502         import mimetypes, mimetools
 503         try:
 504             from cStringIO import StringIO
 505         except ImportError:
 506             from StringIO import StringIO
 507         host, path = splithost(url)
 508         if not host: raise IOError, ('ftp error', 'no host given')
 509         host, port = splitport(host)
 510         user, host = splituser(host)
 511         if user: user, passwd = splitpasswd(user)
 512         else: passwd = None
 513         host = unquote(host)
 514         user = unquote(user or '')
 515         passwd = unquote(passwd or '')
 516         host = socket.gethostbyname(host)
 517         if not port:
 518             import ftplib
 519             port = ftplib.FTP_PORT
 520         else:
 521             port = int(port)
 522         path, attrs = splitattr(path)
 523         path = unquote(path)
 524         dirs = path.split('/')
 525         dirs, file = dirs[:-1], dirs[-1]
 526         if dirs and not dirs[0]: dirs = dirs[1:]
 527         if dirs and not dirs[0]: dirs[0] = '/'
 528         key = user, host, port, '/'.join(dirs)
 529         # XXX thread unsafe!
 530         if len(self.ftpcache) > MAXFTPCACHE:
 531             # Prune the cache, rather arbitrarily
 532             for k in self.ftpcache.keys():
 533                 if k != key:
 534                     v = self.ftpcache[k]
 535                     del self.ftpcache[k]
 536                     v.close()
 537         try:
 538             if not key in self.ftpcache:
 539                 self.ftpcache[key] = \
 540                     ftpwrapper(user, passwd, host, port, dirs)
 541             if not file: type = 'D'
 542             else: type = 'I'
 543             for attr in attrs:
 544                 attr, value = splitvalue(attr)
 545                 if attr.lower() == 'type' and \
 546                    value in ('a', 'A', 'i', 'I', 'd', 'D'):
 547                     type = value.upper()
 548             (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
 549             mtype = mimetypes.guess_type("ftp:" + url)[0]
 550             headers = ""
 551             if mtype:
 552                 headers += "Content-Type: %s\n" % mtype
 553             if retrlen is not None and retrlen >= 0:
 554                 headers += "Content-Length: %d\n" % retrlen
 555             headers = mimetools.Message(StringIO(headers))
 556             return addinfourl(fp, headers, "ftp:" + url)
 557         except ftperrors(), msg:
 558             raise IOError, ('ftp error', msg), sys.exc_info()[2]
 559
 560     def open_data(self, url, data=None):
 561         """Use "data" URL."""
 562         if not isinstance(url, str):
 563             raise IOError, ('data error', 'proxy support for data protocol currently not implemented')
 564         # ignore POSTed data
 565         #
 566         # syntax of data URLs:
 567         # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
 568         # mediatype := [ type "/" subtype ] *( ";" parameter )
 569         # data      := *urlchar
 570         # parameter := attribute "=" value
 571         import mimetools
 572         try:
 573             from cStringIO import StringIO
 574         except ImportError:
 575             from StringIO import StringIO
 576         try:
 577             [type, data] = url.split(',', 1)
 578         except ValueError:
 579             raise IOError, ('data error', 'bad data URL')
 580         if not type:
 581             type = 'text/plain;charset=US-ASCII'
 582         semi = type.rfind(';')
 583         if semi >= 0 and '=' not in type[semi:]:
 584             encoding = type[semi+1:]
 585             type = type[:semi]
 586         else:
 587             encoding = ''
 588         msg = []
 589         msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT',
 590                                             time.gmtime(time.time())))
 591         msg.append('Content-type: %s' % type)
 592         if encoding == 'base64':
 593             import base64
 594             data = base64.decodestring(data)
 595         else:
 596             data = unquote(data)
 597         msg.append('Content-Length: %d' % len(data))
 598         msg.append('')
 599         msg.append(data)
 600         msg = '\n'.join(msg)
 601         f = StringIO(msg)
 602         headers = mimetools.Message(f, 0)
 603         #f.fileno = None     # needed for addinfourl
 604         return addinfourl(f, headers, url)
 605
 606
 607 class FancyURLopener(URLopener):
 608     """Derived class with handlers for errors we can handle (perhaps)."""
 609
 610     def __init__(self, *args, **kwargs):
 611         URLopener.__init__(self, *args, **kwargs)
 612         self.auth_cache = {}
 613         self.tries = 0
 614         self.maxtries = 10
 615
 616     def http_error_default(self, url, fp, errcode, errmsg, headers):
 617         """Default error handling -- don't raise an exception."""
 618         return addinfourl(fp, headers, "http:" + url, errcode)
 619
 620     def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
 621         """Error 302 -- relocated (temporarily)."""
 622         self.tries += 1
 623         if self.maxtries and self.tries >= self.maxtries:
 624             if hasattr(self, "http_error_500"):
 625                 meth = self.http_error_500
 626             else:
 627                 meth = self.http_error_default
 628             self.tries = 0
 629             return meth(url, fp, 500,
 630                         "Internal Server Error: Redirect Recursion", headers)
 631         result = self.redirect_internal(url, fp, errcode, errmsg, headers,
 632                                         data)
 633         self.tries = 0
 634         return result
 635
 636     def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
 637         if 'location' in headers:
 638             newurl = headers['location']
 639         elif 'uri' in headers:
 640             newurl = headers['uri']
 641         else:
 642             return
 643         void = fp.read()
 644         fp.close()
 645         # In case the server sent a relative URL, join with original:
 646         newurl = basejoin(self.type + ":" + url, newurl)
 647         return self.open(newurl)
 648
 649     def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
 650         """Error 301 -- also relocated (permanently)."""
 651         return self.http_error_302(url, fp, errcode, errmsg, headers, data)
 652
 653     def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
 654         """Error 303 -- also relocated (essentially identical to 302)."""
 655         return self.http_error_302(url, fp, errcode, errmsg, headers, data)
 656
 657     def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
 658         """Error 307 -- relocated, but turn POST into error."""
 659         if data is None:
 660             return self.http_error_302(url, fp, errcode, errmsg, headers, data)
 661         else:
 662             return self.http_error_default(url, fp, errcode, errmsg, headers)
 663
 664     def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
 665         """Error 401 -- authentication required.
 666         This function supports Basic authentication only."""
 667         if not 'www-authenticate' in headers:
 668             URLopener.http_error_default(self, url, fp,
 669                                          errcode, errmsg, headers)
 670         stuff = headers['www-authenticate']
 671         import re
 672         match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
 673         if not match:
 674             URLopener.http_error_default(self, url, fp,
 675                                          errcode, errmsg, headers)
 676         scheme, realm = match.groups()
 677         if scheme.lower() != 'basic':
 678             URLopener.http_error_default(self, url, fp,
 679                                          errcode, errmsg, headers)
 680         name = 'retry_' + self.type + '_basic_auth'
 681         if data is None:
 682             return getattr(self,name)(url, realm)
 683         else:
 684             return getattr(self,name)(url, realm, data)
 685
 686     def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
 687         """Error 407 -- proxy authentication required.
 688         This function supports Basic authentication only."""
 689         if not 'proxy-authenticate' in headers:
 690             URLopener.http_error_default(self, url, fp,
 691                                          errcode, errmsg, headers)
 692         stuff = headers['proxy-authenticate']
 693         import re
 694         match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
 695         if not match:
 696             URLopener.http_error_default(self, url, fp,
 697                                          errcode, errmsg, headers)
 698         scheme, realm = match.groups()
 699         if scheme.lower() != 'basic':
 700             URLopener.http_error_default(self, url, fp,
 701                                          errcode, errmsg, headers)
 702         name = 'retry_proxy_' + self.type + '_basic_auth'
 703         if data is None:
 704             return getattr(self,name)(url, realm)
 705         else:
 706             return getattr(self,name)(url, realm, data)
 707
 708     def retry_proxy_http_basic_auth(self, url, realm, data=None):
 709         host, selector = splithost(url)
 710         newurl = 'http://' + host + selector
 711         proxy = self.proxies['http']
 712         urltype, proxyhost = splittype(proxy)
 713         proxyhost, proxyselector = splithost(proxyhost)
 714         i = proxyhost.find('@') + 1
 715         proxyhost = proxyhost[i:]
 716         user, passwd = self.get_user_passwd(proxyhost, realm, i)
 717         if not (user or passwd): return None
 718         proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
 719         self.proxies['http'] = 'http://' + proxyhost + proxyselector
 720         if data is None:
 721             return self.open(newurl)
 722         else:
 723             return self.open(newurl, data)
 724
 725     def retry_proxy_https_basic_auth(self, url, realm, data=None):
 726         host, selector = splithost(url)
 727         newurl = 'https://' + host + selector
 728         proxy = self.proxies['https']
 729         urltype, proxyhost = splittype(proxy)
 730         proxyhost, proxyselector = splithost(proxyhost)
 731         i = proxyhost.find('@') + 1
 732         proxyhost = proxyhost[i:]
 733         user, passwd = self.get_user_passwd(proxyhost, realm, i)
 734         if not (user or passwd): return None
 735         proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
 736         self.proxies['https'] = 'https://' + proxyhost + proxyselector
 737         if data is None:
 738             return self.open(newurl)
 739         else:
 740             return self.open(newurl, data)
 741
 742     def retry_http_basic_auth(self, url, realm, data=None):
 743         host, selector = splithost(url)
 744         i = host.find('@') + 1
 745         host = host[i:]
 746         user, passwd = self.get_user_passwd(host, realm, i)
 747         if not (user or passwd): return None
 748         host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
 749         newurl = 'http://' + host + selector
 750         if data is None:
 751             return self.open(newurl)
 752         else:
 753             return self.open(newurl, data)
 754
 755     def retry_https_basic_auth(self, url, realm, data=None):
 756         host, selector = splithost(url)
 757         i = host.find('@') + 1
 758         host = host[i:]
 759         user, passwd = self.get_user_passwd(host, realm, i)
 760         if not (user or passwd): return None
 761         host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
 762         newurl = 'https://' + host + selector
 763         if data is None:
 764             return self.open(newurl)
 765         else:
 766             return self.open(newurl, data)
 767
 768     def get_user_passwd(self, host, realm, clear_cache=0):
 769         key = realm + '@' + host.lower()
 770         if key in self.auth_cache:
 771             if clear_cache:
 772                 del self.auth_cache[key]
 773             else:
 774                 return self.auth_cache[key]
 775         user, passwd = self.prompt_user_passwd(host, realm)
 776         if user or passwd: self.auth_cache[key] = (user, passwd)
 777         return user, passwd
 778
 779     def prompt_user_passwd(self, host, realm):
 780         """Override this in a GUI environment!"""
 781         import getpass
 782         try:
 783             user = raw_input("Enter username for %s at %s: " % (realm,
 784                                                                 host))
 785             passwd = getpass.getpass("Enter password for %s in %s at %s: " %
 786                 (user, realm, host))
 787             return user, passwd
 788         except KeyboardInterrupt:
 789             print
 790             return None, None
 791
 792
 793 # Utility functions
 794
 795 _localhost = None
 796 def localhost():
 797     """Return the IP address of the magic hostname 'localhost'."""
 798     global _localhost
 799     if _localhost is None:
 800         _localhost = socket.gethostbyname('localhost')
 801     return _localhost
 802
 803 _thishost = None
 804 def thishost():
 805     """Return the IP address of the current host."""
 806     global _thishost
 807     if _thishost is None:
 808         _thishost = socket.gethostbyname(socket.gethostname())
 809     return _thishost
 810
 811 _ftperrors = None
 812 def ftperrors():
 813     """Return the set of errors raised by the FTP class."""
 814     global _ftperrors
 815     if _ftperrors is None:
 816         import ftplib
 817         _ftperrors = ftplib.all_errors
 818     return _ftperrors
 819
 820 _noheaders = None
 821 def noheaders():
 822     """Return an empty mimetools.Message object."""
 823     global _noheaders
 824     if _noheaders is None:
 825         import mimetools
 826         try:
 827             from cStringIO import StringIO
 828         except ImportError:
 829             from StringIO import StringIO
 830         _noheaders = mimetools.Message(StringIO(), 0)
 831         _noheaders.fp.close()   # Recycle file descriptor
 832     return _noheaders
 833
 834
 835 # Utility classes
 836
 837 class ftpwrapper:
 838     """Class used by open_ftp() for cache of open FTP connections."""
 839
 840     def __init__(self, user, passwd, host, port, dirs,
 841                  timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
 842         self.user = user
 843         self.passwd = passwd
 844         self.host = host
 845         self.port = port
 846         self.dirs = dirs
 847         self.timeout = timeout
 848         self.init()
 849
 850     def init(self):
 851         import ftplib
 852         self.busy = 0
 853         self.ftp = ftplib.FTP()
 854         self.ftp.connect(self.host, self.port, self.timeout)
 855         self.ftp.login(self.user, self.passwd)
 856         for dir in self.dirs:
 857             self.ftp.cwd(dir)
 858
 859     def retrfile(self, file, type):
 860         import ftplib
 861         self.endtransfer()
 862         if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
 863         else: cmd = 'TYPE ' + type; isdir = 0
 864         try:
 865             self.ftp.voidcmd(cmd)
 866         except ftplib.all_errors:
 867             self.init()
 868             self.ftp.voidcmd(cmd)
 869         conn = None
 870         if file and not isdir:
 871             # Try to retrieve as a file
 872             try:
 873                 cmd = 'RETR ' + file
 874                 conn = self.ftp.ntransfercmd(cmd)
 875             except ftplib.error_perm, reason:
 876                 if str(reason)[:3] != '550':
 877                     raise IOError, ('ftp error', reason), sys.exc_info()[2]
 878         if not conn:
 879             # Set transfer mode to ASCII!
 880             self.ftp.voidcmd('TYPE A')
 881             # Try a directory listing. Verify that directory exists.
 882             if file:
 883                 pwd = self.ftp.pwd()
 884                 try:
 885                     try:
 886                         self.ftp.cwd(file)
 887                     except ftplib.error_perm, reason:
 888                         raise IOError, ('ftp error', reason), sys.exc_info()[2]
 889                 finally:
 890                     self.ftp.cwd(pwd)
 891                 cmd = 'LIST ' + file
 892             else:
 893                 cmd = 'LIST'
 894             conn = self.ftp.ntransfercmd(cmd)
 895         self.busy = 1
 896         # Pass back both a suitably decorated object and a retrieval length
 897         return (addclosehook(conn[0].makefile('rb'),
 898                              self.endtransfer), conn[1])
 899     def endtransfer(self):
 900         if not self.busy:
 901             return
 902         self.busy = 0
 903         try:
 904             self.ftp.voidresp()
 905         except ftperrors():
 906             pass
 907
 908     def close(self):
 909         self.endtransfer()
 910         try:
 911             self.ftp.close()
 912         except ftperrors():
 913             pass
 914
 915 class addbase:
 916     """Base class for addinfo and addclosehook."""
 917
 918     def __init__(self, fp):
 919         self.fp = fp
 920         self.read = self.fp.read
 921         self.readline = self.fp.readline
 922         if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
 923         if hasattr(self.fp, "fileno"):
 924             self.fileno = self.fp.fileno
 925         else:
 926             self.fileno = lambda: None
 927         if hasattr(self.fp, "__iter__"):
 928             self.__iter__ = self.fp.__iter__
 929             if hasattr(self.fp, "next"):
 930                 self.next = self.fp.next
 931
 932     def __repr__(self):
 933         return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
 934                                              id(self), self.fp)
 935
 936     def close(self):
 937         self.read = None
 938         self.readline = None
 939         self.readlines = None
 940         self.fileno = None
 941         if self.fp: self.fp.close()
 942         self.fp = None
 943
 944 class addclosehook(addbase):
 945     """Class to add a close hook to an open file."""
 946
 947     def __init__(self, fp, closehook, *hookargs):
 948         addbase.__init__(self, fp)
 949         self.closehook = closehook
 950         self.hookargs = hookargs
 951
 952     def close(self):
 953         addbase.close(self)
 954         if self.closehook:
 955             self.closehook(*self.hookargs)
 956             self.closehook = None
 957             self.hookargs = None
 958
 959 class addinfo(addbase):
 960     """class to add an info() method to an open file."""
 961
 962     def __init__(self, fp, headers):
 963         addbase.__init__(self, fp)
 964         self.headers = headers
 965
 966     def info(self):
 967         return self.headers
 968
 969 class addinfourl(addbase):
 970     """class to add info() and geturl() methods to an open file."""
 971
 972     def __init__(self, fp, headers, url, code=None):
 973         addbase.__init__(self, fp)
 974         self.headers = headers
 975         self.url = url
 976         self.code = code
 977
 978     def info(self):
 979         return self.headers
 980
 981     def getcode(self):
 982         return self.code
 983
 984     def geturl(self):
 985         return self.url
 986
 987
 988 # Utilities to parse URLs (most of these return None for missing parts):
 989 # unwrap('<URL:type://host/path>') --> 'type://host/path'
 990 # splittype('type:opaquestring') --> 'type', 'opaquestring'
 991 # splithost('//host[:port]/path') --> 'host[:port]', '/path'
 992 # splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
 993 # splitpasswd('user:passwd') -> 'user', 'passwd'
 994 # splitport('host:port') --> 'host', 'port'
 995 # splitquery('/path?query') --> '/path', 'query'
 996 # splittag('/path#tag') --> '/path', 'tag'
 997 # splitattr('/path;attr1=value1;attr2=value2;...') ->
 998 #   '/path', ['attr1=value1', 'attr2=value2', ...]
 999 # splitvalue('attr=value') --> 'attr', 'value'
1000 # unquote('abc%20def') -> 'abc def'
1001 # quote('abc def') -> 'abc%20def')
1002
1003 try:
1004     unicode
1005 except NameError:
1006     def _is_unicode(x):
1007         return 0
1008 else:
1009     def _is_unicode(x):
1010         return isinstance(x, unicode)
1011
1012 def toBytes(url):
1013     """toBytes(u"URL") --> 'URL'."""
1014     # Most URL schemes require ASCII. If that changes, the conversion
1015     # can be relaxed
1016     if _is_unicode(url):
1017         try:
1018             url = url.encode("ASCII")
1019         except UnicodeError:
1020             raise UnicodeError("URL " + repr(url) +
1021                                " contains non-ASCII characters")
1022     return url
1023
1024 def unwrap(url):
1025     """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
1026     url = url.strip()
1027     if url[:1] == '<' and url[-1:] == '>':
1028         url = url[1:-1].strip()
1029     if url[:4] == 'URL:': url = url[4:].strip()
1030     return url
1031
1032 _typeprog = None
1033 def splittype(url):
1034     """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
1035     global _typeprog
1036     if _typeprog is None:
1037         import re
1038         _typeprog = re.compile('^([^/:]+):')
1039
1040     match = _typeprog.match(url)
1041     if match:
1042         scheme = match.group(1)
1043         return scheme.lower(), url[len(scheme) + 1:]
1044     return None, url
1045
1046 _hostprog = None
1047 def splithost(url):
1048     """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
1049     global _hostprog
1050     if _hostprog is None:
1051         import re
1052         _hostprog = re.compile('^//([^/?]*)(.*)$')
1053
1054     match = _hostprog.match(url)
1055     if match: return match.group(1, 2)
1056     return None, url
1057
1058 _userprog = None
1059 def splituser(host):
1060     """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
1061     global _userprog
1062     if _userprog is None:
1063         import re
1064         _userprog = re.compile('^(.*)@(.*)$')
1065
1066     match = _userprog.match(host)
1067     if match: return map(unquote, match.group(1, 2))
1068     return None, host
1069
1070 _passwdprog = None
1071 def splitpasswd(user):
1072     """splitpasswd('user:passwd') -> 'user', 'passwd'."""
1073     global _passwdprog
1074     if _passwdprog is None:
1075         import re
1076         _passwdprog = re.compile('^([^:]*):(.*)$',re.S)
1077
1078     match = _passwdprog.match(user)
1079     if match: return match.group(1, 2)
1080     return user, None
1081
1082 # splittag('/path#tag') --> '/path', 'tag'
1083 _portprog = None
1084 def splitport(host):
1085     """splitport('host:port') --> 'host', 'port'."""
1086     global _portprog
1087     if _portprog is None:
1088         import re
1089         _portprog = re.compile('^(.*):([0-9]+)$')
1090
1091     match = _portprog.match(host)
1092     if match: return match.group(1, 2)
1093     return host, None
1094
1095 _nportprog = None
1096 def splitnport(host, defport=-1):
1097     """Split host and port, returning numeric port.
1098     Return given default port if no ':' found; defaults to -1.
1099     Return numerical port if a valid number are found after ':'.
1100     Return None if ':' but not a valid number."""
1101     global _nportprog
1102     if _nportprog is None:
1103         import re
1104         _nportprog = re.compile('^(.*):(.*)$')
1105
1106     match = _nportprog.match(host)
1107     if match:
1108         host, port = match.group(1, 2)
1109         try:
1110             if not port: raise ValueError, "no digits"
1111             nport = int(port)
1112         except ValueError:
1113             nport = None
1114         return host, nport
1115     return host, defport
1116
1117 _queryprog = None
1118 def splitquery(url):
1119     """splitquery('/path?query') --> '/path', 'query'."""
1120     global _queryprog
1121     if _queryprog is None:
1122         import re
1123         _queryprog = re.compile('^(.*)\?([^?]*)$')
1124
1125     match = _queryprog.match(url)
1126     if match: return match.group(1, 2)
1127     return url, None
1128
1129 _tagprog = None
1130 def splittag(url):
1131     """splittag('/path#tag') --> '/path', 'tag'."""
1132     global _tagprog
1133     if _tagprog is None:
1134         import re
1135         _tagprog = re.compile('^(.*)#([^#]*)$')
1136
1137     match = _tagprog.match(url)
1138     if match: return match.group(1, 2)
1139     return url, None
1140
1141 def splitattr(url):
1142     """splitattr('/path;attr1=value1;attr2=value2;...') ->
1143         '/path', ['attr1=value1', 'attr2=value2', ...]."""
1144     words = url.split(';')
1145     return words[0], words[1:]
1146
1147 _valueprog = None
1148 def splitvalue(attr):
1149     """splitvalue('attr=value') --> 'attr', 'value'."""
1150     global _valueprog
1151     if _valueprog is None:
1152         import re
1153         _valueprog = re.compile('^([^=]*)=(.*)$')
1154
1155     match = _valueprog.match(attr)
1156     if match: return match.group(1, 2)
1157     return attr, None
1158
1159 # urlparse contains a duplicate of this method to avoid a circular import.  If
1160 # you update this method, also update the copy in urlparse.  This code
1161 # duplication does not exist in Python3.
1162
1163 _hexdig = '0123456789ABCDEFabcdef'
1164 _hextochr = dict((a + b, chr(int(a + b, 16)))
1165                  for a in _hexdig for b in _hexdig)
1166
1167 def unquote(s):
1168     """unquote('abc%20def') -> 'abc def'."""
1169     res = s.split('%')
1170     # fastpath
1171     if len(res) == 1:
1172         return s
1173     s = res[0]
1174     for item in res[1:]:
1175         try:
1176             s += _hextochr[item[:2]] + item[2:]
1177         except KeyError:
1178             s += '%' + item
1179         except UnicodeDecodeError:
1180             s += unichr(int(item[:2], 16)) + item[2:]
1181     return s
1182
1183 def unquote_plus(s):
1184     """unquote('%7e/abc+def') -> '~/abc def'"""
1185     s = s.replace('+', ' ')
1186     return unquote(s)
1187
1188 always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
1189                'abcdefghijklmnopqrstuvwxyz'
1190                '0123456789' '_.-')
1191 _safe_map = {}
1192 for i, c in zip(xrange(256), str(bytearray(xrange(256)))):
1193     _safe_map[c] = c if (i < 128 and c in always_safe) else '%{:02X}'.format(i)
1194 _safe_quoters = {}
1195
1196 def quote(s, safe='/'):
1197     """quote('abc def') -> 'abc%20def'
1198
1199     Each part of a URL, e.g. the path info, the query, etc., has a
1200     different set of reserved characters that must be quoted.
1201
1202     RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
1203     the following reserved characters.
1204
1205     reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
1206                   "$" | ","
1207
1208     Each of these characters is reserved in some component of a URL,
1209     but not necessarily in all of them.
1210
1211     By default, the quote function is intended for quoting the path
1212     section of a URL.  Thus, it will not encode '/'.  This character
1213     is reserved, but in typical usage the quote function is being
1214     called on a path where the existing slash characters are used as
1215     reserved characters.
1216     """
1217     # fastpath
1218     if not s:
1219         return s
1220     cachekey = (safe, always_safe)
1221     try:
1222         (quoter, safe) = _safe_quoters[cachekey]
1223     except KeyError:
1224         safe_map = _safe_map.copy()
1225         safe_map.update([(c, c) for c in safe])
1226         quoter = safe_map.__getitem__
1227         safe = always_safe + safe
1228         _safe_quoters[cachekey] = (quoter, safe)
1229     if not s.rstrip(safe):
1230         return s
1231     return ''.join(map(quoter, s))
1232
1233 def quote_plus(s, safe=''):
1234     """Quote the query fragment of a URL; replacing ' ' with '+'"""
1235     if ' ' in s:
1236         s = quote(s, safe + ' ')
1237         return s.replace(' ', '+')
1238     return quote(s, safe)
1239
1240 def urlencode(query, doseq=0):
1241     """Encode a sequence of two-element tuples or dictionary into a URL query string.
1242
1243     If any values in the query arg are sequences and doseq is true, each
1244     sequence element is converted to a separate parameter.
1245
1246     If the query arg is a sequence of two-element tuples, the order of the
1247     parameters in the output will match the order of parameters in the
1248     input.
1249     """
1250
1251     if hasattr(query,"items"):
1252         # mapping objects
1253         query = query.items()
1254     else:
1255         # it's a bother at times that strings and string-like objects are
1256         # sequences...
1257         try:
1258             # non-sequence items should not work with len()
1259             # non-empty strings will fail this
1260             if len(query) and not isinstance(query[0], tuple):
1261                 raise TypeError
1262             # zero-length sequences of all types will get here and succeed,
1263             # but that's a minor nit - since the original implementation
1264             # allowed empty dicts that type of behavior probably should be
1265             # preserved for consistency
1266         except TypeError:
1267             ty,va,tb = sys.exc_info()
1268             raise TypeError, "not a valid non-string sequence or mapping object", tb
1269
1270     l = []
1271     if not doseq:
1272         # preserve old behavior
1273         for k, v in query:
1274             k = quote_plus(str(k))
1275             v = quote_plus(str(v))
1276             l.append(k + '=' + v)
1277     else:
1278         for k, v in query:
1279             k = quote_plus(str(k))
1280             if isinstance(v, str):
1281                 v = quote_plus(v)
1282                 l.append(k + '=' + v)
1283             elif _is_unicode(v):
1284                 # is there a reasonable way to convert to ASCII?
1285                 # encode generates a string, but "replace" or "ignore"
1286                 # lose information and "strict" can raise UnicodeError
1287                 v = quote_plus(v.encode("ASCII","replace"))
1288                 l.append(k + '=' + v)
1289             else:
1290                 try:
1291                     # is this a sufficient test for sequence-ness?
1292                     len(v)
1293                 except TypeError:
1294                     # not a sequence
1295                     v = quote_plus(str(v))
1296                     l.append(k + '=' + v)
1297                 else:
1298                     # loop over the sequence
1299                     for elt in v:
1300                         l.append(k + '=' + quote_plus(str(elt)))
1301     return '&'.join(l)
1302
1303 # Proxy handling
1304 def getproxies_environment():
1305     """Return a dictionary of scheme -> proxy server URL mappings.
1306
1307     Scan the environment for variables named <scheme>_proxy;
1308     this seems to be the standard convention.  If you need a
1309     different way, you can pass a proxies dictionary to the
1310     [Fancy]URLopener constructor.
1311
1312     """
1313     proxies = {}
1314     for name, value in os.environ.items():
1315         name = name.lower()
1316         if value and name[-6:] == '_proxy':
1317             proxies[name[:-6]] = value
1318     return proxies
1319
1320 def proxy_bypass_environment(host):
1321     """Test if proxies should not be used for a particular host.
1322
1323     Checks the environment for a variable named no_proxy, which should
1324     be a list of DNS suffixes separated by commas, or '*' for all hosts.
1325     """
1326     no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '')
1327     # '*' is special case for always bypass
1328     if no_proxy == '*':
1329         return 1
1330     # strip port off host
1331     hostonly, port = splitport(host)
1332     # check if the host ends with any of the DNS suffixes
1333     for name in no_proxy.split(','):
1334         if name and (hostonly.endswith(name) or host.endswith(name)):
1335             return 1
1336     # otherwise, don't bypass
1337     return 0
1338
1339
1340 if sys.platform == 'darwin':
1341     from _scproxy import _get_proxy_settings, _get_proxies
1342
1343     def proxy_bypass_macosx_sysconf(host):
1344         """
1345         Return True iff this host shouldn't be accessed using a proxy
1346
1347         This function uses the MacOSX framework SystemConfiguration
1348         to fetch the proxy information.
1349         """
1350         import re
1351         import socket
1352         from fnmatch import fnmatch
1353
1354         hostonly, port = splitport(host)
1355
1356         def ip2num(ipAddr):
1357             parts = ipAddr.split('.')
1358             parts = map(int, parts)
1359             if len(parts) != 4:
1360                 parts = (parts + [0, 0, 0, 0])[:4]
1361             return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
1362
1363         proxy_settings = _get_proxy_settings()
1364
1365         # Check for simple host names:
1366         if '.' not in host:
1367             if proxy_settings['exclude_simple']:
1368                 return True
1369
1370         hostIP = None
1371
1372         for value in proxy_settings.get('exceptions', ()):
1373             # Items in the list are strings like these: *.local, 169.254/16
1374             if not value: continue
1375
1376             m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
1377             if m is not None:
1378                 if hostIP is None:
1379                     try:
1380                         hostIP = socket.gethostbyname(hostonly)
1381                         hostIP = ip2num(hostIP)
1382                     except socket.error:
1383                         continue
1384
1385                 base = ip2num(m.group(1))
1386                 mask = m.group(2)
1387                 if mask is None:
1388                     mask = 8 * (m.group(1).count('.') + 1)
1389
1390                 else:
1391                     mask = int(mask[1:])
1392                     mask = 32 - mask
1393
1394                 if (hostIP >> mask) == (base >> mask):
1395                     return True
1396
1397             elif fnmatch(host, value):
1398                 return True
1399
1400         return False
1401
1402     def getproxies_macosx_sysconf():
1403         """Return a dictionary of scheme -> proxy server URL mappings.
1404
1405         This function uses the MacOSX framework SystemConfiguration
1406         to fetch the proxy information.
1407         """
1408         return _get_proxies()
1409
1410     def proxy_bypass(host):
1411         if getproxies_environment():
1412             return proxy_bypass_environment(host)
1413         else:
1414             return proxy_bypass_macosx_sysconf(host)
1415
1416     def getproxies():
1417         return getproxies_environment() or getproxies_macosx_sysconf()
1418
1419 elif os.name == 'nt':
1420     def getproxies_registry():
1421         """Return a dictionary of scheme -> proxy server URL mappings.
1422
1423         Win32 uses the registry to store proxies.
1424
1425         """
1426         proxies = {}
1427         try:
1428             import _winreg
1429         except ImportError:
1430             # Std module, so should be around - but you never know!
1431             return proxies
1432         try:
1433             internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1434                 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1435             proxyEnable = _winreg.QueryValueEx(internetSettings,
1436                                                'ProxyEnable')[0]
1437             if proxyEnable:
1438                 # Returned as Unicode but problems if not converted to ASCII
1439                 proxyServer = str(_winreg.QueryValueEx(internetSettings,
1440                                                        'ProxyServer')[0])
1441                 if '=' in proxyServer:
1442                     # Per-protocol settings
1443                     for p in proxyServer.split(';'):
1444                         protocol, address = p.split('=', 1)
1445                         # See if address has a type:// prefix
1446                         import re
1447                         if not re.match('^([^/:]+)://', address):
1448                             address = '%s://%s' % (protocol, address)
1449                         proxies[protocol] = address
1450                 else:
1451                     # Use one setting for all protocols
1452                     if proxyServer[:5] == 'http:':
1453                         proxies['http'] = proxyServer
1454                     else:
1455                         proxies['http'] = 'http://%s' % proxyServer
1456                         proxies['ftp'] = 'ftp://%s' % proxyServer
1457             internetSettings.Close()
1458         except (WindowsError, ValueError, TypeError):
1459             # Either registry key not found etc, or the value in an
1460             # unexpected format.
1461             # proxies already set up to be empty so nothing to do
1462             pass
1463         return proxies
1464
1465     def getproxies():
1466         """Return a dictionary of scheme -> proxy server URL mappings.
1467
1468         Returns settings gathered from the environment, if specified,
1469         or the registry.
1470
1471         """
1472         return getproxies_environment() or getproxies_registry()
1473
1474     def proxy_bypass_registry(host):
1475         try:
1476             import _winreg
1477             import re
1478         except ImportError:
1479             # Std modules, so should be around - but you never know!
1480             return 0
1481         try:
1482             internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1483                 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1484             proxyEnable = _winreg.QueryValueEx(internetSettings,
1485                                                'ProxyEnable')[0]
1486             proxyOverride = str(_winreg.QueryValueEx(internetSettings,
1487                                                      'ProxyOverride')[0])
1488             # ^^^^ Returned as Unicode but problems if not converted to ASCII
1489         except WindowsError:
1490             return 0
1491         if not proxyEnable or not proxyOverride:
1492             return 0
1493         # try to make a host list from name and IP address.
1494         rawHost, port = splitport(host)
1495         host = [rawHost]
1496         try:
1497             addr = socket.gethostbyname(rawHost)
1498             if addr != rawHost:
1499                 host.append(addr)
1500         except socket.error:
1501             pass
1502         try:
1503             fqdn = socket.getfqdn(rawHost)
1504             if fqdn != rawHost:
1505                 host.append(fqdn)
1506         except socket.error:
1507             pass
1508         # make a check value list from the registry entry: replace the
1509         # '<local>' string by the localhost entry and the corresponding
1510         # canonical entry.
1511         proxyOverride = proxyOverride.split(';')
1512         # now check if we match one of the registry values.
1513         for test in proxyOverride:
1514             if test == '<local>':
1515                 if '.' not in rawHost:
1516                     return 1
1517             test = test.replace(".", r"\.")     # mask dots
1518             test = test.replace("*", r".*")     # change glob sequence
1519             test = test.replace("?", r".")      # change glob char
1520             for val in host:
1521                 # print "%s <--> %s" %( test, val )
1522                 if re.match(test, val, re.I):
1523                     return 1
1524         return 0
1525
1526     def proxy_bypass(host):
1527         """Return a dictionary of scheme -> proxy server URL mappings.
1528
1529         Returns settings gathered from the environment, if specified,
1530         or the registry.
1531
1532         """
1533         if getproxies_environment():
1534             return proxy_bypass_environment(host)
1535         else:
1536             return proxy_bypass_registry(host)
1537
1538 else:
1539     # By default use environment variables
1540     getproxies = getproxies_environment
1541     proxy_bypass = proxy_bypass_environment
1542
1543 # Test and time quote() and unquote()
1544 def test1():
1545     s = ''
1546     for i in range(256): s = s + chr(i)
1547     s = s*4
1548     t0 = time.time()
1549     qs = quote(s)
1550     uqs = unquote(qs)
1551     t1 = time.time()
1552     if uqs != s:
1553         print 'Wrong!'
1554     print repr(s)
1555     print repr(qs)
1556     print repr(uqs)
1557     print round(t1 - t0, 3), 'sec'
1558
1559
1560 def reporthook(blocknum, blocksize, totalsize):
1561     # Report during remote transfers
1562     print "Block number: %d, Block size: %d, Total size: %d" % (
1563         blocknum, blocksize, totalsize)
1564
1565 # Test program
1566 def test(args=[]):
1567     if not args:
1568         args = [
1569             '/etc/passwd',
1570             'file:/etc/passwd',
1571             'file://localhost/etc/passwd',
1572             'ftp://ftp.gnu.org/pub/README',
1573             'http://www.python.org/index.html',
1574             ]
1575         if hasattr(URLopener, "open_https"):
1576             args.append('https://synergy.as.cmu.edu/~geek/')
1577     try:
1578         for url in args:
1579             print '-'*10, url, '-'*10
1580             fn, h = urlretrieve(url, None, reporthook)
1581             print fn
1582             if h:
1583                 print '======'
1584                 for k in h.keys(): print k + ':', h[k]
1585                 print '======'
1586             with open(fn, 'rb') as fp:
1587                 data = fp.read()
1588             if '\r' in data:
1589                 table = string.maketrans("", "")
1590                 data = data.translate(table, "\r")
1591             print data
1592             fn, h = None, None
1593         print '-'*40
1594     finally:
1595         urlcleanup()
1596
1597 def main():
1598     import getopt, sys
1599     try:
1600         opts, args = getopt.getopt(sys.argv[1:], "th")
1601     except getopt.error, msg:
1602         print msg
1603         print "Use -h for help"
1604         return
1605     t = 0
1606     for o, a in opts:
1607         if o == '-t':
1608             t = t + 1
1609         if o == '-h':
1610             print "Usage: python urllib.py [-t] [url ...]"
1611             print "-t runs self-test;",
1612             print "otherwise, contents of urls are printed"
1613             return
1614     if t:
1615         if t > 1:
1616             test1()
1617         test(args)
1618     else:
1619         if not args:
1620             print "Use -h for help"
1621         for url in args:
1622             print urlopen(url).read(),
1623
1624 # Run test program when run as a script
1625 if __name__ == '__main__':
1626     main()