blockfinder

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # For the people of Smubworld!
   5 import urllib2
   6 import os
   7 import time
   8 import getopt
   9 import sys
  10 from math import ceil, log
  11 import sqlite3
  12 import hashlib
  13 import gzip
  14 from xml.dom import minidom
  15 import socket
  16 __program__ = 'blockfinder'
  17 __url__ = 'http://github.com/ioerror/blockfinder/'
  18 ___author__ = 'Jacob Appelbaum <jacob@appelbaum.net>, dave b. <db@d1b.org>'
  19 __copyright__ = 'Copyright (c) 2010'
  20 __license__ = 'See LICENSE for licensing information'
  21 __version__ = '3.1415'
  22
  23 try:
  24     import GeoIP
  25 except ImportError:
  26      GeoIP = None
  27
  28 try:
  29     from future import antigravity
  30 except ImportError:
  31     antigravity = None
  32
  33 class BlockFinderError(Exception):
  34     pass
  35
  36 def update_progress_bar(percent_done, caption=""):
  37     """Write a progress bar to the console"""
  38     rows, columns = map(int, os.popen('stty size', 'r').read().split())
  39     width = columns - 4 - len(caption)
  40     sys.stdout.write("[%s>%s] %s\x1b[G" % (
  41                                     "=" * int(percent_done*width),
  42                                     "." * (width - int(percent_done * width)),
  43                                     caption) )
  44     sys.stdout.flush()
  45
  46 # XXX TODO:allow the use of a proxy
  47 # Set up a proper Request object, set the user agent and if desired, a proxy
  48 def fetch(url, useragent):
  49     """ Fetch (with progress meter) and return the contents of a url. """
  50     req = urllib2.Request(url)
  51     req.add_header('User-agent', useragent)
  52     #req.set_proxy(host, type)
  53     fetcher = urllib2.urlopen(req)
  54     length_header = fetcher.headers.get("content-length")
  55     if length_header == None:
  56         raise Exception("Missing content-length header in reply from server.")
  57     length = int(length_header)
  58     print "Fetching ", str (round(float(length/1024),2)) , " kilobytes"
  59     ret = ""
  60     t_start = time.time()
  61     while True:
  62         t_delta = time.time() - t_start
  63         if t_delta == 0:
  64             t_delta = 1
  65         update_progress_bar(
  66             float(len(ret)) / length,
  67             "%.2f K/s" % (len(ret) / 1024 / t_delta) )
  68         tmp = fetcher.read(1024)
  69         if len(tmp) == 0:
  70             if len(ret) != length:
  71                 raise Exception("Expected %s bytes, only received %s" % (
  72                                 len(ret), length ))
  73             print ""
  74             return ret
  75         ret += tmp
  76
  77 def create_blockfinder_cache_dir(cache_dir):
  78     if not os.path.exists(cache_dir):
  79         if verbose:
  80             print "Initializing the cache directory..."
  81         os.mkdir(cache_dir)
  82
  83 def write_to_a_text_file(file_loc, data):
  84     f = open(file_loc, 'w')
  85     f.write(data)
  86     f.close()
  87
  88 def cache_delegation(cache_dir, delegation_url, useragent):
  89     """ Attempt to cache the contents of a delegation url in our cache dir. """
  90     delegation = ""
  91     print "Fetching " + delegation_url
  92     delegation = fetch(delegation_url,useragent)
  93     tmp = delegation_url.split('/')
  94     delegation_file = str(cache_dir) + str(tmp[-1])
  95     try:
  96         write_to_a_text_file(delegation_file, delegation)
  97         return True
  98     except Exception, e:
  99         print repr(e)
 100         return False
 101
 102 def cache_is_dated(cache_dir, cached_files):
 103     """ Returns True if the mtime of any files in cache dir is > 24hrs."""
 104     try:
 105         os.stat(cache_dir)
 106     except OSError, e:
 107         print "\nDid you initialize the cache directory?\n"
 108         raise e
 109     for file in cached_files:
 110         fstat = os.stat(cache_dir + file)
 111         if (time.time() - fstat.st_mtime) > 86400:
 112             return True
 113     return False
 114
 115 def create_sql_database(cache_dir):
 116     """ Creates a new sqlite database.
 117          If there is a previous sqlite database it will be deleted. """
 118     conn = sqlite3.connect(cache_dir + "sqlitedb")
 119     cursor = conn.cursor()
 120     for table in ["ipv4", "ipv6", "asn"]:
 121         cursor.execute("""drop table if exists """ + table)
 122     cursor.execute("""create table asn(registry text, cc text, start text, value INTEGER, date text, status text)""")
 123     cursor.execute("""create table ipv4(registry text, cc text, start text, value INTEGER, date text, status text)""")
 124     cursor.execute("""create table ipv6(registry text, cc text, start text, value INTEGER, date text, status text)""")
 125     cursor.execute("""create table if not exists lir_record(cc text, start text, value INTEGER, type INTEGER)""")
 126     conn.commit()
 127     cursor.close()
 128
 129 def insert_into_sql_database(delegations,cache_dir):
 130     """ inserts delegation information into the sqlite database"""
 131     conn = sqlite3.connect(cache_dir +"sqlitedb")
 132     cursor = conn.cursor()
 133     table = ""
 134     for delegation in delegations:
 135         for entry in delegation:
 136             registry = str(entry['registry'])
 137             if not registry.isdigit() and str (entry['cc']) !="*":
 138                 if entry['type'] == "ipv6":
 139                     table = "ipv6"
 140                 if entry['type'] == "ipv4":
 141                     table = "ipv4"
 142                 if entry['type'] == "asn":
 143                     table = "asn"
 144                 text = """INSERT INTO """ + table + """ ( registry, cc, start, value, date,status) VALUES (?,?,?,?,?,?)"""
 145                 data = [entry['registry'], entry['cc'], entry['start'], entry['value'], entry['date'], entry['status'] ]
 146                 cursor.execute(text, data  )
 147     conn.commit()
 148     cursor.close()
 149
 150 def get_total_delegations_from_db(cache_dir):
 151     """ Returns the total count of the number of entries in the ipv4, ipv6 and asn table """
 152     conn = sqlite3.connect(cache_dir +"sqlitedb")
 153     cursor = conn.cursor()
 154     count = 0
 155     table_names = ["ipv4", "ipv6", "asn"]
 156     for table in table_names:
 157         cursor.execute("""select count (*)  from """ + table)
 158         count += int (cursor.fetchone()[0] )
 159     cursor.close()
 160     return count
 161
 162 def get_possible_match_entries(cc,cache_dir):
 163     """ Get the count of 'possible' matching delegation entries"""
 164     conn = sqlite3.connect(cache_dir +"sqlitedb")
 165     cursor = conn.cursor()
 166     count = 0
 167     table_names =["ipv4", "ipv6", "asn"]
 168     for table in table_names:
 169         cursor.execute("""select count (*)  from """ + table + """ where cc=?""",cc)
 170         count += int (cursor.fetchone()[0] )
 171     cursor.close()
 172     return count
 173
 174 def use_sql_database(request, cc, cache_dir):
 175     """ Use the sqlite database that is created after fetching delegations
 176         to output information for a given request """
 177     conn = sqlite3.connect(cache_dir + "sqlitedb")
 178     cursor = conn.cursor()
 179     if verbose:
 180         print "We have %d entries in our delegation cache." %get_total_delegations_from_db(cache_dir)
 181     text ="""select start,value from """ + request + """ where cc=?"""
 182     cc = (cc,)
 183     cursor.execute(text,cc)
 184     result = []
 185     for row in cursor:
 186         if request == "ipv4":
 187             result.append(str(row[0]) + "/" + str(calculate_ipv4_subnet(int(row[1]))))
 188         elif request == "ipv6":
 189             result.append(str(row[0]) + "/" + str(int(row[1])))
 190         else:
 191             result.append(str(int(row[0])))
 192     result.sort()
 193     if verbose:
 194         result.append("We found %d possible entries in our delegation cache." % get_possible_match_entries(cc, cache_dir) )
 195         cursor.execute("""select count(*) from """ + request + """ where cc=?""", cc )
 196         result.append("We found %d matching entries in our delegation cache." % int (cursor.fetchone()[0] ) )
 197     cursor.close()
 198     return result
 199
 200 def get_md5_from_delegation_md5_file(cache_dir, delegation_file):
 201     """ Returns the md5sum from the delegation md5 file
 202         if it doesn't exist it returns an empty string"""
 203     checksum = ""
 204     try:
 205         f = open(cache_dir + delegation_file + ".md5", "r")
 206         checksum = f.read()
 207         f.close()
 208         if delegation_file == "delegated-afrinic-latest":
 209             pos =  checksum.find(" ")
 210             checksum = str (checksum[:pos])
 211         else:
 212             pos = checksum.find("=") +2
 213             checksum = str (checksum[pos:-1])
 214     except Exception, e:
 215         print repr(e)
 216     return checksum
 217
 218 def verify_delegation_file(cache_dir, delegation_file):
 219     """ compares the delegation file md5sum to that of the provided md5sum
 220        returns True if they match otherwise returns False """
 221     checksum = ""
 222     checksum_of_file = ""
 223     try:
 224         f = open(cache_dir + delegation_file, "rb")
 225         checksum_of_file = str (hashlib.md5(f.read()).hexdigest() )
 226         f.close()
 227     except Exception, e:
 228         print repr(e)
 229     checksum = get_md5_from_delegation_md5_file(cache_dir,delegation_file)
 230     if checksum != checksum_of_file:
 231         return False
 232     if checksum == checksum_of_file and checksum != "":
 233         return True
 234     return False
 235
 236 def verify_cache(cache_dir, delegation_files):
 237     """ if in verbose mode prints the result of checking the checksum of the
 238         delegation files """
 239     for file in delegation_files:
 240         if verbose:
 241             print "verifying " + file
 242         if verify_delegation_file(cache_dir,file):
 243             if verbose:
 244                 print "the md5 checksum of " + file + " *matches* the provided checksum"
 245         else:
 246             if verbose:
 247                 print "the md5 checksum of " + file + " does *not* match the provided checksum"
 248
 249 def update_delegation_cache(cache_dir, delegation_urls, useragent):
 250     """ Fetch multiple delegation urls and cache the contents. """
 251     print "Updating delegation cache..."
 252
 253     for url in delegation_urls.split():
 254         cache_delegation(cache_dir, url + ".md5",useragent)
 255         if verify_delegation_file(cache_dir, url.rpartition('/')[-1]):
 256             pass
 257         else:
 258             cache_delegation(cache_dir, url,useragent)
 259
 260 def update_lir_delegation_cache(cache_dir, delegation_urls, useragent):
 261     """ Fetch multiple LIR delegation urls and cache the contents. """
 262     print "Updating LIR delegation cache..."
 263     for url in delegation_urls.split():
 264         cache_delegation(cache_dir, url,useragent)
 265     unpack_a_delegation_cache(cache_dir, delegation_urls, "LIR")
 266
 267
 268 def extract_data_from_gzip_file(gzip_file_loc, extract_file_loc):
 269         gzip_file = gzip.open(gzip_file_loc, 'rb')
 270         gunzipped_data = gzip_file.read()
 271         gzip_file.close()
 272         gunzipped_file = open(extract_file_loc, 'w')
 273         gunzipped_file.writelines(gunzipped_data)
 274         gunzipped_file.close()
 275
 276 def unpack_a_delegation_cache(cache_dir, delegation_urls, del_type=""):
 277     """ Unpack the fetched LIR delegation files into the blockfinder cache. """
 278     # This probably should unlink the gzip'ed file if we care about space...
 279     for url in delegation_urls.split():
 280         gzip_filename = url.rpartition('/')[-1]
 281         gunziped_filename = gzip_filename.rpartition('.')[0]
 282         if verbose:
 283             print "Unpacking " + del_type + "file " + gzip_filename + " into our cache as " + gunziped_filename
 284         extract_data_from_gzip_file(cache_dir + gzip_filename, cache_dir + gunziped_filename)
 285
 286 def update_geoip_cache(cache_dir, geoip_urls, useragent):
 287     """ Fetch country level resolution GeoIP files from a given url and cache
 288     the contents. Unpack it if it's compressed. """
 289     print "Updating GeoIP cache..."
 290     for url in geoip_urls.split():
 291         cache_delegation(cache_dir, url, useragent)
 292     unpack_a_delegation_cache(cache_dir, geoip_urls, "GeoIP")
 293
 294 def load_delegation(delegation_file):
 295     """ Load, parse and store the delegation file contents as a list. """
 296     keys = "registry cc type start value date status"
 297     try:
 298         f = open(delegation_file, "r")
 299         delegations = [ dict((k,v) for k,v in zip(keys.split(), line.split("|")))
 300                         for line in f.readlines() if not line.startswith("#")]
 301         f.close()
 302         return delegations
 303     except OSError, e:
 304         print repr(e)
 305
 306 def load_all_delegations(cache_dir, delegation_urls):
 307     """ Load all delegations into memory. """
 308     delegations = []
 309     for url in delegation_urls.split():
 310         filename = url.rpartition('/')[-1]
 311         if verbose:
 312             print "Attempting to load delegation file into memory: " + filename
 313         delegations.append(load_delegation(cache_dir + filename))
 314     return delegations
 315
 316 def calculate_ipv4_subnet(host_count):
 317     """ XXX: this will return incorrect values for anything above a /4
 318         e.g. calculate_ipv4_subnet(536870912) will return 2 instead of 3
 319     """
 320     return 32 - int(ceil(log(host_count,2)))
 321
 322 def download_country_code_file(cache_dir, useragent):
 323     """ Download and save the latest opencountrycode XML file """
 324     # Google frontend will not return content-length for some reason...
 325     url = "http://opencountrycodes.appspot.com/xml"
 326     xml = urllib2.urlopen(url).read()
 327     write_to_a_text_file(cache_dir + "countrycodes.xml", xml)
 328
 329 def build_country_code_dictionary(cache_dir):
 330     """ Return a dictionary mapping country name to the country code"""
 331     map_co = {}
 332     xml_file = str(cache_dir) + "countrycodes.xml"
 333     clist = minidom.parse(xml_file)
 334     for country in clist.getElementsByTagName("country"):
 335         code = country.attributes["code"]
 336         name = country.attributes["name"]
 337         map_co[name.value] = code.value
 338     return map_co
 339
 340 def get_name_from_country_code(cache_dir, cc_code):
 341     map_co = build_country_code_dictionary(cache_dir)
 342     country_name = [(key, value) for (key, value) in map_co.items() if value == cc_code]
 343     if len(country_name) > 0:
 344         return country_name[0][0]
 345
 346 def get_country_code_from_name(cache_dir, country_name):
 347     """ Return the country code for a given country name. """
 348     map_co = build_country_code_dictionary(cache_dir)
 349     cc_code = [map_co[key] for key in map_co.keys() if key.upper().startswith(country_name.upper())]
 350     if len(cc_code) > 0:
 351         return cc_code[0]
 352
 353 def ip_address_to_dec(ip_addr):
 354     ipar = ip_addr.split('.')
 355     a = ['','','','']
 356     for i in range(4):
 357         a[i] = hex(int(ipar[i]))[2:]
 358         if(int(ipar[i]) <= 15):
 359             a[i] = """0""" + a[i]
 360
 361     total = '0x'+a[0]+a[1]+a[2]+a[3]
 362     decimal = int(total,16)
 363     return decimal
 364
 365 def ipv4_address_valid(ip_addr):
 366     ipv4arr = ip_addr.split('.')
 367     if len(ipv4arr) == 4:
 368         for items in ipv4arr:
 369             if int(items) > 255:
 370                 return False
 371         return True
 372     else:
 373         return False
 374
 375 def geoip_lookup(cache_dir, ip_addr):
 376     # This would work with the CVS version of the GeoIP code
 377     # However, MaxMind hasn't done a release in a long time.
 378     # http://geoip.cvs.sourceforge.net/viewvc/geoip/python/test_v6.py?revision=1.1&view=markup
 379     #        gi = GeoIP.open(cache_dir + "GeoIPv6.dat",GeoIP.GEOIP_STANDARD)
 380     #        cc = gi.country_code_by_addr_v6(ip_addr)
 381     #        cc_name = gi.country_name_by_addr_v6(ip_addr)
 382     gi = GeoIP.open(cache_dir + "GeoIP.dat",GeoIP.GEOIP_STANDARD)
 383     cc = gi.country_code_by_addr(ip_addr)
 384     cc_name = gi.country_name_by_addr(ip_addr)
 385     return cc, cc_name
 386
 387
 388 def rir_or_lir_lookup(ip_addr,lookup_type, cache_dir):
 389     ipv4arr = ip_addr.split('.')
 390     result = []
 391     conn = sqlite3.connect(cache_dir + "sqlitedb")
 392     cursor = conn.cursor()
 393     if lookup_type == 'rir':
 394         cursor.execute('select cc, start, value from ipv4 WHERE start LIKE ?', ( ipv4arr[0] + "." + ipv4arr[1] + ".%",))
 395     else:
 396         cursor.execute('select * from lir_record WHERE start LIKE ? and type=4', (ipv4arr[0] + "." + ipv4arr[1] + ".%",))
 397     row = cursor.fetchone()
 398
 399     if row is None:
 400         if lookup_type == "rir":
 401             cursor.execute('select cc, start, value from ipv4 WHERE start LIKE ? ', (ipv4arr[0] + ".%",))
 402         else:
 403             cursor.execute('select * from lir_record WHERE start LIKE ? and type=4', (ipv4arr[0] + ".%",))
 404         row = cursor.fetchone()
 405
 406     while(row is not None):
 407         if (ip_address_to_dec(row[1]) <= ip_address_to_dec(ip_addr) <= (ip_address_to_dec(row[1]) + row[2])):
 408             result.append(row[0])
 409             result.append(get_name_from_country_code(cache_dir, row[0]))
 410             cursor.close()
 411             return result
 412         row = cursor.fetchone()
 413     cursor.close()
 414
 415 def rir_lookup(ip_addr, cache_dir):
 416     return rir_or_lir_lookup(ip_addr, "rir", cache_dir)
 417
 418 def lir_lookup(ip_addr, cache_dir):
 419     return rir_or_lir_lookup(ip_addr, "lir", cache_dir)
 420
 421 def lookup_ip_address(ip_addr, cache_dir):
 422     """ Return the country code and name for a given ip address. Attempts to
 423         use GeoIP if available."""
 424
 425     ip_addr = socket.getaddrinfo(ip_addr, 80)[0][4][0]
 426     if not ipv4_address_valid(ip_addr):
 427         raise BlockFinderError('Invalid ip address!')
 428     rir_cc = ""
 429     print "Reverse lookup for: " + ip_addr
 430     if GeoIP:
 431         geoip_cc, geoip_cc_name = geoip_lookup(cache_dir, ip_addr)
 432         print "GeoIP country code: " + str(geoip_cc)
 433         print "GeoIP country name: " + str(geoip_cc_name)
 434
 435     rir = rir_lookup(ip_addr, cache_dir)
 436
 437     if rir:
 438         rir_cc = rir[0]
 439         print 'RIR country code:', rir[0]
 440         print 'RIR country:', rir[1]
 441     else:
 442         print 'Not found in RIR db'
 443
 444     lir = lir_lookup(ip_addr, cache_dir)
 445     if lir:
 446         print 'LIR country code:', lir[0]
 447         print 'LIR country :', lir[1]
 448
 449     if GeoIP:
 450         if geoip_cc != rir_cc:
 451             print "It appears that the RIR data conflicts with the GeoIP data"
 452             print "The GeoIP data is likely closer to being correct due to " \
 453                   "sub-delegation issues with LIR databases"
 454
 455 def return_first_ip_and_number_in_inetnum(line):
 456     start_ip = line.split("-")[0].strip()
 457     end_ip = line.split("-")[1].strip()
 458     num_ips = 1 + (ip_address_to_dec(end_ip) - ip_address_to_dec(start_ip) )
 459     return start_ip, num_ips
 460
 461 def create_or_replace_lir_table_in_db(cache_dir):
 462     conn = sqlite3.connect(cache_dir + "sqlitedb")
 463     cursor = conn.cursor()
 464     cursor.execute("""drop table if exists lir_record """)
 465     cursor.execute("""create table if not exists lir_record(cc text, start text, value INTEGER, type INTEGER)""")
 466     conn.commit()
 467     cursor.close()
 468
 469 def extract_info_from_lir_file_and_insert_into_sqlite(cache_dir, filename):
 470     block = []
 471     country = ""
 472     entry = False
 473     conn = sqlite3.connect(cache_dir + "sqlitedb")
 474     cursor =  conn.cursor()
 475     insert_text = """insert into lir_record (cc, start, value, type) VALUES (?,?,?,?)"""
 476     version = ""
 477     for line in open(cache_dir + filename, "r"):
 478         line = line.replace("\n", "")
 479         if line == "":
 480             entry = False
 481             country, block, version = "", [], ""
 482         elif not entry and "inetnum:" in line:
 483             try:
 484                 line = line.replace("inetnum:", "").strip()
 485                 start_ip, num_ips = return_first_ip_and_number_in_inetnum(line)
 486                 block = [start_ip, num_ips]
 487                 entry = True
 488                 version = "4"
 489             except Exception, e:
 490                 print e
 491         elif not entry and "inet6num:" in line:
 492             try:
 493                 block = line.replace("inet6num:", "").strip().split("/")
 494                 entry = True
 495                 version = "6"
 496             except Exception, e:
 497                 print e
 498         elif entry and "country:" in line:
 499             country = line.replace("country:", "").strip()
 500             data = (country, block[0], block[1], version )
 501             cursor.execute(insert_text, data)
 502     conn.commit()
 503     cursor.close()
 504
 505 def create_db_and_insert_delegation_into_db(cache_dir, delegation_urls):
 506         create_sql_database(cache_dir)
 507         delegations = load_all_delegations(cache_dir, delegation_urls)
 508         insert_into_sql_database(delegations, cache_dir)
 509
 510
 511 def usage():
 512     """ Print usage information. """
 513     print >> sys.stderr, """
 514 blockfinder  [-c DIR] -i
 515 blockfinder [options] -t COUNTRY
 516
 517 The first form initializes the local cache. The second form queries it.
 518
 519 Understood options (not all of which are implemented yet):
 520     -h, --help          Show this help and exit
 521     -v                  Be verbose
 522     -c, --cachedir DIR  Set the cache directory
 523     -u, --useragent
 524     -p, --progress
 525     -o, --output FILE
 526     -4, --ipv4          Search IPv4 allocations
 527     -6, --ipv6          Search IPv6 allocation
 528     -a, --asn           Search ASN allocations
 529     -t, --nation-state CC  Set the country to search (given as a two-letter code)
 530     -n, --country-name "Costa Rica"  Set country to search (full name)
 531     -x, --hack-the-internet          Hack the internet
 532     -r, --reverse-lookup    Return the county name for the specified IP
 533     -i, --initialize-delegation    Initialize or update delegation information
 534     -l, --initialize-lir    Initialize or update lir information
 535     -d, --reload-delegation    Use existing delegation files to update the database
 536
 537 At least one of -t or -i is required, and when in -t mode, at least one of -4,
 538 -6, and -a is required in order to do anything sensible.
 539 """
 540
 541 def main():
 542     """ Where the magic starts. """
 543     try:
 544         opts, args = getopt.getopt(sys.argv[1:], "dlxvhc:u:pso:46at:n:ir:",
 545                                    ["reload-delegation", "initialize-lir", "hack-the-internet",
 546                                    "verbose", "help", "cachedir=", "useragent=", "progress",
 547                                    "silent", "output=", "ipv4", "ipv6","asn", "nation-state=",
 548                                    "country-name", "initialize-delegation","reverse-lookup"])
 549     except getopt.GetoptError, err:
 550         print str(err)
 551         usage()
 552         sys.exit(2)
 553
 554     global verbose
 555     verbose = False
 556     output = None
 557     silent = True
 558     cache_dir = str(os.path.expanduser('~')) + "/.blockfinder/"
 559     update_delegations = False
 560     delegation_urls = """
 561         ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest
 562         ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
 563         ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
 564         ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
 565         ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
 566     """
 567     geoip_country_urls = """http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 568     http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz"""
 569
 570     lir_urls = """ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz
 571     ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz"""
 572     delegation_files = []
 573     for url in delegation_urls.split():
 574         filename = url.rpartition('/')
 575         delegation_files.append(filename[-1])
 576     update_delegations = False
 577     update_lir = False
 578     reload_delegation_db = False
 579     requests = []
 580     country = ""
 581     useragent = "Mozilla/5.0"
 582     ipaddress = ""
 583
 584     create_blockfinder_cache_dir(cache_dir)
 585     if not os.path.exists(cache_dir + "countrycodes.xml"):
 586         try:
 587             download_country_code_file(cache_dir,useragent)
 588         except Exception, e:
 589             print repr(e)
 590
 591     for o, a in opts:
 592         if o in ("-x", "--hack-the-internet"):
 593             print "all your bases are belong to us!"
 594             sys.exit(4)
 595         if o == "-v":
 596             verbose = True
 597         elif o in ("-h", "--help"):
 598             usage()
 599             sys.exit()
 600         elif o in ("-c", "--cachedir"):
 601             cache_dir = a
 602         elif o in ("-u", "--useragent"):
 603             useragent = a
 604         elif o in ("-p", "--progress"):
 605             progress = True
 606         elif o in ("-s", "--silent"):
 607             silent = True
 608         elif o in ("-o", "--output"):
 609             output = a
 610         elif o in ("-4", "--ipv4"):
 611             requests.append("ipv4")
 612         elif o in ("-6", "--ipv6"):
 613             requests.append("ipv6")
 614         elif o in ("-a", "--asn"):
 615             requests.append("asn")
 616         # XXX TODO: This should be a positional argument as it's the only manditory one...
 617         elif o in ("-r", "--reverse-lookup"):
 618             ipaddress = a
 619             requests.append("reverse")
 620         elif o in ("-t", "--nation-state"):
 621             country = a.upper()
 622         elif o in ("-n", "--country-name"):
 623             country = get_country_code_from_name(cache_dir, a)
 624         elif o in ("-i", "--initialize-delegations"):
 625             update_delegations = True
 626         elif o in ("-l", "--initialize-lir"):
 627             update_lir = True
 628         elif o in ("-d", "--reload-delegation"):
 629             reload_delegation_db = True
 630         else:
 631             print "Unhandled option; Sorry!"
 632             sys.exit(3)
 633
 634     if reload_delegation_db:
 635         create_db_and_insert_delegation_into_db(cache_dir, delegation_urls)
 636         sys.exit(0)
 637
 638     # Update and quit.
 639     if update_delegations:
 640         if GeoIP:
 641             update_geoip_cache(cache_dir,geoip_country_urls,useragent)
 642         update_delegation_cache(cache_dir,delegation_urls,useragent)
 643         if verbose:
 644             verify_cache(cache_dir, delegation_files)
 645         create_db_and_insert_delegation_into_db(cache_dir, delegation_urls)
 646         if not update_lir:
 647             sys.exit(0)
 648     if update_lir:
 649         update_lir_delegation_cache(cache_dir,lir_urls,useragent)
 650         print "Extracting and inserting information from the lir files can take up to 5 minutes"
 651         create_or_replace_lir_table_in_db(cache_dir)
 652         for file in "ripe.db.inetnum ripe.db.inet6num".split():
 653             extract_info_from_lir_file_and_insert_into_sqlite(cache_dir, file)
 654         sys.exit(0)
 655
 656     if not requests:
 657         print "Nothing to do. Have you requested anything?"
 658         print "Example usage: blockfinder -v --ipv4 -t mm"
 659         sys.exit(1)
 660
 661     if ipaddress:
 662         lookup_ip_address(ipaddress,cache_dir)
 663         sys.exit(0)
 664     if not country:
 665         print "It appears your search did not match a country."
 666         sys.exit(1)
 667     # Check our cache age and warn if it's aged
 668     if cache_is_dated(cache_dir, delegation_files) and verbose:
 669         print "Your delegation cache is older than 24 hours; you probably want to update it."
 670     if verbose:
 671         print "Using country code: %s" % country
 672
 673     for request in requests:
 674         try:
 675             print " \n".join(use_sql_database(request, country, cache_dir))
 676         except Exception, e:
 677             print repr(e)
 678
 679 if __name__ == "__main__":
 680     main()