blockfinder

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # For the people of Smubworld!
   5 import urllib2
   6 import os
   7 import time
   8 import optparse
   9 import sys
  10 import sqlite3
  11 import hashlib
  12 import gzip
  13 import ipaddr
  14 from ConfigParser import SafeConfigParser
  15
  16 __program__ = 'blockfinder'
  17 __url__ = 'https://github.com/ioerror/blockfinder/'
  18 __author__ = 'Jacob Appelbaum <jacob@appelbaum.net>, David <db@d1b.org>'
  19 __copyright__ = 'Copyright (c) 2010'
  20 __license__ = 'See LICENSE for licensing information'
  21 __version__ = '3.1415'
  22
  23 try:
  24     import GeoIP
  25 except ImportError:
  26      GeoIP = None
  27
  28 try:
  29     from future import antigravity
  30 except ImportError:
  31     antigravity = None
  32
  33 class DatabaseCache:
  34     def __init__(self, cache_dir, verbose=False):
  35         self.cache_dir = cache_dir
  36         self.verbose = verbose
  37         self.cursor = None
  38         self.conn = None
  39
  40         self.db_version = "0.0.1"
  41
  42     def connect_to_database(self):
  43         if not os.path.exists(self.cache_dir):
  44             if self.verbose:
  45                 print "Initializing the cache directory..."
  46             os.mkdir(self.cache_dir)
  47         self.conn = sqlite3.connect(self.cache_dir + "sqlitedb")
  48         self.cursor = self.conn.cursor()
  49
  50     def __get_default_config_file_obj(self):
  51         open_flags = 'rb+'
  52         file_path = os.path.join(self.cache_dir, 'db.cfg')
  53         if not os.path.exists(file_path):
  54             open_flags = 'wb+'
  55         return open(file_path, open_flags)
  56
  57     def _get_db_config(self, file_obj=None):
  58         """ Returns the database configuration object from the provided
  59             file_obj if it is provided, otherwise uses the default database
  60             configuration file.
  61         """
  62         if file_obj is None:
  63             file_obj = self.__get_default_config_file_obj()
  64         config = SafeConfigParser()
  65         config.readfp(file_obj)
  66         file_obj.close()
  67         return config
  68
  69     def set_db_version(self, file_obj=None):
  70         """ set the database version """
  71         if file_obj is None:
  72             file_obj = self.__get_default_config_file_obj()
  73
  74         config = self._get_db_config()
  75         if not config.has_section('db'):
  76             config.add_section('db')
  77         config.set('db', 'version', self.db_version)
  78         config.write(file_obj)
  79         file_obj.close()
  80
  81     def get_db_version(self):
  82         """ Returns the database version """
  83         config = self._get_db_config()
  84         if not config.has_section('db'):
  85             return None
  86         return config.get('db', 'version')
  87
  88     def commit_and_close_database(self):
  89         self.conn.commit()
  90         self.cursor.close()
  91
  92     def create_sql_database(self):
  93         """ Creates a new sqlite database.
  94             Existing delegation entries are dropped prior to inserting
  95             'newer' delegations. """
  96         sql = ('DROP TABLE IF EXISTS delegations; '
  97                'CREATE TABLE delegations(registry TEXT, cc TEXT, '
  98                'start TEXT, value INTEGER, date TEXT, status TEXT, '
  99                'type TEXT); '
 100                'CREATE TABLE IF NOT EXISTS lir_record(cc TEXT, '
 101                'start TEXT, value INTEGER, type INTEGER)')
 102         self.cursor.executescript(sql)
 103         self.conn.commit()
 104
 105     def insert_into_sql_database(self, rows):
 106         """ Inserts delegation information into the sqlite database. """
 107         sql = ('INSERT INTO delegations (registry, cc, start, value, '
 108                'date, status, type) VALUES (?, ?, ?, ?, ?, ?, ?)')
 109         self.cursor.executemany(sql, rows)
 110         self.conn.commit()
 111
 112     def _get_total_delegations_from_db(self):
 113         """ Returns the total count of the number of entries in the ipv4,
 114             ipv6 and asn table. """
 115         sql = 'SELECT COUNT(*) FROM delegations'
 116         self.cursor.execute(sql)
 117         return int(self.cursor.fetchone()[0])
 118
 119     def _get_possible_match_entries(self, cc):
 120         """ Get the count of 'possible' matching delegation entries. """
 121         sql = 'SELECT COUNT(*) FROM delegations WHERE cc = ?'
 122         self.cursor.execute(sql, cc)
 123         return int(self.cursor.fetchone()[0])
 124
 125     def use_sql_database(self, request, cc):
 126         """ Use the sqlite database that is created after fetching
 127             delegations to output information for a given request. """
 128         if self.verbose:
 129             print "We have %d entries in our delegation cache." % \
 130                     self._get_total_delegations_from_db()
 131         sql = ('SELECT start, value FROM delegations WHERE type = ? '
 132                'AND cc = ?')
 133         cc = (cc,)
 134         self.cursor.execute(sql, (request, cc[0]))
 135         result = []
 136         for row in self.cursor:
 137             if request == "ipv4":
 138                 start_ipaddr = ipaddr.IPv4Address(str(row[0]))
 139                 end_ipaddr = start_ipaddr + int(row[1]) - 1
 140                 result += [str(x) for x in \
 141                         ipaddr.summarize_address_range( \
 142                         start_ipaddr, end_ipaddr)]
 143             elif request == "ipv6":
 144                 result.append(str(row[0]) + "/" + str(int(row[1])))
 145             else:
 146                 result.append(str(int(row[0])))
 147         result.sort()
 148         if self.verbose:
 149             result.append("We found %d possible entries in our "
 150                   "delegation cache." % \
 151                   self._get_possible_match_entries(cc))
 152             sql = ('SELECT COUNT(*) FROM delegations WHERE cc = ? '
 153                    'AND type = ?')
 154             self.cursor.execute(sql, (cc[0], request))
 155             result.append("We found %d matching entries in our "
 156                     "delegation cache." % int(self.cursor.fetchone()[0]))
 157         return result
 158
 159     def _rir_or_lir_lookup_ipv4(self, ip_addr, lookup_type):
 160         ipv4arr = ip_addr.split('.')
 161         if lookup_type == 'rir':
 162             sql = ('SELECT cc, start, value FROM delegations '
 163                    'WHERE type = "ipv4" AND start LIKE ?')
 164             self.cursor.execute(sql,
 165                     (ipv4arr[0] + "." + ipv4arr[1] + ".%",))
 166         else:
 167             sql = ('SELECT cc, start, value FROM lir_record '
 168                    'WHERE start LIKE ? AND type = 4')
 169             self.cursor.execute(sql,
 170                     (ipv4arr[0] + "." + ipv4arr[1] + ".%",))
 171         row = self.cursor.fetchone()
 172         if row is None:
 173             if lookup_type == "rir":
 174                 sql = ('SELECT cc, start, value FROM delegations '
 175                        'WHERE type = "ipv4" AND start LIKE ?')
 176                 self.cursor.execute(sql, (ipv4arr[0] + ".%",))
 177             else:
 178                 sql = ('SELECT cc, start, value FROM lir_record '
 179                        'WHERE start LIKE ? AND type = 4')
 180                 self.cursor.execute(sql, (ipv4arr[0] + ".%",))
 181             row = self.cursor.fetchone()
 182         lookup_ipaddr = ipaddr.IPv4Address(ip_addr)
 183         while row is not None:
 184             start_ipaddr = ipaddr.IPv4Address(str(row[1]))
 185             end_ipaddr = start_ipaddr + int(row[2]) - 1
 186             if start_ipaddr <= lookup_ipaddr and \
 187                     lookup_ipaddr <= end_ipaddr:
 188                 return row[0]
 189             row = self.cursor.fetchone()
 190
 191     def rir_lookup(self, ip_addr):
 192         return self._rir_or_lir_lookup_ipv4(ip_addr, "rir")
 193
 194     def lir_lookup(self, ip_addr):
 195         return self._rir_or_lir_lookup_ipv4(ip_addr, "lir")
 196
 197     def asn_lookup(self, asn):
 198         sql = ('SELECT cc FROM delegations WHERE type = "asn" AND '
 199                'start LIKE ?')
 200         self.cursor.execute(sql, (asn,))
 201         row = self.cursor.fetchone()
 202         if row:
 203             return row[0]
 204
 205     def rir_or_lir_lookup_ipv6(self, ip_addr, ip_query, type_q):
 206         if type_q == "RIR":
 207             sql = ('SELECT cc, start, value FROM delegations '
 208                    'WHERE type = "ipv6" AND start like ?')
 209             self.cursor.execute(sql, (ip_query,))
 210         else:
 211             sql = ('SELECT cc, start, value FROM lir_record '
 212                    'WHERE type = 6 AND start LIKE ?')
 213             self.cursor.execute(sql, (ip_query,))
 214         lookup_ipaddr = ipaddr.IPv6Address(ip_addr)
 215         for row in self.cursor:
 216             network = ipaddr.IPv6Network(row[1] + "/" + str(row[2]))
 217             if lookup_ipaddr in network:
 218                 return row[0]
 219
 220     def create_or_replace_lir_table_in_db(self):
 221         sql = 'DROP TABLE IF EXISTS lir_record'
 222         self.cursor.execute(sql)
 223         sql = ('CREATE TABLE IF NOT EXISTS lir_record(cc TEXT, '
 224                'start TEXT, value INTEGER, type INTEGER)')
 225         self.cursor.execute(sql)
 226         self.conn.commit()
 227
 228     def insert_lir_delegation(self, data):
 229         sql = ('INSERT INTO lir_record (cc, start, value, type) '
 230                'VALUES (?, ?, ?, ?)')
 231         self.cursor.execute(sql, data)
 232         self.conn.commit()
 233
 234 class DownloaderParser:
 235     def __init__(self, cache_dir, database_cache, user_agent, \
 236             verbose=False):
 237         self.cache_dir = cache_dir
 238         self.database_cache = database_cache
 239         self.user_agent = user_agent
 240         self.verbose = verbose
 241
 242     def update_progress_bar(self, percent_done, caption=""):
 243         """Write a progress bar to the console"""
 244         rows, columns = map(int, \
 245                 os.popen('stty size', 'r').read().split())
 246         width = columns - 4 - len(caption)
 247         sys.stdout.write("[%s>%s] %s\x1b[G" % (
 248                 "=" * int(percent_done*width),
 249                 "." * (width - int(percent_done * width)), caption))
 250         sys.stdout.flush()
 251
 252     # XXX TODO:allow the use of a proxy
 253     # Set up a proper Request object, set the user agent and if desired,
 254     # a proxy
 255     def fetch(self, url):
 256         """ Fetch (with progress meter) and return the contents of a
 257             url. """
 258         req = urllib2.Request(url)
 259         req.add_header('User-Agent', self.user_agent)
 260         #req.set_proxy(host, type)
 261         fetcher = urllib2.urlopen(req)
 262         length_header = fetcher.headers.get("Content-Length")
 263         if length_header == None:
 264             """ The server did not provide a Content-Length header. """
 265             length_header = -1
 266         length = int(length_header)
 267         print "Fetching ", str(round(float(length/1024),2)), " kilobytes"
 268         ret = ""
 269         t_start = time.time()
 270         while True:
 271             t_delta = time.time() - t_start
 272             if t_delta == 0:
 273                 t_delta = 1
 274             if length_header != -1:
 275                 self.update_progress_bar(float(len(ret)) / length,
 276                     "%.2f K/s" % (len(ret) / 1024 / t_delta))
 277             tmp = fetcher.read(1024)
 278             if len(tmp) == 0:
 279                 if len(ret) != length and length_header != -1:
 280                     raise Exception("Expected %s bytes, only received " \
 281                             "%s" % (len(ret), length))
 282                 print ""
 283                 return ret
 284             ret += tmp
 285
 286     def write_to_a_text_file(self, file_loc, data):
 287         f = open(file_loc, 'w')
 288         f.write(data)
 289         f.close()
 290
 291     def extract_data_from_gzip_file(self, gzip_file_loc, \
 292                 extract_file_loc):
 293         gzip_file = gzip.open(gzip_file_loc, 'rb')
 294         gunzipped_file = open(extract_file_loc, 'w')
 295         while True:
 296             gunzipped_data = gzip_file.read(1024)
 297             if gunzipped_data == "":
 298                 break
 299             gunzipped_file.writelines(gunzipped_data)
 300         gzip_file.close()
 301         gunzipped_file.close()
 302
 303     def read_data_from_binary_file(self, fname):
 304         f = open(fname, 'rb')
 305         data = f.read()
 306         f.close()
 307         return data
 308
 309     def create_blockfinder_cache_dir(self):
 310         if not os.path.exists(self.cache_dir):
 311             if self.verbose:
 312                 print "Initializing the cache directory..."
 313             os.mkdir(self.cache_dir)
 314
 315     def cache_delegation(self, delegation_url):
 316         """ Attempt to cache the contents of a delegation url in our
 317             cache dir. """
 318         delegation = ""
 319         print "Fetching " + delegation_url
 320         delegation = self.fetch(delegation_url)
 321         tmp = delegation_url.split('/')
 322         delegation_file = str(self.cache_dir) + str(tmp[-1])
 323         try:
 324             self.write_to_a_text_file(delegation_file, delegation)
 325             return True
 326         except Exception, e:
 327             print repr(e)
 328             return False
 329
 330     def cache_is_dated(self, cached_files):
 331         """ Returns True if the mtime of any files in cache dir is
 332             > 24 hours. """
 333         try:
 334             os.stat(self.cache_dir)
 335         except OSError, e:
 336             print "\nDid you initialize the cache directory?\n"
 337             raise e
 338         for file in cached_files:
 339             fstat = os.stat(self.cache_dir + file)
 340             if (time.time() - fstat.st_mtime) > 86400:
 341                 return True
 342         return False
 343
 344     def get_md5_from_delegation_md5_file(self, delegation_file):
 345         """ Returns the md5sum from the delegation md5 file
 346             if it doesn't exist it returns an empty string"""
 347         checksum = ""
 348         try:
 349             f = open(self.cache_dir + delegation_file + ".md5", "r")
 350             checksum = f.read()
 351             f.close()
 352             if "=" in checksum:
 353                 pos = checksum.find("=") +2
 354                 checksum = str(checksum[pos:-1])
 355         except Exception, e:
 356             print repr(e)
 357         return checksum
 358
 359     def verify_delegation_file(self, delegation_file):
 360         """ Compares the delegation file md5sum to that of the provided
 361             md5sum, returns True if they match otherwise returns
 362             False. """
 363         checksum = ""
 364         checksum_of_file = ""
 365         try:
 366             data = self.read_data_from_binary_file(self.cache_dir + \
 367                     delegation_file)
 368             checksum_of_file = str(hashlib.md5(data).hexdigest())
 369         except Exception, e:
 370             print repr(e)
 371         checksum = self.get_md5_from_delegation_md5_file(delegation_file)
 372         if checksum != checksum_of_file:
 373             return False
 374         if checksum == checksum_of_file and checksum != "":
 375             return True
 376         return False
 377
 378     def verify_cache(self, delegation_files):
 379         """ If in verbose mode prints the result of checking the checksum
 380             of the delegation files. """
 381         for file in delegation_files:
 382             if self.verbose:
 383                 print "verifying " + file
 384             if self.verify_delegation_file(file):
 385                 if self.verbose:
 386                     print "the md5 checksum of " + file + \
 387                             " *matches* the provided checksum"
 388             else:
 389                 if self.verbose:
 390                     print "the md5 checksum of " + file + \
 391                             " does *not* match the provided checksum"
 392
 393     def update_delegation_cache(self, delegation_urls):
 394         """ Fetch multiple delegation urls and cache the contents. """
 395         print "Updating delegation cache..."
 396         for url in delegation_urls.split():
 397             self.cache_delegation(url + ".md5")
 398             if self.verify_delegation_file(url.rpartition('/')[-1]):
 399                 pass
 400             else:
 401                 self.cache_delegation(url)
 402
 403     def update_lir_delegation_cache(self, delegation_urls):
 404         """ Fetch multiple LIR delegation urls and cache the contents. """
 405         print "Updating LIR delegation cache..."
 406         for url in delegation_urls.split():
 407             self.cache_delegation(url)
 408         self.unpack_a_delegation_cache(delegation_urls, "LIR")
 409
 410     def unpack_a_delegation_cache(self, delegation_urls, del_type=""):
 411         """ Unpack the fetched LIR delegation files into the blockfinder
 412             cache. """
 413         # This probably should unlink the gzip'ed file if we care about
 414         # space...
 415         for url in delegation_urls.split():
 416             gzip_filename = url.rpartition('/')[-1]
 417             gunziped_filename = gzip_filename.rpartition('.')[0]
 418             if self.verbose:
 419                 print "Unpacking " + del_type + "file " + \
 420                         gzip_filename + " into our cache as " + \
 421                         gunziped_filename
 422             self.extract_data_from_gzip_file(self.cache_dir + \
 423                     gzip_filename, self.cache_dir + gunziped_filename)
 424
 425     def update_geoip_cache(self, geoip_urls):
 426         """ Fetch country level resolution GeoIP files from a given url
 427             and cache the contents. Unpack it if it's compressed. """
 428         print "Updating GeoIP cache..."
 429         for url in geoip_urls.split():
 430             self.cache_delegation(url)
 431         self.unpack_a_delegation_cache(geoip_urls, "GeoIP")
 432
 433     def load_delegation(self, delegation_file):
 434         """ Load, parse and store the delegation file contents as a
 435             list. """
 436         keys = "registry cc type start value date status"
 437         try:
 438             f = open(delegation_file, "r")
 439             delegations = [dict((k,v) for k,v in zip(keys.split(), \
 440                     line.strip().split("|"))) \
 441                     for line in f.readlines() if not line.startswith("#")]
 442             f.close()
 443             return delegations
 444         except OSError, e:
 445             print repr(e)
 446
 447     def load_all_delegations(self, delegation_urls):
 448         """ Load all delegations into memory. """
 449         delegations = []
 450         for url in delegation_urls.split():
 451             filename = url.rpartition('/')[-1]
 452             if self.verbose:
 453                 print "Attempting to load delegation file into " \
 454                         + "memory: " + filename
 455             delegations.append(self.load_delegation(self.cache_dir + \
 456                     filename))
 457         return delegations
 458
 459     def download_country_code_file(self):
 460         """ Download and save the latest opencountrycode
 461             TXT(';'-separated) file """
 462         url = "http://www.iso.org/iso/list-en1-semic-3.txt"
 463         print "Fetching " + url
 464         text_content = self.fetch(url)
 465         self.write_to_a_text_file(self.cache_dir + "countrycodes.txt", \
 466                 text_content)
 467
 468     def extract_info_from_lir_file_and_insert_into_sqlite(self, filename):
 469         block = []
 470         country = ""
 471         entry = False
 472         version = ""
 473         for line in open(self.cache_dir + filename, "r"):
 474             line = line.replace("\n", "")
 475             if line == "":
 476                 entry = False
 477                 country, block, version = "", [], ""
 478             elif not entry and "inetnum:" in line:
 479                 try:
 480                     line = line.replace("inetnum:", "").strip()
 481                     start_addr = line.split("-")[0].strip()
 482                     end_addr = line.split("-")[1].strip()
 483                     start_num = int(ipaddr.IPv4Address(start_addr))
 484                     end_num = int(ipaddr.IPv4Address(end_addr))
 485                     num_ips = end_num - start_num + 1
 486                     block = [start_addr, num_ips]
 487                     entry = True
 488                     version = "4"
 489                 except Exception, e:
 490                     if self.verbose:
 491                         print repr(e), line
 492             elif not entry and "inet6num:" in line:
 493                 try:
 494                     block = line.replace("inet6num:", \
 495                             "").strip().split("/")
 496                     entry = True
 497                     version = "6"
 498                 except Exception, e:
 499                     if self.verbose:
 500                         print repr(e), line
 501             elif entry and "country:" in line:
 502                 country = line.replace("country:", "").strip()
 503                 data = (country, block[0], block[1], version)
 504                 self.database_cache.insert_lir_delegation(data)
 505
 506     def create_db_and_insert_delegation_into_db(self, delegation_urls):
 507         self.database_cache.create_sql_database()
 508         delegations = self.load_all_delegations(delegation_urls)
 509         rows = []
 510         for delegation in delegations:
 511             for entry in delegation:
 512                 registry = str(entry['registry'])
 513                 if not registry.isdigit() and str(entry['cc']) != "*":
 514                     temp_row = [entry['registry'], entry['cc'], \
 515                             entry['start'], entry['value'], \
 516                             entry['date'], entry['status'], entry['type']]
 517                     rows.append(temp_row)
 518         self.database_cache.insert_into_sql_database(rows)
 519
 520 class Lookup:
 521     def __init__(self, cache_dir, database_cache, verbose=False):
 522         self.cache_dir = cache_dir
 523         self.database_cache = database_cache
 524         self.verbose = verbose
 525         self.map_co = None
 526         self.build_country_code_dictionary()
 527
 528     def build_country_code_dictionary(self):
 529         """ Return a dictionary mapping country name to the country
 530             code. """
 531         if not os.path.exists(self.cache_dir + "countrycodes.txt"):
 532             return
 533         self.map_co = {}
 534         txt_file = str(self.cache_dir) + "countrycodes.txt"
 535         for line in open(txt_file, 'r'):
 536             line = line.replace("\n", "").replace("\r", "")
 537             if line.startswith("This list states the country"):
 538                 continue
 539             if line == "" or ";" not in line:
 540                 continue
 541             name, code = line.split(";")
 542             """ capitalize the individual parts of the country name """
 543             name = ' '.join([part.capitalize() for part in \
 544                     name.split(" ")])
 545             self.map_co[name] = code
 546
 547     def knows_country_names(self):
 548         return self.map_co is not None
 549
 550     def get_name_from_country_code(self, cc_code):
 551         if not self.knows_country_names():
 552             return
 553         country_name = [(key, value) for (key, value) in \
 554                 self.map_co.items() if value == cc_code]
 555         if len(country_name) > 0:
 556             return country_name[0][0]
 557
 558     def get_country_code_from_name(self, country_name):
 559         """ Return the country code for a given country name. """
 560         if not self.knows_country_names():
 561             return
 562         cc_code = [self.map_co[key] for key in self.map_co.keys() if \
 563                 key.upper().startswith(country_name.upper())]
 564         if len(cc_code) > 0:
 565             return cc_code[0]
 566
 567     def geoip_lookup(self, ip_addr):
 568         # This would work with the CVS version of the GeoIP code
 569         # However, MaxMind hasn't done a release in a long time.
 570         # http://geoip.cvs.sourceforge.net/viewvc/geoip/python/\
 571         # test_v6.py?revision=1.1&view=markup
 572         #        gi = GeoIP.open(self.cache_dir + \
 573         #                "GeoIPv6.dat", GeoIP.GEOIP_STANDARD)
 574         #        cc = gi.country_code_by_addr_v6(ip_addr)
 575         #        cc_name = gi.country_name_by_addr_v6(ip_addr)
 576         gi = GeoIP.open(self.cache_dir + "GeoIP.dat", \
 577                 GeoIP.GEOIP_STANDARD)
 578         cc = gi.country_code_by_addr(ip_addr)
 579         cc_name = gi.country_name_by_addr(ip_addr)
 580         return cc, cc_name
 581
 582     def lookup_ipv6_address(self, ip_addr):
 583         print "Reverse lookup for: " + ip_addr
 584         split_addr = ip_addr.split(":")
 585         for i in ["RIR", "LIR"]:
 586             ip_query = ip_addr.split(":")[0] + ":" + \
 587                     ip_addr.split(":")[1] + "%"
 588             cc = self.database_cache.rir_or_lir_lookup_ipv6(ip_addr, \
 589                     ip_query, i)
 590             if cc:
 591                 print i, "country code:", cc
 592                 cn = self.get_name_from_country_code(cc)
 593                 if cn:
 594                     print i, "country name:", cn
 595             else:
 596                 ip_query = ip_addr.split(":")[0] + ":%"
 597                 cc = self.database_cache.rir_or_lir_lookup_ipv6(ip_addr, \
 598                         ip_query, i)
 599                 print i, "country code:", cc
 600                 cn = self.get_name_from_country_code(cc)
 601                 if cn:
 602                     print i, "country name:", cn
 603
 604     def lookup_ipv4_address(self, ip_addr):
 605         print "Reverse lookup for: " + ip_addr
 606         if GeoIP:
 607             geoip_cc, geoip_cc_name = self.geoip_lookup(ip_addr)
 608             print "GeoIP country code: " + str(geoip_cc)
 609             print "GeoIP country name: " + str(geoip_cc_name)
 610         rir_cc = self.database_cache.rir_lookup(ip_addr)
 611         if rir_cc:
 612             print 'RIR country code:', rir_cc
 613             rir_cn = self.get_name_from_country_code(rir_cc)
 614             if rir_cn:
 615                 print 'RIR country:', rir_cn
 616         else:
 617             print 'Not found in RIR db'
 618         lir_cc = self.database_cache.lir_lookup(ip_addr)
 619         if lir_cc:
 620             print 'LIR country code:', lir_cc
 621             lir_cn = self.get_name_from_country_code(lir_cc)
 622             if lir_cn:
 623                 print 'LIR country:', lir_cn
 624         if GeoIP:
 625             if geoip_cc != rir_cc:
 626                 print "It appears that the RIR data conflicts with the " \
 627                         "GeoIP data.  The GeoIP data is likely closer " \
 628                         "to being correct due to sub-delegation issues " \
 629                         "with LIR databases."
 630
 631     def lookup_ip_address(self, ip_addr):
 632         """ Return the country code and name for a given ip address.
 633             Attempts to use GeoIP if available. """
 634         try:
 635             lookup_ipaddr = ipaddr.IPAddress(ip_addr)
 636             if isinstance(lookup_ipaddr, ipaddr.IPv4Address):
 637                 self.lookup_ipv4_address(ip_addr)
 638             elif isinstance(lookup_ipaddr, ipaddr.IPv6Address):
 639                 self.lookup_ipv6_address(ip_addr)
 640             else:
 641                 print "Did not recognize '%s' as either IPv4 or IPv6 " \
 642                         "address." % ip_addr
 643         except ValueError, e:
 644             print "'%s' is not a valid IP address." % ip_addr
 645
 646     def asn_lookup(self, asn):
 647         asn_cc = self.database_cache.asn_lookup(asn)
 648         if asn_cc:
 649             print "AS country code: %s" % asn_cc
 650             asn_cn = self.get_name_from_country_code(asn_cc)
 651             if asn_cn:
 652                 print "AS country name: %s" % asn_cn
 653         else:
 654             print "AS%s not found!" % asn
 655
 656     def fetch_rir_blocks_by_country(self, request, country):
 657         return self.database_cache.use_sql_database(request, country)
 658
 659 def main():
 660     """ Where the magic starts. """
 661     usage = "Usage: %prog [options]\n\n" \
 662             "Example: %prog -v -t mm"
 663     parser = optparse.OptionParser(usage)
 664     parser.add_option("-v", "--verbose", action="store_true", \
 665             dest="verbose", help = "be verbose", default=False)
 666     parser.add_option("-c", "--cache-dir", action="store", dest="dir", \
 667             help="set cache directory [default: %default]", \
 668             default=str(os.path.expanduser('~')) + "/.blockfinder/")
 669     parser.add_option("--user-agent", action="store", dest="ua", \
 670             help=('provide a User-Agent which will be used when '
 671                   'fetching delegation files [default: "%default"]'), \
 672             default="Mozilla/5.0")
 673     parser.add_option("-x", "--hack-the-internet", action="store_true", \
 674             dest="hack_the_internet", help=optparse.SUPPRESS_HELP)
 675     group = optparse.OptionGroup(parser, "Cache modes",
 676             "Pick at most one of these modes to initialize or update " \
 677             "the local cache.  May not be combined with lookup modes.")
 678     group.add_option("-i", "--init-rir", \
 679             action="store_true", dest="init_del", \
 680             help="initialize or update delegation information")
 681     group.add_option("-d", "--reload-rir", action="store_true", \
 682             dest="reload_del", \
 683             help="use existing delegation files to update the database")
 684     group.add_option("-l", "--init-lir", action="store_true", \
 685             dest="init_lir",
 686             help=("initialize or update lir information; can take up to "
 687                   "5 minutes"))
 688     group.add_option("-z", "--reload-lir", action="store_true",
 689             dest="reload_lir", \
 690             help=("use existing lir files to update the database; can "
 691                   "take up to 5 minutes"))
 692     group.add_option("-o", "--download-cc", action="store_true",
 693             dest="download_cc", help="download country codes file")
 694     parser.add_option_group(group)
 695     group = optparse.OptionGroup(parser, "Lookup modes",
 696             "Pick at most one of these modes to look up data in the " \
 697             "local cache.  May not be combined with cache modes.")
 698     group.add_option("-4", "--ipv4", action="store", dest="ipv4", \
 699             help=("look up country code and name for the specified IPv4 "
 700                   "address"))
 701     group.add_option("-6", "--ipv6", action="store", dest="ipv6", \
 702             help=("look up country code and name for the specified IPv6 "
 703                   "address"))
 704     group.add_option("-a", "--asn", action="store", dest="asn", \
 705             help="look up country code and name for the specified ASN")
 706     group.add_option("-t", "--code", action="store", dest="cc", \
 707             help=("look up all allocations in the delegation cache for "
 708                   "the specified two-letter country code"))
 709     group.add_option("-n", "--name", action="store", dest="cn", \
 710             help=("look up all allocations in the delegation cache for "
 711                   "the specified full country name"))
 712     parser.add_option_group(group)
 713     group = optparse.OptionGroup(parser, "Network modes")
 714     (options, args) = parser.parse_args()
 715     if options.hack_the_internet:
 716         print "all your bases are belong to us!"
 717         sys.exit(0)
 718     options_dict = vars(options)
 719     modes = 0
 720     for mode in ["init_del", "init_lir", "reload_del", "reload_lir",
 721                  "download_cc", "ipv4", "ipv6", "asn", "cc", "cn"]:
 722         if options_dict.has_key(mode) and options_dict.get(mode):
 723             modes += 1
 724     if modes > 1:
 725         parser.error("only 1 cache or lookup mode allowed")
 726     elif modes == 0:
 727         parser.error("must provide 1 cache or lookup mode")
 728     database_cache = DatabaseCache(options.dir, options.verbose)
 729     database_cache.connect_to_database()
 730     downloader_parser = DownloaderParser(options.dir, database_cache, \
 731             options.ua)
 732     lookup = Lookup(options.dir, database_cache)
 733     delegation_urls = """
 734         ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest
 735         ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
 736         ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
 737         ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
 738         ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
 739     """
 740     geoip_country_urls = """http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 741     http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz"""
 742     lir_urls = """ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz
 743     ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz"""
 744     delegation_files = []
 745     for url in delegation_urls.split():
 746         filename = url.rpartition('/')
 747         delegation_files.append(filename[-1])
 748     downloader_parser.create_blockfinder_cache_dir()
 749     if options.ipv4 or options.ipv6 or options.asn or options.cc \
 750             or options.cn:
 751         if downloader_parser.cache_is_dated(delegation_files):
 752             print "Your delegation cache is older than 24 hours; you " \
 753                     "probably want to update it."
 754     if options.asn:
 755         lookup.asn_lookup(options.asn)
 756     elif options.ipv4:
 757         lookup.lookup_ip_address(options.ipv4)
 758     elif options.ipv6:
 759         lookup.lookup_ip_address(options.ipv6)
 760     elif options.cc or options.cn:
 761         country = None
 762         if options.cc:
 763             country = options.cc.upper()
 764         elif not lookup.knows_country_names():
 765             print "Need to download country codes first before looking " \
 766                     "up countries by name."
 767         else:
 768             country = lookup.get_country_code_from_name(options.cn)
 769             if not country:
 770                 print "It appears your search did not match a country."
 771         if country:
 772             for request in ["ipv4", "ipv6", "asn"]:
 773                 print "\n".join(lookup.fetch_rir_blocks_by_country(\
 774                         request, country))
 775     elif options.init_del or options.reload_del:
 776         if options.init_del:
 777             if GeoIP:
 778                 downloader_parser.update_geoip_cache(geoip_country_urls)
 779             downloader_parser.update_delegation_cache(delegation_urls)
 780             if options.verbose:
 781                 lookup.verify_cache(delegation_files)
 782         downloader_parser.create_db_and_insert_delegation_into_db(\
 783                 delegation_urls)
 784     elif options.init_lir or options.reload_lir:
 785         if options.init_lir:
 786             downloader_parser.update_lir_delegation_cache(lir_urls)
 787         print "Extracting and inserting information from the lir files " \
 788                 "can take up to 5 minutes"
 789         database_cache.create_or_replace_lir_table_in_db()
 790         for fname in "ripe.db.inetnum ripe.db.inet6num".split():
 791             downloader_parser.extract_info_from_lir_file_and_insert_into_sqlite(fname)
 792     elif options.download_cc:
 793         downloader_parser.download_country_code_file()
 794     database_cache.commit_and_close_database()
 795
 796 if __name__ == "__main__":
 797     main()
 798