blockfinder

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # For the people of Smubworld!
   5 import urllib2
   6 import os
   7 import time
   8 import optparse
   9 import sys
  10 import sqlite3
  11 import hashlib
  12 import gzip
  13 import ConfigParser
  14 import zipfile
  15
  16 try:
  17     import ipaddr
  18 except ImportError:
  19     from embedded_ipaddr import ipaddr
  20
  21 is_win32 = (sys.platform == "win32")
  22
  23 __program__ = 'blockfinder'
  24 __url__ = 'https://github.com/ioerror/blockfinder/'
  25 __author__ = 'Jacob Appelbaum <jacob@appelbaum.net>, David <db@d1b.org>'
  26 __copyright__ = 'Copyright (c) 2010'
  27 __license__ = 'See LICENSE for licensing information'
  28 __version__ = '3.1415'
  29
  30 try:
  31     from future import antigravity
  32 except ImportError:
  33     antigravity = None
  34
  35
  36 class DatabaseCache:
  37     def __init__(self, cache_dir, verbose=False):
  38         self.cache_dir = cache_dir
  39         self.verbose = verbose
  40         self.cursor = None
  41         self.conn = None
  42         self.db_version = "0.0.4"
  43         self.db_path = os.path.join(self.cache_dir + "sqlitedb")
  44
  45     def erase_database(self):
  46         """ Erase the database file. """
  47         if os.path.exists(self.db_path):
  48             os.remove(self.db_path)
  49
  50     def connect_to_database(self):
  51         """ Connect to the database cache, possibly after creating it if
  52             it doesn't exist yet, or after making sure an existing
  53             database cache has the correct version.  Return True if a
  54             connection could be established, False otherwise. """
  55         if not os.path.exists(self.cache_dir):
  56             if self.verbose:
  57                 print "Initializing the cache directory..."
  58             os.mkdir(self.cache_dir)
  59         if os.path.exists(self.db_path):
  60             cache_version = self.get_db_version()
  61             if not cache_version:
  62                 cache_version = "0.0.1"
  63             if cache_version != self.db_version:
  64                 print("The existing database cache uses version %s, "
  65                         "not the expected %s." % (cache_version,
  66                         self.db_version))
  67                 return False
  68         self.conn = sqlite3.connect(self.db_path)
  69         self.cursor = self.conn.cursor()
  70         self.create_assignments_table()
  71         return True
  72
  73     def __get_default_config_file_obj(self):
  74         open_flags = 'rb+'
  75         file_path = os.path.join(self.cache_dir, 'db.cfg')
  76         if not os.path.exists(file_path):
  77             open_flags = 'wb+'
  78         return open(file_path, open_flags)
  79
  80     def _get_db_config(self, file_obj=None):
  81         """ Return the database configuration object from the provided
  82             file_obj if provided, otherwise from the default database
  83             configuration file. """
  84         if file_obj is None:
  85             file_obj = self.__get_default_config_file_obj()
  86         config = ConfigParser.SafeConfigParser()
  87         config.readfp(file_obj)
  88         file_obj.close()
  89         return config
  90
  91     def set_db_version(self, file_obj=None):
  92         """ Set the database version string in the config file. """
  93         if file_obj is None:
  94             file_obj = self.__get_default_config_file_obj()
  95         config = self._get_db_config()
  96         if not config.has_section('db'):
  97             config.add_section('db')
  98         config.set('db', 'version', self.db_version)
  99         config.write(file_obj)
 100         file_obj.close()
 101
 102     def get_db_version(self):
 103         """ Read and return the database version string from the config
 104             file. """
 105         config = self._get_db_config()
 106         if not config.has_section('db'):
 107             return None
 108         return config.get('db', 'version')
 109
 110     def commit_and_close_database(self):
 111         self.conn.commit()
 112         self.cursor.close()
 113
 114     def create_assignments_table(self):
 115         """ Create the assignments table that stores all assignments from
 116             IPv4/IPv6/ASN to country code.  Blocks are stored as first hex
 117             of and first hex after the assignment.  Numbers are stored
 118             as hex strings, because SQLite's INTEGER type only holds up to
 119             63 unsigned bits, which is not enough to store a /64 IPv6
 120             block.  Hex strings have leading zeros, with IPv6 addresses
 121             being 33 hex characters long and IPv4 addresses and ASN being
 122             9 hex characters long.  The first number after an assignment
 123             range is stored instead of the last number in the range to
 124             facilitate comparisons with neighboring ranges. """
 125         sql = ('CREATE TABLE IF NOT EXISTS assignments(start_hex TEXT, '
 126                'next_start_hex TEXT, num_type TEXT, country_code TEXT, '
 127                'source_type TEXT, source_name TEXT)')
 128         self.cursor.execute(sql)
 129         self.conn.commit()
 130
 131     def delete_assignments(self, source_type):
 132         """ Delete all assignments from the database cache matching a
 133             given source type ("rir", "lir", etc.). """
 134         sql = 'DELETE FROM assignments WHERE source_type = ?'
 135         self.cursor.execute(sql, (source_type, ))
 136         self.conn.commit()
 137
 138     def insert_assignment(self, start_num, end_num, num_type,
 139             country_code, source_type, source_name):
 140         """ Insert an assignment into the database cache, without
 141             commiting after the insertion. """
 142         sql = ('INSERT INTO assignments (start_hex, next_start_hex, '
 143                'num_type, country_code, source_type, source_name) '
 144                'VALUES (?, ?, ?, ?, ?, ?)')
 145         if num_type == 'ipv6':
 146             start_hex = '%033x' % start_num
 147             next_start_hex = '%033x' % (end_num + 1)
 148         else:
 149             start_hex = '%09x' % start_num
 150             next_start_hex = '%09x' % (end_num + 1)
 151         self.cursor.execute(sql, (start_hex, next_start_hex, num_type,
 152                 country_code, source_type, source_name))
 153
 154     def commit_changes(self):
 155         """ Commit changes, e.g., after inserting assignments into the
 156             database cache. """
 157         self.conn.commit()
 158
 159     def fetch_assignments(self, num_type, country_code):
 160         """ Fetch all assignments from the database cache matching the
 161             given number type ("asn", "ipv4", or "ipv6") and country code.
 162             The result is a sorted list of tuples containing (start_num,
 163             end_num). """
 164         sql = ('SELECT start_hex, next_start_hex FROM assignments '
 165                'WHERE num_type = ? AND country_code = ? '
 166                'ORDER BY start_hex')
 167         self.cursor.execute(sql, (num_type, country_code))
 168         result = []
 169         for row in self.cursor:
 170             result.append((long(row[0], 16), long(row[1], 16) - 1))
 171         return result
 172
 173     def fetch_country_code(self, num_type, source_type, lookup_num):
 174         """ Fetch the country code from the database cache that is
 175             assigned to the given number (e.g., IPv4 address in decimal
 176             notation), number type (e.g., "ipv4"), and source type (e.g.,
 177             "rir"). """
 178         sql = ('SELECT country_code FROM assignments WHERE num_type = ? '
 179                'AND source_type = ? AND start_hex <= ? '
 180                'AND next_start_hex > ?')
 181         if num_type == 'ipv6':
 182             lookup_hex = '%033x' % long(lookup_num)
 183         else:
 184             lookup_hex = '%09x' % long(lookup_num)
 185         self.cursor.execute(sql, (num_type, source_type, lookup_hex,
 186                 lookup_hex))
 187         row = self.cursor.fetchone()
 188         if row:
 189             return row[0]
 190
 191     def fetch_country_blocks_in_other_sources(self, first_country_code):
 192         """ Fetch all assignments matching the given country code, then look
 193             up to which country code(s) the same number ranges are assigned in
 194             other source types.  Return 8-tuples containing (1) first source
 195             type, (2) first and (3) last number of the assignment in the first
 196             source type, (4) second source type, (5) first and (6) last number
 197             of the assignment in the second source type, (7) country code in
 198             the second source type, and (8) number type. """
 199         sql = ('SELECT first.source_type, first.start_hex, '
 200                'first.next_start_hex, second.source_type, '
 201                'second.start_hex, second.next_start_hex, '
 202                'second.country_code, first.num_type '
 203                'FROM assignments AS first '
 204                'JOIN assignments AS second '
 205                'WHERE first.country_code = ? '
 206                'AND first.start_hex <= second.next_start_hex '
 207                'AND first.next_start_hex >= second.start_hex '
 208                'AND first.num_type = second.num_type '
 209                'ORDER BY first.source_type, first.start_hex, '
 210                'second.source_type, second.start_hex')
 211         self.cursor.execute(sql, (first_country_code, ))
 212         result = []
 213         for row in self.cursor:
 214             result.append((str(row[0]), long(row[1], 16),
 215                     long(row[2], 16) - 1, str(row[3]), long(row[4], 16),
 216                     long(row[5], 16) - 1, str(row[6]), str(row[7])))
 217         return result
 218
 219
 220 class DownloaderParser:
 221     def __init__(self, cache_dir, database_cache, user_agent,
 222             verbose=False):
 223         self.cache_dir = cache_dir
 224         self.database_cache = database_cache
 225         self.user_agent = user_agent
 226         self.verbose = verbose
 227
 228     MAXMIND_URLS = """
 229         http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
 230         http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz
 231     """
 232
 233     RIR_URLS = """
 234         ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest
 235         ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
 236         ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
 237         ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
 238         ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
 239     """
 240
 241     LIR_URLS = """
 242         ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz
 243         ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz
 244     """
 245
 246     COUNTRY_CODE_URL = "http://www.iso.org/iso/list-en1-semic-3.txt"
 247
 248     def download_maxmind_files(self):
 249         """ Download all LIR delegation urls. """
 250         for maxmind_url in self.MAXMIND_URLS.split():
 251             self._download_to_cache_dir(maxmind_url)
 252
 253     def download_rir_files(self):
 254         """ Download all RIR delegation files including md5 checksum. """
 255         for rir_url in self.RIR_URLS.split():
 256             rir_md5_url = rir_url + '.md5'
 257             self._download_to_cache_dir(rir_url)
 258             self._download_to_cache_dir(rir_md5_url)
 259
 260     def download_lir_files(self):
 261         """ Download all LIR delegation urls. """
 262         for lir_url in self.LIR_URLS.split():
 263             self._download_to_cache_dir(lir_url)
 264
 265     def download_country_code_file(self):
 266         """ Download and save the latest semicolon-separated open country
 267             codes file. """
 268         self._download_to_cache_dir(self.COUNTRY_CODE_URL)
 269
 270     def _download_to_cache_dir(self, url):
 271         """ Fetch a resource (with progress bar) and store contents to the
 272             local cache directory under the file name given in the URL. """
 273         if not os.path.exists(self.cache_dir):
 274             if self.verbose:
 275                 print "Initializing the cache directory..."
 276             os.mkdir(self.cache_dir)
 277         filename = url.split('/')[-1]
 278         req = urllib2.Request(url)
 279         if self.user_agent:
 280             req.add_header('User-Agent', self.user_agent)
 281         # TODO Allow use of a proxy.
 282         #req.set_proxy(host, type)
 283         fetcher = urllib2.urlopen(req)
 284         length_header = fetcher.headers.get("Content-Length")
 285         expected_bytes = -1
 286         if length_header:
 287             expected_bytes = int(length_header)
 288             print("Fetching %d kilobytes" %
 289                     round(float(expected_bytes / 1024), 2))
 290         download_started = time.time()
 291         output_file = open(os.path.join(self.cache_dir, filename), "wb")
 292         received_bytes, seconds_elapsed = 0, 0
 293         while True:
 294             seconds_elapsed = time.time() - download_started
 295             if expected_bytes >= 0:
 296                 self._update_progress_bar(received_bytes, expected_bytes,
 297                         seconds_elapsed)
 298             chunk = fetcher.read(1024)
 299             if len(chunk) == 0:
 300                 if expected_bytes >= 0 and received_bytes != expected_bytes:
 301                     print("Expected %s bytes, only received %s" %
 302                             (expected_bytes, received_bytes))
 303                 print ""
 304                 break
 305             received_bytes += len(chunk)
 306             output_file.write(chunk)
 307         output_file.close()
 308
 309     def _update_progress_bar(self, received_bytes, expected_bytes,
 310             seconds_elapsed):
 311         """ Write a progress bar to the console. """
 312         if is_win32:
 313           rows    = 100 # use some WinCon function for these?
 314           columns = 80  # but not really important.
 315           EOL = "\r"
 316         else:
 317           rows, columns = map(int, os.popen('stty size', 'r').read().split())
 318           EOL = "\x1b[G"
 319         if seconds_elapsed == 0:
 320             seconds_elapsed = 1
 321         percent_done = float(received_bytes) / float(expected_bytes)
 322         caption = "%.2f K/s" % (received_bytes / 1024 / seconds_elapsed)
 323         width = columns - 4 - len(caption)
 324         sys.stdout.write("[%s>%s] %s%s" % (
 325                 "=" * int(percent_done * width),
 326                 "." * (width - int(percent_done * width)), caption, EOL))
 327         sys.stdout.flush()
 328
 329     def check_rir_file_mtimes(self):
 330         """ Return True if the mtime of any RIR file in our cache directory
 331             is > 24 hours, False otherwise. """
 332         if not os.path.exists(self.cache_dir):
 333             return False
 334         for rir_url in self.RIR_URLS.split():
 335             rir_path = os.path.join(self.cache_dir,
 336                     rir_url.split('/')[-1])
 337             if os.path.exists(rir_path):
 338                 rir_stat = os.stat(rir_path)
 339                 if (time.time() - rir_stat.st_mtime) > 86400:
 340                     return True
 341         return False
 342
 343     def verify_rir_files(self):
 344         """ Compute md5 checksums of all RIR files, compare them to the
 345             provided .md5 files, and return True if the two checksums match,
 346             or False otherwise. """
 347         for rir_url in self.RIR_URLS.split():
 348             rir_path = os.path.join(self.cache_dir,
 349                     rir_url.split('/')[-1])
 350             rir_md5_path = os.path.join(self.cache_dir,
 351                     rir_url.split('/')[-1] + '.md5')
 352             if not os.path.exists(rir_md5_path) or \
 353                     not os.path.exists(rir_path):
 354                 continue
 355             rir_md5_file = open(rir_md5_path, 'r')
 356             expected_checksum = rir_md5_file.read()
 357             rir_md5_file.close()
 358             if "=" in expected_checksum:
 359                 expected_checksum = expected_checksum.split("=")[-1].strip()
 360             if expected_checksum == "":
 361                 continue
 362             computed_checksum = ""
 363             rir_file = open(rir_path, 'rb')
 364             rir_data = rir_file.read()
 365             rir_file.close()
 366             computed_checksum = str(hashlib.md5(rir_data).hexdigest())
 367             if expected_checksum != computed_checksum:
 368                 print("The computed md5 checksum of %s, %s, does *not* "
 369                         "match the provided checksum %s!" %
 370                         (rir_path, computed_checksum, expected_checksum))
 371
 372     def parse_maxmind_files(self, maxmind_urls=None):
 373         """ Parse locally cached MaxMind files and insert assignments to the
 374             local database cache, overwriting any existing MaxMind
 375             assignments. """
 376         if not maxmind_urls:
 377             maxmind_urls = self.MAXMIND_URLS.split()
 378         self.database_cache.delete_assignments('maxmind')
 379         for maxmind_url in maxmind_urls:
 380             maxmind_path = os.path.join(self.cache_dir,
 381                     maxmind_url.split('/')[-1])
 382             if not os.path.exists(maxmind_path):
 383                 print "Unable to find %s." % maxmind_path
 384                 continue
 385             if maxmind_path.endswith('.zip'):
 386                 maxmind_zip_path = zipfile.ZipFile(maxmind_path)
 387                 for contained_filename in maxmind_zip_path.namelist():
 388                     content = maxmind_zip_path.read(contained_filename)
 389                     self._parse_maxmind_content(content, 'maxmind',
 390                             'maxmind')
 391             elif maxmind_path.endswith('.gz'):
 392                 content = gzip.open(maxmind_path).read()
 393                 self._parse_maxmind_content(content, 'maxmind', 'maxmind')
 394         self.database_cache.commit_changes()
 395
 396     def import_maxmind_file(self, maxmind_path):
 397         self.database_cache.delete_assignments(maxmind_path)
 398         if not os.path.exists(maxmind_path):
 399             print "Unable to find %s." % maxmind_path
 400             return
 401         content = open(maxmind_path).read()
 402         self._parse_maxmind_content(content, maxmind_path, maxmind_path)
 403         self.database_cache.commit_changes()
 404
 405     def _parse_maxmind_content(self, content, source_type, source_name):
 406         keys = ['start_str', 'end_str', 'start_num', 'end_num',
 407                  'country_code', 'country_name']
 408         for line in content.split('\n'):
 409             if len(line.strip()) == 0 or line.startswith("#"):
 410                 continue
 411             line = line.replace('"', '').replace(' ', '').strip()
 412             parts = line.split(',')
 413             entry = dict((k, v) for k, v in zip(keys, parts))
 414             start_num = int(entry['start_num'])
 415             end_num = int(entry['end_num'])
 416             country_code = str(entry['country_code'])
 417             start_ipaddr = ipaddr.IPAddress(entry['start_str'])
 418             if isinstance(start_ipaddr, ipaddr.IPv4Address):
 419                 num_type = 'ipv4'
 420             else:
 421                 num_type = 'ipv6'
 422             self.database_cache.insert_assignment(start_num, end_num,
 423                     num_type, country_code, source_type, source_name)
 424
 425     def parse_rir_files(self, rir_urls=None):
 426         """ Parse locally cached RIR files and insert assignments to the local
 427             database cache, overwriting any existing RIR assignments. """
 428         if not rir_urls:
 429             rir_urls = self.RIR_URLS.split()
 430         self.database_cache.delete_assignments('rir')
 431         keys = "registry country_code type start value date status"
 432         for rir_url in rir_urls:
 433             rir_path = os.path.join(self.cache_dir,
 434                     rir_url.split('/')[-1])
 435             if not os.path.exists(rir_path):
 436                 print "Unable to find %s." % rir_path
 437                 continue
 438             for line in open(rir_path, 'r'):
 439                 if line.startswith("#"):
 440                     continue
 441                 entry = dict((k, v) for k, v in
 442                         zip(keys.split(), line.strip().split("|")))
 443                 source_name = str(entry['registry'])
 444                 country_code = str(entry['country_code'])
 445                 if source_name.isdigit() or country_code == "*":
 446                     continue
 447                 num_type = entry['type']
 448                 if num_type == 'asn':
 449                     start_num = end_num = int(entry['start'])
 450                 elif num_type == 'ipv4':
 451                     start_num = int(ipaddr.IPAddress(entry['start']))
 452                     end_num = start_num + long(entry['value']) - 1
 453                 elif num_type == 'ipv6':
 454                     network_str = entry['start'] + '/' + entry['value']
 455                     network_ipaddr = ipaddr.IPv6Network(network_str)
 456                     start_num = int(network_ipaddr.network)
 457                     end_num = int(network_ipaddr.broadcast)
 458                 self.database_cache.insert_assignment(start_num,
 459                         end_num, num_type, country_code, 'rir',
 460                         source_name)
 461         self.database_cache.commit_changes()
 462
 463     def parse_lir_files(self, lir_urls=None):
 464         """ Parse locally cached LIR files and insert assignments to the local
 465             database cache, overwriting any existing LIR assignments. """
 466         if not lir_urls:
 467             lir_urls = self.LIR_URLS.split()
 468         self.database_cache.delete_assignments('lir')
 469         for lir_url in lir_urls:
 470             lir_path = os.path.join(self.cache_dir,
 471                     lir_url.split('/')[-1])
 472             if not os.path.exists(lir_path):
 473                 print "Unable to find %s." % lir_path
 474                 continue
 475             if lir_path.endswith('.gz'):
 476                 lir_file = gzip.open(lir_path)
 477             else:
 478                 lir_file = open(lir_path)
 479             start_num = 0
 480             end_num = 0
 481             country_code = ""
 482             entry = False
 483             num_type = ""
 484             for line in lir_file:
 485                 line = line.replace("\n", "")
 486                 if line == "":
 487                     entry = False
 488                     start_num, end_num, country_code, num_type = 0, 0, "", ""
 489                 elif not entry and "inetnum:" in line:
 490                     try:
 491                         line = line.replace("inetnum:", "").strip()
 492                         start_str = line.split("-")[0].strip()
 493                         end_str = line.split("-")[1].strip()
 494                         start_num = int(ipaddr.IPv4Address(start_str))
 495                         end_num = int(ipaddr.IPv4Address(end_str))
 496                         entry = True
 497                         num_type = 'ipv4'
 498                     except Exception, e:
 499                         if self.verbose:
 500                             print repr(e), line
 501                 elif not entry and "inet6num:" in line:
 502                     try:
 503                         network_str = line.replace("inet6num:", "").strip()
 504                         network_ipaddr = ipaddr.IPv6Network(network_str)
 505                         start_num = int(network_ipaddr.network)
 506                         end_num = int(network_ipaddr.broadcast)
 507                         entry = True
 508                         num_type = 'ipv6'
 509                     except Exception, e:
 510                         if self.verbose:
 511                             print repr(e), line
 512                 elif entry and "country:" in line:
 513                     country_code = line.replace("country:", "").strip()
 514                     self.database_cache.insert_assignment(start_num,
 515                             end_num, num_type, country_code, 'lir', 'ripencc')
 516         self.database_cache.commit_changes()
 517
 518
 519 class Lookup:
 520     def __init__(self, cache_dir, database_cache, verbose=False):
 521         self.cache_dir = cache_dir
 522         self.database_cache = database_cache
 523         self.verbose = verbose
 524         self.map_co = None
 525         self.build_country_code_dictionary()
 526
 527     def build_country_code_dictionary(self):
 528         """ Return a dictionary mapping country name to the country
 529             code. """
 530         country_code_path = os.path.join(self.cache_dir,
 531                 'list-en1-semic-3.txt')
 532         if not os.path.exists(country_code_path):
 533             return
 534         self.map_co = {}
 535         for line in open(country_code_path):
 536             if line == "" or line.startswith("Country ") or ";" not in line:
 537                 continue
 538             country_name, country_code = line.strip().split(";")
 539             country_name = ' '.join([part.capitalize() for part in \
 540                     country_name.split(" ")])
 541             self.map_co[country_name] = country_code
 542
 543     def knows_country_names(self):
 544         return self.map_co is not None
 545
 546     def get_name_from_country_code(self, cc_code):
 547         if not self.knows_country_names():
 548             return
 549         country_name = [(key, value) for (key, value) in \
 550                 self.map_co.items() if value == cc_code]
 551         if len(country_name) > 0:
 552             return country_name[0][0]
 553
 554     def get_country_code_from_name(self, country_name):
 555         """ Return the country code for a given country name. """
 556         if not self.knows_country_names():
 557             return
 558         cc_code = [self.map_co[key] for key in self.map_co.keys() if \
 559                 key.upper().startswith(country_name.upper())]
 560         if len(cc_code) > 0:
 561             return cc_code[0]
 562
 563     def lookup_ipv6_address(self, lookup_ipaddr):
 564         print "Reverse lookup for: " + str(lookup_ipaddr)
 565         for source_type in ['maxmind', 'rir', 'lir']:
 566             cc = self.database_cache.fetch_country_code('ipv6',
 567                     source_type, int(lookup_ipaddr))
 568             if cc:
 569                 print source_type.upper(), "country code:", cc
 570                 cn = self.get_name_from_country_code(cc)
 571                 if cn:
 572                     print source_type.upper(), "country name:", cn
 573
 574     def lookup_ipv4_address(self, lookup_ipaddr):
 575         print "Reverse lookup for: " + str(lookup_ipaddr)
 576         maxmind_cc = self.database_cache.fetch_country_code('ipv4', 'maxmind',
 577                 int(lookup_ipaddr))
 578         if maxmind_cc:
 579             print 'MaxMind country code:', maxmind_cc
 580             maxmind_cn = self.get_name_from_country_code(maxmind_cc)
 581             if maxmind_cn:
 582                 print 'MaxMind country name:', maxmind_cn
 583         rir_cc = self.database_cache.fetch_country_code('ipv4', 'rir',
 584                 int(lookup_ipaddr))
 585         if rir_cc:
 586             print 'RIR country code:', rir_cc
 587             rir_cn = self.get_name_from_country_code(rir_cc)
 588             if rir_cn:
 589                 print 'RIR country name:', rir_cn
 590         else:
 591             print 'Not found in RIR db'
 592         lir_cc = self.database_cache.fetch_country_code('ipv4', 'lir',
 593                 int(lookup_ipaddr))
 594         if lir_cc:
 595             print 'LIR country code:', lir_cc
 596             lir_cn = self.get_name_from_country_code(lir_cc)
 597             if lir_cn:
 598                 print 'LIR country name:', lir_cn
 599         if maxmind_cc and maxmind_cc != rir_cc:
 600             print("It appears that the RIR data conflicts with MaxMind's "
 601                     "data.  MaxMind's data is likely closer to being "
 602                     "correct due to sub-delegation issues with LIR databases.")
 603
 604     def lookup_ip_address(self, lookup_str):
 605         """ Return the country code and name for a given ip address. """
 606         try:
 607             lookup_ipaddr = ipaddr.IPAddress(lookup_str)
 608             if isinstance(lookup_ipaddr, ipaddr.IPv4Address):
 609                 self.lookup_ipv4_address(lookup_ipaddr)
 610             elif isinstance(lookup_ipaddr, ipaddr.IPv6Address):
 611                 self.lookup_ipv6_address(lookup_ipaddr)
 612             else:
 613                 print("Did not recognize '%s' as either IPv4 or IPv6 "
 614                         "address." % lookup_str)
 615         except ValueError, e:
 616             print "'%s' is not a valid IP address." % lookup_str
 617
 618     def asn_lookup(self, asn):
 619         asn_cc = self.database_cache.fetch_country_code('asn', 'rir', asn)
 620         if asn_cc:
 621             print "AS country code: %s" % asn_cc
 622             asn_cn = self.get_name_from_country_code(asn_cc)
 623             if asn_cn:
 624                 print "AS country name: %s" % asn_cn
 625         else:
 626             print "AS%s not found!" % asn
 627
 628     def fetch_rir_blocks_by_country(self, request, country):
 629         result = []
 630         for (start_num, end_num) in \
 631                 self.database_cache.fetch_assignments(request, country):
 632             if request == "ipv4" or request == "ipv6":
 633                 start_ipaddr = ipaddr.IPAddress(start_num)
 634                 end_ipaddr = ipaddr.IPAddress(end_num)
 635                 result += [str(x) for x in
 636                         ipaddr.summarize_address_range(
 637                         start_ipaddr, end_ipaddr)]
 638             else:
 639                 result.append(str(start_num))
 640         return result
 641
 642     def lookup_countries_in_different_source(self, first_country_code):
 643         """ Look up all assignments matching the given country code, then
 644             look up to which country code(s) the same number ranges are
 645             assigned in other source types.  Print out the result showing
 646             similarities and differences. """
 647         print("\nLegend:\n"
 648                "  '<' = found assignment range with country code '%s'\n"
 649                "  '>' = overlapping assignment range with same country code\n"
 650                "  '*' = overlapping assignment range, first conflict\n"
 651                "  '#' = overlapping assignment range, second conflict and "
 652                "beyond\n  ' ' = neighboring assignment range") % (
 653                first_country_code, )
 654         results = self.database_cache.fetch_country_blocks_in_other_sources(
 655                 first_country_code)
 656         prev_first_source_type = ''
 657         prev_first_start_num = -1
 658         cur_second_country_codes = []
 659         for (first_source_type, first_start_num, first_end_num,
 660                 second_source_type, second_start_num, second_end_num,
 661                 second_country_code, num_type) in results:
 662             if first_source_type != prev_first_source_type:
 663                 print "\nAssignments in '%s':" % (first_source_type, )
 664             prev_first_source_type = first_source_type
 665             if first_start_num != prev_first_start_num:
 666                 cur_second_country_codes = []
 667                 print ""
 668             prev_first_start_num = first_start_num
 669             marker = ''
 670             if second_end_num >= first_start_num and \
 671                     second_start_num <= first_end_num:
 672                 if first_country_code != second_country_code and \
 673                         second_country_code not in cur_second_country_codes:
 674                     cur_second_country_codes.append(second_country_code)
 675                 if first_source_type == second_source_type:
 676                     marker = '<'
 677                 elif len(cur_second_country_codes) == 0:
 678                     marker = '>'
 679                 elif len(cur_second_country_codes) == 1:
 680                     marker = '*'
 681                 else:
 682                     marker = '#'
 683             if num_type.startswith("ip") and \
 684                     second_start_num == second_end_num:
 685                 second_range = "%s" % (ipaddr.IPAddress(second_start_num), )
 686             elif num_type.startswith("ip") and \
 687                     second_start_num < second_end_num:
 688                 second_range = "%s-%s" % (ipaddr.IPAddress(second_start_num),
 689                         ipaddr.IPAddress(second_end_num))
 690             elif second_start_num < second_end_num:
 691                 second_range = "AS%d-%d" % (second_start_num, second_end_num)
 692             else:
 693                 second_range = "AS%d" % (second_start_num, )
 694             print "%1s %s %s %s" % (marker, second_country_code, second_range,
 695                     second_source_type, )
 696
 697 def split_callback(option, opt, value, parser):
 698     split_value = value.split(':')
 699     setattr(parser.values, option.dest, split_value[0])
 700     if len(split_value) > 1 and split_value[1] != '':
 701         setattr(parser.values, 'type_filter', split_value[1])
 702
 703 def main():
 704     """ Where the magic starts. """
 705     usage = ("Usage: %prog [options]\n\n"
 706             "Example: %prog -v -t mm")
 707     parser = optparse.OptionParser(usage)
 708     parser.add_option("-v", "--verbose", action="store_true",
 709             dest="verbose", help = "be verbose", default=False)
 710     parser.add_option("-c", "--cache-dir", action="store", dest="dir",
 711             help="set cache directory [default: %default]",
 712             default=str(os.path.expanduser('~')) + "/.blockfinder/")
 713     parser.add_option("--user-agent", action="store", dest="ua",
 714             help=('provide a User-Agent which will be used when '
 715                   'fetching delegation files [default: "%default"]'),
 716             default="Mozilla/5.0 (Windows NT 6.1; rv:10.0) Gecko/20100101 Firefox/10.0)")
 717     parser.add_option("-x", "--hack-the-internet", action="store_true",
 718             dest="hack_the_internet", help=optparse.SUPPRESS_HELP)
 719     group = optparse.OptionGroup(parser, "Cache modes",
 720             "Pick at most one of these modes to initialize or update "
 721             "the local cache.  May not be combined with lookup modes.")
 722     group.add_option("-m", "--init-maxmind", action="store_true",
 723             dest="init_maxmind",
 724             help="initialize or update MaxMind GeoIP database")
 725     group.add_option("-g", "--reload-maxmind", action="store_true",
 726             dest="reload_maxmind",
 727             help=("update cache from existing MaxMind GeoIP database"))
 728     group.add_option("-r", "--import-maxmind", action="store",
 729             dest="import_maxmind", metavar="FILE",
 730             help=("import the specified MaxMind GeoIP database file into "
 731                   "the database cache using its file name as source "
 732                   "name"))
 733     group.add_option("-i", "--init-rir",
 734             action="store_true", dest="init_del",
 735             help="initialize or update delegation information")
 736     group.add_option("-d", "--reload-rir", action="store_true",
 737             dest="reload_del",
 738             help="use existing delegation files to update the database")
 739     group.add_option("-l", "--init-lir", action="store_true",
 740             dest="init_lir",
 741             help=("initialize or update lir information; can take up to "
 742                   "5 minutes"))
 743     group.add_option("-z", "--reload-lir", action="store_true",
 744             dest="reload_lir",
 745             help=("use existing lir files to update the database; can "
 746                   "take up to 5 minutes"))
 747     group.add_option("-o", "--download-cc", action="store_true",
 748             dest="download_cc", help="download country codes file")
 749     group.add_option("-e", "--erase-cache", action="store_true",
 750             dest="erase_cache", help="erase the local database cache")
 751     parser.add_option_group(group)
 752     group = optparse.OptionGroup(parser, "Lookup modes",
 753             "Pick at most one of these modes to look up data in the "
 754             "local cache.  May not be combined with cache modes.")
 755     group.add_option("-4", "--ipv4", action="store", dest="ipv4",
 756             help=("look up country code and name for the specified IPv4 "
 757                   "address"))
 758     group.add_option("-6", "--ipv6", action="store", dest="ipv6",
 759             help=("look up country code and name for the specified IPv6 "
 760                   "address"))
 761     group.add_option("-a", "--asn", action="store", dest="asn",
 762             help="look up country code and name for the specified ASN")
 763     group.add_option("-t", "--code", action="callback", dest="cc",
 764             callback=split_callback, metavar="CC[:type]", type="str",
 765             help=("look up all allocations (or only those for number "
 766                   "type 'ipv4', 'ipv6', or 'asn' if provided) in the "
 767                   "delegation cache for the specified two-letter country "
 768                   "code"))
 769     group.add_option("-n", "--name", action="callback", dest="cn",
 770             callback=split_callback, metavar="CN[:type]", type="str",
 771             help=("look up all allocations (or only those for number "
 772                   "type 'ipv4', 'ipv6', or 'asn' if provided) in the "
 773                   "delegation cache for the specified full country name"))
 774     group.add_option("-p", "--compare", action="store", dest="compare",
 775             metavar="CC",
 776             help=("compare assignments to the specified country code "
 777                   "with overlapping assignments in other data sources; "
 778                   "can take some time and produce some long output"))
 779     group.add_option("-w", "--what-country", action="store", dest="what_cc",
 780             help=("look up country name for specified country code"))
 781     parser.add_option_group(group)
 782     group = optparse.OptionGroup(parser, "Network modes")
 783     (options, args) = parser.parse_args()
 784     if options.hack_the_internet:
 785         print "all your bases are belong to us!"
 786         sys.exit(0)
 787     options_dict = vars(options)
 788     modes = 0
 789     for mode in ["init_maxmind", "reload_maxmind", "import_maxmind",
 790                  "init_del", "init_lir", "reload_del", "reload_lir",
 791                  "download_cc", "erase_cache", "ipv4", "ipv6", "asn",
 792                  "cc", "cn", "compare", "what_cc"]:
 793         if options_dict.has_key(mode) and options_dict.get(mode):
 794             modes += 1
 795     if modes > 1:
 796         parser.error("only 1 cache or lookup mode allowed")
 797     elif modes == 0:
 798         parser.error("must provide 1 cache or lookup mode")
 799     database_cache = DatabaseCache(options.dir, options.verbose)
 800     if options.erase_cache:
 801         database_cache.erase_database()
 802         sys.exit(0)
 803     if not database_cache.connect_to_database():
 804         print "Could not connect to database."
 805         print("You may need to erase it using -e and then reload it "
 806                 "using -d/-z.  Exiting.")
 807         sys.exit(1)
 808     database_cache.set_db_version()
 809     downloader_parser = DownloaderParser(options.dir, database_cache,
 810             options.ua)
 811     lookup = Lookup(options.dir, database_cache)
 812     if options.ipv4 or options.ipv6 or options.asn or options.cc \
 813             or options.cn or options.compare:
 814         if downloader_parser.check_rir_file_mtimes():
 815             print("Your cached RIR files are older than 24 hours; you "
 816                     "probably want to update them.")
 817     if options.asn:
 818         lookup.asn_lookup(options.asn)
 819     elif options.ipv4:
 820         lookup.lookup_ip_address(options.ipv4)
 821     elif options.ipv6:
 822         lookup.lookup_ip_address(options.ipv6)
 823     elif options.cc or options.cn or options.what_cc:
 824         country = None
 825         if options.cc:
 826             country = options.cc.upper()
 827         elif not lookup.knows_country_names():
 828             print("Need to download country codes first before looking "
 829                   "up countries by name.")
 830         elif options.what_cc:
 831             country = options.what_cc.upper()
 832             country_name = lookup.get_name_from_country_code(country)
 833             if country_name:
 834                 print("Hmm...%s? That would be %s."
 835                       % (options.what_cc, country_name))
 836                 sys.exit(0)
 837             else:
 838                 print("Hmm, %s? We're not sure either. Are you sure that's "
 839                       "a country code?" % options.what_cc)
 840                 sys.exit(1)
 841         else:
 842             country = lookup.get_country_code_from_name(options.cn)
 843             if not country:
 844                 print "It appears your search did not match a country."
 845         if country:
 846             types = ["ipv4", "ipv6", "asn"]
 847             if hasattr(options, 'type_filter') and options.type_filter.lower() in types:
 848                 types = [options.type_filter.lower()]
 849             for request in types:
 850                 print "\n".join(lookup.fetch_rir_blocks_by_country(\
 851                         request, country))
 852     elif options.compare:
 853         print("Comparing assignments with overlapping assignments in other "
 854                "data sources...")
 855         lookup.lookup_countries_in_different_source(options.compare)
 856     elif options.init_maxmind or options.reload_maxmind:
 857         if options.init_maxmind:
 858             print "Downloading Maxmind GeoIP files..."
 859             downloader_parser.download_maxmind_files()
 860         print "Importing Maxmind GeoIP files..."
 861         downloader_parser.parse_maxmind_files()
 862     elif options.import_maxmind:
 863         print "Importing Maxmind GeoIP files..."
 864         downloader_parser.import_maxmind_file(options.import_maxmind)
 865     elif options.init_del or options.reload_del:
 866         if options.init_del:
 867             print "Downloading RIR files..."
 868             downloader_parser.download_rir_files()
 869             print "Verifying RIR files..."
 870             downloader_parser.verify_rir_files()
 871         print "Importing RIR files..."
 872         downloader_parser.parse_rir_files()
 873     elif options.init_lir or options.reload_lir:
 874         if options.init_lir:
 875             print "Downloading LIR delegation files..."
 876             downloader_parser.download_lir_files()
 877         print "Importing LIR files..."
 878         downloader_parser.parse_lir_files()
 879     elif options.download_cc:
 880         print "Downloading country code file..."
 881         downloader_parser.download_country_code_file()
 882     database_cache.commit_and_close_database()
 883
 884 if __name__ == "__main__":
 885     main()
 886