Merge pull request #78 from aagbsn/fix/77_str_decode_arguments

[blockfinder.git] / blockfinder

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# For the people of Smubworld!
import os
import time
import optparse
import sys
import sqlite3
import hashlib
import gzip
import zipfile
import re

if sys.version_info[0] >= 3:
    import configparser
    import ipaddress as ipaddr
    from urllib.request import (urlopen, Request)
    from urllib.error import URLError
    long = int
else:
    import ConfigParser as configparser
    from urllib2 import (urlopen, Request, URLError)
    from embedded_ipaddr import ipaddr
    ipaddr.ip_address = ipaddr.IPAddress

is_win32 = (sys.platform == "win32")

__program__ = 'blockfinder'
__url__ = 'https://github.com/ioerror/blockfinder/'
__author__ = 'Jacob Appelbaum <jacob@appelbaum.net>, David <db@d1b.org>'
__copyright__ = 'Copyright (c) 2010'
__license__ = 'See LICENSE for licensing information'
__version__ = '3.1415'

try:
    from future import antigravity
except ImportError:
    antigravity = None


class DatabaseCache:
    def __init__(self, cache_dir, verbose=False):
        self.cache_dir = cache_dir
        self.verbose = verbose
        self.cursor = None
        self.conn = None
        self.db_version = "0.0.4"
        self.db_path = os.path.join(self.cache_dir + "sqlitedb")

    def erase_database(self):
        """ Erase the database file. """
        if os.path.exists(self.db_path):
            os.remove(self.db_path)

    def connect_to_database(self):
        """ Connect to the database cache, possibly after creating it if
            it doesn't exist yet, or after making sure an existing
            database cache has the correct version.  Return True if a
            connection could be established, False otherwise. """
        if not os.path.exists(self.cache_dir):
            if self.verbose:
                print("Initializing the cache directory...")
            os.mkdir(self.cache_dir)
        if os.path.exists(self.db_path):
            cache_version = self.get_db_version()
            if not cache_version:
                cache_version = "0.0.1"
            if cache_version != self.db_version:
                print(("The existing database cache uses version %s, "
                        "not the expected %s." % (cache_version,
                        self.db_version)))
                return False
        self.conn = sqlite3.connect(self.db_path)
        self.cursor = self.conn.cursor()
        self.create_assignments_table()
        return True

    def __get_default_config_file_obj(self):
        open_flags = 'r+'
        file_path = os.path.join(self.cache_dir, 'db.cfg')
        if not os.path.exists(file_path):
            open_flags = 'w+'
        return open(file_path, open_flags)

    def _get_db_config(self, file_obj=None):
        """ Return the database configuration object from the provided
            file_obj if provided, otherwise from the default database
            configuration file. """
        if file_obj is None:
            file_obj = self.__get_default_config_file_obj()
        config = configparser.SafeConfigParser()
        config.readfp(file_obj)
        file_obj.close()
        return config

    def set_db_version(self, file_obj=None):
        """ Set the database version string in the config file. """
        if file_obj is None:
            file_obj = self.__get_default_config_file_obj()
        config = self._get_db_config()
        if not config.has_section('db'):
            config.add_section('db')
        config.set('db', 'version', self.db_version)
        config.write(file_obj)
        file_obj.close()

    def get_db_version(self):
        """ Read and return the database version string from the config
            file. """
        config = self._get_db_config()
        if not config.has_section('db'):
            return None
        return config.get('db', 'version')

    def commit_and_close_database(self):
        self.conn.commit()
        self.cursor.close()

    def create_assignments_table(self):
        """ Create the assignments table that stores all assignments from
            IPv4/IPv6/ASN to country code.  Blocks are stored as first hex
            of and first hex after the assignment.  Numbers are stored
            as hex strings, because SQLite's INTEGER type only holds up to
            63 unsigned bits, which is not enough to store a /64 IPv6
            block.  Hex strings have leading zeros, with IPv6 addresses
            being 33 hex characters long and IPv4 addresses and ASN being
            9 hex characters long.  The first number after an assignment
            range is stored instead of the last number in the range to
            facilitate comparisons with neighboring ranges. """
        sql = ('CREATE TABLE IF NOT EXISTS assignments(start_hex TEXT, '
               'next_start_hex TEXT, num_type TEXT, country_code TEXT, '
               'source_type TEXT, source_name TEXT)')
        self.cursor.execute(sql)
        self.conn.commit()

    def delete_assignments(self, source_type):
        """ Delete all assignments from the database cache matching a
            given source type ("rir", "lir", etc.). """
        sql = 'DELETE FROM assignments WHERE source_type = ?'
        self.cursor.execute(sql, (source_type, ))
        self.conn.commit()

    def insert_assignment(self, start_num, end_num, num_type,
            country_code, source_type, source_name):
        """ Insert an assignment into the database cache, without
            commiting after the insertion. """
        sql = ('INSERT INTO assignments (start_hex, next_start_hex, '
               'num_type, country_code, source_type, source_name) '
               'VALUES (?, ?, ?, ?, ?, ?)')
        if num_type == 'ipv6':
            start_hex = '%033x' % start_num
            next_start_hex = '%033x' % (end_num + 1)
        else:
            start_hex = '%09x' % start_num
            next_start_hex = '%09x' % (end_num + 1)
        self.cursor.execute(sql, (start_hex, next_start_hex, num_type,
                country_code, source_type, source_name))

    def commit_changes(self):
        """ Commit changes, e.g., after inserting assignments into the
            database cache. """
        self.conn.commit()

    def fetch_assignments(self, num_type, country_code):
        """ Fetch all assignments from the database cache matching the
            given number type ("asn", "ipv4", or "ipv6") and country code.
            The result is a sorted list of tuples containing (start_num,
            end_num). """
        sql = ('SELECT start_hex, next_start_hex FROM assignments '
               'WHERE num_type = ? AND country_code = ? '
               'ORDER BY start_hex')
        self.cursor.execute(sql, (num_type, country_code))
        result = []
        for row in self.cursor:
            result.append((long(row[0], 16), long(row[1], 16) - 1))
        return result

    def fetch_country_code(self, num_type, source_type, lookup_num):
        """ Fetch the country code from the database cache that is
            assigned to the given number (e.g., IPv4 address in decimal
            notation), number type (e.g., "ipv4"), and source type (e.g.,
            "rir"). """
        sql = ('SELECT country_code FROM assignments WHERE num_type = ? '
               'AND source_type = ? AND start_hex <= ? '
               'AND next_start_hex > ?')
        if num_type == 'ipv6':
            lookup_hex = '%033x' % long(lookup_num)
        else:
            lookup_hex = '%09x' % long(lookup_num)
        self.cursor.execute(sql, (num_type, source_type, lookup_hex,
                lookup_hex))
        row = self.cursor.fetchone()
        if row:
            return row[0]

    def fetch_country_blocks_in_other_sources(self, first_country_code):
        """ Fetch all assignments matching the given country code, then look
            up to which country code(s) the same number ranges are assigned in
            other source types.  Return 8-tuples containing (1) first source
            type, (2) first and (3) last number of the assignment in the first
            source type, (4) second source type, (5) first and (6) last number
            of the assignment in the second source type, (7) country code in
            the second source type, and (8) number type. """
        sql = ('SELECT first.source_type, first.start_hex, '
               'first.next_start_hex, second.source_type, '
               'second.start_hex, second.next_start_hex, '
               'second.country_code, first.num_type '
               'FROM assignments AS first '
               'JOIN assignments AS second '
               'WHERE first.country_code = ? '
               'AND first.start_hex <= second.next_start_hex '
               'AND first.next_start_hex >= second.start_hex '
               'AND first.num_type = second.num_type '
               'ORDER BY first.source_type, first.start_hex, '
               'second.source_type, second.start_hex')
        self.cursor.execute(sql, (first_country_code, ))
        result = []
        for row in self.cursor:
            result.append((str(row[0]), long(row[1], 16),
                    long(row[2], 16) - 1, str(row[3]), long(row[4], 16),
                    long(row[5], 16) - 1, str(row[6]), str(row[7])))
        return result


class DownloaderParser:
    def __init__(self, cache_dir, database_cache, user_agent,
            verbose=False):
        self.cache_dir = cache_dir
        self.database_cache = database_cache
        self.user_agent = user_agent
        self.verbose = verbose

    MAXMIND_URLS = """
        http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
        http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz
    """

    RIR_URLS = """
        ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest
        ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
        ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
        ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
        ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
    """

    LIR_URLS = """
        ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz
        ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz
    """

    COUNTRY_CODE_URL = "http://www.iso.org/iso/home/standards/country_codes/country_names_and_code_elements_txt-temp.htm"

    def download_maxmind_files(self):
        """ Download all LIR delegation urls. """
        for maxmind_url in self.MAXMIND_URLS.split():
            self._download_to_cache_dir(maxmind_url)

    def download_rir_files(self):
        """ Download all RIR delegation files including md5 checksum. """
        for rir_url in self.RIR_URLS.split():
            rir_md5_url = rir_url + '.md5'
            self._download_to_cache_dir(rir_url)
            self._download_to_cache_dir(rir_md5_url)

    def download_lir_files(self):
        """ Download all LIR delegation urls. """
        for lir_url in self.LIR_URLS.split():
            self._download_to_cache_dir(lir_url)

    def download_country_code_file(self):
        """ Download and save the latest semicolon-separated open country
            codes file. """
        self._download_to_cache_dir(self.COUNTRY_CODE_URL)

    def _download_to_cache_dir(self, url):
        """ Fetch a resource (with progress bar) and store contents to the
            local cache directory under the file name given in the URL. """
        if not os.path.exists(self.cache_dir):
            if self.verbose:
                print("Initializing the cache directory...")
            os.mkdir(self.cache_dir)
        filename = url.split('/')[-1]
        if self.verbose:
            print(url)
        req = Request(url)
        if self.user_agent:
            req.add_header('User-Agent', self.user_agent)
        # TODO Allow use of a proxy.
        #req.set_proxy(host, type)
        try:
            fetcher = urlopen(req)
        except URLError as err:
            msg = "An error occurred while attempting to cache file from:"
            print(("%s\n\t%s\n\t%s" % (msg, url, str(err))))
            return
        length_header = fetcher.headers.get("Content-Length")
        expected_bytes = -1
        if length_header:
            expected_bytes = int(length_header)
            print(("Fetching %d kilobytes" %
                    round(float(expected_bytes / 1024), 2)))
        download_started = time.time()
        output_file = open(os.path.join(self.cache_dir, filename), "wb")
        received_bytes, seconds_elapsed = 0, 0
        while True:
            seconds_elapsed = time.time() - download_started
            if expected_bytes >= 0:
                self._update_progress_bar(received_bytes, expected_bytes,
                        seconds_elapsed)
            chunk = fetcher.read(1024)
            if len(chunk) == 0:
                if expected_bytes >= 0 and received_bytes != expected_bytes:
                    print(("Expected %s bytes, only received %s" %
                            (expected_bytes, received_bytes)))
                print("")
                break
            received_bytes += len(chunk)
            output_file.write(chunk)
        output_file.close()

    def _update_progress_bar(self, received_bytes, expected_bytes,
            seconds_elapsed):
        """ Write a progress bar to the console. """
        if is_win32:
          rows    = 100 # use some WinCon function for these?
          columns = 80  # but not really important.
          EOL = "\r"
        else:
          rows, columns = list(map(int, os.popen('stty size', 'r'
              ).read().split()))
          EOL = "\x1b[G"
        if seconds_elapsed == 0:
            seconds_elapsed = 1
        percent_done = float(received_bytes) / float(expected_bytes)
        caption = "%.2f K/s" % (received_bytes / 1024 / seconds_elapsed)
        width = columns - 4 - len(caption)
        sys.stdout.write("[%s>%s] %s%s" % (
                "=" * int(percent_done * width),
                "." * (width - int(percent_done * width)), caption, EOL))
        sys.stdout.flush()

    def check_rir_file_mtimes(self):
        """ Return True if the mtime of any RIR file in our cache directory
            is > 24 hours, False otherwise. """
        if not os.path.exists(self.cache_dir):
            return False
        for rir_url in self.RIR_URLS.split():
            rir_path = os.path.join(self.cache_dir,
                    rir_url.split('/')[-1])
            if os.path.exists(rir_path):
                rir_stat = os.stat(rir_path)
                if (time.time() - rir_stat.st_mtime) > 86400:
                    return True
        return False

    def verify_rir_files(self):
        """ Compute md5 checksums of all RIR files, compare them to the
            provided .md5 files, and return True if the two checksums match,
            or False otherwise. """
        for rir_url in self.RIR_URLS.split():
            rir_path = os.path.join(self.cache_dir,
                    rir_url.split('/')[-1])
            rir_md5_path = os.path.join(self.cache_dir,
                    rir_url.split('/')[-1] + '.md5')
            if not os.path.exists(rir_md5_path) or \
                    not os.path.exists(rir_path):
                continue
            rir_md5_file = open(rir_md5_path, 'r')
            expected_checksum = rir_md5_file.read()
            rir_md5_file.close()
            if "=" in expected_checksum:
                expected_checksum = expected_checksum.split("=")[-1].strip()
            elif expected_checksum == "":
                if self.verbose:
                    print("No checksum... skipping verification...")
                continue
            else:
                regex = re.compile("[a-f0-9]{32}")
                regres = regex.findall(expected_checksum)
                if len(regres) > 1:
                    print("Error: mutiple checksum found")
                elif len(regres) < 1:
                    print("Error: no checksum found")
                else:
                    expected_checksum = regres[0]
            computed_checksum = ""
            rir_file = open(rir_path, 'rb')
            rir_data = rir_file.read()
            rir_file.close()
            computed_checksum = str(hashlib.md5(rir_data).hexdigest())
            if expected_checksum != computed_checksum:
                print(("The computed md5 checksum of %s, %s, does *not* "
                        "match the provided checksum %s!" %
                        (rir_path, computed_checksum, expected_checksum)))

    def parse_maxmind_files(self, maxmind_urls=None):
        """ Parse locally cached MaxMind files and insert assignments to the
            local database cache, overwriting any existing MaxMind
            assignments. """
        if not maxmind_urls:
            maxmind_urls = self.MAXMIND_URLS.split()
        self.database_cache.delete_assignments('maxmind')
        for maxmind_url in maxmind_urls:
            maxmind_path = os.path.join(self.cache_dir,
                    maxmind_url.split('/')[-1])
            if not os.path.exists(maxmind_path):
                print("Unable to find %s." % maxmind_path)
                continue
            if maxmind_path.endswith('.zip'):
                maxmind_zip_path = zipfile.ZipFile(maxmind_path)
                for contained_filename in maxmind_zip_path.namelist():
                    content = maxmind_zip_path.read(contained_filename)
                    self._parse_maxmind_content(content, 'maxmind',
                            'maxmind')
            elif maxmind_path.endswith('.gz'):
                content = gzip.open(maxmind_path).read()
                self._parse_maxmind_content(content, 'maxmind', 'maxmind')
        self.database_cache.commit_changes()

    def import_maxmind_file(self, maxmind_path):
        self.database_cache.delete_assignments(maxmind_path)
        if not os.path.exists(maxmind_path):
            print("Unable to find %s." % maxmind_path)
            return
        content = open(maxmind_path).read()
        self._parse_maxmind_content(content, maxmind_path, maxmind_path)
        self.database_cache.commit_changes()

    def _parse_maxmind_content(self, content, source_type, source_name):
        keys = ['start_str', 'end_str', 'start_num', 'end_num',
                 'country_code', 'country_name']
        for line in content.decode('utf-8').split('\n'):
            if len(line.strip()) == 0 or line.startswith("#"):
                continue
            line = line.replace('"', '').replace(' ', '').strip()
            parts = line.split(',')
            entry = dict((k, v) for k, v in zip(keys, parts))
            start_num = int(entry['start_num'])
            end_num = int(entry['end_num'])
            country_code = str(entry['country_code'])
            start_ipaddr = ipaddr.ip_address(entry['start_str'])
            if isinstance(start_ipaddr, ipaddr.IPv4Address):
                num_type = 'ipv4'
            else:
                num_type = 'ipv6'
            self.database_cache.insert_assignment(start_num, end_num,
                    num_type, country_code, source_type, source_name)

    def parse_rir_files(self, rir_urls=None):
        """ Parse locally cached RIR files and insert assignments to the local
            database cache, overwriting any existing RIR assignments. """
        if not rir_urls:
            rir_urls = self.RIR_URLS.split()
        self.database_cache.delete_assignments('rir')
        keys = "registry country_code type start value date status"
        for rir_url in rir_urls:
            rir_path = os.path.join(self.cache_dir,
                    rir_url.split('/')[-1])
            if not os.path.exists(rir_path):
                print("Unable to find %s." % rir_path)
                continue
            for line in open(rir_path, 'r'):
                if line.startswith("#"):
                    continue
                entry = dict((k, v) for k, v in
                        zip(keys.split(), line.strip().split("|")))
                source_name = str(entry['registry'])
                country_code = str(entry['country_code'])
                if source_name.replace(".", "", 1).isdigit() or country_code == "*":
                    continue
                num_type = entry['type']
                if num_type == 'asn':
                    start_num = end_num = int(entry['start'])
                elif num_type == 'ipv4':
                    start_num = int(ipaddr.IPv4Address(entry['start']))
                    end_num = start_num + int(entry['value']) - 1
                elif num_type == 'ipv6':
                    network_str = entry['start'] + '/' + entry['value']
                    network_ipaddr = ipaddr.IPv6Network(network_str)
                    start_num = int(network_ipaddr.network_address)
                    end_num = int(network_ipaddr.broadcast_address)
                self.database_cache.insert_assignment(start_num,
                        end_num, num_type, country_code, 'rir',
                        source_name)
        self.database_cache.commit_changes()

    def parse_lir_files(self, lir_urls=None):
        """ Parse locally cached LIR files and insert assignments to the local
            database cache, overwriting any existing LIR assignments. """
        if not lir_urls:
            lir_urls = self.LIR_URLS.split()
        self.database_cache.delete_assignments('lir')
        for lir_url in lir_urls:
            lir_path = os.path.join(self.cache_dir,
                    lir_url.split('/')[-1])
            if not os.path.exists(lir_path):
                print("Unable to find %s." % lir_path)
                continue
            if lir_path.endswith('.gz'):
                lir_file = gzip.open(lir_path)
            else:
                lir_file = open(lir_path)
            start_num = 0
            end_num = 0
            country_code = ""
            entry = False
            num_type = ""
            for line in lir_file:
                line = line.decode('utf-8', 'ignore').replace("\n", "")
                if line == "":
                    entry = False
                    start_num, end_num, country_code, num_type = 0, 0, "", ""
                elif not entry and "inetnum:" in line:
                    try:
                        line = line.replace("inetnum:", "").strip()
                        start_str = line.split("-")[0].strip()
                        end_str = line.split("-")[1].strip()
                        start_num = int(ipaddr.IPv4Address(start_str))
                        end_num = int(ipaddr.IPv4Address(end_str))
                        entry = True
                        num_type = 'ipv4'
                    except Exception as e:
                        if self.verbose:
                            print(repr(e), line)
                elif not entry and "inet6num:" in line:
                    try:
                        network_str = line.replace("inet6num:", "").strip()
                        network_ipaddr = ipaddr.IPv6Network(network_str)
                        start_num = int(network_ipaddr.network_address)
                        end_num = int(network_ipaddr.broadcast_address)
                        entry = True
                        num_type = 'ipv6'
                    except Exception as e:
                        if self.verbose:
                            print(repr(e), line)
                elif entry and "country:" in line:
                    country_code = line.replace("country:", "").strip()
                    self.database_cache.insert_assignment(start_num,
                            end_num, num_type, country_code, 'lir', 'ripencc')
        self.database_cache.commit_changes()


class Lookup:
    def __init__(self, cache_dir, database_cache, verbose=False):
        self.cache_dir = cache_dir
        self.database_cache = database_cache
        self.verbose = verbose
        self.map_co = None
        self.build_country_code_dictionary()

    def build_country_code_dictionary(self):
        """ Return a dictionary mapping country name to the country
            code. """
        country_code_path = os.path.join(self.cache_dir,
                'country_names_and_code_elements_txt-temp.htm')
        if not os.path.exists(country_code_path):
            return
        self.map_co = {}
        for line in open(country_code_path):
            if line == "" or line.startswith("Country ") or ";" not in line:
                continue
            country_name, country_code = line.strip().split(";")
            country_name = ' '.join([part.capitalize() for part in \
                    country_name.split(" ")])
            self.map_co[country_name] = country_code

    def knows_country_names(self):
        return self.map_co is not None

    def get_name_from_country_code(self, cc_code):
        if not self.knows_country_names():
            return
        country_name = [(key, value) for (key, value) in \
                list(self.map_co.items()) if value == cc_code]
        if len(country_name) > 0:
            return country_name[0][0]

    def get_country_code_from_name(self, country_name):
        """ Return the country code for a given country name. """
        if not self.knows_country_names():
            return
        cc_code = [self.map_co[key] for key in list(self.map_co.keys()) if \
                key.upper().startswith(country_name.upper())]
        if len(cc_code) > 0:
            return cc_code[0]

    def lookup_ipv6_address(self, lookup_ipaddr):
        print("Reverse lookup for: " + str(lookup_ipaddr))
        for source_type in ['maxmind', 'rir', 'lir']:
            cc = self.database_cache.fetch_country_code('ipv6',
                    source_type, int(lookup_ipaddr))
            if cc:
                print(source_type.upper(), "country code:", cc)
                cn = self.get_name_from_country_code(cc)
                if cn:
                    print(source_type.upper(), "country name:", cn)

    def lookup_ipv4_address(self, lookup_ipaddr):
        print("Reverse lookup for: " + str(lookup_ipaddr))
        maxmind_cc = self.database_cache.fetch_country_code('ipv4', 'maxmind',
                int(lookup_ipaddr))
        if maxmind_cc:
            print('MaxMind country code:', maxmind_cc)
            maxmind_cn = self.get_name_from_country_code(maxmind_cc)
            if maxmind_cn:
                print('MaxMind country name:', maxmind_cn)
        rir_cc = self.database_cache.fetch_country_code('ipv4', 'rir',
                int(lookup_ipaddr))
        if rir_cc:
            print('RIR country code:', rir_cc)
            rir_cn = self.get_name_from_country_code(rir_cc)
            if rir_cn:
                print('RIR country name:', rir_cn)
        else:
            print('Not found in RIR db')
        lir_cc = self.database_cache.fetch_country_code('ipv4', 'lir',
                int(lookup_ipaddr))
        if lir_cc:
            print('LIR country code:', lir_cc)
            lir_cn = self.get_name_from_country_code(lir_cc)
            if lir_cn:
                print('LIR country name:', lir_cn)
        if maxmind_cc and maxmind_cc != rir_cc:
            print("It appears that the RIR data conflicts with MaxMind's "
                    "data.  MaxMind's data is likely closer to being "
                    "correct due to sub-delegation issues with LIR databases.")

    def lookup_ip_address(self, lookup_str):
        """ Return the country code and name for a given ip address. """
        try:
            lookup_ipaddr = ipaddr.ip_address(lookup_str)
            if isinstance(lookup_ipaddr, ipaddr.IPv4Address):
                self.lookup_ipv4_address(lookup_ipaddr)
            elif isinstance(lookup_ipaddr, ipaddr.IPv6Address):
                self.lookup_ipv6_address(lookup_ipaddr)
            else:
                print(("Did not recognize '%s' as either IPv4 or IPv6 "
                        "address." % lookup_str))
        except ValueError as e:
            print("'%s' is not a valid IP address." % lookup_str)

    def asn_lookup(self, asn):
        asn_cc = self.database_cache.fetch_country_code('asn', 'rir', asn)
        if asn_cc:
            print("AS country code: %s" % asn_cc)
            asn_cn = self.get_name_from_country_code(asn_cc)
            if asn_cn:
                print("AS country name: %s" % asn_cn)
        else:
            print("AS%s not found!" % asn)

    def fetch_rir_blocks_by_country(self, request, country):
        result = []
        for (start_num, end_num) in \
                self.database_cache.fetch_assignments(request, country):
            if request == "ipv4" or request == "ipv6":
                start_ipaddr = ipaddr.ip_address(start_num)
                end_ipaddr = ipaddr.ip_address(end_num)
                result += [str(x) for x in
                        ipaddr.summarize_address_range(
                        start_ipaddr, end_ipaddr)]
            else:
                result.append(str(start_num))
        return result

    def lookup_countries_in_different_source(self, first_country_code):
        """ Look up all assignments matching the given country code, then
            look up to which country code(s) the same number ranges are
            assigned in other source types.  Print out the result showing
            similarities and differences. """
        print(("\nLegend:\n"
               "  '<' = found assignment range with country code '%s'\n"
               "  '>' = overlapping assignment range with same country code\n"
               "  '*' = overlapping assignment range, first conflict\n"
               "  '#' = overlapping assignment range, second conflict and "
               "beyond\n  ' ' = neighboring assignment range") % (
               first_country_code, ))
        results = self.database_cache.fetch_country_blocks_in_other_sources(
                first_country_code)
        prev_first_source_type = ''
        prev_first_start_num = -1
        cur_second_country_codes = []
        for (first_source_type, first_start_num, first_end_num,
                second_source_type, second_start_num, second_end_num,
                second_country_code, num_type) in results:
            if first_source_type != prev_first_source_type:
                print("\nAssignments in '%s':" % (first_source_type, ))
            prev_first_source_type = first_source_type
            if first_start_num != prev_first_start_num:
                cur_second_country_codes = []
                print("")
            prev_first_start_num = first_start_num
            marker = ''
            if second_end_num >= first_start_num and \
                    second_start_num <= first_end_num:
                if first_country_code != second_country_code and \
                        second_country_code not in cur_second_country_codes:
                    cur_second_country_codes.append(second_country_code)
                if first_source_type == second_source_type:
                    marker = '<'
                elif len(cur_second_country_codes) == 0:
                    marker = '>'
                elif len(cur_second_country_codes) == 1:
                    marker = '*'
                else:
                    marker = '#'
            if num_type.startswith("ip") and \
                    second_start_num == second_end_num:
                second_range = "%s" % (ipaddr.ip_address(second_start_num), )
            elif num_type.startswith("ip") and \
                    second_start_num < second_end_num:
                second_range = "%s-%s" % (ipaddr.ip_address(second_start_num),
                        ipaddr.ip_address(second_end_num))
            elif second_start_num < second_end_num:
                second_range = "AS%d-%d" % (second_start_num, second_end_num)
            else:
                second_range = "AS%d" % (second_start_num, )
            print("%1s %s %s %s" % (marker, second_country_code, second_range,
                    second_source_type, ))

def split_callback(option, opt, value, parser):
    split_value = value.split(':')
    setattr(parser.values, option.dest, split_value[0])
    if len(split_value) > 1 and split_value[1] != '':
        setattr(parser.values, 'type_filter', split_value[1])

def main():
    """ Where the magic starts. """
    usage = ("Usage: %prog [options]\n\n"
            "Example: %prog -v -t mm")
    parser = optparse.OptionParser(usage)
    parser.add_option("-v", "--verbose", action="store_true",
            dest="verbose", help = "be verbose", default=False)
    parser.add_option("-c", "--cache-dir", action="store", dest="dir",
            help="set cache directory [default: %default]",
            default=str(os.path.expanduser('~')) + "/.blockfinder/")
    parser.add_option("--user-agent", action="store", dest="ua",
            help=('provide a User-Agent which will be used when '
                  'fetching delegation files [default: "%default"]'),
            default="Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0")
    parser.add_option("-x", "--hack-the-internet", action="store_true",
            dest="hack_the_internet", help=optparse.SUPPRESS_HELP)
    group = optparse.OptionGroup(parser, "Cache modes",
            "Pick at most one of these modes to initialize or update "
            "the local cache.  May not be combined with lookup modes.")
    group.add_option("-m", "--init-maxmind", action="store_true",
            dest="init_maxmind",
            help="initialize or update MaxMind GeoIP database")
    group.add_option("-g", "--reload-maxmind", action="store_true",
            dest="reload_maxmind",
            help=("update cache from existing MaxMind GeoIP database"))
    group.add_option("-r", "--import-maxmind", action="store",
            dest="import_maxmind", metavar="FILE",
            help=("import the specified MaxMind GeoIP database file into "
                  "the database cache using its file name as source "
                  "name"))
    group.add_option("-i", "--init-rir",
            action="store_true", dest="init_del",
            help="initialize or update delegation information")
    group.add_option("-d", "--reload-rir", action="store_true",
            dest="reload_del",
            help="use existing delegation files to update the database")
    group.add_option("-l", "--init-lir", action="store_true",
            dest="init_lir",
            help=("initialize or update lir information; can take up to "
                  "5 minutes"))
    group.add_option("-z", "--reload-lir", action="store_true",
            dest="reload_lir",
            help=("use existing lir files to update the database; can "
                  "take up to 5 minutes"))
    group.add_option("-o", "--download-cc", action="store_true",
            dest="download_cc", help="download country codes file")
    group.add_option("-e", "--erase-cache", action="store_true",
            dest="erase_cache", help="erase the local database cache")
    parser.add_option_group(group)
    group = optparse.OptionGroup(parser, "Lookup modes",
            "Pick at most one of these modes to look up data in the "
            "local cache.  May not be combined with cache modes.")
    group.add_option("-4", "--ipv4", action="store", dest="ipv4",
            help=("look up country code and name for the specified IPv4 "
                  "address"))
    group.add_option("-6", "--ipv6", action="store", dest="ipv6",
            help=("look up country code and name for the specified IPv6 "
                  "address"))
    group.add_option("-a", "--asn", action="store", dest="asn",
            help="look up country code and name for the specified ASN")
    group.add_option("-t", "--code", action="callback", dest="cc",
            callback=split_callback, metavar="CC[:type]", type="str",
            help=("look up all allocations (or only those for number "
                  "type 'ipv4', 'ipv6', or 'asn' if provided) in the "
                  "delegation cache for the specified two-letter country "
                  "code"))
    group.add_option("-n", "--name", action="callback", dest="cn",
            callback=split_callback, metavar="CN[:type]", type="str",
            help=("look up all allocations (or only those for number "
                  "type 'ipv4', 'ipv6', or 'asn' if provided) in the "
                  "delegation cache for the specified full country name"))
    group.add_option("-p", "--compare", action="store", dest="compare",
            metavar="CC",
            help=("compare assignments to the specified country code "
                  "with overlapping assignments in other data sources; "
                  "can take some time and produce some long output"))
    group.add_option("-w", "--what-country", action="store", dest="what_cc",
            help=("look up country name for specified country code"))
    parser.add_option_group(group)
    group = optparse.OptionGroup(parser, "Network modes")
    (options, args) = parser.parse_args()
    if options.hack_the_internet:
        print("all your bases are belong to us!")
        sys.exit(0)
    options_dict = vars(options)
    modes = 0
    for mode in ["init_maxmind", "reload_maxmind", "import_maxmind",
                 "init_del", "init_lir", "reload_del", "reload_lir",
                 "download_cc", "erase_cache", "ipv4", "ipv6", "asn",
                 "cc", "cn", "compare", "what_cc"]:
        if mode in options_dict and options_dict.get(mode):
            modes += 1
    if modes > 1:
        parser.error("only 1 cache or lookup mode allowed")
    elif modes == 0:
        parser.error("must provide 1 cache or lookup mode")
    database_cache = DatabaseCache(options.dir, options.verbose)
    if options.erase_cache:
        database_cache.erase_database()
        sys.exit(0)
    if not database_cache.connect_to_database():
        print("Could not connect to database.")
        print("You may need to erase it using -e and then reload it "
                "using -d/-z.  Exiting.")
        sys.exit(1)
    database_cache.set_db_version()
    downloader_parser = DownloaderParser(options.dir, database_cache,
            options.ua)
    lookup = Lookup(options.dir, database_cache)
    if options.ipv4 or options.ipv6 or options.asn or options.cc \
            or options.cn or options.compare:
        if downloader_parser.check_rir_file_mtimes():
            print("Your cached RIR files are older than 24 hours; you "
                    "probably want to update them.")
    if options.asn:
        lookup.asn_lookup(options.asn)
    elif options.ipv4:
        lookup.lookup_ip_address(options.ipv4)
    elif options.ipv6:
        lookup.lookup_ip_address(options.ipv6)
    elif options.cc or options.cn or options.what_cc:
        country = None
        if options.cc:
            country = options.cc.upper()
        elif not lookup.knows_country_names():
            print("Need to download country codes first before looking "
                  "up countries by name.")
        elif options.what_cc:
            country = options.what_cc.upper()
            country_name = lookup.get_name_from_country_code(country)
            if country_name:
                print(("Hmm...%s? That would be %s."
                      % (options.what_cc, country_name)))
                sys.exit(0)
            else:
                print(("Hmm, %s? We're not sure either. Are you sure that's "
                      "a country code?" % options.what_cc))
                sys.exit(1)
        else:
            country = lookup.get_country_code_from_name(options.cn)
            if not country:
                print("It appears your search did not match a country.")
        if country:
            types = ["ipv4", "ipv6", "asn"]
            if hasattr(options, 'type_filter') and options.type_filter.lower() in types:
                types = [options.type_filter.lower()]
            for request in types:
                print("\n".join(lookup.fetch_rir_blocks_by_country(\
                        request, country)))
    elif options.compare:
        print("Comparing assignments with overlapping assignments in other "
               "data sources...")
        lookup.lookup_countries_in_different_source(options.compare)
    elif options.init_maxmind or options.reload_maxmind:
        if options.init_maxmind:
            print("Downloading Maxmind GeoIP files...")
            downloader_parser.download_maxmind_files()
        print("Importing Maxmind GeoIP files...")
        downloader_parser.parse_maxmind_files()
    elif options.import_maxmind:
        print("Importing Maxmind GeoIP files...")
        downloader_parser.import_maxmind_file(options.import_maxmind)
    elif options.init_del or options.reload_del:
        if options.init_del:
            print("Downloading RIR files...")
            downloader_parser.download_rir_files()
            print("Verifying RIR files...")
            downloader_parser.verify_rir_files()
        print("Importing RIR files...")
        downloader_parser.parse_rir_files()
    elif options.init_lir or options.reload_lir:
        if options.init_lir:
            print("Downloading LIR delegation files...")
            downloader_parser.download_lir_files()
        print("Importing LIR files...")
        downloader_parser.parse_lir_files()
    elif options.download_cc:
        print("Downloading country code file...")
        downloader_parser.download_country_code_file()
    database_cache.commit_and_close_database()

if __name__ == "__main__":
    main()