blockfinder

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # For the people of Smubworld!
   5 import urllib2
   6 import os
   7 import time
   8 import getopt
   9 import sys
  10 from math import *
  11 import sqlite3
  12 import hashlib
  13 __program__ = 'blockfinder'
  14 __url__ = 'http://github.com/ioerror/blockfinder/'
  15 __author__ = 'Jacob Appelbaum <jacob@appelbaum.net>'
  16 __copyright__ = 'Copyright (c) 2009'
  17 __license__ = 'See LICENSE for licensing information'
  18 __version__ = '3.141'
  19
  20 try:
  21     from future import antigravity
  22 except ImportError:
  23      antigravity = None
  24
  25 def update_progress_bar(percent_done, caption=""):
  26     """Write a progress bar to the console"""
  27     rows, columns = map(int, os.popen('stty size', 'r').read().split())
  28     width = columns - 4 - len(caption)
  29     sys.stdout.write("[%s>%s] %s\x1b[G" % (
  30                                     "=" * int(percent_done*width),
  31                                     "." * (width - int(percent_done * width)),
  32                                     caption) )
  33     sys.stdout.flush()
  34
  35 # XXX TODO:allow the use of a proxy
  36 # Set up a proper Request object, set the user agent and if desired, a proxy
  37 def fetch(url,useragent):
  38     """ Fetch (with progress meter) and return the contents of a url. """
  39     req =urllib2.Request(url)
  40     req.add_header('User-agent', useragent)
  41     #req.set_proxy(host, type)
  42     fetcher = urllib2.urlopen(req)
  43     length = int(fetcher.headers.get("content-length"))
  44     if not length:
  45         raise Exception("Missing content-length header in reply from server.")
  46     print "Fetching ", str (round(float(length/1024),2)) , " kilobytes"
  47     ret = ""
  48     t_start = time.time()
  49     while True:
  50         t_delta = time.time() - t_start
  51         update_progress_bar(
  52             float(len(ret)) / length,
  53             "%.2f K/s" % (len(ret) / 1024 / t_delta) )
  54         tmp = fetcher.read(1024)
  55         if len(tmp) == 0:
  56             if len(ret) != length:
  57                 raise Exception("Expected %s bytes, only received %s" % (
  58                                 len(ret), length ))
  59             print ""
  60             return ret
  61         ret += tmp
  62
  63 def cache_delegation(cache_dir, delegation_url, useragent):
  64     """ Attempt to cache the contents of a delegation url in our cache dir. """
  65     try:
  66         os.stat(cache_dir)
  67     except OSError, e:
  68         if e.errno == 2:
  69             if verbose:
  70                 print "Initializing the cache directory..."
  71             os.mkdir(cache_dir)
  72         else:
  73              raise e
  74     delegation = ""
  75     print "Fetching " + delegation_url
  76     delegation = fetch(delegation_url,useragent)
  77     tmp = delegation_url.split('/')
  78     delegation_file = str(cache_dir) + str(tmp[-1])
  79     try:
  80         f = open(delegation_file, 'w')
  81         f.write(delegation)
  82         f.close()
  83         return True
  84     except Exception, e:
  85         print repr(e)
  86         return False
  87
  88 def cache_is_dated(cache_dir, cached_files):
  89     """ Returns True if the mtime of any files in cache dir is > 24hrs."""
  90     try:
  91         os.stat(cache_dir)
  92     except OSError, e:
  93         print "\nDid you initialize the cache directory?\n"
  94         raise e
  95     for file in cached_files:
  96         fstat = os.stat(cache_dir + file)
  97         if (time.time() - fstat.st_mtime) > 86400:
  98             return True
  99     return False
 100
 101 def create_sql_database(cache_dir):
 102     """creates a sqlite database and if there is already an existing one it deletes it.
 103     ftp://ftp.arin.net/pub/stats/arin/README"""
 104     try:
 105         os.remove(cache_dir +"sqlitedb")
 106     except:
 107         pass
 108     conn = sqlite3.connect(cache_dir +"sqlitedb")
 109     cursor = conn.cursor()
 110     cursor.execute("""create table asn(registry text, cc text, start text, value INTEGER, date text, status text)""")
 111     cursor.execute("""create table ipv4(registry text, cc text, start text, value INTEGER, date text, status text)""")
 112     cursor.execute("""create table ipv6(registry text, cc text, start text, value INTEGER, date text, status text)""")
 113     conn.commit()
 114     cursor.close()
 115
 116 def insert_into_sql_database(delegations,cache_dir):
 117     """ inserts delegations into the sqlite database"""
 118     conn = sqlite3.connect(cache_dir +"sqlitedb")
 119     cursor = conn.cursor()
 120     table =""
 121     for delegation in delegations:
 122         for entry in delegation:
 123             registry = str(entry['registry'])
 124             if registry.isdigit() ==False and str (entry['cc']) !="*":
 125                 if  entry['type'] == "ipv6":
 126                     table = "ipv6"
 127                 if entry['type'] == "ipv4":
 128                     table = "ipv4"
 129                 if entry['type'] == "asn":
 130                     table = "asn"
 131                 text = """INSERT INTO """ + table + """ ( registry, cc, start, value, date,status) VALUES (?,?,?,?,?,?)"""
 132                 data = [entry['registry'], entry['cc'], entry['start'], entry['value'], entry['date'], entry['status'] ]
 133                 cursor.execute(text, data  )
 134     conn.commit()
 135     cursor.close()
 136
 137 def get_total_delegations_from_db(cache_dir):
 138     """returns count of the n.o. of entries in the ipv4 +ipv6 + asn tables"""
 139     conn = sqlite3.connect(cache_dir +"sqlitedb")
 140     cursor = conn.cursor()
 141     count =0
 142     table_names =["ipv4", "ipv6", "asn"]
 143     for table in table_names:
 144         cursor.execute("""select count (*)  from """ + table)
 145         count = count + int (cursor.fetchone()[0] )
 146     cursor.close()
 147     return count
 148
 149 def get_possible_match_entries(cc,cache_dir):
 150     """ get the count of 'possible' matching delegation entries"""
 151     conn = sqlite3.connect(cache_dir +"sqlitedb")
 152     cursor = conn.cursor()
 153     count =0
 154     table_names =["ipv4", "ipv6", "asn"]
 155     for table in table_names:
 156         cursor.execute("""select count (*)  from """ + table + """ where cc=?""",cc)
 157         count = count + int (cursor.fetchone()[0] )
 158     cursor.close()
 159     return count
 160
 161 def use_sql_database(request, cc, cache_dir):
 162     """now with added magic!"""
 163     conn = sqlite3.connect(cache_dir +"sqlitedb")
 164     cursor = conn.cursor()
 165     if verbose:
 166         print "We have %d entries in our delegation cache." %get_total_delegations_from_db(cache_dir)
 167     text ="""select start,value from """ + request + """ where cc=?"""
 168     cc = (cc,)
 169     cursor.execute(text,cc)
 170     for row in cursor:
 171         if request == "ipv4":
 172             print str(row[0]) + "/" + str(calculate_ipv4_subnet(int(row[1])))
 173         elif request == "ipv6":
 174             print str(row[0]) + "/" + str(int(row[1]))
 175         else:
 176             print str(int(row[0]) )
 177     if verbose:
 178          print "We found %d possible entries in our delegation cache." % get_possible_match_entries(cc, cache_dir)
 179          cursor.execute("""select count(*) from """ + request + """ where cc=?""", cc )
 180          print "We found %d matching entries in our delegation cache." % int (cursor.fetchone()[0] )
 181     cursor.close()
 182
 183 def get_md5_from_delegation_md5_file(cache_dir, delegation_file):
 184     """ Returns the md5sum from the delegation md5 file (if it exists)"""
 185     checksum =""
 186     try:
 187         f = open(cache_dir + delegation_file +".md5", "r")
 188         checksum = f.read()
 189         f.close()
 190         if delegation_file == "delegated-afrinic-latest":
 191             pos =  checksum.find(" ")
 192             assert pos <len(checksum)
 193             checksum = str ( checksum[:pos] )
 194         else:
 195             pos =  checksum.find("=") +2
 196             assert pos <len(checksum)
 197             checksum = str ( checksum[pos:-1] )
 198     except Exception, e:
 199             print repr(e)
 200     return checksum
 201
 202 def verify_delegation_file(cache_dir, delegation_file):
 203     """compares the delegation file md5sum to that of the provided md5sum
 204        returns True if they match otherwise returns False"""
 205     checksum = ""
 206     checksum_of_file=""
 207     try:
 208         f = open(cache_dir + delegation_file, "rb")
 209         checksum_of_file = str (hashlib.md5(f.read()).hexdigest() )
 210         f.close()
 211     except Exception, e:
 212         print repr(e)
 213     checksum = get_md5_from_delegation_md5_file(cache_dir,delegation_file)
 214     if checksum != checksum_of_file:
 215         return False
 216     if checksum == checksum_of_file and checksum!="":
 217         return True
 218
 219 def verify_cache(cache_dir, delegation_files):
 220     """ if in verbose mode prints the result of checking the checksum of the
 221         delegation files """
 222     for file in delegation_files:
 223         if verbose:
 224             print "verifying " + file
 225         if verify_delegation_file(cache_dir,file):
 226             if verbose:
 227                 print "the md5 checksum of " + file + " *matches* the provided checksum"
 228         else:
 229             if verbose:
 230                 print "the md5 checksum of " + file + " does *not* match the provided checksum"
 231
 232 def update_delegation_cache(cache_dir, delegation_urls, useragent):
 233     """ Fetch multiple delegation urls and cache the contents. """
 234     print "Updating delegation cache..."
 235     for url in delegation_urls.split():
 236         cache_delegation(cache_dir, url+".md5",useragent)
 237         if verify_delegation_file(cache_dir, url.rpartition('/')[-1]):
 238             pass
 239         else:
 240             cache_delegation(cache_dir, url,useragent)
 241     return True
 242
 243 def load_delegation(delegation_file):
 244     """ Load, parse and store the delegation file contents as a list. """
 245     keys = "registry cc type start value date status"
 246     try:
 247         f = open(delegation_file, "r")
 248         delegations = [ dict((k,v) for k,v in zip(keys.split(), line.split("|")))
 249                         for line in f.readlines() if not line.startswith("#")]
 250         f.close()
 251         return delegations
 252     except OSError, e:
 253         print repr(e)
 254
 255 def load_all_delegations(cache_dir, delegation_urls):
 256     """ Load all delegations into memory. """
 257     delegations = []
 258     for url in delegation_urls.split():
 259         filename = url.rpartition('/')[-1]
 260         if verbose:
 261             print "Attempting to load delegation file into memory: " + filename
 262         delegations.append(load_delegation(cache_dir + filename))
 263     return delegations
 264
 265 def calculate_ipv4_subnet(host_count):
 266     return 32 - int(floor(log(host_count,2)))
 267
 268 def usage():
 269     """ Print usage information. """
 270     print >> sys.stderr, """
 271 blockfinder  [-c DIR] -i
 272 blockfinder [options] -t COUNTRY
 273
 274 The first form initializes the local cache. The second form queries it.
 275
 276 Understood options (not all of which are implemented yet):
 277     -h, --help          Show this help and exit
 278     -v                  Be verbose
 279     -c, --cachedir DIR  Set the cache directory
 280     -u, --useragent
 281     -p, --progress
 282     -o, --output FILE
 283     -4, --ipv4          Search IPv4 allocations
 284     -6, --ipv6          Search IPv6 allocation
 285     -a, --asn           Search ASN allocations
 286     -t, --nationstate CC  Set the country to search (given as a two-letter code)
 287
 288 At least one of -t or -i is required, and when in -t mode, at least one of -4,
 289 -6, and -a is required in order to do anything sensible.
 290 """
 291
 292 def main():
 293     """ Where the magic starts. """
 294     try:
 295         opts, args = getopt.getopt(sys.argv[1:],
 296                                    "vhc:u:pso:46at:i",
 297                                    ["verbose", "help", "cachedir=", "useragent=", "progress",
 298                                     "silent", "output=", "ipv4", "ipv6", "asn", "country=",
 299                                     "initialize-delegation"])
 300     except getopt.GetoptError, err:
 301         print str(err)
 302         usage()
 303         sys.exit(2)
 304
 305     global verbose
 306     verbose = False
 307     output = None
 308     silent = True
 309     cache_dir = str(os.path.expanduser('~')) + "/.blockfinder/"
 310     update_delegations = False
 311     delegation_urls="""
 312         ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest
 313         ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
 314         ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
 315         ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
 316         ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
 317     """
 318     delegation_files=[]
 319     for url in delegation_urls.split():
 320         filename = url.rpartition('/')
 321         delegation_files.append(filename[-1])
 322     update_delegations = False
 323     requests = []
 324     country = ""
 325     useragent = "Mozilla/5.0"
 326     for o, a in opts:
 327         if o == "-v":
 328             verbose = True
 329         elif o in ("-h", "--help"):
 330             usage()
 331             sys.exit()
 332         elif o in ("-c", "--cachedir"):
 333             cache_dir = a
 334         elif o in ("-u", "--useragent"):
 335             useragent=a
 336         elif o in ("-p", "--progress"):
 337             progress = True
 338         elif o in ("-s", "--silent"):
 339             silent = True
 340         elif o in ("-o", "--output"):
 341             output = a
 342         elif o in ("-4", "--ipv4"):
 343             requests.append("ipv4")
 344         elif o in ("-6", "--ipv6"):
 345             requests.append("ipv6")
 346         elif o in ("-a", "--asn"):
 347             requests.append("asn")
 348         # XXX TODO: This should be a positional argument as it's the only manditory one...
 349         elif o in ("-t", "--nation-state"):
 350             country = a.upper()
 351         elif o in ("-i", "--initialize-delegations"):
 352             update_delegations = True
 353         else:
 354             assert False, "Unhandled option; Sorry!"
 355
 356     # Update and quit.
 357     if update_delegations:
 358         update_delegation_cache(cache_dir,delegation_urls,useragent)
 359         if verbose:
 360             verify_cache(cache_dir, delegation_files)
 361         delegations = load_all_delegations(cache_dir, delegation_urls)
 362         create_sql_database(cache_dir)
 363         insert_into_sql_database(delegations, cache_dir)
 364         sys.exit(0)
 365     if not requests:
 366         print "Nothing to do. Have you requested anything?"
 367         print "Example usage: blockfinder -v --ipv4 -t mm"
 368         sys.exit(1)
 369     # Check our cache age and warn if it's aged
 370     total_delegations = 0
 371     if cache_is_dated(cache_dir, delegation_files) and verbose:
 372         print "Your delegation cache is older than 24 hours; you probably want to update it."
 373     for request in requests:
 374         use_sql_database(request, country, cache_dir)
 375
 376 if __name__ == "__main__":
 377     main()