2 # -*- coding: utf-8 -*-
4 # For the people of Smubworld!
10 from math
import ceil
, log
14 from xml
.dom
import minidom
16 __program__
= 'blockfinder'
17 __url__
= 'http://github.com/ioerror/blockfinder/'
18 ___author__
= 'Jacob Appelbaum <jacob@appelbaum.net>, dave b. <db@d1b.org>'
19 __copyright__
= 'Copyright (c) 2010'
20 __license__
= 'See LICENSE for licensing information'
21 __version__
= '3.1415'
29 from future
import antigravity
33 class BlockFinderError(Exception):
36 def update_progress_bar(percent_done
, caption
=""):
37 """Write a progress bar to the console"""
38 rows
, columns
= map(int, os
.popen('stty size', 'r').read().split())
39 width
= columns
- 4 - len(caption
)
40 sys
.stdout
.write("[%s>%s] %s\x1b[G" % (
41 "=" * int(percent_done
*width
),
42 "." * (width
- int(percent_done
* width
)),
46 # XXX TODO:allow the use of a proxy
47 # Set up a proper Request object, set the user agent and if desired, a proxy
48 def fetch(url
, useragent
):
49 """ Fetch (with progress meter) and return the contents of a url. """
50 req
= urllib2
.Request(url
)
51 req
.add_header('User-agent', useragent
)
52 #req.set_proxy(host, type)
53 fetcher
= urllib2
.urlopen(req
)
54 length_header
= fetcher
.headers
.get("content-length")
55 if length_header
== None:
56 raise Exception("Missing content-length header in reply from server.")
57 length
= int(length_header
)
58 print "Fetching ", str (round(float(length
/1024),2)) , " kilobytes"
62 t_delta
= time
.time() - t_start
66 float(len(ret
)) / length
,
67 "%.2f K/s" % (len(ret
) / 1024 / t_delta
) )
68 tmp
= fetcher
.read(1024)
70 if len(ret
) != length
:
71 raise Exception("Expected %s bytes, only received %s" % (
77 def create_blockfinder_cache_dir(cache_dir
):
78 if not os
.path
.exists(cache_dir
):
80 print "Initializing the cache directory..."
83 def write_to_a_text_file(file_loc
, data
):
84 f
= open(file_loc
, 'w')
88 def cache_delegation(cache_dir
, delegation_url
, useragent
):
89 """ Attempt to cache the contents of a delegation url in our cache dir. """
91 print "Fetching " + delegation_url
92 delegation
= fetch(delegation_url
,useragent
)
93 tmp
= delegation_url
.split('/')
94 delegation_file
= str(cache_dir
) + str(tmp
[-1])
96 write_to_a_text_file(delegation_file
, delegation
)
102 def cache_is_dated(cache_dir
, cached_files
):
103 """ Returns True if the mtime of any files in cache dir is > 24hrs."""
107 print "\nDid you initialize the cache directory?\n"
109 for file in cached_files
:
110 fstat
= os
.stat(cache_dir
+ file)
111 if (time
.time() - fstat
.st_mtime
) > 86400:
115 def create_sql_database(cache_dir
):
116 """ Creates a new sqlite database.
117 If there is a previous sqlite database it will be deleted. """
118 conn
= sqlite3
.connect(cache_dir
+ "sqlitedb")
119 cursor
= conn
.cursor()
120 for table
in ["ipv4", "ipv6", "asn"]:
121 cursor
.execute("""drop table if exists """ + table
)
122 cursor
.execute("""create table asn(registry text, cc text, start text, value INTEGER, date text, status text)""")
123 cursor
.execute("""create table ipv4(registry text, cc text, start text, value INTEGER, date text, status text)""")
124 cursor
.execute("""create table ipv6(registry text, cc text, start text, value INTEGER, date text, status text)""")
125 cursor
.execute("""create table if not exists lir_record(cc text, start text, value INTEGER, type INTEGER)""")
129 def insert_into_sql_database(delegations
,cache_dir
):
130 """ inserts delegation information into the sqlite database"""
131 conn
= sqlite3
.connect(cache_dir
+"sqlitedb")
132 cursor
= conn
.cursor()
134 for delegation
in delegations
:
135 for entry
in delegation
:
136 registry
= str(entry
['registry'])
137 if not registry
.isdigit() and str (entry
['cc']) !="*":
138 if entry
['type'] == "ipv6":
140 if entry
['type'] == "ipv4":
142 if entry
['type'] == "asn":
144 text
= """INSERT INTO """ + table
+ """ ( registry, cc, start, value, date,status) VALUES (?,?,?,?,?,?)"""
145 data
= [entry
['registry'], entry
['cc'], entry
['start'], entry
['value'], entry
['date'], entry
['status'] ]
146 cursor
.execute(text
, data
)
150 def get_total_delegations_from_db(cache_dir
):
151 """ Returns the total count of the number of entries in the ipv4, ipv6 and asn table """
152 conn
= sqlite3
.connect(cache_dir
+"sqlitedb")
153 cursor
= conn
.cursor()
155 table_names
= ["ipv4", "ipv6", "asn"]
156 for table
in table_names
:
157 cursor
.execute("""select count (*) from """ + table
)
158 count
+= int (cursor
.fetchone()[0] )
162 def get_possible_match_entries(cc
,cache_dir
):
163 """ Get the count of 'possible' matching delegation entries"""
164 conn
= sqlite3
.connect(cache_dir
+"sqlitedb")
165 cursor
= conn
.cursor()
167 table_names
=["ipv4", "ipv6", "asn"]
168 for table
in table_names
:
169 cursor
.execute("""select count (*) from """ + table
+ """ where cc=?""",cc
)
170 count
+= int (cursor
.fetchone()[0] )
174 def use_sql_database(request
, cc
, cache_dir
):
175 """ Use the sqlite database that is created after fetching delegations
176 to output information for a given request """
177 conn
= sqlite3
.connect(cache_dir
+ "sqlitedb")
178 cursor
= conn
.cursor()
180 print "We have %d entries in our delegation cache." %get
_total
_delegations
_from
_db
(cache_dir
)
181 text
="""select start,value from """ + request
+ """ where cc=?"""
183 cursor
.execute(text
,cc
)
186 if request
== "ipv4":
187 result
.append(str(row
[0]) + "/" + str(calculate_ipv4_subnet(int(row
[1]))))
188 elif request
== "ipv6":
189 result
.append(str(row
[0]) + "/" + str(int(row
[1])))
191 result
.append(str(int(row
[0])))
194 result
.append("We found %d possible entries in our delegation cache." % get_possible_match_entries(cc
, cache_dir
) )
195 cursor
.execute("""select count(*) from """ + request
+ """ where cc=?""", cc
)
196 result
.append("We found %d matching entries in our delegation cache." % int (cursor
.fetchone()[0] ) )
200 def get_md5_from_delegation_md5_file(cache_dir
, delegation_file
):
201 """ Returns the md5sum from the delegation md5 file
202 if it doesn't exist it returns an empty string"""
205 f
= open(cache_dir
+ delegation_file
+ ".md5", "r")
208 if delegation_file
== "delegated-afrinic-latest":
209 pos
= checksum
.find(" ")
210 checksum
= str (checksum
[:pos
])
212 pos
= checksum
.find("=") +2
213 checksum
= str (checksum
[pos
:-1])
218 def verify_delegation_file(cache_dir
, delegation_file
):
219 """ compares the delegation file md5sum to that of the provided md5sum
220 returns True if they match otherwise returns False """
222 checksum_of_file
= ""
224 f
= open(cache_dir
+ delegation_file
, "rb")
225 checksum_of_file
= str (hashlib
.md5(f
.read()).hexdigest() )
229 checksum
= get_md5_from_delegation_md5_file(cache_dir
,delegation_file
)
230 if checksum
!= checksum_of_file
:
232 if checksum
== checksum_of_file
and checksum
!= "":
236 def verify_cache(cache_dir
, delegation_files
):
237 """ if in verbose mode prints the result of checking the checksum of the
239 for file in delegation_files
:
241 print "verifying " + file
242 if verify_delegation_file(cache_dir
,file):
244 print "the md5 checksum of " + file + " *matches* the provided checksum"
247 print "the md5 checksum of " + file + " does *not* match the provided checksum"
249 def update_delegation_cache(cache_dir
, delegation_urls
, useragent
):
250 """ Fetch multiple delegation urls and cache the contents. """
251 print "Updating delegation cache..."
253 for url
in delegation_urls
.split():
254 cache_delegation(cache_dir
, url
+ ".md5",useragent
)
255 if verify_delegation_file(cache_dir
, url
.rpartition('/')[-1]):
258 cache_delegation(cache_dir
, url
,useragent
)
260 def update_lir_delegation_cache(cache_dir
, delegation_urls
, useragent
):
261 """ Fetch multiple LIR delegation urls and cache the contents. """
262 print "Updating LIR delegation cache..."
263 for url
in delegation_urls
.split():
264 cache_delegation(cache_dir
, url
,useragent
)
265 unpack_a_delegation_cache(cache_dir
, delegation_urls
, "LIR")
268 def extract_data_from_gzip_file(gzip_file_loc
, extract_file_loc
):
269 gzip_file
= gzip
.open(gzip_file_loc
, 'rb')
270 gunzipped_data
= gzip_file
.read()
272 gunzipped_file
= open(extract_file_loc
, 'w')
273 gunzipped_file
.writelines(gunzipped_data
)
274 gunzipped_file
.close()
276 def unpack_a_delegation_cache(cache_dir
, delegation_urls
, del_type
=""):
277 """ Unpack the fetched LIR delegation files into the blockfinder cache. """
278 # This probably should unlink the gzip'ed file if we care about space...
279 for url
in delegation_urls
.split():
280 gzip_filename
= url
.rpartition('/')[-1]
281 gunziped_filename
= gzip_filename
.rpartition('.')[0]
283 print "Unpacking " + del_type
+ "file " + gzip_filename
+ " into our cache as " + gunziped_filename
284 extract_data_from_gzip_file(cache_dir
+ gzip_filename
, cache_dir
+ gunziped_filename
)
286 def update_geoip_cache(cache_dir
, geoip_urls
, useragent
):
287 """ Fetch country level resolution GeoIP files from a given url and cache
288 the contents. Unpack it if it's compressed. """
289 print "Updating GeoIP cache..."
290 for url
in geoip_urls
.split():
291 cache_delegation(cache_dir
, url
, useragent
)
292 unpack_a_delegation_cache(cache_dir
, geoip_urls
, "GeoIP")
294 def load_delegation(delegation_file
):
295 """ Load, parse and store the delegation file contents as a list. """
296 keys
= "registry cc type start value date status"
298 f
= open(delegation_file
, "r")
299 delegations
= [ dict((k
,v
) for k
,v
in zip(keys
.split(), line
.split("|")))
300 for line
in f
.readlines() if not line
.startswith("#")]
306 def load_all_delegations(cache_dir
, delegation_urls
):
307 """ Load all delegations into memory. """
309 for url
in delegation_urls
.split():
310 filename
= url
.rpartition('/')[-1]
312 print "Attempting to load delegation file into memory: " + filename
313 delegations
.append(load_delegation(cache_dir
+ filename
))
316 def calculate_ipv4_subnet(host_count
):
317 """ XXX: this will return incorrect values for anything above a /4
318 e.g. calculate_ipv4_subnet(536870912) will return 2 instead of 3
320 return 32 - int(ceil(log(host_count
,2)))
322 def download_country_code_file(cache_dir
, useragent
):
323 """ Download and save the latest opencountrycode XML file """
324 # Google frontend will not return content-length for some reason...
325 url
= "http://opencountrycodes.appspot.com/xml"
326 xml
= urllib2
.urlopen(url
).read()
327 write_to_a_text_file(cache_dir
+ "countrycodes.xml", xml
)
329 def build_country_code_dictionary(cache_dir
):
330 """ Return a dictionary mapping country name to the country code"""
332 xml_file
= str(cache_dir
) + "countrycodes.xml"
333 clist
= minidom
.parse(xml_file
)
334 for country
in clist
.getElementsByTagName("country"):
335 code
= country
.attributes
["code"]
336 name
= country
.attributes
["name"]
337 map_co
[name
.value
] = code
.value
340 def get_name_from_country_code(cache_dir
, cc_code
):
341 map_co
= build_country_code_dictionary(cache_dir
)
342 country_name
= [(key
, value
) for (key
, value
) in map_co
.items() if value
== cc_code
]
343 if len(country_name
) > 0:
344 return country_name
[0][0]
346 def get_country_code_from_name(cache_dir
, country_name
):
347 """ Return the country code for a given country name. """
348 map_co
= build_country_code_dictionary(cache_dir
)
349 cc_code
= [map_co
[key
] for key
in map_co
.keys() if key
.upper().startswith(country_name
.upper())]
353 def ip_address_to_dec(ip_addr
):
354 ipar
= ip_addr
.split('.')
357 a
[i
] = hex(int(ipar
[i
]))[2:]
358 if(int(ipar
[i
]) <= 15):
359 a
[i
] = """0""" + a
[i
]
361 total
= '0x'+a
[0]+a
[1]+a
[2]+a
[3]
362 decimal
= int(total
,16)
365 def ipv4_address_valid(ip_addr
):
366 ipv4arr
= ip_addr
.split('.')
367 if len(ipv4arr
) == 4:
368 for items
in ipv4arr
:
375 def geoip_lookup(cache_dir
, ip_addr
):
376 # This would work with the CVS version of the GeoIP code
377 # However, MaxMind hasn't done a release in a long time.
378 # http://geoip.cvs.sourceforge.net/viewvc/geoip/python/test_v6.py?revision=1.1&view=markup
379 # gi = GeoIP.open(cache_dir + "GeoIPv6.dat",GeoIP.GEOIP_STANDARD)
380 # cc = gi.country_code_by_addr_v6(ip_addr)
381 # cc_name = gi.country_name_by_addr_v6(ip_addr)
382 gi
= GeoIP
.open(cache_dir
+ "GeoIP.dat",GeoIP
.GEOIP_STANDARD
)
383 cc
= gi
.country_code_by_addr(ip_addr
)
384 cc_name
= gi
.country_name_by_addr(ip_addr
)
388 def rir_or_lir_lookup(ip_addr
,lookup_type
, cache_dir
):
389 ipv4arr
= ip_addr
.split('.')
391 conn
= sqlite3
.connect(cache_dir
+ "sqlitedb")
392 cursor
= conn
.cursor()
393 if lookup_type
== 'rir':
394 cursor
.execute('select cc, start, value from ipv4 WHERE start LIKE ?', ( ipv4arr
[0] + "." + ipv4arr
[1] + ".%",))
396 cursor
.execute('select * from lir_record WHERE start LIKE ? and type=4', (ipv4arr
[0] + "." + ipv4arr
[1] + ".%",))
397 row
= cursor
.fetchone()
400 if lookup_type
== "rir":
401 cursor
.execute('select cc, start, value from ipv4 WHERE start LIKE ? ', (ipv4arr
[0] + ".%",))
403 cursor
.execute('select * from lir_record WHERE start LIKE ? and type=4', (ipv4arr
[0] + ".%",))
404 row
= cursor
.fetchone()
406 while(row
is not None):
407 if (ip_address_to_dec(row
[1]) <= ip_address_to_dec(ip_addr
) <= (ip_address_to_dec(row
[1]) + row
[2])):
408 result
.append(row
[0])
409 result
.append(get_name_from_country_code(cache_dir
, row
[0]))
412 row
= cursor
.fetchone()
415 def rir_lookup(ip_addr
, cache_dir
):
416 return rir_or_lir_lookup(ip_addr
, "rir", cache_dir
)
418 def lir_lookup(ip_addr
, cache_dir
):
419 return rir_or_lir_lookup(ip_addr
, "lir", cache_dir
)
421 def lookup_ip_address(ip_addr
, cache_dir
):
422 """ Return the country code and name for a given ip address. Attempts to
423 use GeoIP if available."""
425 ip_addr
= socket
.getaddrinfo(ip_addr
, 80)[0][4][0]
426 if not ipv4_address_valid(ip_addr
):
427 raise BlockFinderError('Invalid ip address!')
429 print "Reverse lookup for: " + ip_addr
431 geoip_cc
, geoip_cc_name
= geoip_lookup(cache_dir
, ip_addr
)
432 print "GeoIP country code: " + str(geoip_cc
)
433 print "GeoIP country name: " + str(geoip_cc_name
)
435 rir
= rir_lookup(ip_addr
, cache_dir
)
439 print 'RIR country code:', rir
[0]
440 print 'RIR country:', rir
[1]
442 print 'Not found in RIR db'
444 lir
= lir_lookup(ip_addr
, cache_dir
)
446 print 'LIR country code:', lir
[0]
447 print 'LIR country :', lir
[1]
450 if geoip_cc
!= rir_cc
:
451 print "It appears that the RIR data conflicts with the GeoIP data"
452 print "The GeoIP data is likely closer to being correct due to " \
453 "sub-delegation issues with LIR databases"
455 def return_first_ip_and_number_in_inetnum(line
):
456 start_ip
= line
.split("-")[0].strip()
457 end_ip
= line
.split("-")[1].strip()
458 num_ips
= 1 + (ip_address_to_dec(end_ip
) - ip_address_to_dec(start_ip
) )
459 return start_ip
, num_ips
461 def create_or_replace_lir_table_in_db(cache_dir
):
462 conn
= sqlite3
.connect(cache_dir
+ "sqlitedb")
463 cursor
= conn
.cursor()
464 cursor
.execute("""drop table if exists lir_record """)
465 cursor
.execute("""create table if not exists lir_record(cc text, start text, value INTEGER, type INTEGER)""")
469 def extract_info_from_lir_file_and_insert_into_sqlite(cache_dir
, filename
):
473 conn
= sqlite3
.connect(cache_dir
+ "sqlitedb")
474 cursor
= conn
.cursor()
475 insert_text
= """insert into lir_record (cc, start, value, type) VALUES (?,?,?,?)"""
477 for line
in open(cache_dir
+ filename
, "r"):
478 line
= line
.replace("\n", "")
481 country
, block
, version
= "", [], ""
482 elif not entry
and "inetnum:" in line
:
484 line
= line
.replace("inetnum:", "").strip()
485 start_ip
, num_ips
= return_first_ip_and_number_in_inetnum(line
)
486 block
= [start_ip
, num_ips
]
491 elif not entry
and "inet6num:" in line
:
493 block
= line
.replace("inet6num:", "").strip().split("/")
498 elif entry
and "country:" in line
:
499 country
= line
.replace("country:", "").strip()
500 data
= (country
, block
[0], block
[1], version
)
501 cursor
.execute(insert_text
, data
)
505 def create_db_and_insert_delegation_into_db(cache_dir
, delegation_urls
):
506 create_sql_database(cache_dir
)
507 delegations
= load_all_delegations(cache_dir
, delegation_urls
)
508 insert_into_sql_database(delegations
, cache_dir
)
512 """ Print usage information. """
513 print >> sys
.stderr
, """
514 blockfinder [-c DIR] -i
515 blockfinder [options] -t COUNTRY
517 The first form initializes the local cache. The second form queries it.
519 Understood options (not all of which are implemented yet):
520 -h, --help Show this help and exit
522 -c, --cachedir DIR Set the cache directory
526 -4, --ipv4 Search IPv4 allocations
527 -6, --ipv6 Search IPv6 allocation
528 -a, --asn Search ASN allocations
529 -t, --nation-state CC Set the country to search (given as a two-letter code)
530 -n, --country-name "Costa Rica" Set country to search (full name)
531 -x, --hack-the-internet Hack the internet
532 -r, --reverse-lookup Return the county name for the specified IP
533 -i, --initialize-delegation Initialize or update delegation information
534 -l, --initialize-lir Initialize or update lir information
535 -d, --reload-delegation Use existing delegation files to update the database
537 At least one of -t or -i is required, and when in -t mode, at least one of -4,
538 -6, and -a is required in order to do anything sensible.
542 """ Where the magic starts. """
544 opts
, args
= getopt
.getopt(sys
.argv
[1:], "dlxvhc:u:pso:46at:n:ir:",
545 ["reload-delegation", "initialize-lir", "hack-the-internet",
546 "verbose", "help", "cachedir=", "useragent=", "progress",
547 "silent", "output=", "ipv4", "ipv6","asn", "nation-state=",
548 "country-name", "initialize-delegation","reverse-lookup"])
549 except getopt
.GetoptError
, err
:
558 cache_dir
= str(os
.path
.expanduser('~')) + "/.blockfinder/"
559 update_delegations
= False
560 delegation_urls
= """
561 ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest
562 ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
563 ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
564 ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
565 ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
567 geoip_country_urls
= """http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
568 http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz"""
570 lir_urls
= """ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz
571 ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz"""
572 delegation_files
= []
573 for url
in delegation_urls
.split():
574 filename
= url
.rpartition('/')
575 delegation_files
.append(filename
[-1])
576 update_delegations
= False
578 reload_delegation_db
= False
581 useragent
= "Mozilla/5.0"
584 create_blockfinder_cache_dir(cache_dir
)
585 if not os
.path
.exists(cache_dir
+ "countrycodes.xml"):
587 download_country_code_file(cache_dir
,useragent
)
592 if o
in ("-x", "--hack-the-internet"):
593 print "all your bases are belong to us!"
597 elif o
in ("-h", "--help"):
600 elif o
in ("-c", "--cachedir"):
602 elif o
in ("-u", "--useragent"):
604 elif o
in ("-p", "--progress"):
606 elif o
in ("-s", "--silent"):
608 elif o
in ("-o", "--output"):
610 elif o
in ("-4", "--ipv4"):
611 requests
.append("ipv4")
612 elif o
in ("-6", "--ipv6"):
613 requests
.append("ipv6")
614 elif o
in ("-a", "--asn"):
615 requests
.append("asn")
616 # XXX TODO: This should be a positional argument as it's the only manditory one...
617 elif o
in ("-r", "--reverse-lookup"):
619 requests
.append("reverse")
620 elif o
in ("-t", "--nation-state"):
622 elif o
in ("-n", "--country-name"):
623 country
= get_country_code_from_name(cache_dir
, a
)
624 elif o
in ("-i", "--initialize-delegations"):
625 update_delegations
= True
626 elif o
in ("-l", "--initialize-lir"):
628 elif o
in ("-d", "--reload-delegation"):
629 reload_delegation_db
= True
631 print "Unhandled option; Sorry!"
634 if reload_delegation_db
:
635 create_db_and_insert_delegation_into_db(cache_dir
, delegation_urls
)
639 if update_delegations
:
641 update_geoip_cache(cache_dir
,geoip_country_urls
,useragent
)
642 update_delegation_cache(cache_dir
,delegation_urls
,useragent
)
644 verify_cache(cache_dir
, delegation_files
)
645 create_db_and_insert_delegation_into_db(cache_dir
, delegation_urls
)
649 update_lir_delegation_cache(cache_dir
,lir_urls
,useragent
)
650 print "Extracting and inserting information from the lir files can take up to 5 minutes"
651 create_or_replace_lir_table_in_db(cache_dir
)
652 for file in "ripe.db.inetnum ripe.db.inet6num".split():
653 extract_info_from_lir_file_and_insert_into_sqlite(cache_dir
, file)
657 print "Nothing to do. Have you requested anything?"
658 print "Example usage: blockfinder -v --ipv4 -t mm"
662 lookup_ip_address(ipaddress
,cache_dir
)
665 print "It appears your search did not match a country."
667 # Check our cache age and warn if it's aged
668 if cache_is_dated(cache_dir
, delegation_files
) and verbose
:
669 print "Your delegation cache is older than 24 hours; you probably want to update it."
671 print "Using country code: %s" % country
673 for request
in requests
:
675 print " \n".join(use_sql_database(request
, country
, cache_dir
))
679 if __name__
== "__main__":