2 # -*- coding: utf-8 -*-
4 # For the people of Smubworld!
17 if sys
.version_info
[0] >= 3:
19 import ipaddress
as ipaddr
20 from urllib
.request
import (urlopen
, Request
)
21 from urllib
.error
import URLError
24 import ConfigParser
as configparser
25 from urllib2
import (urlopen
, Request
, URLError
)
27 from embedded_ipaddr
import ipaddr
28 ipaddr
.ip_address
= ipaddr
.IPAddress
32 is_win32
= (sys
.platform
== "win32")
34 __program__
= 'blockfinder'
35 __url__
= 'https://github.com/ioerror/blockfinder/'
36 __author__
= 'Jacob Appelbaum <jacob@appelbaum.net>, David <db@d1b.org>'
37 __copyright__
= 'Copyright (c) 2010'
38 __license__
= 'See LICENSE for licensing information'
39 __version__
= '3.14159'
42 from future
import antigravity
48 def __init__(self
, cache_dir
, verbose
=False):
49 self
.cache_dir
= cache_dir
50 self
.verbose
= verbose
53 self
.db_version
= "0.0.4"
54 self
.db_path
= os
.path
.join(self
.cache_dir
+ "sqlitedb")
56 def erase_database(self
):
57 """ Erase the database file. """
58 if os
.path
.exists(self
.db_path
):
59 os
.remove(self
.db_path
)
61 def connect_to_database(self
):
62 """ Connect to the database cache, possibly after creating it if
63 it doesn't exist yet, or after making sure an existing
64 database cache has the correct version. Return True if a
65 connection could be established, False otherwise. """
66 if not os
.path
.exists(self
.cache_dir
):
68 print("Initializing the cache directory...")
69 os
.mkdir(self
.cache_dir
)
70 if os
.path
.exists(self
.db_path
):
71 cache_version
= self
.get_db_version()
73 cache_version
= "0.0.1"
74 if cache_version
!= self
.db_version
:
75 print(("The existing database cache uses version %s, "
76 "not the expected %s." % (cache_version
,
79 self
.conn
= sqlite3
.connect(self
.db_path
)
80 self
.cursor
= self
.conn
.cursor()
81 self
.create_assignments_table()
82 self
.create_asn_description_table()
83 self
.create_asn_assignments_table()
86 def __get_default_config_file_obj(self
):
88 file_path
= os
.path
.join(self
.cache_dir
, 'db.cfg')
89 if not os
.path
.exists(file_path
):
91 return open(file_path
, open_flags
)
93 def _get_db_config(self
, file_obj
=None):
94 """ Return the database configuration object from the provided
95 file_obj if provided, otherwise from the default database
96 configuration file. """
98 file_obj
= self
.__get
_default
_config
_file
_obj
()
99 config
= configparser
.SafeConfigParser()
100 config
.readfp(file_obj
)
104 def set_db_version(self
, file_obj
=None):
105 """ Set the database version string in the config file. """
107 file_obj
= self
.__get
_default
_config
_file
_obj
()
108 config
= self
._get
_db
_config
()
109 if not config
.has_section('db'):
110 config
.add_section('db')
111 config
.set('db', 'version', self
.db_version
)
112 config
.write(file_obj
)
115 def get_db_version(self
):
116 """ Read and return the database version string from the config
118 config
= self
._get
_db
_config
()
119 if not config
.has_section('db'):
121 return config
.get('db', 'version')
123 def commit_and_close_database(self
):
127 def create_assignments_table(self
):
128 """ Create the assignments table that stores all assignments from
129 IPv4/IPv6/ASN to country code. Blocks are stored as first hex
130 of and first hex after the assignment. Numbers are stored
131 as hex strings, because SQLite's INTEGER type only holds up to
132 63 unsigned bits, which is not enough to store a /64 IPv6
133 block. Hex strings have leading zeros, with IPv6 addresses
134 being 33 hex characters long and IPv4 addresses and ASN being
135 9 hex characters long. The first number after an assignment
136 range is stored instead of the last number in the range to
137 facilitate comparisons with neighboring ranges. """
138 sql
= ('CREATE TABLE IF NOT EXISTS assignments(start_hex TEXT, '
139 'next_start_hex TEXT, num_type TEXT, country_code TEXT, '
140 'source_type TEXT, source_name TEXT)')
141 self
.cursor
.execute(sql
)
144 def create_asn_description_table(self
):
145 """ Create the assignments table that stores all the descriptions
146 associated with ASNs. """
147 sql
= ('CREATE TABLE IF NOT EXISTS asn_descriptions(as_num INT, '
148 'source_name TEXT, description TEXT)')
149 self
.cursor
.execute(sql
)
150 sql
= ('CREATE INDEX IF NOT EXISTS DescriptionsByASN ON '
151 'asn_descriptions ( as_num )')
152 self
.cursor
.execute(sql
)
155 def create_asn_assignments_table(self
):
156 """ Create the assignments table that stores the assignments from
158 #XXX: IPv6 not yet supported. (Not available from routeviews?)
159 sql
= ('CREATE TABLE IF NOT EXISTS asn_assignments(start_hex TEXT, '
160 'next_start_hex TEXT, num_type TEXT, as_num INT, '
161 'source_type TEXT, source_name TEXT, PRIMARY KEY(start_hex, '
163 self
.cursor
.execute(sql
)
164 sql
= ('CREATE INDEX IF NOT EXISTS ASNEntriesByStartHex on '
165 'asn_assignments ( start_hex )')
166 self
.cursor
.execute(sql
)
169 def delete_assignments(self
, source_type
):
170 """ Delete all assignments from the database cache matching a
171 given source type ("rir", "lir", etc.). """
172 sql
= 'DELETE FROM assignments WHERE source_type = ?'
173 self
.cursor
.execute(sql
, (source_type
, ))
176 def delete_asn_descriptions(self
):
177 """ Delete all asn descriptions from the database cache. """
178 sql
= 'DELETE FROM asn_descriptions'
179 self
.cursor
.execute(sql
)
182 def delete_asn_assignments(self
):
183 """ Delete all the bgp netblock to as entries """
184 sql
= 'DELETE FROM asn_assignments'
185 self
.cursor
.execute(sql
)
188 def insert_assignment(self
, start_num
, end_num
, num_type
,
189 country_code
, source_type
, source_name
):
190 """ Insert an assignment into the database cache, without
191 commiting after the insertion. """
192 sql
= ('INSERT INTO assignments (start_hex, next_start_hex, '
193 'num_type, country_code, source_type, source_name) '
194 'VALUES (?, ?, ?, ?, ?, ?)')
195 if num_type
== 'ipv6':
196 start_hex
= '%033x' % start_num
197 next_start_hex
= '%033x' % (end_num
+ 1)
199 start_hex
= '%09x' % start_num
200 next_start_hex
= '%09x' % (end_num
+ 1)
201 self
.cursor
.execute(sql
, (start_hex
, next_start_hex
, num_type
,
202 country_code
, source_type
, source_name
))
204 def insert_asn_description(self
, asn
, source_name
, description
):
205 sql
= ('INSERT INTO asn_descriptions (as_num, source_name, description) '
207 self
.cursor
.execute(sql
, (asn
, source_name
, unicode(description
)))
209 def insert_asn_assignment(self
, start_num
, end_num
, num_type
, asn
,
210 source_type
, source_name
):
211 #XXX: This is sqlite specific syntax
212 sql
= ('INSERT OR IGNORE INTO asn_assignments (start_hex, '
213 'next_start_hex, num_type, as_num, source_type, source_name) '
214 'VALUES (?, ?, ?, ?, ?, ?)')
215 if num_type
== 'ipv6':
216 start_hex
= '%033x' % start_num
217 next_start_hex
= '%033x' % (end_num
+ 1)
219 start_hex
= '%09x' % start_num
220 next_start_hex
= '%09x' % (end_num
+ 1)
221 self
.cursor
.execute(sql
, (start_hex
, next_start_hex
, num_type
, asn
,
222 source_type
, source_name
))
224 def commit_changes(self
):
225 """ Commit changes, e.g., after inserting assignments into the
229 def fetch_assignments(self
, num_type
, country_code
):
230 """ Fetch all assignments from the database cache matching the
231 given number type ("asn", "ipv4", or "ipv6") and country code.
232 The result is a sorted list of tuples containing (start_num,
234 sql
= ('SELECT start_hex, next_start_hex FROM assignments '
235 'WHERE num_type = ? AND country_code = ? '
236 'ORDER BY start_hex')
237 self
.cursor
.execute(sql
, (num_type
, country_code
))
239 for row
in self
.cursor
:
240 result
.append((long(row
[0], 16), long(row
[1], 16) - 1))
243 def fetch_country_code(self
, num_type
, source_type
, lookup_num
):
244 """ Fetch the country code from the database cache that is
245 assigned to the given number (e.g., IPv4 address in decimal
246 notation), number type (e.g., "ipv4"), and source type (e.g.,
248 sql
= ('SELECT country_code FROM assignments WHERE num_type = ? '
249 'AND source_type = ? AND start_hex <= ? '
250 'AND next_start_hex > ?')
251 if num_type
== 'ipv6':
252 lookup_hex
= '%033x' % long(lookup_num
)
254 lookup_hex
= '%09x' % long(lookup_num
)
255 self
.cursor
.execute(sql
, (num_type
, source_type
, lookup_hex
,
257 row
= self
.cursor
.fetchone()
261 def fetch_country_blocks_in_other_sources(self
, first_country_code
):
262 """ Fetch all assignments matching the given country code, then look
263 up to which country code(s) the same number ranges are assigned in
264 other source types. Return 8-tuples containing (1) first source
265 type, (2) first and (3) last number of the assignment in the first
266 source type, (4) second source type, (5) first and (6) last number
267 of the assignment in the second source type, (7) country code in
268 the second source type, and (8) number type. """
269 sql
= ('SELECT first.source_type, first.start_hex, '
270 'first.next_start_hex, second.source_type, '
271 'second.start_hex, second.next_start_hex, '
272 'second.country_code, first.num_type '
273 'FROM assignments AS first '
274 'JOIN assignments AS second '
275 'WHERE first.country_code = ? '
276 'AND first.start_hex <= second.next_start_hex '
277 'AND first.next_start_hex >= second.start_hex '
278 'AND first.num_type = second.num_type '
279 'ORDER BY first.source_type, first.start_hex, '
280 'second.source_type, second.start_hex')
281 self
.cursor
.execute(sql
, (first_country_code
, ))
283 for row
in self
.cursor
:
284 result
.append((str(row
[0]), long(row
[1], 16),
285 long(row
[2], 16) - 1, str(row
[3]), long(row
[4], 16),
286 long(row
[5], 16) - 1, str(row
[6]), str(row
[7])))
289 def fetch_org_by_ip_address(self
, lookup_str
, num_type
):
290 if num_type
== 'ipv4':
291 lookup_hex
= '%09x' % long(int(lookup_str
))
293 lookup_hex
= '%033x' % long(int(lookup_str
))
294 sql
= ('SELECT asn_descriptions.as_num, asn_descriptions.description, '
295 'asn_assignments.start_hex, asn_assignments.next_start_hex '
296 'FROM asn_descriptions JOIN asn_assignments ON '
297 'asn_assignments.as_num = asn_descriptions.as_num '
298 'WHERE num_type = ? AND start_hex <= ? AND next_start_hex > ?')
299 self
.cursor
.execute(sql
, (num_type
, lookup_hex
, lookup_hex
))
300 row
= self
.cursor
.fetchall()
304 def fetch_org_by_ip_range(self
, lookup_start
, lookup_end
, num_type
):
305 if num_type
== 'ipv4':
306 lookup_start_hex
= '%09x' % long(int(lookup_start
))
307 lookup_end_hex
= '%09x' % long(int(lookup_end
))
309 lookup_start_hex
= '%033x' % long(int(lookup_start
))
310 lookup_end_hex
= '%033x' % long(int(lookup_end
))
312 sql
= ('SELECT asn_descriptions.as_num, asn_descriptions.description, '
313 'asn_assignments.start_hex, asn_assignments.next_start_hex '
314 'FROM asn_descriptions JOIN asn_assignments ON '
315 'asn_assignments.as_num = asn_descriptions.as_num '
316 'WHERE num_type = ? AND start_hex >= ? AND next_start_hex <= ?')
317 self
.cursor
.execute(sql
, (num_type
, lookup_start_hex
, lookup_end_hex
))
318 row
= self
.cursor
.fetchall()
322 def _concatenate_and_write(self
, records
, write_function
=None, record_filter
=None, bits
=32):
326 start_hex
, next_start_hex
, record
= \
327 long(row
[0], 16), long(row
[1], 16), str(row
[2])
328 nb
= bits
- int(log(next_start_hex
- start_hex
, 2))
329 net
= ipaddr
.IPNetwork("%s/%d" %
330 (ipaddr
.IPAddress(start_hex
),nb
))
331 if callable(record_filter
):
332 record
= record_filter(record
)
336 # Concatenate adjacent blocks of the same country
337 if netblocks
and netblocks
[-1][1] == record
:
338 pn
= netblocks
[-1][0]
339 nb
= bits
- int(log(int(net
.network
) + \
340 int(net
.numhosts
) - int(pn
.network
), 2))
341 netblocks
[-1] = (ipaddr
.IPNetwork("%s/%d" % \
342 (pn
.network
, nb
)), record
)
344 # if the adjacent blocks aren't the same country,
345 # write the last block out to csv and add the new block
346 # to the list for possible concatenation
348 prev_n
,prev_record
= netblocks
.pop()
350 write_function(prev_n
,prev_record
)
351 netblocks
.append((net
,record
))
353 # this is the base case
355 netblocks
.append((net
,record
))
357 def export_asn(self
, filename
, num_type
):
358 """ Export assignments to the CSV format used to build the geoip-database asn lookup """
359 sql
= ('SELECT start_hex, next_start_hex, as_num '
360 'FROM asn_assignments WHERE num_type = ? ORDER BY start_hex')
361 self
.cursor
.execute(sql
, (num_type
,))
363 f
= open(filename
, 'w')
365 print("Unable to open %s" % filename
)
368 def write_csv_line(network
, asn
):
370 f
.write(""""%s","%s","%d","%d","%s"\n""" % (network
.network
,
371 network
.broadcast
, int(network
.network
),
372 int(network
.broadcast
), asn
))
373 if num_type
== 'ipv6': ip_bits
= 128
374 elif num_type
== 'ipv4': ip_bits
= 32
377 self
._concatenate
_and
_write
(self
.cursor
, write_function
=write_csv_line
,
381 def export_geoip(self
, lookup
, filename
, num_type
):
382 """ Export assignments to the CSV format used to build the
383 geoip-database package """
385 sql
= ('SELECT start_hex, next_start_hex, country_code '
386 'FROM assignments WHERE num_type = ? ORDER BY start_hex')
387 self
.cursor
.execute(sql
, (num_type
,))
390 f
= open(filename
, 'w')
392 print("Unable to open %s" % filename
)
395 def write_csv_line(network
, country_code
):
396 country_name
= lookup
.get_name_from_country_code(country_code
)
398 country_name
= country_name
.split("#")[0].strip() #Drop comments
399 f
.write(""""%s","%s","%d","%d","%s","%s"\n""" % (network
.network
,
400 network
.broadcast
, int(network
.network
),
401 int(network
.broadcast
), country_code
, country_name
))
403 if num_type
== 'ipv6': ip_bits
= 128
404 elif num_type
== 'ipv4': ip_bits
= 32
407 self
._concatenate
_and
_write
(self
.cursor
, write_function
=write_csv_line
,
408 record_filter
=str.upper
, bits
=ip_bits
)
411 class DownloaderParser
:
412 def __init__(self
, cache_dir
, database_cache
, user_agent
,
414 self
.cache_dir
= cache_dir
415 self
.database_cache
= database_cache
416 self
.user_agent
= user_agent
417 self
.verbose
= verbose
420 http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
421 http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz
425 ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest
426 ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
427 ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
428 ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
429 ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest
433 ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz
434 ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz
437 COUNTRY_CODE_URL
= "http://www.iso.org/iso/home/standards/country_codes/country_names_and_code_elements_txt-temp.htm"
439 ASN_DESCRIPTION_URL
= "http://www.cidr-report.org/as2.0/autnums.html"
441 ASN_ASSIGNMENT_URLS
= """
442 http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2
445 def download_maxmind_files(self
):
446 """ Download all LIR delegation urls. """
447 for maxmind_url
in self
.MAXMIND_URLS
.split():
448 self
._download
_to
_cache
_dir
(maxmind_url
)
450 def download_rir_files(self
):
451 """ Download all RIR delegation files including md5 checksum. """
452 for rir_url
in self
.RIR_URLS
.split():
453 rir_md5_url
= rir_url
+ '.md5'
454 self
._download
_to
_cache
_dir
(rir_url
)
455 self
._download
_to
_cache
_dir
(rir_md5_url
)
457 def download_lir_files(self
):
458 """ Download all LIR delegation urls. """
459 for lir_url
in self
.LIR_URLS
.split():
460 self
._download
_to
_cache
_dir
(lir_url
)
462 def download_country_code_file(self
):
463 """ Download and save the latest semicolon-separated open country
465 self
._download
_to
_cache
_dir
(self
.COUNTRY_CODE_URL
)
467 def download_asn_description_file(self
):
468 """ Download and save the latest ASN to Name report from
470 self
._download
_to
_cache
_dir
(self
.ASN_DESCRIPTION_URL
)
472 def download_asn_assignment_files(self
):
473 """ Download and save the latest routing snapshots. """
474 for assignment_url
in self
.ASN_ASSIGNMENT_URLS
.split():
475 self
._download
_to
_cache
_dir
(assignment_url
)
477 def _download_to_cache_dir(self
, url
):
478 """ Fetch a resource (with progress bar) and store contents to the
479 local cache directory under the file name given in the URL. """
480 if not os
.path
.exists(self
.cache_dir
):
482 print("Initializing the cache directory...")
483 os
.mkdir(self
.cache_dir
)
484 filename
= url
.split('/')[-1]
489 req
.add_header('User-Agent', self
.user_agent
)
490 # TODO Allow use of a proxy.
491 #req.set_proxy(host, type)
493 fetcher
= urlopen(req
)
494 except URLError
as err
:
495 msg
= "An error occurred while attempting to cache file from:"
496 print(("%s\n\t%s\n\t%s" % (msg
, url
, str(err
))))
498 length_header
= fetcher
.headers
.get("Content-Length")
501 expected_bytes
= int(length_header
)
502 print(("Fetching %d kilobytes" %
503 round(float(expected_bytes
/ 1024), 2)))
504 download_started
= time
.time()
505 output_file
= open(os
.path
.join(self
.cache_dir
, filename
), "wb")
506 received_bytes
, seconds_elapsed
= 0, 0
508 seconds_elapsed
= time
.time() - download_started
509 if expected_bytes
>= 0:
510 self
._update
_progress
_bar
(received_bytes
, expected_bytes
,
512 chunk
= fetcher
.read(1024)
514 if expected_bytes
>= 0 and received_bytes
!= expected_bytes
:
515 print(("Expected %s bytes, only received %s" %
516 (expected_bytes
, received_bytes
)))
519 received_bytes
+= len(chunk
)
520 output_file
.write(chunk
)
523 def _update_progress_bar(self
, received_bytes
, expected_bytes
,
525 """ Write a progress bar to the console. """
527 rows
= 100 # use some WinCon function for these?
528 columns
= 80 # but not really important.
531 rows
, columns
= list(map(int, os
.popen('stty size', 'r'
534 if seconds_elapsed
== 0:
536 percent_done
= float(received_bytes
) / float(expected_bytes
)
537 caption
= "%.2f K/s" % (received_bytes
/ 1024 / seconds_elapsed
)
538 width
= columns
- 4 - len(caption
)
539 sys
.stdout
.write("[%s>%s] %s%s" % (
540 "=" * int(percent_done
* width
),
541 "." * (width
- int(percent_done
* width
)), caption
, EOL
))
544 def check_rir_file_mtimes(self
):
545 """ Return True if the mtime of any RIR file in our cache directory
546 is > 24 hours, False otherwise. """
547 if not os
.path
.exists(self
.cache_dir
):
549 for rir_url
in self
.RIR_URLS
.split():
550 rir_path
= os
.path
.join(self
.cache_dir
,
551 rir_url
.split('/')[-1])
552 if os
.path
.exists(rir_path
):
553 rir_stat
= os
.stat(rir_path
)
554 if (time
.time() - rir_stat
.st_mtime
) > 86400:
558 def verify_rir_files(self
):
559 """ Compute md5 checksums of all RIR files, compare them to the
560 provided .md5 files, and return True if the two checksums match,
561 or False otherwise. """
562 for rir_url
in self
.RIR_URLS
.split():
563 rir_path
= os
.path
.join(self
.cache_dir
,
564 rir_url
.split('/')[-1])
565 rir_md5_path
= os
.path
.join(self
.cache_dir
,
566 rir_url
.split('/')[-1] + '.md5')
567 if not os
.path
.exists(rir_md5_path
) or \
568 not os
.path
.exists(rir_path
):
570 rir_md5_file
= open(rir_md5_path
, 'r')
571 expected_checksum
= rir_md5_file
.read()
573 if "=" in expected_checksum
:
574 expected_checksum
= expected_checksum
.split("=")[-1].strip()
575 elif expected_checksum
== "":
577 print("No checksum... skipping verification...")
580 regex
= re
.compile("[a-f0-9]{32}")
581 regres
= regex
.findall(expected_checksum
)
583 print("Error: mutiple checksum found")
584 elif len(regres
) < 1:
585 print("Error: no checksum found")
587 expected_checksum
= regres
[0]
588 computed_checksum
= ""
589 rir_file
= open(rir_path
, 'rb')
590 rir_data
= rir_file
.read()
592 computed_checksum
= str(hashlib
.md5(rir_data
).hexdigest())
593 if expected_checksum
!= computed_checksum
:
594 print(("The computed md5 checksum of %s, %s, does *not* "
595 "match the provided checksum %s!" %
596 (rir_path
, computed_checksum
, expected_checksum
)))
598 def parse_maxmind_files(self
, maxmind_urls
=None):
599 """ Parse locally cached MaxMind files and insert assignments to the
600 local database cache, overwriting any existing MaxMind
603 maxmind_urls
= self
.MAXMIND_URLS
.split()
604 self
.database_cache
.delete_assignments('maxmind')
605 for maxmind_url
in maxmind_urls
:
606 maxmind_path
= os
.path
.join(self
.cache_dir
,
607 maxmind_url
.split('/')[-1])
608 if not os
.path
.exists(maxmind_path
):
609 print("Unable to find %s." % maxmind_path
)
611 if maxmind_path
.endswith('.zip'):
612 maxmind_zip_path
= zipfile
.ZipFile(maxmind_path
)
613 for contained_filename
in maxmind_zip_path
.namelist():
614 content
= maxmind_zip_path
.read(contained_filename
)
615 self
._parse
_maxmind
_content
(content
, 'maxmind',
617 elif maxmind_path
.endswith('.gz'):
618 content
= gzip
.open(maxmind_path
).read()
619 self
._parse
_maxmind
_content
(content
, 'maxmind', 'maxmind')
620 self
.database_cache
.commit_changes()
622 def import_maxmind_file(self
, maxmind_path
):
623 self
.database_cache
.delete_assignments(maxmind_path
)
624 if not os
.path
.exists(maxmind_path
):
625 print("Unable to find %s." % maxmind_path
)
627 content
= open(maxmind_path
).read()
628 self
._parse
_maxmind
_content
(content
, maxmind_path
, maxmind_path
)
629 self
.database_cache
.commit_changes()
631 def _parse_maxmind_content(self
, content
, source_type
, source_name
):
632 keys
= ['start_str', 'end_str', 'start_num', 'end_num',
633 'country_code', 'country_name']
634 for line
in content
.decode('utf-8').split('\n'):
635 if len(line
.strip()) == 0 or line
.startswith("#"):
637 line
= line
.replace('"', '').replace(' ', '').strip()
638 parts
= line
.split(',')
639 entry
= dict((k
, v
) for k
, v
in zip(keys
, parts
))
640 start_num
= int(entry
['start_num'])
641 end_num
= int(entry
['end_num'])
642 country_code
= str(entry
['country_code'])
643 start_ipaddr
= ipaddr
.ip_address(entry
['start_str'])
644 if isinstance(start_ipaddr
, ipaddr
.IPv4Address
):
648 self
.database_cache
.insert_assignment(start_num
, end_num
,
649 num_type
, country_code
, source_type
, source_name
)
651 def parse_rir_files(self
, rir_urls
=None):
652 """ Parse locally cached RIR files and insert assignments to the local
653 database cache, overwriting any existing RIR assignments. """
655 rir_urls
= self
.RIR_URLS
.split()
656 self
.database_cache
.delete_assignments('rir')
657 keys
= "registry country_code type start value date status"
658 for rir_url
in rir_urls
:
659 rir_path
= os
.path
.join(self
.cache_dir
,
660 rir_url
.split('/')[-1])
661 if not os
.path
.exists(rir_path
):
662 print("Unable to find %s." % rir_path
)
664 for line
in open(rir_path
, 'r'):
665 if line
.startswith("#"):
667 entry
= dict((k
, v
) for k
, v
in
668 zip(keys
.split(), line
.strip().split("|")))
669 source_name
= str(entry
['registry'])
670 country_code
= str(entry
['country_code'])
671 if source_name
.replace(".", "", 1).isdigit() or country_code
== "*":
673 num_type
= entry
['type']
674 if num_type
== 'asn':
675 start_num
= end_num
= int(entry
['start'])
676 elif num_type
== 'ipv4':
677 start_num
= int(ipaddr
.IPv4Address(entry
['start']))
678 end_num
= start_num
+ int(entry
['value']) - 1
679 elif num_type
== 'ipv6':
680 network_str
= entry
['start'] + '/' + entry
['value']
681 network_ipaddr
= ipaddr
.IPv6Network(network_str
)
682 start_num
= int(network_ipaddr
.network_address
)
683 end_num
= int(network_ipaddr
.broadcast_address
)
684 self
.database_cache
.insert_assignment(start_num
,
685 end_num
, num_type
, country_code
, 'rir',
687 self
.database_cache
.commit_changes()
689 def parse_lir_files(self
, lir_urls
=None):
690 """ Parse locally cached LIR files and insert assignments to the local
691 database cache, overwriting any existing LIR assignments. """
693 lir_urls
= self
.LIR_URLS
.split()
694 self
.database_cache
.delete_assignments('lir')
695 for lir_url
in lir_urls
:
696 lir_path
= os
.path
.join(self
.cache_dir
,
697 lir_url
.split('/')[-1])
698 if not os
.path
.exists(lir_path
):
699 print("Unable to find %s." % lir_path
)
701 if lir_path
.endswith('.gz'):
702 lir_file
= gzip
.open(lir_path
)
704 lir_file
= open(lir_path
)
710 for line
in lir_file
:
711 line
= line
.decode('utf-8', 'ignore').replace("\n", "")
714 start_num
, end_num
, country_code
, num_type
= 0, 0, "", ""
715 elif not entry
and "inetnum:" in line
:
717 line
= line
.replace("inetnum:", "").strip()
718 start_str
= line
.split("-")[0].strip()
719 end_str
= line
.split("-")[1].strip()
720 start_num
= int(ipaddr
.IPv4Address(start_str
))
721 end_num
= int(ipaddr
.IPv4Address(end_str
))
724 except Exception as e
:
727 elif not entry
and "inet6num:" in line
:
729 network_str
= line
.replace("inet6num:", "").strip()
730 network_ipaddr
= ipaddr
.IPv6Network(network_str
)
731 start_num
= int(network_ipaddr
.network_address
)
732 end_num
= int(network_ipaddr
.broadcast_address
)
735 except Exception as e
:
738 elif entry
and "country:" in line
:
739 country_code
= line
.replace("country:", "").strip()
740 self
.database_cache
.insert_assignment(start_num
,
741 end_num
, num_type
, country_code
, 'lir', 'ripencc')
742 self
.database_cache
.commit_changes()
744 def parse_asn_description_file(self
, asn_description_url
=None):
745 """ Parse locally cached ASN to Description mappings and insert
746 mappings to the local database cache, overwriting any existing ASN
747 to Name assignments. """
748 if not asn_description_url
:
749 asn_description_url
= self
.ASN_DESCRIPTION_URL
750 self
.database_cache
.delete_asn_descriptions()
751 asn_description_path
= os
.path
.join(self
.cache_dir
,
752 asn_description_url
.split('/')[-1])
753 asn_descriptions
= open(asn_description_path
)
754 source_name
= 'cidr_report'
755 skiplen
= len('<a href="/cgi-bin/as-report?as=AS')
756 for line
in asn_descriptions
:
758 asn
, _name
= line
[skiplen
:].split('&view=2.0')
759 description
= _name
.split('</a>')[1].strip()
760 self
.database_cache
.insert_asn_description(asn
, source_name
,
764 self
.database_cache
.commit_changes()
765 asn_descriptions
.close()
767 def parse_asn_assignment_files(self
, asn_assignment_urls
=None):
768 if not asn_assignment_urls
:
769 asn_assignment_urls
= self
.ASN_ASSIGNMENT_URLS
.split()
770 self
.database_cache
.delete_asn_assignments()
771 for asn_assignment_url
in asn_assignment_urls
:
772 asn_assignment_path
= os
.path
.join(self
.cache_dir
,
773 asn_assignment_url
.split('/')[-1])
774 if not os
.path
.exists(asn_assignment_path
):
775 print("Unable to find %s." % asn_assignment_path
)
777 if asn_assignment_path
.endswith('.bz2'):
778 b
= bz2
.BZ2File(asn_assignment_path
)
780 if line
.startswith("*"):
782 netblock
, path
= l
[1], l
[6:-1]
783 nexthop
, metric
, locprf
, weight
= l
[2],l
[3],l
[4],l
[5]
785 network
= ipaddr
.IPNetwork(netblock
)
786 #XXX add support for other sources too
787 source_type
= 'bgp_snapshot'
788 source_name
= 'routeviews'
790 if isinstance(network
, ipaddr
.IPv4Network
): num_type
= "ipv4"
791 else: num_type
= "ivp6"
793 self
.database_cache
.insert_asn_assignment(int(network
.network
),
794 int(network
.broadcast
), num_type
, path
[-1],
795 source_type
, source_name
)
798 def __init__(self
, cache_dir
, database_cache
, verbose
=False):
799 self
.cache_dir
= cache_dir
800 self
.database_cache
= database_cache
801 self
.verbose
= verbose
803 self
.build_country_code_dictionary()
805 def build_country_code_dictionary(self
):
806 """ Return a dictionary mapping country name to the country
808 country_code_path
= os
.path
.join(self
.cache_dir
,
809 'country_names_and_code_elements_txt-temp.htm')
810 if not os
.path
.exists(country_code_path
):
813 for line
in open(country_code_path
):
814 if line
== "" or line
.startswith("Country ") or ";" not in line
:
816 country_name
, country_code
= line
.strip().split(";")
817 country_name
= ' '.join([part
.capitalize() for part
in \
818 country_name
.split(" ")])
819 self
.map_co
[country_name
] = country_code
821 def knows_country_names(self
):
822 return self
.map_co
is not None
824 def get_name_from_country_code(self
, cc_code
):
825 if not self
.knows_country_names():
827 country_name
= [(key
, value
) for (key
, value
) in \
828 list(self
.map_co
.items()) if value
== cc_code
]
829 if len(country_name
) > 0:
830 return country_name
[0][0]
832 def get_country_code_from_name(self
, country_name
):
833 """ Return the country code for a given country name. """
834 if not self
.knows_country_names():
836 cc_code
= [self
.map_co
[key
] for key
in list(self
.map_co
.keys()) if \
837 key
.upper().startswith(country_name
.upper())]
841 def lookup_ipv6_address(self
, lookup_ipaddr
):
842 print("Reverse lookup for: " + str(lookup_ipaddr
))
843 for source_type
in ['maxmind', 'rir', 'lir']:
844 cc
= self
.database_cache
.fetch_country_code('ipv6',
845 source_type
, int(lookup_ipaddr
))
847 print(source_type
.upper(), "country code:", cc
)
848 cn
= self
.get_name_from_country_code(cc
)
850 print(source_type
.upper(), "country name:", cn
)
852 def lookup_ipv4_address(self
, lookup_ipaddr
):
853 print("Reverse lookup for: " + str(lookup_ipaddr
))
854 maxmind_cc
= self
.database_cache
.fetch_country_code('ipv4', 'maxmind',
857 print('MaxMind country code:', maxmind_cc
)
858 maxmind_cn
= self
.get_name_from_country_code(maxmind_cc
)
860 print('MaxMind country name:', maxmind_cn
)
861 rir_cc
= self
.database_cache
.fetch_country_code('ipv4', 'rir',
864 print('RIR country code:', rir_cc
)
865 rir_cn
= self
.get_name_from_country_code(rir_cc
)
867 print('RIR country name:', rir_cn
)
869 print('Not found in RIR db')
870 lir_cc
= self
.database_cache
.fetch_country_code('ipv4', 'lir',
873 print('LIR country code:', lir_cc
)
874 lir_cn
= self
.get_name_from_country_code(lir_cc
)
876 print('LIR country name:', lir_cn
)
877 if maxmind_cc
and maxmind_cc
!= rir_cc
:
878 print("It appears that the RIR data conflicts with MaxMind's "
879 "data. MaxMind's data is likely closer to being "
880 "correct due to sub-delegation issues with LIR databases.")
882 def lookup_ip_address(self
, lookup_str
):
883 """ Return the country code and name for a given ip address. """
885 lookup_ipaddr
= ipaddr
.ip_address(lookup_str
)
886 if isinstance(lookup_ipaddr
, ipaddr
.IPv4Address
):
887 self
.lookup_ipv4_address(lookup_ipaddr
)
888 elif isinstance(lookup_ipaddr
, ipaddr
.IPv6Address
):
889 self
.lookup_ipv6_address(lookup_ipaddr
)
891 print(("Did not recognize '%s' as either IPv4 or IPv6 "
892 "address." % lookup_str
))
893 except ValueError as e
:
894 print("'%s' is not a valid IP address." % lookup_str
)
896 def asn_lookup(self
, asn
):
897 asn_cc
= self
.database_cache
.fetch_country_code('asn', 'rir', asn
)
899 print("AS country code: %s" % asn_cc
)
900 asn_cn
= self
.get_name_from_country_code(asn_cc
)
902 print("AS country name: %s" % asn_cn
)
904 print("AS%s not found!" % asn
)
906 def fetch_rir_blocks_by_country(self
, request
, country
):
908 for (start_num
, end_num
) in \
909 self
.database_cache
.fetch_assignments(request
, country
):
910 if request
== "ipv4" or request
== "ipv6":
911 start_ipaddr
= ipaddr
.ip_address(start_num
)
912 end_ipaddr
= ipaddr
.ip_address(end_num
)
913 result
+= [str(x
) for x
in
914 ipaddr
.summarize_address_range(
915 start_ipaddr
, end_ipaddr
)]
917 result
.append(str(start_num
))
920 def lookup_countries_in_different_source(self
, first_country_code
):
921 """ Look up all assignments matching the given country code, then
922 look up to which country code(s) the same number ranges are
923 assigned in other source types. Print out the result showing
924 similarities and differences. """
926 " '<' = found assignment range with country code '%s'\n"
927 " '>' = overlapping assignment range with same country code\n"
928 " '*' = overlapping assignment range, first conflict\n"
929 " '#' = overlapping assignment range, second conflict and "
930 "beyond\n ' ' = neighboring assignment range") % (
931 first_country_code
, ))
932 results
= self
.database_cache
.fetch_country_blocks_in_other_sources(
934 prev_first_source_type
= ''
935 prev_first_start_num
= -1
936 cur_second_country_codes
= []
937 for (first_source_type
, first_start_num
, first_end_num
,
938 second_source_type
, second_start_num
, second_end_num
,
939 second_country_code
, num_type
) in results
:
940 if first_source_type
!= prev_first_source_type
:
941 print("\nAssignments in '%s':" % (first_source_type
, ))
942 prev_first_source_type
= first_source_type
943 if first_start_num
!= prev_first_start_num
:
944 cur_second_country_codes
= []
946 prev_first_start_num
= first_start_num
948 if second_end_num
>= first_start_num
and \
949 second_start_num
<= first_end_num
:
950 if first_country_code
!= second_country_code
and \
951 second_country_code
not in cur_second_country_codes
:
952 cur_second_country_codes
.append(second_country_code
)
953 if first_source_type
== second_source_type
:
955 elif len(cur_second_country_codes
) == 0:
957 elif len(cur_second_country_codes
) == 1:
961 if num_type
.startswith("ip") and \
962 second_start_num
== second_end_num
:
963 second_range
= "%s" % (ipaddr
.ip_address(second_start_num
), )
964 elif num_type
.startswith("ip") and \
965 second_start_num
< second_end_num
:
966 second_range
= "%s-%s" % (ipaddr
.ip_address(second_start_num
),
967 ipaddr
.ip_address(second_end_num
))
968 elif second_start_num
< second_end_num
:
969 second_range
= "AS%d-%d" % (second_start_num
, second_end_num
)
971 second_range
= "AS%d" % (second_start_num
, )
972 print("%1s %s %s %s" % (marker
, second_country_code
, second_range
,
973 second_source_type
, ))
975 def _get_network_string_from_range(self
, end
, start
, bits
=32):
976 start
, end
= int(start
,16), int(end
,16)
977 netbits
= bits
- int(log(end
- start
,2))
978 return ipaddr
.IPNetwork("%s/%d" % (ipaddr
.IPAddress(start
), netbits
))
980 def lookup_org_by_ip(self
, lookup_str
):
981 """ Return the ASN and AS Description by IP """
983 lookup_ipaddr
= ipaddr
.IPAddress(lookup_str
)
984 if isinstance(lookup_ipaddr
, ipaddr
.IPv4Address
):
987 elif isinstance(lookup_ipaddr
, ipaddr
.IPv6Address
):
992 rs
= self
.database_cache
.fetch_org_by_ip_address(lookup_ipaddr
, num_type
)
994 network
= self
._get
_network
_string
_from
_range
(r
[3],r
[2], bits
=len_bits
)
995 print("%s in %s announced by AS%s - %s" % (lookup_str
, network
, r
[0], r
[1]))
997 print("'%s' is not a valid IP address." % lookup_str
)
999 print("Did not find any matching announcements containing %s." % lookup_str
)
1001 def lookup_org_by_range(self
, start_range
, end_range
):
1002 output_str
= "%s announced by AS%s - %s"
1004 a
= ipaddr
.IPAddress(start_range
)
1005 b
= ipaddr
.IPAddress(end_range
)
1006 if isinstance(a
, ipaddr
.IPv4Address
) and isinstance(b
, ipaddr
.IPv4Address
):
1009 elif isinstance(a
, ipaddr
.IPv6Address
) and isinstance(b
, ipaddr
.IPv6Address
):
1014 rs
= self
.database_cache
.fetch_org_by_ip_range(min(a
,b
), max(a
,b
), num_type
)
1016 network
= self
._get
_network
_string
_from
_range
(r
[3],r
[2], bits
=len_bits
)
1017 print(output_str
% (network
, r
[0], r
[1]))
1019 print("%s %s is not a valid IP range." % (start_range
, end_range
))
1021 print("Did not find any matching announcements in range %s %s." % (start_range
, end_range
))
1023 def split_callback(option
, opt
, value
, parser
):
1024 split_value
= value
.split(':')
1025 setattr(parser
.values
, option
.dest
, split_value
[0])
1026 if len(split_value
) > 1 and split_value
[1] != '':
1027 setattr(parser
.values
, 'type_filter', split_value
[1])
1030 """ Where the magic starts. """
1031 usage
= ("Usage: %prog [options]\n\n"
1032 "Example: %prog -v -t mm")
1033 parser
= optparse
.OptionParser(usage
)
1034 parser
.add_option("-v", "--verbose", action
="store_true",
1035 dest
="verbose", help = "be verbose", default
=False)
1036 parser
.add_option("-c", "--cache-dir", action
="store", dest
="dir",
1037 help="set cache directory [default: %default]",
1038 default
=str(os
.path
.expanduser('~')) + "/.blockfinder/")
1039 parser
.add_option("--user-agent", action
="store", dest
="ua",
1040 help=('provide a User-Agent which will be used when '
1041 'fetching delegation files [default: "%default"]'),
1042 default
="Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0")
1043 parser
.add_option("-x", "--hack-the-internet", action
="store_true",
1044 dest
="hack_the_internet", help=optparse
.SUPPRESS_HELP
)
1045 group
= optparse
.OptionGroup(parser
, "Cache modes",
1046 "Pick at most one of these modes to initialize or update "
1047 "the local cache. May not be combined with lookup modes.")
1048 group
.add_option("-m", "--init-maxmind", action
="store_true",
1049 dest
="init_maxmind",
1050 help="initialize or update MaxMind GeoIP database")
1051 group
.add_option("-g", "--reload-maxmind", action
="store_true",
1052 dest
="reload_maxmind",
1053 help=("update cache from existing MaxMind GeoIP database"))
1054 group
.add_option("-r", "--import-maxmind", action
="store",
1055 dest
="import_maxmind", metavar
="FILE",
1056 help=("import the specified MaxMind GeoIP database file into "
1057 "the database cache using its file name as source "
1059 group
.add_option("-i", "--init-rir",
1060 action
="store_true", dest
="init_del",
1061 help="initialize or update delegation information")
1062 group
.add_option("-d", "--reload-rir", action
="store_true",
1064 help="use existing delegation files to update the database")
1065 group
.add_option("-l", "--init-lir", action
="store_true",
1067 help=("initialize or update lir information; can take up to "
1069 group
.add_option("-z", "--reload-lir", action
="store_true",
1071 help=("use existing lir files to update the database; can "
1072 "take up to 5 minutes"))
1073 group
.add_option("-o", "--download-cc", action
="store_true",
1074 dest
="download_cc", help="download country codes file")
1075 group
.add_option("-e", "--erase-cache", action
="store_true",
1076 dest
="erase_cache", help="erase the local database cache")
1077 group
.add_option("-j", "--init-asn-descriptions", action
="store_true",
1078 dest
="init_asn_descriptions",
1079 help=("initialize or update asn description information"))
1080 group
.add_option("-k", "--reload-asn-descriptions", action
="store_true",
1081 dest
="reload_asn_descriptions",
1082 help=("Use existing asn descriptions to update database"))
1083 group
.add_option("-y", "--init-asn-assignments", action
="store_true",
1084 dest
="init_asn_assignments",
1085 help=("initialize or update asn assignment information"))
1086 group
.add_option("-u", "--reload-asn-assignments", action
="store_true",
1087 dest
="reload_asn_assignments",
1088 help=("Use existing asn assignments to update database"))
1089 parser
.add_option_group(group
)
1090 group
= optparse
.OptionGroup(parser
, "Lookup modes",
1091 "Pick at most one of these modes to look up data in the "
1092 "local cache. May not be combined with cache modes.")
1093 group
.add_option("-4", "--ipv4", action
="store", dest
="ipv4",
1094 help=("look up country code and name for the specified IPv4 "
1096 group
.add_option("-6", "--ipv6", action
="store", dest
="ipv6",
1097 help=("look up country code and name for the specified IPv6 "
1099 group
.add_option("-a", "--asn", action
="store", dest
="asn",
1100 help="look up country code and name for the specified ASN")
1101 group
.add_option("-t", "--code", action
="callback", dest
="cc",
1102 callback
=split_callback
, metavar
="CC[:type]", type="str",
1103 help=("look up all allocations (or only those for number "
1104 "type 'ipv4', 'ipv6', or 'asn' if provided) in the "
1105 "delegation cache for the specified two-letter country "
1107 group
.add_option("-n", "--name", action
="callback", dest
="cn",
1108 callback
=split_callback
, metavar
="CN[:type]", type="str",
1109 help=("look up all allocations (or only those for number "
1110 "type 'ipv4', 'ipv6', or 'asn' if provided) in the "
1111 "delegation cache for the specified full country name"))
1112 group
.add_option("-p", "--compare", action
="store", dest
="compare",
1114 help=("compare assignments to the specified country code "
1115 "with overlapping assignments in other data sources; "
1116 "can take some time and produce some long output"))
1117 group
.add_option("-w", "--what-country", action
="store", dest
="what_cc",
1118 help=("look up country name for specified country code"))
1119 group
.add_option("--lookup-org-by-ip", "--lookup-org-by-ip",
1120 action
="store", dest
="lookup_org_by_ip",
1121 help=("look up ASN and AS Description for an IP address"))
1122 group
.add_option("--lookup-org-by-range", "--lookup-org-by-range",
1123 action
="store_true", dest
="lookup_org_by_range",
1124 help=("look up announced networks in a range of addresses; "
1125 "requires --range-start and --range-end to be set"))
1126 group
.add_option("--range-start", "--range-start",
1127 action
="store", dest
="range_start",
1128 help=("Specify the start of a range of addresses"))
1129 group
.add_option("--range-end", "--range-end",
1130 action
="store", dest
="range_end",
1131 help=("Specify the end of a range of addresses"))
1132 parser
.add_option_group(group
)
1133 group
= optparse
.OptionGroup(parser
, "Export modes")
1134 group
.add_option("--export-geoip", "--export-geoip", action
="store_true",
1136 help=("export the lookup database to GeoIPCountryWhois.csv and "
1137 "v6.csv files in the format used to build the debian "
1138 "package geoip-database"))
1139 group
.add_option("--geoip-v4-file", "--geoip-v4-file", action
="store",
1140 dest
="geoip_v4_filename",
1141 help=("The filename to write the IPv4 GeoIP dataset to"))
1142 group
.add_option("--geoip-v6-file", "--geoip-v6-file", action
="store",
1143 dest
="geoip_v6_filename",
1144 help=("The filename to write the IPv6 GeoIP dataset to"))
1145 group
.add_option("--geoip-asn-file", "--geoip-asn-file", action
="store",
1146 dest
="geoip_asn_filename",
1147 help=("The filename to write the IPv4 GeoIP ASNum dataset to"))
1148 parser
.add_option_group(group
)
1150 group
= optparse
.OptionGroup(parser
, "Network modes")
1151 (options
, args
) = parser
.parse_args()
1152 if options
.hack_the_internet
:
1153 print("all your bases are belong to us!")
1155 options_dict
= vars(options
)
1157 for mode
in ["init_maxmind", "reload_maxmind", "import_maxmind",
1158 "init_del", "init_lir", "reload_del", "reload_lir",
1159 "download_cc", "erase_cache", "ipv4", "ipv6", "asn",
1160 "cc", "cn", "compare", "what_cc", "init_asn_descriptions",
1161 "reload_asn_descriptions", "init_asn_assignments",
1162 "reload_asn_assignments", "lookup_org_by_ip",
1163 "lookup_org_by_range", "export"]:
1164 if mode
in options_dict
and options_dict
.get(mode
):
1167 parser
.error("only 1 cache or lookup mode allowed")
1169 parser
.error("must provide 1 cache or lookup mode")
1170 database_cache
= DatabaseCache(options
.dir, options
.verbose
)
1171 if options
.erase_cache
:
1172 database_cache
.erase_database()
1174 if not database_cache
.connect_to_database():
1175 print("Could not connect to database.")
1176 print("You may need to erase it using -e and then reload it "
1177 "using -d/-z. Exiting.")
1179 database_cache
.set_db_version()
1180 downloader_parser
= DownloaderParser(options
.dir, database_cache
,
1182 lookup
= Lookup(options
.dir, database_cache
)
1183 if options
.ipv4
or options
.ipv6
or options
.asn
or options
.cc \
1184 or options
.cn
or options
.compare
:
1185 if downloader_parser
.check_rir_file_mtimes():
1186 print("Your cached RIR files are older than 24 hours; you "
1187 "probably want to update them.")
1189 lookup
.asn_lookup(options
.asn
)
1190 elif options
.lookup_org_by_ip
:
1191 lookup
.lookup_org_by_ip(options
.lookup_org_by_ip
)
1192 elif options
.lookup_org_by_range
:
1193 if not (options
.range_start
and options
.range_end
):
1194 print("You must specify the start and end addresses; "
1195 "see --range-start and --range-end")
1197 lookup
.lookup_org_by_range(options
.range_start
, options
.range_end
)
1199 lookup
.lookup_ip_address(options
.ipv4
)
1201 lookup
.lookup_ip_address(options
.ipv6
)
1202 elif options
.cc
or options
.cn
or options
.what_cc
:
1205 country
= options
.cc
.upper()
1206 elif not lookup
.knows_country_names():
1207 print("Need to download country codes first before looking "
1208 "up countries by name.")
1209 elif options
.what_cc
:
1210 country
= options
.what_cc
.upper()
1211 country_name
= lookup
.get_name_from_country_code(country
)
1213 print(("Hmm...%s? That would be %s."
1214 % (options
.what_cc
, country_name
)))
1217 print(("Hmm, %s? We're not sure either. Are you sure that's "
1218 "a country code?" % options
.what_cc
))
1221 country
= lookup
.get_country_code_from_name(options
.cn
)
1223 print("It appears your search did not match a country.")
1225 types
= ["ipv4", "ipv6", "asn"]
1226 if hasattr(options
, 'type_filter') and options
.type_filter
.lower() in types
:
1227 types
= [options
.type_filter
.lower()]
1228 for request
in types
:
1229 print("\n".join(lookup
.fetch_rir_blocks_by_country(\
1231 elif options
.compare
:
1232 print("Comparing assignments with overlapping assignments in other "
1234 lookup
.lookup_countries_in_different_source(options
.compare
)
1235 elif options
.init_maxmind
or options
.reload_maxmind
:
1236 if options
.init_maxmind
:
1237 print("Downloading Maxmind GeoIP files...")
1238 downloader_parser
.download_maxmind_files()
1239 print("Importing Maxmind GeoIP files...")
1240 downloader_parser
.parse_maxmind_files()
1241 elif options
.import_maxmind
:
1242 print("Importing Maxmind GeoIP files...")
1243 downloader_parser
.import_maxmind_file(options
.import_maxmind
)
1244 elif options
.init_del
or options
.reload_del
:
1245 if options
.init_del
:
1246 print("Downloading RIR files...")
1247 downloader_parser
.download_rir_files()
1248 print("Verifying RIR files...")
1249 downloader_parser
.verify_rir_files()
1250 print("Importing RIR files...")
1251 downloader_parser
.parse_rir_files()
1252 elif options
.init_lir
or options
.reload_lir
:
1253 if options
.init_lir
:
1254 print("Downloading LIR delegation files...")
1255 downloader_parser
.download_lir_files()
1256 print("Importing LIR files...")
1257 downloader_parser
.parse_lir_files()
1258 elif options
.download_cc
:
1259 print("Downloading country code file...")
1260 downloader_parser
.download_country_code_file()
1261 elif options
.init_asn_descriptions
or options
.reload_asn_descriptions
:
1262 if options
.init_asn_descriptions
:
1263 print("Downloading ASN Descriptions...")
1264 downloader_parser
.download_asn_description_file()
1265 print("Importing ASN Descriptions...")
1266 downloader_parser
.parse_asn_description_file()
1267 elif options
.init_asn_assignments
or options
.reload_asn_assignments
:
1268 if options
.init_asn_assignments
:
1269 print("Downloading ASN Assignments...")
1270 downloader_parser
.download_asn_assignment_files()
1271 print("Importing ASN Assignments...")
1272 downloader_parser
.parse_asn_assignment_files()
1273 elif options
.export
:
1274 v4_file
= options
.geoip_v4_filename
or "GeoIPCountryWhois.csv"
1275 v6_file
= options
.geoip_v6_filename
or "v6.csv"
1276 asn_file
= options
.geoip_asn_filename
or "GeoIPASNum.csv"
1277 print("Exporting GeoIP IPv4 to %s" % v4_file
)
1278 database_cache
.export_geoip(lookup
, v4_file
, 'ipv4')
1279 print("Exporting GeoIP IPv6 to %s" % v6_file
)
1280 database_cache
.export_geoip(lookup
, v6_file
, 'ipv6')
1281 print("Exporting GeoIP IPv4 ASNum to %s" % asn_file
)
1282 database_cache
.export_asn(asn_file
, 'ipv4')
1284 #print("Exporting GeoIP IPv6 ASNum to %s" % asn_file)
1285 #database_cache.export_geoip(asn_file, 'ipv6')
1286 database_cache
.commit_and_close_database()
1288 if __name__
== "__main__":