python/jben/dict_downloader.py

   1 # -*- coding: utf-8 -*-
   2
   3 from __future__ import absolute_import
   4
   5 import urllib2, sys, os, random
   6 from jben.preferences import Preferences
   7
   8 # The following is shamelessly copied from:
   9 # http://ftp.monash.edu.au/pub/nihongo/.message
  10 #old_static_mirror_list = [
  11 #    "http://ftp.monash.edu.au/pub/nihongo", # Original FTP archive
  12 #    "http://japanology.arts.kuleuven.ac.be/mirrors/monash", # Belgium
  13 #    "ftp://ftp.nrc.ca/pub/packages/nihongo", # Canada (fast site)
  14 #    "http://www.bcit-broadcast.com/monash", # Canada (http only)
  15 #    "ftp://ftp.sedl.org/pub/mirrors/nihongo", # US(Texas)
  16 #    "ftp://ftp.net.usf.edu/pub/monash", # IS (Florida)
  17 #    "ftp://ftp.u-aizu.ac.jp/pub/SciEng/nihongo/ftp.cc.monash.edu.au", # Japan
  18 #    "ftp://ftp.funet.fi/pub/culture/japan/mirrors/monash", # Finland
  19 #    "ftp://ftp.uni-duisburg.de/Mirrors/ftp.monash.edu.au/pub/nihongo" # Germany
  20 #    ]
  21
  22 # Based upon checking the mirrors myself, this is the list I'll use
  23 # for J-Ben.
  24 static_mirror_list = [
  25     # Official archive
  26     # (redirects to ringtail.its.monash.edu.au/pub/nihongo, same IP address)
  27     "http://ftp.monash.edu.au/pub/nihongo",
  28     # Up to date mirrors
  29     "ftp://ftp.nrc.ca/pub/packages/nihongo", # Canada (fast site)
  30     "ftp://ftp.net.usf.edu/pub/monash/pub/nihongo", # IS (Florida)
  31     "ftp://ftp.uni-duisburg.de/Mirrors/ftp.monash.edu.au/pub/nihongo", # Germany
  32     # *Probably* up to date (but can't tell since I can't see file mtimes)
  33     "http://japanology.arts.kuleuven.ac.be/mirrors/monash", # Belgium
  34     "http://www.bcit-broadcast.com/monash", # Canada (http only)
  35     # old links
  36     #"ftp://ftp.u-aizu.ac.jp/pub/SciEng/nihongo/ftp.cc.monash.edu.au", # Japan
  37     #"ftp://ftp.funet.fi/pub/culture/japan/mirrors/monash", # Finland
  38     # bad links
  39     #"ftp://ftp.sedl.org/pub/mirrors/nihongo", # US(Texas)  - "not available"
  40     ]
  41
  42 def get_mirror_list(from_inet=False,
  43                     mirror="http://ftp.monash.edu.au/pub/nihongo"):
  44     """Grabs a mirror list from a valid mirror of Jim Breen's FTP archive."""
  45
  46     # NOTE: although this function does what it says, the mirror list
  47     # is not dependable for our purposes.  Not all mirrors are up to
  48     # date, and one mirror appears to be down.  I advise not using
  49     # from_inet for the time being.
  50
  51     if from_inet:
  52         try:
  53             f = urllib2.urlopen("%s/%s" % (mirror, ".message"))
  54             if f:
  55                 mirrors = ["http://ftp.monash.edu.au/pub/nihongo"]
  56                 for line in f:
  57                     if line[0] == ".":
  58                         line = line[1:].strip().replace("\t", " ")
  59                         l = [s.strip() for s in line.split(" ", 1)]
  60                         url, desc = l
  61                         url = url.rstrip('/')
  62                         mirrors.append(url)
  63                 return mirrors
  64         except urllib2.URLError, e:
  65             print >> sys.stderr, e
  66             pass
  67
  68         return []
  69     else:
  70         return static_mirror_list
  71
  72 def download_dict(fname):
  73     mirrors = get_mirror_list(from_inet=False)
  74     dpath = p.get_dict_path()
  75     print "Using output dict path:", dpath
  76
  77     def get_next_mirror():
  78         i = random.randrange(0, len(mirrors))
  79         return mirrors.pop(i)
  80
  81     mirror = get_next_mirror()
  82
  83     while True:
  84         url = "%s/%s" % (mirror, fname)
  85         target_fname = "%s/%s" % (dpath, fname)
  86         try:
  87             print "Downloading %s to %s..." % (url, target_fname)
  88             resp = urllib2.urlopen(url)
  89             data = resp.read()
  90             resp.close()
  91             if not os.path.exists(dpath):
  92                 os.mkdir(dpath)
  93             ofile = open(target_fname, "wb")
  94             ofile.write(data)
  95             ofile.close()
  96             break
  97         except Exception, e:
  98             print >> sys.stderr, \
  99                   "An error occurred; trying the next mirror."
 100             print >> sys.stderr, "(%s)" % str(e)
 101             mirror = get_next_mirror()
 102
 103
 104 def console_iface():
 105     p.load()
 106
 107     print "The following dictionaries are available:"
 108     dicts = [("edict.gz", "EDICT"),
 109              #("edict2.gz", "EDICT2"),
 110              ("JMdict.gz", "JMdict (full)"),
 111              ("JMdict_e.gz", "JMdict_e (English only)"),
 112              ("kanjidic.gz", "KANJIDIC"),
 113              #("kanjd212.gz", "KANJD212"),
 114              ("kanjidic2.xml.gz", "KANJIDIC2")]
 115     for i, (fname, desc) in enumerate(dicts):
 116         print "\t%d: %s" % (i+1, desc)
 117     sys.stdout.write("Enter the numbers for the dictionaries you want, "
 118                      "separated by spaces: ")
 119     s = sys.stdin.readline()
 120
 121     fnames = []
 122     for d in s.split():
 123         try:
 124             i = int(d) - 1
 125             if i not in range(len(dicts)):
 126                 print >> sys.stderr, "Value %s is out of range." % s
 127                 continue
 128         except ValueError, e:
 129             print >> sys.stderr, "Ignoring invalid value %s" % d
 130             continue
 131         fname = dicts[i][0]
 132         fnames.append(fname)
 133
 134     for fname in fnames:
 135         download_dict(fname)
 136
 137 if __name__ == "__main__":
 138     global p
 139     p = Preferences()
 140     console_iface()