1 # -*- coding: utf-8 -*-
15 class KGSError(RuntimeError):
18 class KGSNotFound(KGSError
):
22 """Object used to fetch data from KGS.
24 The requests and results are cached, real requests are delayed not to
25 contact the server too often.
28 def __init__(self
, cache_dir
, min_delay
=5):
31 cache_dir -- where to save the requests and results, created if does
33 min_delay -- min_delay between requests in seconds
35 # the time of the last request
39 self
.min_delay
= min_delay
41 self
.cache_dir
= cache_dir
43 def fetch_archive_and_extract(self
, player
, year
, month
):
45 Fetches games of @player, from @year and @month, extracts the archive
46 and returns the directory with games.
48 basename
= '%s-%d-%d.tar.gz' % (player
, int(year
), int(month
))
49 url
= 'http://www.gokgs.com/servlet/archives/en_US/' + basename
52 archive_dir
= os
.path
.join(self
.cache_dir
, 'ARCHIVES', str(year
), str(month
))
53 if not os
.path
.isdir(archive_dir
):
54 os
.makedirs(archive_dir
)
56 archive_file
= os
.path
.join(archive_dir
, basename
)
57 if not os
.path
.exists(archive_file
):
59 logging
.info("Fetching KGS archive '%s'"%(url))
62 response
= urllib2
.urlopen(url
)
63 except urllib2
.HTTPError
as e
:
67 raise KGSError("KGS quota exceeded. Please download your latest games manually and upload them as an archive.")
69 except urllib2
.URLError
as e
:
70 raise KGSError("Fetching the KGS archive failed: '%s'\n %s"%(e
, url
))
72 self
.update_last_time()
73 with
open(archive_file
, 'w') as archive
:
74 archive
.write(response
.read())
77 games_dir
= os
.path
.join(self
.cache_dir
, 'GAMES', str(year
), str(month
), str(player
))
78 if not os
.path
.isdir(games_dir
):
79 os
.makedirs(games_dir
)
81 tf
= tarfile
.open(archive_file
)
82 tf
.extractall(games_dir
)
87 def list_games(self
, player
, year
, month
):
88 """ Returns list of tuples
89 [ (playername, rank), ..]
90 there will be two tuples for one game even 19x19 game (white and black player) of
91 @player in @year / @month
93 only regard 19x19 even games
95 ret_games
, ret_active
= self
._player
_archive
(player
, year
, month
)
98 def list_active(self
, player
):
100 Returns list of tuples:
101 [ (player, year, month), ] such that player was active in the year and month.
103 today
= datetime
.datetime
.today()
104 year
, month
= today
.year
, today
.month
106 ret_games
, ret_active
= self
._player
_archive
(player
, year
, month
)
109 raise KGSError("Not an active user '%s'."%player
)
112 def wait_min_delay(self
):
113 diff
= time
.time() - self
.last
114 if diff
< self
.min_delay
:
115 time
.sleep(self
.min_delay
- diff
+ random
.random())
117 def update_last_time(self
):
118 self
.last
= time
.time()
120 def _player_archive(self
, player
, year
, month
):
121 assert re
.match('^[0-9a-zA-Z]*$', player
)
125 tmpname
= tempfile
.mktemp('kgs_fetch')
126 wget_file_dir
= os
.path
.join(self
.cache_dir
, 'LIST', str(year
), str(month
))
127 if not os
.path
.isdir(wget_file_dir
):
128 os
.makedirs(wget_file_dir
)
129 wget_file
= os
.path
.join(wget_file_dir
, player
)
135 [ -e $wget_outfile ] || {
137 wget --user-agent="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20100101 Firefox/15.0.1" \
138 "http://www.gokgs.com/gameArchives.jsp?user=%s&year=%s&month=%s" -O $wget_outfile
142 [ -e $wget_outfile ] || { exit 1; }
145 sed 's/<tr>/\\n&/g' $wget_outfile > $tmp_file
147 # even games in the month
148 sed '/<td>19×19 <\/td>/!d' $tmp_file > ${tmp_file}.games
149 sed -i 's/<td>/\\n&/g' ${tmp_file}.games
150 sed -i -n 's#.*<a href="gameArchives.jsp?user=\([a-zA-Z0-9]*\)">\\1 \[\([0-9]*[kd][?]\{0,1\}\)\]</a>.*#\\1 \\2#p' ${tmp_file}.games
153 sed -i 's/<td>/\\n&/g' $tmp_file
154 sed -n -i 's#.*href="gameArchives.jsp?user=\([a-zA-Z0-9]*\)&year=\([0-9]*\)&month=\([0-9]*\)">.*#\\2 \\3#p' $tmp_file
156 """%( wget_file
, tmpname
, player
, year
, month
)
162 # if the target file does not exist
163 # = it is not cached => we will make a request
164 if not os
.path
.exists(wget_file
):
165 logging
.info("Fetching KGS list of active months for '%s'"%(player))
166 self
.wait_min_delay()
168 retcode
= subprocess
.call(script
, shell
=True)
169 self
.update_last_time()
171 raise RuntimeError("Fetching KGS games failed.")
173 with
open(tmpname
+ '.games', 'r') as fin
:
174 data
= fin
.readlines()
176 os
.unlink(tmpname
+'.games')
181 player
, rank
= line
.split()
182 ret_games
.append((player
, rank
))
184 with
open(tmpname
, 'r') as fin
:
185 data
= fin
.readlines()
192 lyear
, lmonth
= line
.split()
193 ret_active
.append((lyear
, lmonth
))
196 current
= (str(year
), str(month
))
197 if not current
in ret_active
:
198 ret_active
.append(current
)
200 logging
.warn("Current month already in the list of active months..")
202 ret
= (ret_games
, ret_active
)
205 if __name__
== "__main__":
209 print kgs
.list_active('bronislav')