gostyle.py

   1 #!/usr/bin/python
   2 """
   3 This file contains several objects we use to process a pattern files for Go game.
   4 We use it to generate input sets for a neural network (libfann), or if we want to perform a PCA analysis to data.
   5 It generates input vectors from pattern files. See `Const.pat_file_regexp' for file format in regexp.
   6
   7 ===== EXAMPLE PAT FILE =====
   8 4632 (border:3 s:5000003 s:6000049 s:700004a)
   9 3497 (atariescape:0 border:0 ldist:4 lldist:2 s:30011a1)
  10 ...
  11 ===== END =====
  12 """
  13 import re
  14 import sys
  15 import cPickle
  16 import random
  17 from itertools import izip,count
  18
  19 class Const:
  20         """Class used to hold global const variables, such as the pat file format."""
  21         pat_file_regexp = '^\s*(\d+)\s*(.+)$'
  22
  23 def print_vector(vector, where=sys.stdout):
  24         """Helper method for printing vector (list of floats)."""
  25         for x in vector:
  26                 print >> where, x,
  27         print >> where
  28
  29 def print_set_to_file( data, filename):
  30         """
  31         Helper method for printing datasets for neural network.
  32         FORMAT of the file
  33         number_of_pairs len_of_input_vector len_of_output_vector
  34         first_input_vector
  35         first_output_vector
  36         second_input_vector
  37         ...
  38         """
  39         def print_set( data, where):
  40                 print >> where,  len(data), len(data[0][0]), len(data[0][1])
  41                 for i,o in data:
  42                         print_vector(i, where)
  43                         print_vector(o, where)
  44         fout = open(filename, 'w')
  45         print_set(data, fout)
  46         fout.close()
  47
  48 def dump_object_to_file(object, filename):
  49         """Helper function to save an object to file using cPickle module."""
  50         f=open(filename,'w')
  51         cPickle.dump(object, f,-1)
  52         f.close()
  53
  54 def load_object_from_file(filename):
  55         """Helper function to recover an object from file using cPickle module."""
  56         f=open(filename,'r')
  57         object = cPickle.load(f)
  58         f.close()
  59         return object
  60
  61 class VectorGenerator(object):
  62         """Abstract class. When called, returns a vector (list of floats). Has output_dim param."""
  63         def __call__(self):
  64                 raise NotImplementedError
  65
  66 class VectorToVector(VectorGenerator):
  67         """Abstract class. When called with a vector (list of floats), returns a vector. Has input_dim and output_dim params."""
  68         def __call__(self, vector):
  69                 raise NotImplementedError
  70
  71 ###
  72 ###             PCA Analysis
  73 ###  Note that you must have `numpy' and `mdp' python modules, otherwise the PCA will
  74 ###  fallback to doing nothing at all (but doing it completely compatibly with the rest of the code :-).
  75 try:
  76         from numpy import array
  77         import mdp
  78         class PCA(VectorToVector):
  79                 """
  80                 Object performing a PCA analysis on either a given vector (see `__call__' function),
  81                 or on a list of vectors (see `process_list_of_vectors' function).
  82                 """
  83                 def __init__(self, list_of_train_vectors, *args, **kwargs):
  84                         self.pca = mdp.nodes.PCANode(*args, **kwargs)
  85                         self.pca.train(array(list_of_train_vectors))
  86                         self.input_dim = self.pca.input_dim
  87                 def __call__(self, vector):
  88                         return list(self.pca(array([vector]))[0])
  89                 def process_list_of_vectors(self, list_of_vectors):
  90                         return [ list(vec) for vec in self.pca(array(list_of_vectors)) ]
  91
  92 except ImportError, e:
  93         print >>sys.stderr, "Warning: %s. PCA will not work."%(str(e))
  94
  95         class PCA(VectorToVector):
  96                 """Default dummy class for PCA, not very useless."""
  97                 def __init__(self, *args, **kwargs):
  98                         pass
  99                 def __call__(self, list_of_vectors):
 100                         return list_of_vectors
 101                 def process_list_of_vectors(self, list_of_vectors):
 102                         return list_of_vectors
 103
 104
 105 class Compose(VectorGenerator):
 106         """
 107         A class used as a composer of different objects, such as InputVectorGenerator and PCA.
 108         Use this if you want to e.g. generate PCA processed vectors.
 109         """
 110         def __init__(self, vector_generator, vector_to_vector):
 111                 if not isinstance(vector_generator, VectorGenerator):
 112                         raise TypeError
 113                 if not isinstance(vector_to_vector, VectorToVector):
 114                         raise TypeError
 115                 self.vector_generator = vector_generator
 116                 self.vector_to_vector = vector_to_vector
 117         #       if vector_generator.output_dim != vector_to_vector.input_dim:
 118         #               raise RuntimeError("Dimensions of Composed object mismatch.")
 119         def __call__(self, *args, **kwargs):
 120                 return self.vector_to_vector(self.vector_generator(*args, **kwargs))
 121
 122 class OccurenceVectorGenerator(VectorGenerator):
 123         """
 124         A class used to generate input vectors based on a relative number of occurences of some input patterns.
 125         The object is initialized with a file of patterns. It takes the topmost `num_features' patterns.
 126         """
 127         def generate_top_pattern_dict(self):
 128                 rexp=re.compile(Const.pat_file_regexp)
 129                 self.top_pattern_dict = {}
 130                 i = 0
 131                 input_file = open(self.filename)
 132                 for line in input_file:
 133                         if i >= self.output_dim:
 134                                 break
 135                         self.top_pattern_dict[rexp.match(line).group(2)] = i
 136                         i += 1
 137                 input_file.close()
 138
 139         def __init__(self, main_pat_file, num_features):
 140                 self.output_dim = num_features
 141                 self.filename = main_pat_file
 142                 self.generate_top_pattern_dict()
 143
 144         def __call__(self, pat_file):
 145                 vector = [0]*len(self.top_pattern_dict)
 146                 rexp=re.compile(Const.pat_file_regexp)
 147                 i = 0
 148                 input_file = open(pat_file)
 149                 for line in input_file:
 150                         match = rexp.match(line)
 151                         if not match:
 152                                 raise IOError("Wrong file format: " + pat_file)
 153                         if match.group(2) in self.top_pattern_dict:
 154                                 index=self.top_pattern_dict[match.group(2)]
 155                                 vector[index] += int(match.group(1))
 156                                 i += 1
 157                         if i >= len(self.top_pattern_dict):
 158                                 break
 159                 input_file.close()
 160                 if len(vector) != self.output_dim:
 161                         raise RuntimeError
 162                 return vector
 163
 164 class Rescale(VectorToVector):
 165         """Class that rescales vectors to a given interval!"""
 166         def __init__(self, a=-1.0, b=1.0):
 167                 if a > b:
 168                         raise RuntimeError
 169                 self.a = a
 170                 self.tot = b - a #abs(a) + abs(b)
 171         def __call__(self, vector):
 172                 if len(vector) == 0:
 173                         raise RuntimeError
 174                 to_zero = 0 - min(vector)
 175                 maximum = max(vector) + to_zero
 176                 return [ self.tot * (x + to_zero) / maximum + self.a for x in vector ]
 177
 178 class InputVectorGenerator(VectorGenerator):
 179         """
 180         First we generate an occurence vector by OccurenceVectorGenerator.
 181         Then, an input vector is generated as a relative number of occurences of the topmost patterns.
 182         The occurences are mapped so that the most frequently used
 183         one is mapped to 1.0 and the rest is mapped relatively on the scale (1.0,-1.0). See `__call__' function.
 184         """
 185         def __init__(self, *args, **kwargs):
 186                 self.gen = Compose(OccurenceVectorGenerator(*args, **kwargs), Rescale(-1.0, 1.0))
 187         def __call__(self, *args, **kwargs):
 188                 return self.gen(*args, **kwargs)
 189
 190 class Combinator(object):
 191         def __init__(self, num_lincombs = 1, skip_subset_len = [0], max_len = 2):
 192                 self.num_lincombs = num_lincombs
 193                 self.skip_subset_len = skip_subset_len
 194                 self.max_len = max_len
 195         def get_subsets(self, set):
 196                 if len(set) == 0:
 197                         return [[]]
 198                 sub=self.get_subsets(set[1:])
 199                 return sub + filter( lambda x : ( self.max_len==0 or len(x)<=self.max_len ), [ set[:1]+subset  for subset in sub] )
 200         def linear_combination(self, list_of_vectors, coefs):
 201                 if len(list_of_vectors) != len(coefs):
 202                         raise Exception("len(list_of_vectors) != len(coefs)")
 203                 if len(list_of_vectors) == 0:
 204                         return
 205                 len_vec=len(list_of_vectors[0])
 206                 res_vec=[0]*len_vec
 207                 for p in xrange(len(list_of_vectors)):
 208                         for i in xrange(len_vec):
 209                                 res_vec[i] += coefs[p] * list_of_vectors[p][i]
 210                 return res_vec
 211         def get_random_norm_coefs(self, num ):
 212                 coefs=[]
 213                 rnd_nums= [ random.random() for i in xrange(num-1) ] + [1]
 214                 rnd_nums.sort()
 215                 first=0
 216                 for next in rnd_nums:
 217                         coefs.append(next-first)
 218                         first=next
 219                 return coefs
 220         def combine(self, data):
 221                 combinations = []
 222                 for subset in self.get_subsets(range(len(data))):
 223                         if len(subset) in self.skip_subset_len:
 224                                 continue
 225                         input_vectors = [ data[index][0] for index in subset ]
 226                         output_vectors = [ data[index][1] for index in subset ]
 227                         for i in xrange(self.num_lincombs):
 228                                 coefs = self.get_random_norm_coefs(len(subset))
 229                                 combinations += [(self.linear_combination(input_vectors, coefs), self.linear_combination(output_vectors, coefs))]
 230                 return combinations
 231
 232 class PlayerStrategyIdentificator(object):
 233         """Object holding information about default strategies for players."""
 234         def __init__(self, strategy_players):
 235                 self.strategy_players = strategy_players
 236
 237                 self.player_strategy={}
 238                 self.all_players = []
 239                 self.all_strategies = []
 240                 for strategy, players in self.strategy_players.items():
 241                         self.all_strategies += [strategy]
 242                         for player in players:
 243                                 self.all_players += [player]
 244                                 self.player_strategy[player] = strategy
 245
 246         def __call__(self, player_name):
 247                 return self.player_strategy[player_name]
 248
 249 class StrategyOutputVectorGenerator(VectorGenerator):
 250         """
 251         This object generates output vectors for players with strategies specified in `PlayerStrategyIdentificator' object.
 252         It is initialized with a list of strategies `valid_strategies' it shall take into acount.
 253         When called (see `__call__') with a name of a player with a strategy from `valid_strategies' it returns a vector
 254         that corresponds to the strategy like this.
 255         """
 256         def __init__(self, strategy_players, valid_strategies=None):
 257                 self.identificator = PlayerStrategyIdentificator(strategy_players)
 258                 if valid_strategies == None:
 259                         valid_strategies = self.identificator.all_strategies
 260                 index=0
 261                 self.valid_strategy_index={}
 262                 for s in valid_strategies:
 263                         self.valid_strategy_index[s]= index
 264                         index += 1
 265         def __call__(self, player_name):
 266                 try:
 267                         player_strat = self.identificator(player_name)
 268                         player_strat_index = self.valid_strategy_index[player_strat]
 269                         return [ 1.0 if i == player_strat_index else -1.0 for i in xrange(len(self.valid_strategy_index)) ]
 270                 except KeyError:
 271                         return None
 272
 273 class PlanarOutputVectorGenerator(VectorGenerator):
 274         """Class that explicitly returns predefined output vectors for given players."""
 275         def __init__(self, player_vector):
 276                 self.player_vector = player_vector
 277                 self.players = player_vector.keys()
 278         def __call__(self, player_name):
 279                 try:
 280                         return self.player_vector[player_name]
 281                 except KeyError:
 282                         return None
 283
 284 if __name__ == '__main__':
 285         print >>sys.stderr, "This is just a library file..."