utils/misc.py

   1 #coding: utf-8
   2 import unicodedata
   3 from itertools import chain, izip,  count
   4 import hashlib
   5 import random
   6 import logging
   7 import threading
   8 import os
   9 import time
  10 import datetime
  11
  12 def identity(x):
  13     return x
  14
  15 #
  16 # Functional tools
  17 #
  18
  19 def filter_null(iterable):
  20     return [ x for x in iterable if x ]
  21
  22 def filter_both(predicate, iterable):
  23     yes, no = [], []
  24     for i in iterable:
  25         if predicate(i):
  26             yes.append(i)
  27         else:
  28             no.append(i)
  29     return yes, no
  30
  31 def flatten(list_of_lists):
  32     return chain.from_iterable(list_of_lists)
  33
  34 def flatten_twice(list_of_lists_of_lists):
  35     return flatten(flatten( list_of_lists_of_lists ))
  36
  37 def argmax(pairs):
  38     """Given an iterable of pairs (key, value), return the key corresponding to the greatest value."""
  39     return max(pairs, key=lambda x:x[1])[0]
  40
  41 def argmin(pairs):
  42     return min(pairs, key=lambda x:x[1])[0]
  43
  44 def argmax_index(values):
  45     """Given an iterable of values, return the index of the greatest value."""
  46     return argmax(izip(count(), values))
  47
  48 def argmin_index(values):
  49     return argmin(izip(count(), values))
  50
  51 def bucket_by_key(iterable, key_fc):
  52     """
  53     Throws items in @iterable into buckets given by @key_fc function.
  54     e.g.
  55     >>> bucket_by_key([1,2,-3,4,5,6,-7,8,-9], lambda num: 'neg' if num < 0 else 'nonneg')
  56     {'neg': [-3, -7, -9], 'nonneg': [1, 2, 4, 5, 6, 8]}
  57     """
  58     buckets = {}
  59     for item in iterable:
  60         buckets.setdefault(key_fc(item), []).append(item)
  61     return buckets
  62
  63 def first_true_pred(predicates, value):
  64     """Given a list of predicates and a value, return the index of first predicate,
  65     s.t. predicate(value) == True. If no such predicate found, raises IndexError."""
  66     for num, pred in enumerate(predicates):
  67         if pred(value):
  68             return num
  69     raise IndexError
  70
  71 #
  72 # Partial
  73 #
  74
  75 class MyPartial:
  76     """
  77     An alternative implementation of functools partial, allowing to specify args
  78     from the right as well.
  79     """
  80     def __init__(self, func, args=(), keywords={}, right=False):
  81         self.func = func
  82         self.args = args
  83         self.keywords = keywords
  84         self.right = right
  85
  86     def _frepr(self):
  87         return repr(self.func)
  88
  89     def __repr__(self):
  90         return "MyPartial(%s, %s, %s%s)"%(self._frepr(),
  91                                           repr(self.args), repr(self.keywords),
  92                                           ", right=True" if self.right else '')
  93
  94     def _merge_args(self, args_new):
  95         if self.right:
  96             return args_new + tuple(self.args)
  97         return tuple(self.args) + args_new
  98
  99     def _merge_kwargs(self, kwargs_new):
 100         kwargs = self.keywords.copy()
 101         kwargs.update(kwargs_new)
 102         return kwargs
 103
 104     def __call__(self, *args_new, **kwargs_new):
 105         args = self._merge_args(args_new)
 106         kwargs = self._merge_kwargs(kwargs_new)
 107         return self.func(*args, **kwargs)
 108
 109 def partial(f, *args, **kwargs):
 110     """
 111     def minus(a, b):
 112         return a - b
 113
 114     partial(minus, 10)  is like:
 115
 116     lambda b : minus(10, b)
 117     """
 118     return MyPartial(f, args, kwargs)
 119
 120 def partial_right(f, *args, **kwargs):
 121     """
 122     def minus(a, b):
 123         return a - b
 124
 125     partial_right(minus, 10)  is like:
 126
 127     lambda a : minus(a, 10)
 128     """
 129     return MyPartial(f, args, kwargs, right=True)
 130
 131 #
 132 # Type info and conversion
 133 #
 134
 135 def is_conv(x, cls):
 136     try:
 137         cls(x)
 138         return True
 139     except:
 140         return False
 141
 142 def is_int(x):
 143     return is_conv(x, int)
 144
 145 def is_float(x):
 146     return is_conv(x, float)
 147
 148 def is_type(x, type):
 149     try:
 150         return x == type(x)
 151     except:
 152         return False
 153
 154 def is_type_int(x):
 155     return is_type(x, int)
 156
 157 def is_type_float(x):
 158     return is_type(x, float)
 159
 160 #
 161 # Hash utils & random strings
 162 #
 163
 164 def sha256(txt):
 165     return hashlib.sha256(txt).hexdigest()
 166 def sha512(txt):
 167     return hashlib.sha512(txt).hexdigest()
 168
 169 def random_hash(LEN=10):
 170     return str(random.randint(10**(LEN-1),10**LEN-1))
 171
 172 def unique_hash(length=32):
 173     """Returns "unique" hash. (the shorter the length, the less unique it is).
 174     I consider one in 16**32 to be pretty unique. :-) (supposing that sha256 works).
 175
 176     (Am I mistaken??)"""
 177     return sha256( "%.20f %s %d %d"%(time.time(), random_hash(), os.getpid(), threading.current_thread().ident ) ) [:length]
 178
 179 time_based_random_hash =  unique_hash
 180
 181 def tmp_names(base=random_hash(), first_simple=False):
 182     i = 0
 183     if first_simple:
 184         yield "%s"%(base)
 185         i += 1
 186     while True:
 187         yield "%s_%d"%(base, i)
 188         i+=1
 189
 190 #
 191 # Text stuff
 192 #
 193
 194 def encode_utf8(st):
 195     return unicode(st).encode('utf-8')
 196
 197 def remove_accents(istr):
 198     nkfd_form = unicodedata.normalize('NFKD', unicode(istr))
 199     return u"".join([c for c in nkfd_form if not unicodedata.combining(c)])
 200
 201 def unicode2ascii(unistr):
 202     return remove_accents(unistr).encode('ascii', 'ignore')
 203
 204 def pad2len(s, l, padchar):
 205     if len(s) < l:
 206         return s + padchar * (l - len(s))
 207     return s
 208
 209 #
 210 #
 211 #
 212
 213 def iter_files(directory):
 214     for (dirpath, _, filenames) in os.walk(directory):
 215         for name in filenames:
 216                 yield os.path.join(dirpath, name)
 217
 218 def timeit(f, *args, **kwargs):
 219     t0 = time.time()
 220     ret = f(*args, **kwargs)
 221     diff = time.time() - t0
 222     print "TOOK: %.3f sec"%(diff,)
 223     return ret
 224
 225 if __name__ == '__main__':
 226     def test_partial():
 227         def fc ( a, b, c='def C', d='def D' ):
 228             return "a=%d, b=%d, c=%s, d=%s"%(a,b,c,d)
 229
 230         nor = partial(fc, 20, c=10)
 231         zpr = partial_right(fc, 20, c=10)
 232
 233         print "puvodni:", repr(fc)
 234         print "normalni:", repr(nor)
 235         print "zprava:", repr(zpr)
 236
 237         print "normalni(10) = ", nor(10)
 238         print "zprava(10) = ", zpr(10)
 239
 240         nor2 = partial(nor, 10)
 241         print "double:", nor2
 242         print "double()", nor2()
 243
 244         class Fobj:
 245             def __call__(self, a, b):
 246                 return "a=%s b=%s"%(a,b)
 247
 248         ca = partial(Fobj(), 10)
 249         print ca