pygr/classutil.py

   1 import os, sys, tempfile
   2 from weakref import WeakValueDictionary
   3 import dbfile, logger
   4
   5
   6 class FilePopenBase(object):
   7     '''Base class for subprocess.Popen-like class interface that
   8 can be supported on Python 2.3 (without subprocess).  The main goal
   9 is to avoid the pitfalls of Popen.communicate(), which cannot handle
  10 more than a small amount of data, and to avoid the possibility of deadlocks
  11 and the issue of threading, by using temporary files'''
  12     def __init__(self, args, bufsize=0, executable=None,
  13                  stdin=None, stdout=None, stderr=None, *largs, **kwargs):
  14         '''Mimics the interface of subprocess.Popen() with two additions:
  15 - stdinFlag, if passed, gives a flag to add the stdin filename directly
  16 to the command line (rather than passing it by redirecting stdin).
  17 example: stdinFlag="-i" will add the following to the commandline:
  18 -i /path/to/the/file
  19 If set to None, the stdin filename is still appended to the commandline,
  20 but without a preceding flag argument.
  21
  22 -stdoutFlag: exactly the same thing, except for the stdout filename.
  23 '''
  24         self.stdin, self._close_stdin = self._get_pipe_file(stdin, 'stdin')
  25         self.stdout, self._close_stdout = self._get_pipe_file(stdout, 'stdout')
  26         self.stderr, self._close_stderr = self._get_pipe_file(stderr, 'stderr')
  27         kwargs = kwargs.copy() # get a copy we can change
  28         try: # add as a file argument
  29             stdinFlag = kwargs['stdinFlag']
  30             if stdinFlag:
  31                 args.append(stdinFlag)
  32             args.append(self._stdin_path)
  33             del kwargs['stdinFlag']
  34             stdin = None
  35         except KeyError: # just make it read this stream
  36             stdin = self.stdin
  37         try: # add as a file argument
  38             stdoutFlag = kwargs['stdoutFlag']
  39             if stdoutFlag:
  40                 args.append(stdoutFlag)
  41             args.append(self._stdout_path)
  42             del kwargs['stdoutFlag']
  43             stdout = None
  44         except KeyError: # make it write to this stream
  45             stdout = self.stdout
  46         self.args = (args, bufsize, executable, stdin, stdout,
  47                      self.stderr) + largs
  48         self.kwargs = kwargs
  49     def _get_pipe_file(self, ifile, attr):
  50         'create a temp filename instead of a PIPE; save the filename'
  51         if ifile == PIPE: # use temp file instead!
  52             fd, path = tempfile.mkstemp()
  53             setattr(self, '_' + attr + '_path', path)
  54             return os.fdopen(fd,'w+b'), True
  55         elif ifile is not None:
  56             setattr(self, '_' + attr + '_path', ifile.name)
  57         return ifile, False
  58     def _close_file(self, attr):
  59         'close and delete this temp file if still open'
  60         if getattr(self, '_close_' + attr):
  61             getattr(self, attr).close()
  62             setattr(self, '_close_' + attr, False)
  63             os.remove(getattr(self, '_' + attr + '_path'))
  64     def _rewind_for_reading(self, ifile):
  65         if ifile is not None:
  66             ifile.flush()
  67             ifile.seek(0)
  68     def close(self):
  69         """Close any open temp (PIPE) files. """
  70         self._close_file('stdin')
  71         self._close_file('stdout')
  72         self._close_file('stderr')
  73     def __del__(self):
  74         self.close()
  75
  76 try:
  77     import subprocess
  78     PIPE = subprocess.PIPE
  79     class FilePopen(FilePopenBase):
  80         'this subclass uses the subprocess module Popen() functionality'
  81         def wait(self):
  82             self._rewind_for_reading(self.stdin)
  83             p = subprocess.Popen(*self.args, **self.kwargs)
  84             p.wait()
  85             self._close_file('stdin')
  86             self._rewind_for_reading(self.stdout)
  87             self._rewind_for_reading(self.stderr)
  88             return p.returncode
  89
  90 except ImportError:
  91     CSH_REDIRECT = False # SH style redirection is default
  92     import platform
  93     if platform.system() == 'Windows':
  94         def mkarg(arg):
  95             """Very basic quoting of arguments for Windows """
  96             return '"' + arg + '"'
  97     else: # UNIX
  98         from commands import mkarg
  99         try:
 100             if os.environ['SHELL'].endswith('csh'):
 101                 CSH_REDIRECT = True
 102         except KeyError:
 103             pass
 104
 105     class FilePopen(FilePopenBase):
 106         'this subclass fakes subprocess.Popen.wait() using os.system()'
 107         def wait(self):
 108             self._rewind_for_reading(self.stdin)
 109             args = map(mkarg, self.args[0])
 110             if self.args[3]: # redirect stdin
 111                 args += ['<', mkarg(self._stdin_path)]
 112             if self.args[4]: # redirect stdout
 113                 args += ['>', mkarg(self._stdout_path)]
 114             cmd = ' '.join(args)
 115             if self.args[5]: # redirect stderr
 116                 if CSH_REDIRECT:
 117                     cmd = '(%s) >& %s' % (cmd, mkarg(self._stderr_path))
 118                 else:
 119                     cmd = cmd + ' 2> ' + mkarg(self._stderr_path)
 120             returncode = os.system(cmd)
 121             self._close_file('stdin')
 122             self._rewind_for_reading(self.stdout)
 123             self._rewind_for_reading(self.stderr)
 124             return returncode
 125     PIPE = id(FilePopen) # an arbitrary code for identifying this code
 126
 127 def call_subprocess(*popenargs, **kwargs):
 128     'portable interface to subprocess.call(), even if subprocess not available'
 129     p = FilePopen(*popenargs, **kwargs)
 130     return p.wait()
 131
 132
 133 def ClassicUnpickler(cls, state):
 134     'standard python unpickling behavior'
 135     self = cls.__new__(cls)
 136     try:
 137         setstate = self.__setstate__
 138     except AttributeError: # JUST SAVE TO __dict__ AS USUAL
 139         self.__dict__.update(state)
 140     else:
 141         setstate(state)
 142     return self
 143 ClassicUnpickler.__safe_for_unpickling__ = 1
 144
 145
 146 def filename_unpickler(cls,path,kwargs):
 147     'raise IOError if path not readable'
 148     if not os.path.exists(path):
 149         try:# CONVERT TO ABSOLUTE PATH BASED ON SAVED DIRECTORY PATH
 150             path = os.path.normpath(os.path.join(kwargs['curdir'], path))
 151             if not os.path.exists(path):
 152                 raise IOError('unable to open file %s' % path)
 153         except KeyError:
 154             raise IOError('unable to open file %s' % path)
 155     if cls is SourceFileName:
 156         return SourceFileName(path)
 157     raise ValueError('Attempt to unpickle untrusted class ' + cls.__name__)
 158 filename_unpickler.__safe_for_unpickling__ = 1
 159
 160 class SourceFileName(str):
 161     '''store a filepath string, raise IOError on unpickling
 162 if filepath not readable, and complain if the user tries
 163 to pickle a relative path'''
 164     def __reduce__(self):
 165         if not os.path.isabs(str(self)):
 166             print >>sys.stderr,'''
 167 WARNING: You are trying to pickle an object that has a local
 168 file dependency stored only as a relative file path:
 169 %s
 170 This is not a good idea, because anyone working in
 171 a different directory would be unable to unpickle this object,
 172 since it would be unable to find the file using the relative path.
 173 To avoid this problem, SourceFileName is saving the current
 174 working directory path so that it can translate the relative
 175 path to an absolute path.  In the future, please use absolute
 176 paths when constructing objects that you intend to save to worldbase
 177 or pickle!''' % str(self)
 178         return (filename_unpickler,(self.__class__,str(self),
 179                                     dict(curdir=os.getcwd())))
 180
 181 def file_dirpath(filename):
 182     'return path to directory containing filename'
 183     dirname = os.path.dirname(filename)
 184     if dirname=='':
 185         return os.curdir
 186     else:
 187         return dirname
 188
 189 def get_valid_path(*pathTuples):
 190     '''for each tuple in args, build path using os.path.join(),
 191     and return the first path that actually exists, or else None.'''
 192     for t in pathTuples:
 193         mypath = os.path.join(*t)
 194         if os.path.exists(mypath):
 195             return mypath
 196
 197 def search_dirs_for_file(filepath, pathlist=()):
 198     'return successful path based on trying pathlist locations'
 199     if os.path.exists(filepath):
 200         return filepath
 201     b = os.path.basename(filepath)
 202     for s in pathlist: # NOW TRY EACH DIRECTORY IN pathlist
 203         mypath = os.path.join(s,b)
 204         if os.path.exists(mypath):
 205             return mypath
 206     raise IOError('unable to open %s from any location in %s'
 207                   %(filepath,pathlist))
 208
 209 def report_exception():
 210     'print string message from exception to stderr'
 211     import traceback
 212     info = sys.exc_info()[:2]
 213     l = traceback.format_exception_only(info[0],info[1])
 214     print >>sys.stderr,'Warning: caught %s\nContinuing...' % l[0]
 215
 216 def standard_invert(self):
 217     'keep a reference to an inverse mapping, using self._inverseClass'
 218     try:
 219         return self._inverse
 220     except AttributeError:
 221         self._inverse = self._inverseClass(self)
 222         return self._inverse
 223
 224 def lazy_create_invert(klass):
 225     """Create a function to replace __invert__ with a call to a cached object.
 226
 227     lazy_create_invert defines a method that looks up self._inverseObj
 228     and, it it doesn't exist, creates it from 'klass' and then saves it.
 229     The resulting object is then returned as the inverse.  This allows
 230     for one-time lazy creation of a single object per parent class.
 231
 232     """
 233     def invert_fn(self, klass=klass):
 234         try:
 235             return self._inverse
 236         except AttributeError:
 237             # does not exist yet; create & store.
 238             inverseObj = klass(self)
 239             self._inverse = inverseObj
 240             return inverseObj
 241
 242     return invert_fn
 243
 244 def standard_getstate(self):
 245     'get dict of attributes to save, using self._pickleAttrs dictionary'
 246     d={}
 247     for attr,arg in self._pickleAttrs.items():
 248         try:
 249             if isinstance(arg,str):
 250                 d[arg] = getattr(self,attr)
 251             else:
 252                 d[attr] = getattr(self,attr)
 253         except AttributeError:
 254             pass
 255     try: # DON'T SAVE itemClass IF SIMPLY A SHADOW of default itemClass from __class__
 256         if not hasattr(self.__class__,'itemClass') or \
 257            (self.itemClass is not self.__class__.itemClass and
 258             (not hasattr(self.itemClass,'_shadowParent') or
 259              self.itemClass._shadowParent is not self.__class__.itemClass)):
 260             try:
 261                 d['itemClass'] = self.itemClass._shadowParent
 262             except AttributeError:
 263                 d['itemClass'] = self.itemClass
 264         if not hasattr(self.__class__,'itemSliceClass') or \
 265            (self.itemSliceClass is not self.__class__.itemSliceClass and
 266             (not hasattr(self.itemSliceClass,'_shadowParent') or
 267              self.itemSliceClass._shadowParent is not self.__class__.itemSliceClass)):
 268             try:
 269                 d['itemSliceClass'] = self.itemSliceClass._shadowParent
 270             except AttributeError:
 271                 d['itemSliceClass'] = self.itemSliceClass
 272     except AttributeError:
 273         pass
 274     try: # SAVE CUSTOM UNPACKING METHOD
 275         d['unpack_edge'] = self.__dict__['unpack_edge']
 276     except KeyError:
 277         pass
 278     return d
 279
 280
 281 def standard_setstate(self,state):
 282     'apply dict of saved state by passing as kwargs to constructor'
 283     if isinstance(state,list):  # GET RID OF THIS BACKWARDS-COMPATIBILITY CODE!
 284         self.__init__(*state)
 285         print >>sys.stderr,'WARNING: obsolete list pickle %s. Update by resaving!' \
 286               % repr(self)
 287     else:
 288         state['unpicklingMode'] = True # SIGNAL THAT WE ARE UNPICKLING
 289         self.__init__(**state)
 290
 291 def apply_itemclass(self,state):
 292     try:
 293         self.itemClass = state['itemClass']
 294         self.itemSliceClass = state['itemSliceClass']
 295     except KeyError:
 296         pass
 297
 298 def generate_items(items):
 299     'generate o.id,o for o in items'
 300     for o in items:
 301         yield o.id,o
 302
 303 def item_unpickler(db,*args):
 304     'get an item or subslice of a database'
 305     obj = db
 306     for arg in args:
 307         obj = obj[arg]
 308     return obj
 309 item_unpickler.__safe_for_unpickling__ = 1
 310
 311
 312 def item_reducer(self): ############################# SUPPORT FOR PICKLING
 313     'pickle an item of a database just as a reference'
 314     return (item_unpickler, (self.db,self.id))
 315
 316 def shadow_reducer(self):
 317     'pickle shadow class instance using its true class'
 318     shadowClass = self.__class__
 319     trueClass = shadowClass._shadowParent # super() TOTALLY FAILED ME HERE!
 320     self.__class__ = trueClass # FORCE IT TO PICKLE USING trueClass
 321     keepDB = False
 322     if hasattr(shadowClass, 'db') and not hasattr(self, 'db'):
 323         keepDB = True
 324         self.__dict__['db'] = shadowClass.db # retain this attribute!!
 325     if hasattr(trueClass,'__reduce__'): # USE trueClass.__reduce__
 326         result = trueClass.__reduce__(self)
 327     elif hasattr(trueClass,'__getstate__'): # USE trueClass.__getstate__
 328         result = (ClassicUnpickler,(trueClass,self.__getstate__()))
 329     else: # PICKLE __dict__ AS USUAL PYTHON PRACTICE
 330         result = (ClassicUnpickler,(trueClass,self.__dict__))
 331     self.__class__ = shadowClass # RESTORE SHADOW CLASS
 332     if keepDB: # now we can drop the temporary db attribute we added
 333         del self.__dict__['db']
 334     return result
 335
 336
 337 def get_bound_subclass(obj, classattr='__class__', subname=None, factories=(),
 338                        attrDict=None, subclassArgs=None):
 339     'create a subclass specifically for obj to bind its shadow attributes'
 340     targetClass = getattr(obj,classattr)
 341     try:
 342         if targetClass._shadowOwner is obj:
 343             return targetClass # already shadowed, so nothing to do
 344     except AttributeError: # not a shadow class, so just shadow it
 345         pass
 346     else: # someone else's shadow class, so shadow its parent
 347         targetClass = targetClass._shadowParent
 348     if subname is None: # get a name from worldbase ID
 349         try:
 350             subname = obj._persistent_id.split('.')[-1]
 351         except AttributeError:
 352             subname = '__generic__'
 353     class shadowClass(targetClass):
 354         __reduce__ = shadow_reducer
 355         _shadowParent = targetClass # NEED TO KNOW ORIGINAL CLASS
 356         _shadowOwner = obj # need to know who owns it
 357         if attrDict is not None: # add attributes to the class dictionary
 358             locals().update(attrDict)
 359         for f in factories:
 360             f(locals())
 361     if classattr == 'itemClass' or classattr == 'itemSliceClass':
 362         shadowClass.db = obj # the class needs to know its db object
 363     try: # run subclass initializer if present
 364         subclass_init = shadowClass._init_subclass
 365     except AttributeError: # no subclass initializer, so nothing to do
 366         pass
 367     else: # run the subclass initializer
 368         if subclassArgs is None:
 369             subclassArgs = {}
 370         subclass_init(**subclassArgs)
 371     shadowClass.__name__ = targetClass.__name__ + '_' + subname
 372     setattr(obj,classattr,shadowClass) # SHADOW CLASS REPLACES ORIGINAL
 373     return shadowClass
 374
 375 def method_not_implemented(*args,**kwargs):
 376     raise NotImplementedError
 377 def read_only_error(*args, **kwargs):
 378     raise NotImplementedError("read only dict")
 379
 380 def methodFactory(methodList, methodStr, localDict):
 381     'save a method or exec expression for each name in methodList'
 382     for methodName in methodList:
 383         if callable(methodStr):
 384             localDict[methodName] = methodStr
 385         else:
 386             localDict[methodName]=eval(methodStr%methodName)
 387
 388 def open_shelve(filename, mode=None, writeback=False, allowReadOnly=False,
 389                 useHash=False, verbose=True):
 390     '''Alternative to shelve.open with several benefits:
 391 - uses bsddb btree by default instead of bsddb hash, which is very slow
 392   for large databases.  Will automatically fall back to using bsddb hash
 393   for existing hash-based shelve files.  Set useHash=True to force it to use bsddb hash.
 394
 395 - allowReadOnly=True will automatically suppress permissions errors so
 396   user can at least get read-only access to the desired shelve, if no write permission.
 397
 398 - mode=None first attempts to open file in read-only mode, but if the file
 399   does not exist, opens it in create mode.
 400
 401 - raises standard exceptions defined in dbfile: WrongFormatError, PermissionsError,
 402   ReadOnlyError, NoSuchFileError
 403
 404 - avoids generating bogus __del__ warnings as Python shelve.open() does.
 405   '''
 406     if mode=='r': # READ-ONLY MODE, RAISE EXCEPTION IF NOT FOUND
 407         return dbfile.shelve_open(filename, flag=mode, useHash=useHash)
 408     elif mode is None:
 409         try: # 1ST TRY READ-ONLY, BUT IF NOT FOUND CREATE AUTOMATICALLY
 410             return dbfile.shelve_open(filename, 'r', useHash=useHash)
 411         except dbfile.NoSuchFileError:
 412             mode = 'c' # CREATE NEW SHELVE FOR THE USER
 413     try: # CREATION / WRITING: FORCE IT TO WRITEBACK AT close() IF REQUESTED
 414         return dbfile.shelve_open(filename, flag=mode, writeback=writeback,
 415                                   useHash=useHash)
 416     except dbfile.ReadOnlyError:
 417         if allowReadOnly:
 418             d = dbfile.shelve_open(filename, flag='r', useHash=useHash)
 419             if verbose:
 420                 logger.warn('''
 421 Opening shelve file %s in read-only mode because you lack
 422 write permissions to this file.  You will NOT be able to modify the contents
 423 of this shelve dictionary.  To avoid seeing this warning message,
 424 use verbose=False argument for the classutil.open_shelve() function.
 425 ''' % filename)
 426             return d
 427         else:
 428             raise
 429
 430
 431 def get_shelve_or_dict(filename=None,dictClass=None,**kwargs):
 432     if filename is not None:
 433         if dictClass is not None:
 434             return dictClass(filename,**kwargs)
 435         else:
 436             from mapping import IntShelve
 437             return IntShelve(filename,**kwargs)
 438     return {}
 439
 440
 441 class PathSaver(object):
 442     def __init__(self,origPath):
 443         self.origPath = origPath
 444         self.origDir = os.getcwd()
 445     def __str__(self):
 446         if os.access(self.origPath,os.R_OK):
 447             return self.origPath
 448         trypath = os.path.join(self.origDir,self.origPath)
 449         if os.access(trypath,os.R_OK):
 450             return trypath
 451
 452 def override_rich_cmp(localDict):
 453     'create rich comparison methods that just use __cmp__'
 454     mycmp = localDict['__cmp__']
 455     localDict['__lt__'] = lambda self,other: mycmp(self,other)<0
 456     localDict['__le__'] = lambda self,other: mycmp(self,other)<=0
 457     localDict['__eq__'] = lambda self,other: mycmp(self,other)==0
 458     localDict['__ne__'] = lambda self,other: mycmp(self,other)!=0
 459     localDict['__gt__'] = lambda self,other: mycmp(self,other)>0
 460     localDict['__ge__'] = lambda self,other: mycmp(self,other)>=0
 461
 462
 463 class DBAttributeDescr(object):
 464     'obtain an attribute from associated db object'
 465     def __init__(self,attr):
 466         self.attr = attr
 467     def __get__(self,obj,objtype):
 468         return getattr(obj.db,self.attr)
 469
 470 def get_env_or_cwd(envname):
 471     'get the specified environment value or path to current directory'
 472     try:
 473         return os.environ[envname] # USER-SPECIFIED DIRECTORY
 474     except KeyError:
 475         return os.getcwd() # DEFAULT: SAVE IN CURRENT DIRECTORY
 476
 477
 478 class RecentValueDictionary(WeakValueDictionary):
 479     '''keep the most recent n references in a WeakValueDictionary.
 480     This combines the elegant cache behavior of a WeakValueDictionary
 481     (only keep an item in cache if the user is still using it),
 482     with the most common efficiency pattern: locality, i.e.
 483     references to a given thing tend to cluster in time.  Note that
 484     this works *even* if the user is not holding a reference to
 485     the item between requests for it.  Our Most Recent queue will
 486     hold a reference to it, keeping it in the WeakValueDictionary,
 487     until it is bumped by more recent requests.
 488
 489     n: the maximum number of objects to keep in the Most Recent queue,
 490        default value 50.'''
 491     def __init__(self, n=None):
 492         WeakValueDictionary.__init__(self)
 493         if n<1: # user doesn't want any Most Recent value queue
 494             self.__class__ = WeakValueDictionary # revert to regular WVD
 495             return
 496         if isinstance(n, int):
 497             self.n = n # size limit
 498         else:
 499             self.n = 50
 500         self.i = 0 # counter
 501         self._keepDict = {} # most recent queue
 502     def __getitem__(self, k):
 503         v = WeakValueDictionary.__getitem__(self, k) # KeyError if not found
 504         self.keep_this(v)
 505         return v
 506     def keep_this(self, v):
 507         'add v as our most recent ref; drop oldest ref if over size limit'
 508         self._keepDict[v] = self.i # mark as most recent request
 509         self.i += 1
 510         if len(self._keepDict)>self.n: # delete oldest entry
 511             l = self._keepDict.items()
 512             imin = l[0][1]
 513             vmin = l[0][0]
 514             for v,i in l[1:]:
 515                 if i<imin:
 516                     imin = i
 517                     vmin = v
 518             del self._keepDict[vmin]
 519     def __setitem__(self, k, v):
 520         WeakValueDictionary.__setitem__(self, k, v)
 521         self.keep_this(v)
 522     def clear(self):
 523         self._keepDict.clear()
 524         WeakValueDictionary.clear(self)
 525
 526
 527 def make_attribute_interface(d):
 528     """
 529     If 'd' contains int values, use them to index tuples.
 530
 531     If 'd' contains str values, use them to retrieve attributes from an obj.
 532
 533     If d empty, use standard 'getattr'.
 534     """
 535     if len(d):
 536         v = d.values()[0]
 537         if isinstance(v, int):
 538             return AttrFromTuple(d)
 539         elif isinstance(v, str):
 540             return AttrFromObject(d)
 541         raise ValueError('dictionary values must be int or str!')
 542
 543     return getattr
 544
 545 class AttrFromTuple(object):
 546     def __init__(self, attrDict):
 547         self.attrDict = attrDict
 548
 549     def __call__(self, obj, attr, default=None):
 550         'getattr from tuple obj'
 551         try:
 552             return obj[self.attrDict[attr]]
 553         except (IndexError, KeyError):
 554             if default is not None:
 555                 return default
 556         raise AttributeError("object has no attribute '%s'" % attr)
 557
 558 class AttrFromObject(AttrFromTuple):
 559     def __call__(self, obj, attr, default=None):
 560         'getattr with attribute name aliases'
 561         try:
 562             return getattr(obj, self.attrDict[attr])
 563         except KeyError:
 564             try:
 565                 return getattr(obj, attr)
 566             except KeyError:
 567                 if default is not None:
 568                     return default
 569         raise AttributeError("object has no attribute '%s'" % attr)
 570
 571 def kwargs_filter(kwargs, allowed):
 572     'return dictionary of kwargs filtered by list allowed'
 573     d = {}
 574     for arg in allowed:
 575         try:
 576             d[arg] = kwargs[arg]
 577         except KeyError:
 578             pass
 579     return d
 580