python/google/appengine/ext/mapreduce/util.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33 """Utility functions for use with the mapreduce library."""
  34
  35
  36
  37
  38 __all__ = [
  39     "create_datastore_write_config",
  40     "for_name",
  41     "get_short_name",
  42     "handler_for_name",
  43     "is_generator",
  44     "parse_bool",
  45     "total_seconds",
  46     "try_serialize_handler",
  47     "try_deserialize_handler",
  48     ]
  49
  50 import inspect
  51 import pickle
  52 import types
  53
  54 from google.appengine.datastore import datastore_rpc
  55
  56
  57 def total_seconds(td):
  58   """convert a timedelta to seconds.
  59
  60   This is patterned after timedelta.total_seconds, which is only
  61   available in python 27.
  62
  63   Args:
  64     td: a timedelta object.
  65
  66   Returns:
  67     total seconds within a timedelta. Rounded up to seconds.
  68   """
  69   secs = td.seconds + td.days * 24 * 3600
  70   if td.microseconds:
  71     secs += 1
  72   return secs
  73
  74
  75 def for_name(fq_name, recursive=False):
  76   """Find class/function/method specified by its fully qualified name.
  77
  78   Fully qualified can be specified as:
  79     * <module_name>.<class_name>
  80     * <module_name>.<function_name>
  81     * <module_name>.<class_name>.<method_name> (an unbound method will be
  82       returned in this case).
  83
  84   for_name works by doing __import__ for <module_name>, and looks for
  85   <class_name>/<function_name> in module's __dict__/attrs. If fully qualified
  86   name doesn't contain '.', the current module will be used.
  87
  88   Args:
  89     fq_name: fully qualified name of something to find
  90
  91   Returns:
  92     class object.
  93
  94   Raises:
  95     ImportError: when specified module could not be loaded or the class
  96     was not found in the module.
  97   """
  98
  99
 100
 101   fq_name = str(fq_name)
 102   module_name = __name__
 103   short_name = fq_name
 104
 105   if fq_name.rfind(".") >= 0:
 106     (module_name, short_name) = (fq_name[:fq_name.rfind(".")],
 107                                  fq_name[fq_name.rfind(".") + 1:])
 108
 109   try:
 110     result = __import__(module_name, None, None, [short_name])
 111     return result.__dict__[short_name]
 112   except KeyError:
 113
 114
 115
 116
 117
 118     if recursive:
 119       raise
 120     else:
 121       raise ImportError("Could not find '%s' on path '%s'" % (
 122                         short_name, module_name))
 123   except ImportError:
 124
 125
 126     try:
 127       module = for_name(module_name, recursive=True)
 128       if hasattr(module, short_name):
 129         return getattr(module, short_name)
 130       else:
 131
 132         raise KeyError()
 133     except KeyError:
 134       raise ImportError("Could not find '%s' on path '%s'" % (
 135                         short_name, module_name))
 136     except ImportError:
 137
 138
 139       pass
 140
 141
 142     raise
 143
 144
 145 def handler_for_name(fq_name):
 146   """Resolves and instantiates handler by fully qualified name.
 147
 148   First resolves the name using for_name call. Then if it resolves to a class,
 149   instantiates a class, if it resolves to a method - instantiates the class and
 150   binds method to the instance.
 151
 152   Args:
 153     fq_name: fully qualified name of something to find.
 154
 155   Returns:
 156     handler instance which is ready to be called.
 157   """
 158   resolved_name = for_name(fq_name)
 159   if isinstance(resolved_name, (type, types.ClassType)):
 160
 161     return resolved_name()
 162   elif isinstance(resolved_name, types.MethodType):
 163
 164     return getattr(resolved_name.im_class(), resolved_name.__name__)
 165   else:
 166     return resolved_name
 167
 168
 169 def try_serialize_handler(handler):
 170   """Try to serialize map/reduce handler.
 171
 172   Args:
 173     handler: handler function/instance. Handler can be a function or an
 174       instance of a callable class. In the latter case, the handler will
 175       be serialized across slices to allow users to save states.
 176
 177   Returns:
 178     serialized handler string or None.
 179   """
 180   if (isinstance(handler, types.InstanceType) or
 181       (isinstance(handler, object) and
 182        not inspect.isfunction(handler) and
 183        not inspect.ismethod(handler)) and
 184       hasattr(handler, "__call__")):
 185     return pickle.dumps(handler)
 186   return None
 187
 188
 189 def try_deserialize_handler(serialized_handler):
 190   """Reverse function of try_serialize_handler.
 191
 192   Args:
 193     serialized_handler: serialized handler str or None.
 194
 195   Returns:
 196     handler instance or None.
 197   """
 198   if serialized_handler:
 199     return pickle.loads(serialized_handler)
 200
 201
 202 def is_generator(obj):
 203   """Return true if the object is generator or generator function.
 204
 205   Generator function objects provides same attributes as functions.
 206   See isfunction.__doc__ for attributes listing.
 207
 208   Adapted from Python 2.6.
 209
 210   Args:
 211     obj: an object to test.
 212
 213   Returns:
 214     true if the object is generator function.
 215   """
 216   if isinstance(obj, types.GeneratorType):
 217     return True
 218
 219   CO_GENERATOR = 0x20
 220   return bool(((inspect.isfunction(obj) or inspect.ismethod(obj)) and
 221                obj.func_code.co_flags & CO_GENERATOR))
 222
 223
 224 def get_short_name(fq_name):
 225   """Returns the last component of the name."""
 226   return fq_name.split(".")[-1:][0]
 227
 228
 229 def parse_bool(obj):
 230   """Return true if the object represents a truth value, false otherwise.
 231
 232   For bool and numeric objects, uses Python's built-in bool function.  For
 233   str objects, checks string against a list of possible truth values.
 234
 235   Args:
 236     obj: object to determine boolean value of; expected
 237
 238   Returns:
 239     Boolean value according to 5.1 of Python docs if object is not a str
 240       object.  For str objects, return True if str is in TRUTH_VALUE_SET
 241       and False otherwise.
 242     http://docs.python.org/library/stdtypes.html
 243   """
 244   if type(obj) is str:
 245     TRUTH_VALUE_SET = ["true", "1", "yes", "t", "on"]
 246     return obj.lower() in TRUTH_VALUE_SET
 247   else:
 248     return bool(obj)
 249
 250
 251 def create_datastore_write_config(mapreduce_spec):
 252   """Creates datastore config to use in write operations.
 253
 254   Args:
 255     mapreduce_spec: current mapreduce specification as MapreduceSpec.
 256
 257   Returns:
 258     an instance of datastore_rpc.Configuration to use for all write
 259     operations in the mapreduce.
 260   """
 261   force_writes = parse_bool(mapreduce_spec.params.get("force_writes", "false"))
 262   if force_writes:
 263     return datastore_rpc.Configuration(force_writes=force_writes)
 264   else:
 265
 266     return datastore_rpc.Configuration()