recover-unapplied-writes/apply_unapplied_writes.py

   1 #!/usr/bin/python2.4
   2 #
   3 # Copyright 2010 Google Inc.
   4
   5 """Methods to fix unapplied writes.
   6
   7 Unapplied writes are stored in the datastore as entities where the
   8 child node of the entity key has kind __unapplied_write__OriginalKind.
   9
  10 This module contains methods for applying those writes.
  11
  12 Currently it contains methods which work at the datastore low level API.
  13
  14 Sample usage using remote_api:
  15   * Enable remote_api for your application
  16     (see http://code.google.com/appengine/articles/remote_api.html )
  17   * Change to a directory where this module exists.
  18   * Run remote_api_shell.py $YOURAPPID
  19   * In the remote_api_shell, run the following:
  20     import apply_unapplied_writes
  21     apply_unapplied_writes.apply_entity_by_name(YOURKIND, A KEY NAME OR ID)
  22
  23
  24 Sample usage using the map reduce framework:
  25   * Install the map reduce framework in your app.
  26     (see http://code.google.com/p/appengine-mapreduce/ )
  27   * Add this module to your app.
  28   * Add the following to the mapreduce.yaml file:
  29       mapreduce:
  30       - name: "Apply unapplied writes"
  31         mapper:
  32           input_reader: mapreduce.input_readers.DatastoreInputReader
  33           handler: apply_unapplied_writes.apply_model_instance
  34           params:
  35           - name: entity_kind
  36   * Visit http://YOURAPP.appspot.com/mapreduce/ ,
  37   * Select 'Unapplied Entity Map' from the dropdown.
  38   * Enter the kind you wish to apply.
  39   * Click Launch Job.
  40 """
  41
  42 import logging
  43 from google.appengine.api import datastore
  44
  45
  46 UNAPPLIED_WRITE_KIND_PREFIX = '__unapplied_write__'
  47 UNAPPLIED_WRITE_KIND_PREFIX_LEN = len(UNAPPLIED_WRITE_KIND_PREFIX)
  48
  49
  50 def apply_entity(unapplied_entity, delete_unapplied_entity=True):
  51   """Re-write an entity representing an unapplied write  to apply it.
  52
  53   Args:
  54     entity: An app engine datastore entity, typically loaded by datastore.Get.
  55       This will not work for a model instance, e.g. one loaded from db.get.
  56     delete_unapplied_entity: If true, the record of the unapplied write will
  57         be removed from the datastore.
  58   """
  59   key = unapplied_entity.key()
  60   path = unapplied_entity.key().to_path()
  61   kind = path[-2]
  62   if not kind.startswith(UNAPPLIED_WRITE_KIND_PREFIX):
  63     logging.Error("Attempting to apply an already applied write: %r", key)
  64     return
  65   kind = kind[UNAPPLIED_WRITE_KIND_PREFIX_LEN:]
  66   id_or_name = path[-1]
  67   namespace = unapplied_entity.namespace()
  68
  69   # You can insert code here to change id_or_name.
  70
  71   if isinstance(id_or_name, basestring):
  72     entity_to_apply = datastore.Entity(kind, key.parent(), name=id_or_name,
  73                                       namespace=namespace)
  74   elif id_or_name:
  75     entity_to_apply = datastore.Entity(kind, key.parent(), id=id_or_name,
  76                                        namespace=namespace)
  77   else:
  78     entity_to_apply = datastore.Entity(kind, key.parent(),
  79                                        namespace=namespace)
  80   entity_to_apply.update(unapplied_entity)
  81
  82   # You can insert code here to change entity_to_apply.
  83
  84   datastore.Put(entity_to_apply)
  85   if delete_unapplied_entity:
  86     datastore.Delete(unapplied_entity)
  87
  88
  89 def apply_entity_by_name(kind, id_or_name, parent=None,
  90                          delete_unapplied_entity=True):
  91   """Apply an unapplied write for a given kind and id or name.
  92
  93   This will load and apply an unapplied write for the identified
  94   entity.
  95
  96   Args:
  97     kind: The kind of the entity to apply.
  98     id_or_name: The numeric ID or string name of the entity to find and apply.
  99     parent: Parent key for the entity to apply.
 100     delete_unapplied_entity: If true, the record of the unapplied write will
 101         be removed from the datastore.
 102   """
 103
 104   if parent:
 105     path = parent.to_path()
 106   else:
 107     path = []
 108   path += [UNAPPLIED_WRITE_KIND_PREFIX + kind, id_or_name]
 109   key = datastore.Key.from_path(*path)
 110   unapplied_entity = datastore.Get(key)
 111   apply_entity(unapplied_entity, delete_unapplied_entity)
 112
 113
 114 def apply_model_instance(model_instance, delete_unapplied_entity=True):
 115   """Apply an unapplied write from a model instance.
 116
 117   This is a wrapper for apply_entity, suitable for use with the
 118   App Engine mapper framework.
 119
 120   Args:
 121    model_instance. This is typically the result of a db.get or db.Query.get.
 122    delete_unapplied_entity: If true, the record of the unapplied write will
 123         be removed from the datastore.
 124   """
 125   unapplied_entity = model_instance._populate_entity()
 126   apply_entity(unapplied_entity, delete_unapplied_entity)
 127