scripts/stats.py

   1 #!/usr/bin/python2.5
   2 #
   3 # Copyright 2009 the Melange authors.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #   http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 """Starts an interactive shell with statistic helpers.
  18 """
  19
  20 __authors__ = [
  21   '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  22 ]
  23
  24
  25 import cPickle
  26 import datetime
  27 import operator
  28 import sys
  29
  30 import interactive
  31
  32
  33 def dateFetch(queryGen, last=None, batchSize=100):
  34   """Iterator that yields an entity in batches.
  35
  36   Args:
  37     queryGen: should return a Query object
  38     last: used to .filter() for last_modified_on
  39     batchSize: how many entities to retrieve in one datastore call
  40
  41   Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
  42   """
  43
  44   from google.appengine.ext import db
  45
  46    # AppEngine will not fetch more than 1000 results
  47   batchSize = min(batchSize,1000)
  48
  49   query = None
  50   done = False
  51   count = 0
  52
  53   while not done:
  54     print count
  55     query = queryGen()
  56     query.order('last_modified_on')
  57     if last:
  58       query.filter("last_modified_on > ", last)
  59     results = query.fetch(batchSize)
  60     for result in results:
  61       count += 1
  62       yield result
  63     if batchSize > len(results):
  64       done = True
  65     else:
  66       last = results[-1].last_modified_on
  67
  68
  69 def addKey(target, fieldname):
  70   """Adds the key of the specified field.
  71   """
  72
  73   result = target.copy()
  74   result['%s_key' % fieldname] = target[fieldname].key().name()
  75   return result
  76
  77
  78 def getEntities(model):
  79   """Returns all users as dictionary.
  80   """
  81
  82   def wrapped():
  83     gen = lambda: model.all()
  84     it = interactive.deepFetch(gen)
  85
  86     entities = [(i.key().name(), i) for i in it]
  87     return dict(entities)
  88
  89   return wrapped
  90
  91
  92 def getProps(last=None):
  93   """Returns all proposals as a list of dictionaries.
  94   """
  95
  96   key_order = [
  97       'link_id', 'scope_path', 'title', 'abstract', 'content',
  98       'additional_info', '_mentor', 'possible_mentors', 'score',
  99       'status', '_org', 'created_on', 'last_modified_on']
 100
 101   from soc.models.student_proposal import StudentProposal
 102
 103   gen = lambda: StudentProposal.all()
 104
 105   it = dateFetch(gen, last)
 106
 107   proposals = [(i.key().name(), i.toDict(key_order)) for i in it]
 108   if proposals:
 109     last = i.last_modified_on # last modified entity
 110   else:
 111     last = datetime.datetime.now()
 112
 113   return dict(proposals), last
 114
 115
 116 def orgStats(target, orgs):
 117   """Retrieves org stats.
 118   """
 119
 120   from soc.logic import dicts
 121
 122   orgs = [(v.key(), v) for k, v in orgs.iteritems()]
 123   orgs = dict(orgs)
 124
 125   grouped = dicts.groupby(target.values(), '_org')
 126
 127   grouped = [(orgs[k], v) for k, v in grouped.iteritems()]
 128   popularity = [(k.link_id, len(v)) for k, v in grouped]
 129
 130   return dict(grouped), dict(popularity)
 131
 132
 133 def countStudentsWithProposals():
 134   """Retrieves number of Students who have submitted at least one Student Proposal.
 135   """
 136
 137   proposals = getStudentProposals()
 138   students = {}
 139
 140   for proposal_key in proposals.keys():
 141     students[proposals[proposal_key].scope_path] = True
 142
 143   return len(students)
 144
 145
 146 def printPopularity(popularity):
 147   """Prints the popularity for the specified proposals.
 148   """
 149
 150   g = operator.itemgetter(1)
 151
 152   for item in sorted(popularity.iteritems(), key=g, reverse=True):
 153     print "%s: %d" % item
 154
 155
 156 def saveValues(values, saver):
 157   """Saves the specified popularities.
 158   """
 159
 160   import logging
 161   from google.appengine.ext import db
 162
 163   from soc.models.organization import Organization
 164
 165   def txn(key, value):
 166     org = Organization.get_by_key_name(key)
 167     saver(org, value)
 168     org.put()
 169
 170   for key, value in sorted(values.iteritems()):
 171     print key
 172     db.run_in_transaction_custom_retries(10, txn, key, value)
 173
 174   print "done"
 175
 176
 177 def addFollower(follower, proposals, add_public=True, add_private=True):
 178   """Adds a user as follower to the specified proposals.
 179
 180   Args:
 181     follower: the User to add as follower
 182     proposals: a list with the StudnetProposals that should be subscribed to
 183     add_public: whether the user is subscribed to public updates
 184     add_private: whether the user should be subscribed to private updates
 185   """
 186
 187   from soc.models.review_follower import ReviewFollower
 188
 189   result = []
 190
 191   for i in proposals:
 192      properties = {
 193        'user': follower,
 194        'link_id': follower.link_id,
 195        'scope': i,
 196        'scope_path': i.key().name(),
 197        'key_name': '%s/%s' % (i.key().name(), follower.link_id),
 198        'subscribed_public': add_public,
 199        'subscribed_private': add_private,
 200      }
 201
 202      entity = ReviewFollower(**properties)
 203      result.append(entity)
 204
 205   return result
 206
 207
 208 def loadPickle(name):
 209   """Loads a pickle.
 210   """
 211
 212   f = open(name + '.dat')
 213   return cPickle.load(f)
 214
 215
 216 def dumpPickle(target, name):
 217   """Dumps a pickle.
 218   """
 219
 220   f = open("%s.dat" % name, 'w')
 221   cPickle.dump(target, f)
 222
 223
 224 def main(args):
 225   """Main routine.
 226   """
 227
 228   interactive.setup()
 229
 230   from soc.models.organization import Organization
 231   from soc.models.user import User
 232   from soc.models.student import Student
 233   from soc.models.mentor import Mentor
 234   from soc.models.org_admin import OrgAdmin
 235
 236   def slotSaver(org, value):
 237     org.slots = value
 238   def popSaver(org, value):
 239     org.nr_applications = value
 240   def rawSaver(org, value):
 241     org.slots_calculated = value
 242
 243   context = {
 244       'load': loadPickle,
 245       'dump': dumpPickle,
 246       'orgStats': orgStats,
 247       'printPopularity': printPopularity,
 248       'saveValues': saveValues,
 249       'getOrgs': getEntities(Organization),
 250       'getUsers': getEntities(User),
 251       'getStudents': getEntities(Student),
 252       'getMentors': getEntities(Mentor),
 253       'getOrgAdmins': getEntities(OrgAdmin),
 254       'getProps': getProps,
 255       'countStudentsWithProposals': countStudentsWithProposals,
 256       'addFollower': addFollower,
 257       'Organization': Organization,
 258       'User': User,
 259       'Student': Student,
 260       'Mentor': Mentor,
 261       'OrgAdmin': OrgAdmin,
 262       'slotSaver': slotSaver,
 263       'popSaver': popSaver,
 264       'rawSaver': rawSaver,
 265   }
 266
 267   interactive.remote(args, context)
 268
 269 if __name__ == '__main__':
 270   if len(sys.argv) < 2:
 271     print "Usage: %s app_id [host]" % (sys.argv[0],)
 272     sys.exit(1)
 273
 274   main(sys.argv[1:])