3 # Copyright 2009 the Melange authors.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """Starts an interactive shell with statistic helpers.
21 '"Sverre Rabbelier" <sverre@rabbelier.nl>',
34 def dateFetch(queryGen
, last
=None, batchSize
=100):
35 """Iterator that yields an entity in batches.
38 queryGen: should return a Query object
39 last: used to .filter() for last_modified_on
40 batchSize: how many entities to retrieve in one datastore call
42 Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
45 from google
.appengine
.ext
import db
47 # AppEngine will not fetch more than 1000 results
48 batchSize
= min(batchSize
,1000)
57 query
.order('last_modified_on')
59 query
.filter("last_modified_on > ", last
)
60 results
= query
.fetch(batchSize
)
61 for result
in results
:
64 if batchSize
> len(results
):
67 last
= results
[-1].last_modified_on
70 def addKey(target
, fieldname
):
71 """Adds the key of the specified field.
74 result
= target
.copy()
75 result
['%s_key' % fieldname
] = target
[fieldname
].key().name()
79 def getEntities(model
):
80 """Returns all users as dictionary.
84 gen
= lambda: model
.all()
85 it
= interactive
.deepFetch(gen
)
87 entities
= [(i
.key().name(), i
) for i
in it
]
93 def getProps(last
=None):
94 """Returns all proposals as a list of dictionaries.
98 'link_id', 'scope_path', 'title', 'abstract', 'content',
99 'additional_info', '_mentor', 'possible_mentors', 'score',
100 'status', '_org', 'created_on', 'last_modified_on']
102 from soc
.models
.student_proposal
import StudentProposal
104 gen
= lambda: StudentProposal
.all()
106 it
= dateFetch(gen
, last
)
108 proposals
= [(i
.key().name(), i
.toDict(key_order
)) for i
in it
]
110 last
= i
.last_modified_on
# last modified entity
112 last
= datetime
.datetime
.now()
114 return dict(proposals
), last
117 def orgStats(target
, orgs
):
118 """Retrieves org stats.
121 from soc
.logic
import dicts
123 orgs
= [(v
.key(), v
) for k
, v
in orgs
.iteritems()]
126 grouped
= dicts
.groupby(target
.values(), '_org')
128 grouped
= [(orgs
[k
], v
) for k
, v
in grouped
.iteritems()]
129 popularity
= [(k
.link_id
, len(v
)) for k
, v
in grouped
]
131 return dict(grouped
), dict(popularity
)
134 def countStudentsWithProposals():
135 """Retrieves number of Students who have submitted at least one Student Proposal.
138 proposals
= getStudentProposals()
141 for proposal_key
in proposals
.keys():
142 students
[proposals
[proposal_key
].scope_path
] = True
147 def printPopularity(popularity
):
148 """Prints the popularity for the specified proposals.
151 g
= operator
.itemgetter(1)
153 for item
in sorted(popularity
.iteritems(), key
=g
, reverse
=True):
154 print "%s: %d" % item
157 def saveValues(values
, saver
):
158 """Saves the specified popularities.
162 from google
.appengine
.ext
import db
164 from soc
.models
.organization
import Organization
167 org
= Organization
.get_by_key_name(key
)
171 for key
, value
in sorted(values
.iteritems()):
173 db
.run_in_transaction_custom_retries(10, txn
, key
, value
)
178 def addFollower(follower
, proposals
, add_public
=True, add_private
=True):
179 """Adds a user as follower to the specified proposals.
182 follower: the User to add as follower
183 proposals: a list with the StudnetProposals that should be subscribed to
184 add_public: whether the user is subscribed to public updates
185 add_private: whether the user should be subscribed to private updates
188 from soc
.models
.review_follower
import ReviewFollower
195 'link_id': follower
.link_id
,
197 'scope_path': i
.key().name(),
198 'key_name': '%s/%s' % (i
.key().name(), follower
.link_id
),
199 'subscribed_public': add_public
,
200 'subscribed_private': add_private
,
203 entity
= ReviewFollower(**properties
)
204 result
.append(entity
)
209 def convertProposals(org
):
210 """Convert all proposals for the specified organization.
213 org: the organization for which all proposals will be converted
216 from soc
.logic
.models
.student_proposal
import logic
as proposal_logic
217 from soc
.logic
.models
.student_project
import logic
as project_logic
219 proposals
= proposal_logic
.getProposalsToBeAcceptedForOrg(org
)
221 print "accepting %d proposals, with %d slots" % (len(proposals
), org
.slots
)
223 for proposal
in proposals
:
225 'link_id': 't%i' % (int(time
.time()*100)),
226 'scope_path': proposal
.org
.key().id_or_name(),
227 'scope': proposal
.org
,
228 'program': proposal
.program
,
229 'student': proposal
.scope
,
230 'title': proposal
.title
,
231 'abstract': proposal
.abstract
,
232 'mentor': proposal
.mentor
,
235 project
= project_logic
.updateOrCreateFromFields(fields
, silent
=True)
241 proposal_logic
.updateEntityProperties(proposal
, fields
, silent
=True)
244 'status': ['new', 'pending'],
248 querygen
= lambda: proposal_logic
.getQueryForFields(fields
)
249 proposals
= [i
for i
in interactive
.deepFetch(querygen
, batchSize
=10)]
251 print "rejecting %d proposals" % len(proposals
)
254 'status': 'rejected',
257 for proposal
in proposals
:
258 proposal_logic
.updateEntityProperties(proposal
, fields
, silent
=True)
262 """Creates the job that is responsible for sending mails.
265 from soc
.logic
.models
.job
import logic
as job_logic
266 from soc
.logic
.models
.priority_group
import logic
as priority_logic
267 from soc
.logic
.models
.program
import logic
as program_logic
269 program_entity
= program_logic
.getFromKeyName('google/gsoc2009')
271 priority_group
= priority_logic
.getGroup(priority_logic
.EMAIL
)
273 'priority_group': priority_group
,
274 'task_name': 'setupStudentProposalMailing',
275 'key_data': [program_entity
.key()]}
277 job_logic
.updateOrCreateFromFields(job_fields
)
280 def reviveJobs(amount
):
281 """Sets jobs that are stuck in 'aborted' to waiting.
284 amount: the amount of jobs to revive
287 from soc
.models
.job
import Job
289 query
= Job
.all().filter('status', 'aborted')
290 jobs
= query
.fetch(amount
)
296 job
.status
= 'waiting'
298 print "restarted %d" % job
.key().id()
301 def deidleJobs(amount
):
302 """Sets jobs that are stuck in 'started' to waiting.
305 amount: the amount of jobs to deidle
308 from soc
.models
.job
import Job
310 query
= Job
.all().filter('status', 'started')
311 jobs
= query
.fetch(amount
)
317 job
.status
= 'waiting'
319 print "restarted %d" % job
.key().id()
322 def deleteEntities(model
, step_size
=25):
323 """Deletes all entities of the specified type
330 entities
= model
.all().fetch(step_size
)
335 for entity
in entities
:
340 print "deleted %d entities" % count
344 def loadPickle(name
):
348 f
= open(name
+ '.dat')
349 return cPickle
.load(f
)
352 def dumpPickle(target
, name
):
356 f
= open("%s.dat" % name
, 'w')
357 cPickle
.dump(target
, f
)
366 from soc
.models
.organization
import Organization
367 from soc
.models
.user
import User
368 from soc
.models
.student
import Student
369 from soc
.models
.mentor
import Mentor
370 from soc
.models
.org_admin
import OrgAdmin
371 from soc
.models
.job
import Job
372 from soc
.models
.student_proposal
import StudentProposal
373 from soc
.models
.student_project
import StudentProject
375 def slotSaver(org
, value
):
377 def popSaver(org
, value
):
378 org
.nr_applications
= value
379 def rawSaver(org
, value
):
380 org
.slots_calculated
= value
385 'orgStats': orgStats
,
386 'printPopularity': printPopularity
,
387 'saveValues': saveValues
,
388 'getEntities': getEntities
,
389 'deleteEntities': deleteEntities
,
390 'getOrgs': getEntities(Organization
),
391 'getUsers': getEntities(User
),
392 'getStudents': getEntities(Student
),
393 'getMentors': getEntities(Mentor
),
394 'getOrgAdmins': getEntities(OrgAdmin
),
395 'getStudentProjects': getEntities(StudentProject
),
396 'getProps': getProps
,
397 'countStudentsWithProposals': countStudentsWithProposals
,
398 'convertProposals': convertProposals
,
399 'addFollower': addFollower
,
400 'Organization': Organization
,
405 'OrgAdmin': OrgAdmin
,
406 'StudentProject': StudentProject
,
407 'StudentProposal': StudentProposal
,
408 'slotSaver': slotSaver
,
409 'popSaver': popSaver
,
410 'rawSaver': rawSaver
,
411 'startSpam': startSpam
,
412 'reviveJobs': reviveJobs
,
413 'deidleJobs': deidleJobs
,
416 interactive
.remote(args
, context
)
418 if __name__
== '__main__':
419 if len(sys
.argv
) < 2:
420 print "Usage: %s app_id [host]" % (sys
.argv
[0],)