app/shell/shell.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 """
  18 An interactive, stateful AJAX shell that runs Python code on the server.
  19
  20 Part of http://code.google.com/p/google-app-engine-samples/.
  21
  22 May be run as a standalone app or in an existing app as an admin-only handler.
  23 Can be used for system administration tasks, as an interactive way to try out
  24 APIs, or as a debugging aid during development.
  25
  26 The logging, os, sys, db, and users modules are imported automatically.
  27
  28 Interpreter state is stored in the datastore so that variables, function
  29 definitions, and other values in the global and local namespaces can be used
  30 across commands.
  31
  32 To use the shell in your app, copy shell.py, static/*, and templates/* into
  33 your app's source directory. Then, copy the URL handlers from app.yaml into
  34 your app.yaml.
  35
  36 TODO: unit tests!
  37 """
  38
  39 import logging
  40 import new
  41 import os
  42 import pickle
  43 import sys
  44 import traceback
  45 import types
  46 import wsgiref.handlers
  47
  48 from django.template import loader
  49 from google.appengine.api import users
  50 from google.appengine.ext import db
  51 from google.appengine.ext import webapp
  52 from google.appengine.ext.webapp import template
  53
  54 import django.template
  55 import gae_django
  56
  57
  58 # Set to True if stack traces should be shown in the browser, etc.
  59 _DEBUG = True
  60
  61 # The entity kind for shell sessions. Feel free to rename to suit your app.
  62 _SESSION_KIND = '_Shell_Session'
  63
  64 # Types that can't be pickled.
  65 UNPICKLABLE_TYPES = (
  66   types.ModuleType,
  67   types.TypeType,
  68   types.ClassType,
  69   types.FunctionType,
  70   )
  71
  72 # Unpicklable statements to seed new sessions with.
  73 INITIAL_UNPICKLABLES = [
  74   'import logging',
  75   'import os',
  76   'import sys',
  77   'from google.appengine.ext import db',
  78   'from google.appengine.api import users',
  79   ]
  80
  81
  82 class ShellSession(db.Model):
  83   """A shell session. Stores the session's globals.
  84
  85   Each session globals is stored in one of two places:
  86
  87   If the global is picklable, it's stored in the parallel globals and
  88   global_names list properties. (They're parallel lists to work around the
  89   unfortunate fact that the datastore can't store dictionaries natively.)
  90
  91   If the global is not picklable (e.g. modules, classes, and functions), or if
  92   it was created by the same statement that created an unpicklable global,
  93   it's not stored directly. Instead, the statement is stored in the
  94   unpicklables list property. On each request, before executing the current
  95   statement, the unpicklable statements are evaluated to recreate the
  96   unpicklable globals.
  97
  98   The unpicklable_names property stores all of the names of globals that were
  99   added by unpicklable statements. When we pickle and store the globals after
 100   executing a statement, we skip the ones in unpicklable_names.
 101
 102   Using Text instead of string is an optimization. We don't query on any of
 103   these properties, so they don't need to be indexed.
 104   """
 105   global_names = db.ListProperty(db.Text)
 106   globals = db.ListProperty(db.Blob)
 107   unpicklable_names = db.ListProperty(db.Text)
 108   unpicklables = db.ListProperty(db.Text)
 109
 110   def set_global(self, name, value):
 111     """Adds a global, or updates it if it already exists.
 112
 113     Also removes the global from the list of unpicklable names.
 114
 115     Args:
 116       name: the name of the global to remove
 117       value: any picklable value
 118     """
 119     blob = db.Blob(pickle.dumps(value))
 120
 121     if name in self.global_names:
 122       index = self.global_names.index(name)
 123       self.globals[index] = blob
 124     else:
 125       self.global_names.append(db.Text(name))
 126       self.globals.append(blob)
 127
 128     self.remove_unpicklable_name(name)
 129
 130   def remove_global(self, name):
 131     """Removes a global, if it exists.
 132
 133     Args:
 134       name: string, the name of the global to remove
 135     """
 136     if name in self.global_names:
 137       index = self.global_names.index(name)
 138       del self.global_names[index]
 139       del self.globals[index]
 140
 141   def globals_dict(self):
 142     """Returns a dictionary view of the globals.
 143     """
 144     return dict((name, pickle.loads(val))
 145                 for name, val in zip(self.global_names, self.globals))
 146
 147   def add_unpicklable(self, statement, names):
 148     """Adds a statement and list of names to the unpicklables.
 149
 150     Also removes the names from the globals.
 151
 152     Args:
 153       statement: string, the statement that created new unpicklable global(s).
 154       names: list of strings; the names of the globals created by the statement.
 155     """
 156     self.unpicklables.append(db.Text(statement))
 157
 158     for name in names:
 159       self.remove_global(name)
 160       if name not in self.unpicklable_names:
 161         self.unpicklable_names.append(db.Text(name))
 162
 163   def remove_unpicklable_name(self, name):
 164     """Removes a name from the list of unpicklable names, if it exists.
 165
 166     Args:
 167       name: string, the name of the unpicklable global to remove
 168     """
 169     if name in self.unpicklable_names:
 170       self.unpicklable_names.remove(name)
 171
 172
 173 class FrontPageHandler(webapp.RequestHandler):
 174   """Creates a new session and renders the shell.html template.
 175   """
 176
 177   def get(self):
 178     # set up the session. TODO: garbage collect old shell sessions
 179     session_key = self.request.get('session')
 180     if session_key:
 181       session = ShellSession.get(session_key)
 182     else:
 183       # create a new session
 184       session = ShellSession()
 185       session.unpicklables = [db.Text(line) for line in INITIAL_UNPICKLABLES]
 186       session_key = session.put()
 187
 188     template_file = os.path.join(os.path.dirname(__file__), 'templates',
 189                                  'shell.html')
 190     session_url = '/?session=%s' % session_key
 191     vars = { 'server_software': os.environ['SERVER_SOFTWARE'],
 192              'python_version': sys.version,
 193              'session': str(session_key),
 194              'user': users.get_current_user(),
 195              'login_url': users.create_login_url(session_url),
 196              'logout_url': users.create_logout_url(session_url),
 197              }
 198
 199     rendered = loader.render_to_string('shell.html', dictionary=vars)
 200     # rendered = webapp.template.render(template_file, vars, debug=_DEBUG)
 201     self.response.out.write(rendered)
 202
 203
 204 class StatementHandler(webapp.RequestHandler):
 205   """Evaluates a python statement in a given session and returns the result.
 206   """
 207
 208   def get(self):
 209     self.response.headers['Content-Type'] = 'text/plain'
 210
 211     # extract the statement to be run
 212     statement = self.request.get('statement')
 213     if not statement:
 214       return
 215
 216     # the python compiler doesn't like network line endings
 217     statement = statement.replace('\r\n', '\n')
 218
 219     # add a couple newlines at the end of the statement. this makes
 220     # single-line expressions such as 'class Foo: pass' evaluate happily.
 221     statement += '\n\n'
 222
 223     # log and compile the statement up front
 224     try:
 225       logging.info('Compiling and evaluating:\n%s' % statement)
 226       compiled = compile(statement, '<string>', 'single')
 227     except:
 228       self.response.out.write(traceback.format_exc())
 229       return
 230
 231     # create a dedicated module to be used as this statement's __main__
 232     statement_module = new.module('__main__')
 233
 234     # use this request's __builtin__, since it changes on each request.
 235     # this is needed for import statements, among other things.
 236     import __builtin__
 237     statement_module.__builtins__ = __builtin__
 238
 239     # load the session from the datastore
 240     session = ShellSession.get(self.request.get('session'))
 241
 242     # swap in our custom module for __main__. then unpickle the session
 243     # globals, run the statement, and re-pickle the session globals, all
 244     # inside it.
 245     old_main = sys.modules.get('__main__')
 246     try:
 247       sys.modules['__main__'] = statement_module
 248       statement_module.__name__ = '__main__'
 249
 250       # re-evaluate the unpicklables
 251       for code in session.unpicklables:
 252         exec code in statement_module.__dict__
 253
 254       # re-initialize the globals
 255       for name, val in session.globals_dict().items():
 256         try:
 257           statement_module.__dict__[name] = val
 258         except:
 259           msg = 'Dropping %s since it could not be unpickled.\n' % name
 260           self.response.out.write(msg)
 261           logging.warning(msg + traceback.format_exc())
 262           session.remove_global(name)
 263
 264       # run!
 265       old_globals = dict(statement_module.__dict__)
 266       try:
 267         old_stdout = sys.stdout
 268         old_stderr = sys.stderr
 269         try:
 270           sys.stdout = self.response.out
 271           sys.stderr = self.response.out
 272           exec compiled in statement_module.__dict__
 273         finally:
 274           sys.stdout = old_stdout
 275           sys.stderr = old_stderr
 276       except:
 277         self.response.out.write(traceback.format_exc())
 278         return
 279
 280       # extract the new globals that this statement added
 281       new_globals = {}
 282       for name, val in statement_module.__dict__.items():
 283         if name not in old_globals or val != old_globals[name]:
 284           new_globals[name] = val
 285
 286       if True in [isinstance(val, UNPICKLABLE_TYPES)
 287                   for val in new_globals.values()]:
 288         # this statement added an unpicklable global. store the statement and
 289         # the names of all of the globals it added in the unpicklables.
 290         session.add_unpicklable(statement, new_globals.keys())
 291         logging.debug('Storing this statement as an unpicklable.')
 292
 293       else:
 294         # this statement didn't add any unpicklables. pickle and store the
 295         # new globals back into the datastore.
 296         for name, val in new_globals.items():
 297           if not name.startswith('__'):
 298             session.set_global(name, val)
 299
 300     finally:
 301       sys.modules['__main__'] = old_main
 302
 303     session.put()
 304
 305
 306 def main():
 307   """Main program.
 308   """
 309
 310   application = webapp.WSGIApplication(
 311     [('/admin/shell', FrontPageHandler),
 312      ('/admin/shell/shell.do', StatementHandler)], debug=_DEBUG)
 313   wsgiref.handlers.CGIHandler().run(application)
 314
 315
 316 if __name__ == '__main__':
 317   main()