shell/shell.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 """
  18 An interactive, stateful AJAX shell that runs Python code on the server.
  19
  20 Part of http://code.google.com/p/google-app-engine-samples/.
  21
  22 May be run as a standalone app or in an existing app as an admin-only handler.
  23 Can be used for system administration tasks, as an interactive way to try out
  24 APIs, or as a debugging aid during development.
  25
  26 The logging, os, sys, db, and users modules are imported automatically.
  27
  28 Interpreter state is stored in the datastore so that variables, function
  29 definitions, and other values in the global and local namespaces can be used
  30 across commands.
  31
  32 To use the shell in your app, copy shell.py, static/*, and templates/* into
  33 your app's source directory. Then, copy the URL handlers from app.yaml into
  34 your app.yaml.
  35
  36 TODO: unit tests!
  37 """
  38
  39 import logging
  40 import new
  41 import os
  42 import pickle
  43 import sys
  44 import traceback
  45 import types
  46 import wsgiref.handlers
  47
  48 from google.appengine.api import users
  49 from google.appengine.ext import db
  50 from google.appengine.ext import webapp
  51 from google.appengine.ext.webapp import template
  52
  53
  54 # Set to True if stack traces should be shown in the browser, etc.
  55 _DEBUG = True
  56
  57 # The entity kind for shell sessions. Feel free to rename to suit your app.
  58 _SESSION_KIND = '_Shell_Session'
  59
  60 # Types that can't be pickled.
  61 UNPICKLABLE_TYPES = (
  62   types.ModuleType,
  63   types.TypeType,
  64   types.ClassType,
  65   types.FunctionType,
  66   )
  67
  68 # Unpicklable statements to seed new sessions with.
  69 INITIAL_UNPICKLABLES = [
  70   'import logging',
  71   'import os',
  72   'import sys',
  73   'from google.appengine.ext import db',
  74   'from google.appengine.api import users',
  75   'class Foo(db.Expando):\n  pass',
  76   ]
  77
  78
  79 class Session(db.Model):
  80   """A shell session. Stores the session's globals.
  81
  82   Each session globals is stored in one of two places:
  83
  84   If the global is picklable, it's stored in the parallel globals and
  85   global_names list properties. (They're parallel lists to work around the
  86   unfortunate fact that the datastore can't store dictionaries natively.)
  87
  88   If the global is not picklable (e.g. modules, classes, and functions), or if
  89   it was created by the same statement that created an unpicklable global,
  90   it's not stored directly. Instead, the statement is stored in the
  91   unpicklables list property. On each request, before executing the current
  92   statement, the unpicklable statements are evaluated to recreate the
  93   unpicklable globals.
  94
  95   The unpicklable_names property stores all of the names of globals that were
  96   added by unpicklable statements. When we pickle and store the globals after
  97   executing a statement, we skip the ones in unpicklable_names.
  98
  99   Using Text instead of string is an optimization. We don't query on any of
 100   these properties, so they don't need to be indexed.
 101   """
 102   global_names = db.ListProperty(db.Text)
 103   globals = db.ListProperty(db.Blob)
 104   unpicklable_names = db.ListProperty(db.Text)
 105   unpicklables = db.ListProperty(db.Text)
 106
 107   def set_global(self, name, value):
 108     """Adds a global, or updates it if it already exists.
 109
 110     Also removes the global from the list of unpicklable names.
 111
 112     Args:
 113       name: the name of the global to remove
 114       value: any picklable value
 115     """
 116     blob = db.Blob(pickle.dumps(value))
 117
 118     if name in self.global_names:
 119       index = self.global_names.index(name)
 120       self.globals[index] = blob
 121     else:
 122       self.global_names.append(db.Text(name))
 123       self.globals.append(blob)
 124
 125     self.remove_unpicklable_name(name)
 126
 127   def remove_global(self, name):
 128     """Removes a global, if it exists.
 129
 130     Args:
 131       name: string, the name of the global to remove
 132     """
 133     if name in self.global_names:
 134       index = self.global_names.index(name)
 135       del self.global_names[index]
 136       del self.globals[index]
 137
 138   def globals_dict(self):
 139     """Returns a dictionary view of the globals.
 140     """
 141     return dict((name, pickle.loads(val))
 142                 for name, val in zip(self.global_names, self.globals))
 143
 144   def add_unpicklable(self, statement, names):
 145     """Adds a statement and list of names to the unpicklables.
 146
 147     Also removes the names from the globals.
 148
 149     Args:
 150       statement: string, the statement that created new unpicklable global(s).
 151       names: list of strings; the names of the globals created by the statement.
 152     """
 153     self.unpicklables.append(db.Text(statement))
 154
 155     for name in names:
 156       self.remove_global(name)
 157       if name not in self.unpicklable_names:
 158         self.unpicklable_names.append(db.Text(name))
 159
 160   def remove_unpicklable_name(self, name):
 161     """Removes a name from the list of unpicklable names, if it exists.
 162
 163     Args:
 164       name: string, the name of the unpicklable global to remove
 165     """
 166     if name in self.unpicklable_names:
 167       self.unpicklable_names.remove(name)
 168
 169
 170 class FrontPageHandler(webapp.RequestHandler):
 171   """Creates a new session and renders the shell.html template.
 172   """
 173
 174   def get(self):
 175     # set up the session. TODO: garbage collect old shell sessions
 176     session_key = self.request.get('session')
 177     if session_key:
 178       session = Session.get(session_key)
 179     else:
 180       # create a new session
 181       session = Session()
 182       session.unpicklables = [db.Text(line) for line in INITIAL_UNPICKLABLES]
 183       session_key = session.put()
 184
 185     template_file = os.path.join(os.path.dirname(__file__), 'templates',
 186                                  'shell.html')
 187     session_url = '/?session=%s' % session_key
 188     vars = { 'server_software': os.environ['SERVER_SOFTWARE'],
 189              'python_version': sys.version,
 190              'session': str(session_key),
 191              'user': users.get_current_user(),
 192              'login_url': users.create_login_url(session_url),
 193              'logout_url': users.create_logout_url(session_url),
 194              }
 195     rendered = template.render(template_file, vars, debug=_DEBUG)
 196     self.response.out.write(unicode(rendered))
 197
 198
 199 class StatementHandler(webapp.RequestHandler):
 200   """Evaluates a python statement in a given session and returns the result.
 201   """
 202
 203   def get(self):
 204     self.response.headers['Content-Type'] = 'text/plain'
 205
 206     # extract the statement to be run
 207     statement = self.request.get('statement')
 208     if not statement:
 209       return
 210
 211     # the python compiler doesn't like network line endings
 212     statement = statement.replace('\r\n', '\n')
 213
 214     # add a couple newlines at the end of the statement. this makes
 215     # single-line expressions such as 'class Foo: pass' evaluate happily.
 216     statement += '\n\n'
 217
 218     # log and compile the statement up front
 219     try:
 220       logging.info('Compiling and evaluating:\n%s' % statement)
 221       compiled = compile(statement, '<string>', 'single')
 222     except:
 223       self.response.out.write(traceback.format_exc())
 224       return
 225
 226     # create a dedicated module to be used as this statement's __main__
 227     statement_module = new.module('__main__')
 228
 229     # use this request's __builtin__, since it changes on each request.
 230     # this is needed for import statements, among other things.
 231     import __builtin__
 232     statement_module.__builtins__ = __builtin__
 233
 234     # load the session from the datastore
 235     session = Session.get(self.request.get('session'))
 236
 237     # swap in our custom module for __main__. then unpickle the session
 238     # globals, run the statement, and re-pickle the session globals, all
 239     # inside it.
 240     old_main = sys.modules.get('__main__')
 241     try:
 242       sys.modules['__main__'] = statement_module
 243       statement_module.__name__ = '__main__'
 244
 245       # re-evaluate the unpicklables
 246       for code in session.unpicklables:
 247         exec code in statement_module.__dict__
 248
 249       # re-initialize the globals
 250       for name, val in session.globals_dict().items():
 251         try:
 252           statement_module.__dict__[name] = val
 253         except:
 254           msg = 'Dropping %s since it could not be unpickled.\n' % name
 255           self.response.out.write(msg)
 256           logging.warning(msg + traceback.format_exc())
 257           session.remove_global(name)
 258
 259       # run!
 260       old_globals = dict(statement_module.__dict__)
 261       try:
 262         old_stdout = sys.stdout
 263         old_stderr = sys.stderr
 264         try:
 265           sys.stdout = self.response.out
 266           sys.stderr = self.response.out
 267           exec compiled in statement_module.__dict__
 268         finally:
 269           sys.stdout = old_stdout
 270           sys.stderr = old_stderr
 271       except:
 272         self.response.out.write(traceback.format_exc())
 273         return
 274
 275       # extract the new globals that this statement added
 276       new_globals = {}
 277       for name, val in statement_module.__dict__.items():
 278         if name not in old_globals or val != old_globals[name]:
 279           new_globals[name] = val
 280
 281       if True in [isinstance(val, UNPICKLABLE_TYPES)
 282                   for val in new_globals.values()]:
 283         # this statement added an unpicklable global. store the statement and
 284         # the names of all of the globals it added in the unpicklables.
 285         session.add_unpicklable(statement, new_globals.keys())
 286         logging.debug('Storing this statement as an unpicklable.')
 287
 288       else:
 289         # this statement didn't add any unpicklables. pickle and store the
 290         # new globals back into the datastore.
 291         for name, val in new_globals.items():
 292           if not name.startswith('__'):
 293             session.set_global(name, val)
 294
 295     finally:
 296       sys.modules['__main__'] = old_main
 297
 298     session.put()
 299
 300
 301 def main():
 302   application = webapp.WSGIApplication(
 303     [('/', FrontPageHandler),
 304      ('/shell.do', StatementHandler)], debug=_DEBUG)
 305   wsgiref.handlers.CGIHandler().run(application)
 306
 307
 308 if __name__ == '__main__':
 309   main()