From f0f96291bb0f0ca4ef88afd0c1390f4bbaebfc67 Mon Sep 17 00:00:00 2001 From: Pawel Solyga Date: Sun, 24 May 2009 22:29:54 +0200 Subject: [PATCH] Add web based python shell to Melange. It is accessible via http://host/admin/shell url and requires developer rights. Shell project is part of google-app-engine-samples. This commit moves django configuration from main.py to separate gae_django.py module. Shell project has been modified in order to work correctly with django 1.0+. Build script has been updated and includes shell folder and gae_django.py file. http://code.google.com/p/google-app-engine-samples/source/browse/trunk/shell/ --- app/app.yaml.template | 8 ++ app/gae_django.py | 61 ++++++++ app/main.py | 43 +----- app/settings.py | 1 + app/shell/README | 17 +++ app/shell/__init__.py | 0 app/shell/shell.py | 316 +++++++++++++++++++++++++++++++++++++++++ app/shell/static/shell.js | 195 +++++++++++++++++++++++++ app/shell/static/spinner.gif | Bin 0 -> 1514 bytes app/shell/templates/shell.html | 124 ++++++++++++++++ scripts/build.sh | 4 +- 11 files changed, 726 insertions(+), 43 deletions(-) create mode 100644 app/gae_django.py mode change 100644 => 100755 app/settings.py create mode 100644 app/shell/README create mode 100644 app/shell/__init__.py create mode 100644 app/shell/shell.py create mode 100644 app/shell/static/shell.js create mode 100644 app/shell/static/spinner.gif create mode 100644 app/shell/templates/shell.html diff --git a/app/app.yaml.template b/app/app.yaml.template index 0a9f6544..b8455c84 100644 --- a/app/app.yaml.template +++ b/app/app.yaml.template @@ -46,6 +46,14 @@ handlers: - url: /json static_dir: json +- url: /admin/shell.* + script: shell/shell.py + login: admin + +- url: /static + static_dir: shell/static + expiration: 1d + - url: /.* script: main.py diff --git a/app/gae_django.py b/app/gae_django.py new file mode 100644 index 00000000..5455a3d1 --- /dev/null +++ b/app/gae_django.py @@ -0,0 +1,61 @@ +#!/usr/bin/python2.5 +# +# Copyright 2008 the Melange authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module containing Melange Django 1.0+ configuration for Google App Engine. +""" + +import logging +import os +import sys + +__authors__ = [ + # alphabetical order by last name, please + '"Pawel Solyga" ', + ] + + +# Remove the standard version of Django. +for k in [k for k in sys.modules if k.startswith('django')]: + del sys.modules[k] + +# Force sys.path to have our own directory first, in case we want to import +# from it. This lets us replace the built-in Django +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) + +sys.path.insert(0, os.path.abspath('django.zip')) + +# Force Django to reload its settings. +from django.conf import settings +settings._target = None + +# Must set this env var before importing any part of Django +os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' + +import django.core.signals +import django.db + +# Log errors. +def log_exception(*args, **kwds): + """Function used for logging exceptions. + """ + logging.exception('Exception in request:') + +# Log all exceptions detected by Django. +django.core.signals.got_request_exception.connect(log_exception) + +# Unregister the rollback event handler. +django.core.signals.got_request_exception.disconnect( + django.db._rollback_on_exception) diff --git a/app/main.py b/app/main.py index 2e66a633..54b5bd70 100644 --- a/app/main.py +++ b/app/main.py @@ -29,42 +29,7 @@ import sys from google.appengine.ext.webapp import util - -# Remove the standard version of Django. -for k in [k for k in sys.modules if k.startswith('django')]: - del sys.modules[k] - -# Force sys.path to have our own directory first, in case we want to import -# from it. This lets us replace the built-in Django -sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) - -sys.path.insert(0, os.path.abspath('django.zip')) - -ultimate_sys_path = None - -# Force Django to reload its settings. -from django.conf import settings -settings._target = None - -# Must set this env var before importing any part of Django -os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' - -import django.core.handlers.wsgi -import django.core.signals -import django.db - -# Log errors. -def log_exception(*args, **kwds): - """Function used for logging exceptions. - """ - logging.exception('Exception in request:') - -# Log all exceptions detected by Django. -django.core.signals.got_request_exception.connect(log_exception) - -# Unregister the rollback event handler. -django.core.signals.got_request_exception.disconnect( - django.db._rollback_on_exception) +import gae_django def profile_main_as_html(): @@ -117,11 +82,7 @@ def profile_main_as_logs(): def real_main(): """Main program without profiling. """ - global ultimate_sys_path - if ultimate_sys_path is None: - ultimate_sys_path = list(sys.path) - else: - sys.path[:] = ultimate_sys_path + import django.core.handlers.wsgi # Create a Django application for WSGI. application = django.core.handlers.wsgi.WSGIHandler() diff --git a/app/settings.py b/app/settings.py old mode 100644 new mode 100755 index d5610321..f8dd88b1 --- a/app/settings.py +++ b/app/settings.py @@ -100,6 +100,7 @@ TEMPLATE_DIRS = ( os.path.join(ROOT_PATH, 'ghop', 'templates'), os.path.join(ROOT_PATH, 'gsoc', 'templates'), os.path.join(ROOT_PATH, 'soc', 'templates'), + os.path.join(ROOT_PATH, 'shell', 'templates'), ) INSTALLED_APPS = ( diff --git a/app/shell/README b/app/shell/README new file mode 100644 index 00000000..5b0089fe --- /dev/null +++ b/app/shell/README @@ -0,0 +1,17 @@ +An interactive, stateful AJAX shell that runs Python code on the server. + +Part of http://code.google.com/p/google-app-engine-samples/. + +May be run as a standalone app or in an existing app as an admin-only handler. +Can be used for system administration tasks, as an interactive way to try out +APIs, or as a debugging aid during development. + +The logging, os, sys, db, and users modules are imported automatically. + +Interpreter state is stored in the datastore so that variables, function +definitions, and other values in the global and local namespaces can be used +across commands. + +To use the shell in your app, copy shell.py, static/*, and templates/* into +your app's source directory. Then, copy the URL handlers from app.yaml into +your app.yaml. diff --git a/app/shell/__init__.py b/app/shell/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/shell/shell.py b/app/shell/shell.py new file mode 100644 index 00000000..e7632b0e --- /dev/null +++ b/app/shell/shell.py @@ -0,0 +1,316 @@ +#!/usr/bin/python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +An interactive, stateful AJAX shell that runs Python code on the server. + +Part of http://code.google.com/p/google-app-engine-samples/. + +May be run as a standalone app or in an existing app as an admin-only handler. +Can be used for system administration tasks, as an interactive way to try out +APIs, or as a debugging aid during development. + +The logging, os, sys, db, and users modules are imported automatically. + +Interpreter state is stored in the datastore so that variables, function +definitions, and other values in the global and local namespaces can be used +across commands. + +To use the shell in your app, copy shell.py, static/*, and templates/* into +your app's source directory. Then, copy the URL handlers from app.yaml into +your app.yaml. + +TODO: unit tests! +""" + +import logging +import new +import os +import pickle +import sys +import traceback +import types +import wsgiref.handlers + +from django.template import loader +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template + +import gae_django + + +# Set to True if stack traces should be shown in the browser, etc. +_DEBUG = True + +# The entity kind for shell sessions. Feel free to rename to suit your app. +_SESSION_KIND = '_Shell_Session' + +# Types that can't be pickled. +UNPICKLABLE_TYPES = ( + types.ModuleType, + types.TypeType, + types.ClassType, + types.FunctionType, + ) + +# Unpicklable statements to seed new sessions with. +INITIAL_UNPICKLABLES = [ + 'import logging', + 'import os', + 'import sys', + 'from google.appengine.ext import db', + 'from google.appengine.api import users', + ] + + +class ShellSession(db.Model): + """A shell session. Stores the session's globals. + + Each session globals is stored in one of two places: + + If the global is picklable, it's stored in the parallel globals and + global_names list properties. (They're parallel lists to work around the + unfortunate fact that the datastore can't store dictionaries natively.) + + If the global is not picklable (e.g. modules, classes, and functions), or if + it was created by the same statement that created an unpicklable global, + it's not stored directly. Instead, the statement is stored in the + unpicklables list property. On each request, before executing the current + statement, the unpicklable statements are evaluated to recreate the + unpicklable globals. + + The unpicklable_names property stores all of the names of globals that were + added by unpicklable statements. When we pickle and store the globals after + executing a statement, we skip the ones in unpicklable_names. + + Using Text instead of string is an optimization. We don't query on any of + these properties, so they don't need to be indexed. + """ + global_names = db.ListProperty(db.Text) + globals = db.ListProperty(db.Blob) + unpicklable_names = db.ListProperty(db.Text) + unpicklables = db.ListProperty(db.Text) + + def set_global(self, name, value): + """Adds a global, or updates it if it already exists. + + Also removes the global from the list of unpicklable names. + + Args: + name: the name of the global to remove + value: any picklable value + """ + blob = db.Blob(pickle.dumps(value)) + + if name in self.global_names: + index = self.global_names.index(name) + self.globals[index] = blob + else: + self.global_names.append(db.Text(name)) + self.globals.append(blob) + + self.remove_unpicklable_name(name) + + def remove_global(self, name): + """Removes a global, if it exists. + + Args: + name: string, the name of the global to remove + """ + if name in self.global_names: + index = self.global_names.index(name) + del self.global_names[index] + del self.globals[index] + + def globals_dict(self): + """Returns a dictionary view of the globals. + """ + return dict((name, pickle.loads(val)) + for name, val in zip(self.global_names, self.globals)) + + def add_unpicklable(self, statement, names): + """Adds a statement and list of names to the unpicklables. + + Also removes the names from the globals. + + Args: + statement: string, the statement that created new unpicklable global(s). + names: list of strings; the names of the globals created by the statement. + """ + self.unpicklables.append(db.Text(statement)) + + for name in names: + self.remove_global(name) + if name not in self.unpicklable_names: + self.unpicklable_names.append(db.Text(name)) + + def remove_unpicklable_name(self, name): + """Removes a name from the list of unpicklable names, if it exists. + + Args: + name: string, the name of the unpicklable global to remove + """ + if name in self.unpicklable_names: + self.unpicklable_names.remove(name) + + +class FrontPageHandler(webapp.RequestHandler): + """Creates a new session and renders the shell.html template. + """ + + def get(self): + # set up the session. TODO: garbage collect old shell sessions + session_key = self.request.get('session') + if session_key: + session = ShellSession.get(session_key) + else: + # create a new session + session = ShellSession() + session.unpicklables = [db.Text(line) for line in INITIAL_UNPICKLABLES] + session_key = session.put() + + template_file = os.path.join(os.path.dirname(__file__), 'templates', + 'shell.html') + session_url = '/?session=%s' % session_key + vars = { 'server_software': os.environ['SERVER_SOFTWARE'], + 'python_version': sys.version, + 'session': str(session_key), + 'user': users.get_current_user(), + 'login_url': users.create_login_url(session_url), + 'logout_url': users.create_logout_url(session_url), + } + + rendered = loader.render_to_string('shell.html', dictionary=vars) + # rendered = webapp.template.render(template_file, vars, debug=_DEBUG) + self.response.out.write(rendered) + + +class StatementHandler(webapp.RequestHandler): + """Evaluates a python statement in a given session and returns the result. + """ + + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + + # extract the statement to be run + statement = self.request.get('statement') + if not statement: + return + + # the python compiler doesn't like network line endings + statement = statement.replace('\r\n', '\n') + + # add a couple newlines at the end of the statement. this makes + # single-line expressions such as 'class Foo: pass' evaluate happily. + statement += '\n\n' + + # log and compile the statement up front + try: + logging.info('Compiling and evaluating:\n%s' % statement) + compiled = compile(statement, '', 'single') + except: + self.response.out.write(traceback.format_exc()) + return + + # create a dedicated module to be used as this statement's __main__ + statement_module = new.module('__main__') + + # use this request's __builtin__, since it changes on each request. + # this is needed for import statements, among other things. + import __builtin__ + statement_module.__builtins__ = __builtin__ + + # load the session from the datastore + session = ShellSession.get(self.request.get('session')) + + # swap in our custom module for __main__. then unpickle the session + # globals, run the statement, and re-pickle the session globals, all + # inside it. + old_main = sys.modules.get('__main__') + try: + sys.modules['__main__'] = statement_module + statement_module.__name__ = '__main__' + + # re-evaluate the unpicklables + for code in session.unpicklables: + exec code in statement_module.__dict__ + + # re-initialize the globals + for name, val in session.globals_dict().items(): + try: + statement_module.__dict__[name] = val + except: + msg = 'Dropping %s since it could not be unpickled.\n' % name + self.response.out.write(msg) + logging.warning(msg + traceback.format_exc()) + session.remove_global(name) + + # run! + old_globals = dict(statement_module.__dict__) + try: + old_stdout = sys.stdout + old_stderr = sys.stderr + try: + sys.stdout = self.response.out + sys.stderr = self.response.out + exec compiled in statement_module.__dict__ + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + except: + self.response.out.write(traceback.format_exc()) + return + + # extract the new globals that this statement added + new_globals = {} + for name, val in statement_module.__dict__.items(): + if name not in old_globals or val != old_globals[name]: + new_globals[name] = val + + if True in [isinstance(val, UNPICKLABLE_TYPES) + for val in new_globals.values()]: + # this statement added an unpicklable global. store the statement and + # the names of all of the globals it added in the unpicklables. + session.add_unpicklable(statement, new_globals.keys()) + logging.debug('Storing this statement as an unpicklable.') + + else: + # this statement didn't add any unpicklables. pickle and store the + # new globals back into the datastore. + for name, val in new_globals.items(): + if not name.startswith('__'): + session.set_global(name, val) + + finally: + sys.modules['__main__'] = old_main + + session.put() + + +def main(): + """Main program. + """ + + application = webapp.WSGIApplication( + [('/admin/shell', FrontPageHandler), + ('/admin/shell/shell.do', StatementHandler)], debug=_DEBUG) + wsgiref.handlers.CGIHandler().run(application) + + +if __name__ == '__main__': + main() diff --git a/app/shell/static/shell.js b/app/shell/static/shell.js new file mode 100644 index 00000000..67e45e8b --- /dev/null +++ b/app/shell/static/shell.js @@ -0,0 +1,195 @@ +// Copyright 2007 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview + * Javascript code for the interactive AJAX shell. + * + * Part of http://code.google.com/p/google-app-engine-samples/. + * + * Includes a function (shell.runStatement) that sends the current python + * statement in the shell prompt text box to the server, and a callback + * (shell.done) that displays the results when the XmlHttpRequest returns. + * + * Also includes cross-browser code (shell.getXmlHttpRequest) to get an + * XmlHttpRequest. + */ + +/** + * Shell namespace. + * @type {Object} + */ +var shell = {} + +/** + * The shell history. history is an array of strings, ordered oldest to + * newest. historyCursor is the current history element that the user is on. + * + * The last history element is the statement that the user is currently + * typing. When a statement is run, it's frozen in the history, a new history + * element is added to the end of the array for the new statement, and + * historyCursor is updated to point to the new element. + * + * @type {Array} + */ +shell.history = ['']; + +/** + * See {shell.history} + * @type {number} + */ +shell.historyCursor = 0; + +/** + * A constant for the XmlHttpRequest 'done' state. + * @type Number + */ +shell.DONE_STATE = 4; + +/** + * A cross-browser function to get an XmlHttpRequest object. + * + * @return {XmlHttpRequest?} a new XmlHttpRequest + */ +shell.getXmlHttpRequest = function() { + if (window.XMLHttpRequest) { + return new XMLHttpRequest(); + } else if (window.ActiveXObject) { + try { + return new ActiveXObject('Msxml2.XMLHTTP'); + } catch(e) { + return new ActiveXObject('Microsoft.XMLHTTP'); + } + } + + return null; +}; + +/** + * This is the prompt textarea's onkeypress handler. Depending on the key that + * was pressed, it will run the statement, navigate the history, or update the + * current statement in the history. + * + * @param {Event} event the keypress event + * @return {Boolean} false to tell the browser not to submit the form. + */ +shell.onPromptKeyPress = function(event) { + var statement = document.getElementById('statement'); + + if (this.historyCursor == this.history.length - 1) { + // we're on the current statement. update it in the history before doing + // anything. + this.history[this.historyCursor] = statement.value; + } + + // should we pull something from the history? + if (event.shiftKey && event.keyCode == 38 /* up arrow */) { + if (this.historyCursor > 0) { + statement.value = this.history[--this.historyCursor]; + } + return false; + } else if (event.shiftKey && event.keyCode == 40 /* down arrow */) { + if (this.historyCursor < this.history.length - 1) { + statement.value = this.history[++this.historyCursor]; + } + return false; + } else if (!event.altKey) { + // probably changing the statement. update it in the history. + this.historyCursor = this.history.length - 1; + this.history[this.historyCursor] = statement.value; + } + + // should we submit? + var ctrlEnter = (document.getElementById('submit_key').value == 'ctrl-enter'); + if (event.keyCode == 13 /* enter */ && !event.altKey && !event.shiftKey && + event.ctrlKey == ctrlEnter) { + return this.runStatement(); + } +}; + +/** + * The XmlHttpRequest callback. If the request succeeds, it adds the command + * and its resulting output to the shell history div. + * + * @param {XmlHttpRequest} req the XmlHttpRequest we used to send the current + * statement to the server + */ +shell.done = function(req) { + if (req.readyState == this.DONE_STATE) { + var statement = document.getElementById('statement') + statement.className = 'prompt'; + + // add the command to the shell output + var output = document.getElementById('output'); + + output.value += '\n>>> ' + statement.value; + statement.value = ''; + + // add a new history element + this.history.push(''); + this.historyCursor = this.history.length - 1; + + // add the command's result + var result = req.responseText.replace(/^\s*|\s*$/g, ''); // trim whitespace + if (result != '') + output.value += '\n' + result; + + // scroll to the bottom + output.scrollTop = output.scrollHeight; + if (output.createTextRange) { + var range = output.createTextRange(); + range.collapse(false); + range.select(); + } + } +}; + +/** + * This is the form's onsubmit handler. It sends the python statement to the + * server, and registers shell.done() as the callback to run when it returns. + * + * @return {Boolean} false to tell the browser not to submit the form. + */ +shell.runStatement = function() { + var form = document.getElementById('form'); + + // build a XmlHttpRequest + var req = this.getXmlHttpRequest(); + if (!req) { + document.getElementById('ajax-status').innerHTML = + "Your browser doesn't support AJAX. :("; + return false; + } + + req.onreadystatechange = function() { shell.done(req); }; + + // build the query parameter string + var params = ''; + for (i = 0; i < form.elements.length; i++) { + var elem = form.elements[i]; + if (elem.type != 'submit' && elem.type != 'button' && elem.id != 'caret') { + var value = escape(elem.value).replace(/\+/g, '%2B'); // escape ignores + + params += '&' + elem.name + '=' + value; + } + } + + // send the request and tell the user. + document.getElementById('statement').className = 'prompt processing'; + req.open(form.method, form.action + '?' + params, true); + req.setRequestHeader('Content-type', + 'application/x-www-form-urlencoded;charset=UTF-8'); + req.send(null); + + return false; +}; diff --git a/app/shell/static/spinner.gif b/app/shell/static/spinner.gif new file mode 100644 index 0000000000000000000000000000000000000000..3e58d6e8353f4e899e0f97aa6f824f907d584da4 GIT binary patch literal 1514 zcwWtu*;5o(6vn$}db($NHfGpd*f&8DiJC;GXKvq~p6;GoSt?cLA(f}(G4FZG)7Ucv zaYUGb5GzWeR;WruURa8-3B|s;pe%~Gv8kY_EXtgwv@;o-^2$p;S}JbwImd3kwydU|edZf$LCc6N4U zWo31B^~sYbRaI3pGc)V!>r+!x{r&y#HXruCe~BxECLMEQiSP7Z3SS@NxvzwSS=nk_ zDP@N0A)7GjqQ@;`ltT>;$EIR(K43-B&h6y+U19m0D%}rU!Y zJ~Pi7Wo$=HO$08`RMaOm(YYd{OXt&bWfF+40ofIXj&Gf9YnWCUhf#+-NN_kPlq)+$ zk>!v}IO%u6!k}npO8i=Zv=Snw% zVL&B#y*NxZps1sea3C6_8TbN=lxYXF&d8S;V5ua<=xB4-GBb?2^eN@l z*urqhY6)b0%rlGhH#%F_VZ1{g{Ny_pv;g$6+0SUBA%%hxQQ64;9e3Db2@v0nZ;R7se z3+s!}&79!}IhokO^JailhEu$uKp#By5u!d=A1$;vwfXb1JjkRRjbL(oO4PUE8;nem`$kBSFlP iV(~vh+4kcZ`!{o4?w9qr1Ce5@U+5h;1MuGC81pZkHpOoM literal 0 HcwPel00001 diff --git a/app/shell/templates/shell.html b/app/shell/templates/shell.html new file mode 100644 index 00000000..a5a5a10c --- /dev/null +++ b/app/shell/templates/shell.html @@ -0,0 +1,124 @@ + + + + + Interactive Shell + + + + + + +

Interactive server-side Python shell + (original source) +

+

+ Return to main home +

+ + + +
+ + + + + + +
+ +

+ +

+{% if user %} + {{ user.nickname }} + (log out) +{% else %} + log in +{% endif %} + | Shift-Up/Down for history | + + +

+ + + + + + diff --git a/scripts/build.sh b/scripts/build.sh index a54974dd..b1b30333 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -10,8 +10,8 @@ DEFAULT_APP_BUILD=../build DEFAULT_APP_FOLDER="../app" -DEFAULT_APP_FILES="app.yaml cron.yaml index.yaml main.py settings.py urls.py" -DEFAULT_APP_DIRS="soc ghop gsoc feedparser python25src reflistprop jquery ranklist json htmlsanitizer" +DEFAULT_APP_FILES="app.yaml cron.yaml index.yaml main.py settings.py shell.py urls.py gae_django.py" +DEFAULT_APP_DIRS="soc ghop gsoc feedparser python25src reflistprop jquery ranklist shell json htmlsanitizer" DEFAULT_ZIP_FILES="tiny_mce.zip" APP_BUILD=${APP_BUILD:-"${DEFAULT_APP_BUILD}"} -- 2.11.4.GIT