python/google/appengine/tools/download_appstats.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Script for downloading Appstats data using remote_api.
  22
  23 Usage:
  24   %prog [-s HOSTNAME] [-p PATH] [-o OUTPUTFILE] [-j] [-q] [-m] [APPID]
  25
  26 If the -s HOSTNAME flag is not specified, the APPID must be specified.
  27 """
  28
  29
  30
  31 from google.appengine.tools import os_compat
  32
  33 import getpass
  34 import logging
  35 import optparse
  36 import os
  37 import sys
  38
  39 from google.appengine.ext.appstats import loader
  40 from google.appengine.ext.remote_api import remote_api_stub
  41 from google.appengine.tools import appengine_rpc
  42
  43
  44
  45 DEFAULT_PATH_PYTHON = '/_ah/remote_api'
  46 DEFAULT_PATH_JAVA = '/remote_api'
  47 DEFAULT_FILE = 'appstats.pkl'
  48
  49
  50 def auth_func():
  51   return (raw_input('Email: '), getpass.getpass('Password: '))
  52
  53
  54 def download_appstats(servername, appid, path, secure,
  55                       rpc_server_factory, filename, appdir,
  56                       merge, java_application):
  57   """Invoke remote_api to download appstats data."""
  58
  59
  60
  61
  62   if os.path.isdir(appdir):
  63     sys.path.insert(0, appdir)
  64     try:
  65       logging.info('Importing appengine_config from %s', appdir)
  66       import appengine_config
  67     except ImportError, err:
  68       logging.warn('Failed to load appengine_config: %s', err)
  69
  70
  71   remote_api_stub.ConfigureRemoteApi(appid, path, auth_func,
  72                                      servername=servername,
  73                                      save_cookies=True, secure=secure,
  74                                      rpc_server_factory=rpc_server_factory)
  75   remote_api_stub.MaybeInvokeAuthentication()
  76
  77
  78   os.environ['SERVER_SOFTWARE'] = 'Development (remote_api_shell)/1.0'
  79
  80   if not appid:
  81
  82     appid = os.environ['APPLICATION_ID']
  83   download_data(filename, merge, java_application)
  84
  85
  86 def download_data(filename, merge, java_application):
  87   """Download appstats data from memcache."""
  88   oldrecords = []
  89   oldfile = None
  90
  91
  92
  93   if merge:
  94     try:
  95       oldfile = open(filename, 'rb')
  96     except IOError:
  97       logging.info('No file to merge. Creating new file %s',
  98                    filename)
  99   if oldfile:
 100     logging.info('Merging with existing file %s', filename)
 101     oldrecords = loader.UnpickleFromFile(oldfile)
 102     oldfile.close()
 103   if oldrecords:
 104
 105     last_timestamp = oldrecords[0].start_timestamp_milliseconds()
 106     records = loader.FromMemcache(filter_timestamp=last_timestamp,
 107                                   java_application=java_application)
 108   else:
 109     records = loader.FromMemcache(java_application=java_application)
 110
 111   merged_records = records + oldrecords
 112   try:
 113     outfile = open(filename, 'wb')
 114   except IOError:
 115     logging.error('Cannot open %s', filename)
 116     return
 117   loader.PickleToFile(merged_records, outfile)
 118   outfile.close()
 119
 120
 121 def main(argv):
 122   """Parse arguments and run shell."""
 123   parser = optparse.OptionParser(usage=__doc__)
 124   parser.add_option('-s', '--server', dest='server',
 125                     help='The hostname your app is deployed on. '
 126                          'Defaults to <app_id>.appspot.com.')
 127   parser.add_option('-o', '--output', dest='filename', default=DEFAULT_FILE,
 128                     help='The file to which Appstats data must '
 129                          'be downloaded. A .pkl extension is '
 130                          'recommended. Defaults to %s.' % DEFAULT_FILE)
 131   parser.add_option('-p', '--path', dest='path',
 132                     help='The path on the server to the remote_api handler. '
 133                          'Defaults to %s for python and %s for java. '
 134                          % (DEFAULT_PATH_PYTHON, DEFAULT_PATH_JAVA))
 135   parser.add_option('-q', '--quiet',
 136                     action='store_false', dest='verbose', default=True,
 137                     help='do not print download status messages to stdout')
 138   parser.add_option('-j', '--java',
 139                     action='store_true', dest='java_application', default=False,
 140                     help='set this for downloading from a java application')
 141   parser.add_option('-m', '--merge',
 142                     action='store_true', dest='merge', default=False,
 143                     help='if file exists, merge rather than overwrite')
 144   parser.add_option('--secure', dest='secure', action='store_true',
 145                     default=False, help='Use HTTPS when communicating '
 146                                         'with the server.')
 147   parser.add_option('--appdir', dest='appdir', action='store', default='.',
 148                     help='application directory, for finding '
 149                          'appengine_config.py. Defaults to ".".')
 150   (options, args) = parser.parse_args()
 151
 152
 153   if ((not options.server and not args) or len(args) > 2
 154       or (options.path and len(args) > 1)):
 155     parser.print_usage(sys.stderr)
 156     if len(args) > 2:
 157       print >> sys.stderr, 'Unexpected arguments: %s' % args[2:]
 158     elif options.path and len(args) > 1:
 159       print >> sys.stderr, 'Path specified twice.'
 160     sys.exit(1)
 161
 162
 163   servername = options.server
 164   appid = None
 165   if options.java_application:
 166     default_path = DEFAULT_PATH_JAVA
 167   else:
 168     default_path = DEFAULT_PATH_PYTHON
 169   path = options.path or default_path
 170   if args:
 171     if servername:
 172
 173       appid = args[0]
 174     else:
 175
 176       servername = '%s.appspot.com' % args[0]
 177     if len(args) == 2:
 178
 179       path = args[1]
 180   if options.verbose:
 181
 182
 183     logging.getLogger().setLevel(logging.INFO)
 184   download_appstats(servername, appid, path, options.secure,
 185                     appengine_rpc.HttpRpcServer, options.filename,
 186                     options.appdir, options.merge, options.java_application)
 187
 188
 189 if __name__ == '__main__':
 190   main(sys.argv)