python/bulkloader.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20 """Convenience wrapper for starting an appengine tool."""
  21
  22
  23 import os
  24 import re
  25 import sys
  26
  27
  28 if not hasattr(sys, 'version_info'):
  29   sys.stderr.write('Very old versions of Python are not supported. Please '
  30                    'use version 2.5 or greater.\n')
  31   sys.exit(1)
  32 version_tuple = tuple(sys.version_info[:2])
  33 if version_tuple < (2, 5):
  34   sys.stderr.write('Error: Python %d.%d is not supported. Please use '
  35                    'version 2.5 or greater.\n' % version_tuple)
  36   sys.exit(1)
  37
  38
  39 def get_dir_path(sibling):
  40   """Get a path to the directory of this script.
  41
  42   By default, the canonical path (symlinks resolved) will be returned. In some
  43   environments the canonical directory is not sufficient because different
  44   parts of the SDK are referenced by symlinks, including this very module's
  45   file. In this case, the non-canonical path to this file's directory will be
  46   returned (i.e., the directory where the symlink lives, not the directory
  47   where it points).
  48
  49   Args:
  50     sibling: Relative path to a sibiling of this module file. Choose a sibling
  51     that is potentially symlinked into the parent directory.
  52
  53   Returns:
  54     A directory name.
  55
  56   Raises:
  57     ValueError: If no proper path could be determined.
  58   """
  59   if 'GAE_SDK_ROOT' in os.environ:
  60     gae_sdk_root = os.path.abspath(os.environ['GAE_SDK_ROOT'])
  61
  62
  63
  64     os.environ['GAE_SDK_ROOT'] = gae_sdk_root
  65     for dir_path in [gae_sdk_root,
  66                      os.path.join(gae_sdk_root, 'google_appengine')]:
  67       if os.path.exists(os.path.join(dir_path, sibling)):
  68         return dir_path
  69     raise ValueError('GAE_SDK_ROOT %r does not refer to a valid SDK '
  70                      'directory' % gae_sdk_root)
  71   else:
  72     py_file = __file__.replace('.pyc', '.py')
  73     dir_paths = [os.path.abspath(os.path.dirname(os.path.realpath(py_file))),
  74                  os.path.abspath(os.path.dirname(py_file))]
  75     for dir_path in dir_paths:
  76       sibling_path = os.path.join(dir_path, sibling)
  77       if os.path.exists(sibling_path):
  78         return dir_path
  79     raise ValueError('Could not determine SDK root; please set GAE_SDK_ROOT '
  80                      'environment variable.')
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90 DIR_PATH = get_dir_path(os.path.join('lib', 'ipaddr'))
  91 SCRIPT_DIR = os.path.join(DIR_PATH, 'google', 'appengine', 'tools')
  92 GOOGLE_SQL_DIR = os.path.join(
  93     DIR_PATH, 'google', 'storage', 'speckle', 'python', 'tool')
  94
  95 EXTRA_PATHS = [
  96   DIR_PATH,
  97   os.path.join(DIR_PATH, 'lib', 'antlr3'),
  98   os.path.join(DIR_PATH, 'lib', 'django-0.96'),
  99   os.path.join(DIR_PATH, 'lib', 'fancy_urllib'),
 100   os.path.join(DIR_PATH, 'lib', 'ipaddr'),
 101   os.path.join(DIR_PATH, 'lib', 'jinja2-2.6'),
 102   os.path.join(DIR_PATH, 'lib', 'protorpc-1.0'),
 103   os.path.join(DIR_PATH, 'lib', 'PyAMF'),
 104   os.path.join(DIR_PATH, 'lib', 'markupsafe'),
 105   os.path.join(DIR_PATH, 'lib', 'webob_0_9'),
 106   os.path.join(DIR_PATH, 'lib', 'webapp2-2.5.2'),
 107   os.path.join(DIR_PATH, 'lib', 'yaml', 'lib'),
 108   os.path.join(DIR_PATH, 'lib', 'simplejson'),
 109 ]
 110
 111 API_SERVER_EXTRA_PATHS = [
 112   os.path.join(DIR_PATH, 'lib', 'argparse'),
 113 ]
 114 API_SERVER_EXTRA_PATH_SCRIPTS = 'api_server'
 115
 116
 117
 118
 119 ENDPOINTSCFG_EXTRA_PATHS = [
 120   os.path.join(DIR_PATH, 'lib', 'cherrypy'),
 121   os.path.join(DIR_PATH, 'lib', 'concurrent'),
 122   os.path.join(DIR_PATH, 'lib', 'endpoints-1.0'),
 123 ]
 124 ENDPOINTSCFG_EXTRA_PATH_SCRIPTS = 'endpointscfg'
 125
 126
 127 OAUTH_CLIENT_EXTRA_PATHS = [
 128   os.path.join(DIR_PATH, 'lib', 'google-api-python-client'),
 129   os.path.join(DIR_PATH, 'lib', 'httplib2'),
 130   os.path.join(DIR_PATH, 'lib', 'python-gflags'),
 131 ]
 132
 133 OAUTH_CLIENT_EXTRA_PATH_SCRIPTS = '(appcfg|bulkloader)'
 134
 135
 136 GOOGLE_SQL_EXTRA_PATHS = OAUTH_CLIENT_EXTRA_PATHS + [
 137   os.path.join(DIR_PATH, 'lib', 'enum'),
 138   os.path.join(DIR_PATH, 'lib', 'grizzled'),
 139   os.path.join(DIR_PATH, 'lib', 'oauth2'),
 140   os.path.join(DIR_PATH, 'lib', 'prettytable'),
 141   os.path.join(DIR_PATH, 'lib', 'sqlcmd'),
 142 ]
 143
 144 GOOGLE_SQL_EXTRA_PATH_SCRIPTS = 'google_sql'
 145
 146
 147
 148 SCRIPT_EXCEPTIONS = {
 149   "old_dev_appserver.py" : "dev_appserver_main.py"
 150 }
 151
 152 SCRIPT_DIR_EXCEPTIONS = {
 153   'google_sql.py': GOOGLE_SQL_DIR,
 154 }
 155
 156
 157 def fix_sys_path(extra_extra_paths=()):
 158   """Fix the sys.path to include our extra paths."""
 159   extra_paths = EXTRA_PATHS[:]
 160   extra_paths.extend(extra_extra_paths)
 161   sys.path = extra_paths + sys.path
 162
 163
 164 def run_file(file_path, globals_, script_dir=SCRIPT_DIR):
 165   """Execute the file at the specified path with the passed-in globals."""
 166   script_name = os.path.basename(file_path)
 167
 168   if re.match(OAUTH_CLIENT_EXTRA_PATH_SCRIPTS, script_name):
 169     extra_extra_paths = OAUTH_CLIENT_EXTRA_PATHS
 170   elif re.match(GOOGLE_SQL_EXTRA_PATH_SCRIPTS, script_name):
 171     extra_extra_paths = GOOGLE_SQL_EXTRA_PATHS
 172   elif re.match(API_SERVER_EXTRA_PATH_SCRIPTS, script_name):
 173     extra_extra_paths = API_SERVER_EXTRA_PATHS
 174   elif re.match(ENDPOINTSCFG_EXTRA_PATH_SCRIPTS, script_name):
 175     extra_extra_paths = ENDPOINTSCFG_EXTRA_PATHS
 176   else:
 177     extra_extra_paths = []
 178   fix_sys_path(extra_extra_paths)
 179
 180
 181
 182
 183
 184
 185
 186   if 'google' in sys.modules:
 187     del sys.modules['google']
 188
 189   script_name = SCRIPT_EXCEPTIONS.get(script_name, script_name)
 190   script_dir = SCRIPT_DIR_EXCEPTIONS.get(script_name, script_dir)
 191   script_path = os.path.join(script_dir, script_name)
 192   execfile(script_path, globals_)
 193
 194
 195 if __name__ == '__main__':
 196   run_file(__file__, globals())