python/google/appengine/tools/appcfg_java.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Appcfg logic specific to Java apps."""
  18 from __future__ import with_statement
  19
  20 import collections
  21 import os.path
  22 import re
  23 import shutil
  24 import stat
  25 import subprocess
  26 import sys
  27 import tempfile
  28
  29 from google.appengine.datastore import datastore_index
  30 from google.appengine.datastore import datastore_index_xml
  31 from google.appengine.tools import app_engine_web_xml_parser
  32 from google.appengine.tools import backends_xml_parser
  33 from google.appengine.tools import cron_xml_parser
  34 from google.appengine.tools import dispatch_xml_parser
  35 from google.appengine.tools import dos_xml_parser
  36 from google.appengine.tools import jarfile
  37 from google.appengine.tools import queue_xml_parser
  38 from google.appengine.tools import web_xml_parser
  39 from google.appengine.tools import yaml_translator
  40
  41
  42 _CLASSES_JAR_NAME_PREFIX = '_ah_webinf_classes'
  43 _COMPILED_JSP_JAR_NAME_PREFIX = '_ah_compiled_jsps'
  44 _LOCAL_JSPC_CLASS = 'com.google.appengine.tools.development.LocalJspC'
  45 _MAX_COMPILED_JSP_JAR_SIZE = 1024 * 1024 * 5
  46
  47
  48 class Error(Exception):
  49   pass
  50
  51
  52 class ConfigurationError(Error):
  53   """There was a configuration error in the application being uploaded."""
  54   pass
  55
  56
  57 class CompileError(Error):
  58   """There was a compilation error in a JSP file or its generated Java code."""
  59   pass
  60
  61
  62 def IsWarFileWithoutYaml(dir_path):
  63   if os.path.isfile(os.path.join(dir_path, 'app.yaml')):
  64     return False
  65   web_inf = os.path.join(dir_path, 'WEB-INF')
  66   return (os.path.isdir(web_inf) and
  67           set(['appengine-web.xml', 'web.xml']).issubset(os.listdir(web_inf)))
  68
  69
  70 def AddUpdateOptions(parser):
  71   """Adds options specific to the 'update' command on Java apps to 'parser'.
  72
  73   Args:
  74     parser: An instance of OptionsParser.
  75   """
  76   parser.add_option('--retain_upload_dir', action='store_true',
  77                     dest='retain_upload_dir', default=False,
  78                     help='Do not delete temporary (staging) directory used '
  79                     'in uploading Java apps')
  80   parser.add_option('--no_symlinks', action='store_true',
  81                     dest='no_symlinks', default=False,
  82                     help='Do not use symbolic links when making the temporary '
  83                     '(staging) directory for uploading Java apps')
  84   parser.add_option('--compile_encoding', action='store',
  85                     dest='compile_encoding', default='UTF-8',
  86                     help='Set the encoding to be used when compiling Java '
  87                     'source files (default "UTF-8").')
  88   parser.add_option('--disable_jar_jsps', action='store_false',
  89                     dest='jar_jsps', default=True,
  90                     help='Do not jar the classes generated from JSPs.')
  91   parser.add_option('--delete_jsps', action='store_true',
  92                     dest='delete_jsps', default=False,
  93                     help='Delete the JSP source files after compilation.')
  94   parser.add_option('--enable_jar_classes', action='store_true',
  95                     dest='do_jar_classes', default=False,
  96                     help='Jar the WEB-INF/classes content.')
  97   parser.add_option('--enable_jar_splitting', action='store_true',
  98                     dest='do_jar_splitting', default=False,
  99                     help='Split large jar files (> 32M) into smaller '
 100                     'fragments.')
 101   parser.add_option('--jar_splitting_excludes', action='store',
 102                     dest='jar_splitting_exclude_suffixes', default='',
 103                     help='When --enable_jar_splitting is specified and '
 104                     '--jar_splitting_excludes specifies a comma-separated list '
 105                     'of suffixes, a file in a jar whose name ends with one '
 106                     'of the suffixes will not be included in the split jar '
 107                     'fragments.')
 108
 109
 110 class JavaAppUpdate(object):
 111   """Performs Java-specific update configurations."""
 112
 113   _JSP_REGEX = re.compile('.*\\.jspx?')
 114
 115   _xml_parser = collections.namedtuple(
 116       '_xml_parser', ['xml_name', 'yaml_name', 'xml_to_yaml_function'])
 117
 118   _XML_PARSERS = [
 119       _xml_parser('backends.xml', 'backends.yaml',
 120                   backends_xml_parser.GetBackendsYaml),
 121       _xml_parser('cron.xml', 'cron.yaml', cron_xml_parser.GetCronYaml),
 122       _xml_parser('dispatch.xml', 'dispatch.yaml',
 123                   dispatch_xml_parser.GetDispatchYaml),
 124       _xml_parser('dos.xml', 'dos.yaml', dos_xml_parser.GetDosYaml),
 125       _xml_parser('queue.xml', 'queue.yaml', queue_xml_parser.GetQueueYaml),
 126   ]
 127
 128   _XML_VALIDATOR_CLASS = 'com.google.appengine.tools.admin.XmlValidator'
 129
 130   def __init__(self, basepath, options):
 131     self.basepath = os.path.abspath(basepath)
 132     self.options = options
 133     if not hasattr(self.options, 'no_symlinks'):
 134
 135       self.options.no_symlinks = True
 136
 137     java_home, exec_suffix = JavaHomeAndSuffix()
 138     self.java_command = os.path.join(java_home, 'bin', 'java' + exec_suffix)
 139     self.javac_command = os.path.join(java_home, 'bin', 'javac' + exec_suffix)
 140
 141     self._ValidateXmlFiles()
 142
 143     self.app_engine_web_xml = self._ReadAppEngineWebXml()
 144     self.app_engine_web_xml.app_root = self.basepath
 145     if self.options.app_id:
 146       self.app_engine_web_xml.app_id = self.options.app_id
 147     if self.options.version:
 148       self.app_engine_web_xml.version_id = self.options.version
 149     self.web_xml = self._ReadWebXml()
 150
 151   def _ValidateXmlFiles(self):
 152
 153
 154
 155
 156
 157
 158
 159
 160     sdk_dir = os.path.dirname(jarfile.__file__)
 161     xml_validator_jar = os.path.join(
 162         sdk_dir, 'java', 'lib', 'impl', 'libxmlvalidator.jar')
 163     if not os.path.exists(xml_validator_jar):
 164
 165       print >>sys.stderr, ('Not validating XML files because %s does not '
 166                            'exist' % xml_validator_jar)
 167       return
 168     validator_args = []
 169     schema_dir = os.path.join(sdk_dir, 'java', 'docs')
 170     for schema_name in os.listdir(schema_dir):
 171       basename, extension = os.path.splitext(schema_name)
 172       if extension == '.xsd':
 173         schema_file = os.path.join(schema_dir, schema_name)
 174         xml_file = os.path.join(self.basepath, 'WEB-INF', basename + '.xml')
 175         if os.path.exists(xml_file):
 176           validator_args += [xml_file, schema_file]
 177     if validator_args:
 178       command_and_args = [
 179           self.java_command,
 180           '-classpath',
 181           xml_validator_jar,
 182           self._XML_VALIDATOR_CLASS,
 183       ] + validator_args
 184       status = subprocess.call(command_and_args)
 185       if status:
 186
 187         raise ConfigurationError('XML validation failed')
 188
 189   def _ReadAppEngineWebXml(self):
 190     return self._ReadAndParseXml(
 191         basepath=self.basepath,
 192         file_name='appengine-web.xml',
 193         parser=app_engine_web_xml_parser.AppEngineWebXmlParser)
 194
 195   def _ReadWebXml(self, basepath=None):
 196     if not basepath:
 197       basepath = self.basepath
 198     return self._ReadAndParseXml(
 199         basepath=basepath,
 200         file_name='web.xml',
 201         parser=web_xml_parser.WebXmlParser)
 202
 203   def _ReadAndParseXml(self, basepath, file_name, parser):
 204     with open(os.path.join(basepath, 'WEB-INF', file_name)) as file_handle:
 205       return parser().ProcessXml(file_handle.read())
 206
 207   def CreateStagingDirectory(self, tools_dir):
 208     """Creates a staging directory for uploading.
 209
 210     This is where we perform the necessary actions to create an application
 211     directory for the update command to work properly - files are organized
 212     into the static folder, and yaml files are generated where they can be
 213     found later.
 214
 215     Args:
 216       tools_dir: Path to the SDK tools directory
 217         (typically .../google/appengine/tools)
 218
 219     Returns:
 220       The path to a new temporary directory which contains generated yaml files
 221       and a static file directory. For the most part, the rest of the update and
 222       upload flow can resume identically to Python/PHP/Go applications.
 223
 224     Raises:
 225       CompileError: if compilation of JSPs failed.
 226       ConfigurationError: if the app to be staged has a configuration error.
 227       IOError: if there was an I/O problem, for example when scanning jar files.
 228     """
 229     stage_dir = tempfile.mkdtemp(prefix='appcfgpy')
 230     static_dir = os.path.join(stage_dir, '__static__')
 231     os.mkdir(static_dir)
 232     self._CopyOrLink(self.basepath, stage_dir, static_dir, False)
 233     self.app_engine_web_xml.app_root = stage_dir
 234
 235     if self.options.compile_jsps:
 236       self._CompileJspsIfAny(tools_dir, stage_dir)
 237
 238     web_inf = os.path.join(stage_dir, 'WEB-INF')
 239     web_inf_lib = os.path.join(web_inf, 'lib')
 240     api_jar_dict = _FindApiJars(web_inf_lib)
 241     api_versions = set(api_jar_dict.values())
 242     if not api_versions:
 243       api_version = None
 244     elif len(api_versions) == 1:
 245       api_version = api_versions.pop()
 246     else:
 247       raise ConfigurationError('API jars have inconsistent versions: %s' %
 248                                api_jar_dict)
 249
 250
 251     for staged_api_jar in api_jar_dict:
 252       os.remove(staged_api_jar)
 253
 254     appengine_generated = os.path.join(
 255         stage_dir, 'WEB-INF', 'appengine-generated')
 256     self._GenerateAppYaml(stage_dir, api_version, appengine_generated)
 257
 258     app_id = self.options.app_id or self.app_engine_web_xml.app_id
 259     assert app_id, 'Missing app id'
 260
 261
 262     for parser in self._XML_PARSERS:
 263       xml_name = os.path.join(web_inf, parser.xml_name)
 264       if os.path.exists(xml_name):
 265         with open(xml_name) as xml_file:
 266           xml_string = xml_file.read()
 267         yaml_string = parser.xml_to_yaml_function(app_id, xml_string)
 268         yaml_file = os.path.join(appengine_generated, parser.yaml_name)
 269         with open(yaml_file, 'w') as yaml:
 270           yaml.write(yaml_string)
 271
 272
 273
 274
 275
 276
 277     indexes = []
 278     for xml_name in (
 279         'datastore-indexes.xml',
 280         os.path.join('appengine-generated', 'datastore-indexes-auto.xml')):
 281       xml_name = os.path.join(self.basepath, 'WEB-INF', xml_name)
 282       if os.path.exists(xml_name):
 283         with open(xml_name) as xml_file:
 284           xml_string = xml_file.read()
 285         index_definitions = datastore_index_xml.IndexesXmlToIndexDefinitions(
 286             xml_string)
 287         indexes.extend(index_definitions.indexes)
 288     if indexes:
 289       yaml_string = datastore_index.IndexDefinitions(indexes=indexes).ToYAML()
 290       yaml_file = os.path.join(appengine_generated, 'index.yaml')
 291       with open(yaml_file, 'w') as yaml:
 292         yaml.write(yaml_string)
 293
 294     return stage_dir
 295
 296   def GenerateAppYamlString(self, static_file_list, api_version=None):
 297     """Constructs an app.yaml string equivalent to the XML files under WEB-INF.
 298
 299     Args:
 300       static_file_list: a list of strings that are the absolute path names of
 301         static file resources.
 302       api_version: a string that is the Java API version number, or None if
 303         not known or relevant.
 304
 305     Returns:
 306       A string that would have the same effect as the XML files under WEB-INF
 307       if it were the contents of an app.yaml file.
 308     """
 309     return yaml_translator.AppYamlTranslator(
 310         self.app_engine_web_xml,
 311         self.web_xml,
 312         static_file_list,
 313         api_version).GetYaml()
 314
 315   def _GenerateAppYaml(self, stage_dir, api_version, appengine_generated):
 316     """Creates the app.yaml file in WEB-INF/appengine-generated/.
 317
 318     Returns:
 319       The path to the WEB-INF/appengine-generated directory.
 320     """
 321     static_file_list = self._GetStaticFileList(stage_dir)
 322     yaml_str = self.GenerateAppYamlString(static_file_list, api_version)
 323     if not os.path.isdir(appengine_generated):
 324       os.mkdir(appengine_generated)
 325     with open(os.path.join(appengine_generated, 'app.yaml'), 'w') as handle:
 326       handle.write(yaml_str)
 327
 328   def _CopyOrLink(self, source_dir, stage_dir, static_dir, inside_web_inf):
 329     source_dir = os.path.abspath(source_dir)
 330     stage_dir = os.path.abspath(stage_dir)
 331     static_dir = os.path.abspath(static_dir)
 332     for file_name in os.listdir(source_dir):
 333       file_path = os.path.join(source_dir, file_name)
 334
 335       if file_name.startswith('.') or file_name == 'appengine-generated':
 336         continue
 337
 338       if os.path.isdir(file_path):
 339         self._CopyOrLink(
 340             file_path,
 341             os.path.join(stage_dir, file_name),
 342             os.path.join(static_dir, file_name),
 343             inside_web_inf or file_name == 'WEB-INF')
 344       else:
 345         if (inside_web_inf
 346             or self.app_engine_web_xml.IncludesResource(file_path)
 347             or (self.options.compile_jsps
 348                 and file_path.lower().endswith('.jsp'))):
 349           self._CopyOrLinkFile(file_path, os.path.join(stage_dir, file_name))
 350         if (not inside_web_inf
 351             and self.app_engine_web_xml.IncludesStatic(file_path)):
 352           self._CopyOrLinkFile(file_path, os.path.join(static_dir, file_name))
 353
 354   def _CopyOrLinkFile(self, source, dest):
 355
 356     destdir = os.path.dirname(dest)
 357     if not os.path.exists(destdir):
 358       os.makedirs(destdir)
 359     if self._ShouldSplitJar(source):
 360       self._SplitJar(source, destdir)
 361     elif source.endswith('web.xml'):
 362       shutil.copy(source, dest)
 363       os.chmod(dest, os.stat(dest).st_mode | stat.S_IWRITE)
 364
 365     elif self.options.no_symlinks:
 366       shutil.copy(source, dest)
 367     else:
 368       os.symlink(source, dest)
 369
 370   def _MoveDirectoryContents(self, source_dir, dest_dir):
 371     """Move the contents of source_dir to dest_dir, which might not exist.
 372
 373     Raises:
 374       IOError: if the dest_dir hierarchy already contains a file where the
 375         source_dir hierarchy has a file or directory of the same name, or if
 376         the dest_dir hierarchy already contains a directory where the source_dir
 377         hierarchy has a file of the same name.
 378     """
 379     if not os.path.exists(dest_dir):
 380       os.mkdir(dest_dir)
 381     for entry in os.listdir(source_dir):
 382       source_entry = os.path.join(source_dir, entry)
 383       dest_entry = os.path.join(dest_dir, entry)
 384       if os.path.exists(dest_entry):
 385         if os.path.isdir(source_entry) and os.path.isdir(dest_entry):
 386           self._MoveDirectoryContents(source_entry, dest_entry)
 387         else:
 388           raise IOError('Cannot overwrite existing %s' % dest_entry)
 389       else:
 390         shutil.move(source_entry, dest_entry)
 391
 392   _MAX_SIZE = 32 * 1000 * 1000
 393
 394
 395   def _ShouldSplitJar(self, path):
 396     return (path.lower().endswith('.jar') and self.options.do_jar_splitting and
 397             os.path.getsize(path) >= self._MAX_SIZE)
 398
 399   def _SplitJar(self, jar_path, dest_dir):
 400     """Split a source jar into two or more jars in the given dest_dir.
 401
 402     Args:
 403       jar_path: string that is the path to jar to be split. The contents of this
 404         jar will be copied into the output jars, but the jar itself will not be
 405         affected.
 406       dest_dir: directory into which to put the jars that result from splitting
 407         the input jar.
 408
 409     Raises:
 410       IOError: if the jar cannot be split.
 411     """
 412
 413     exclude_suffixes = (
 414         set(self.options.jar_splitting_exclude_suffixes.split(',')) - set(['']))
 415     include = lambda name: not any(name.endswith(s) for s in exclude_suffixes)
 416     jarfile.SplitJar(jar_path, dest_dir, self._MAX_SIZE, include)
 417
 418   @staticmethod
 419   def _GetStaticFileList(staging_dir):
 420     return _FilesMatching(os.path.join(staging_dir, '__static__'))
 421
 422   def _CompileJspsIfAny(self, tools_dir, staging_dir):
 423     """Compiles JSP files, if any, into .class files.."""
 424     if self._MatchingFileExists(self._JSP_REGEX, staging_dir):
 425       gen_dir = tempfile.mkdtemp()
 426       try:
 427         self._CompileJspsWithGenDir(tools_dir, staging_dir, gen_dir)
 428       finally:
 429         shutil.rmtree(gen_dir)
 430
 431   def _CompileJspsWithGenDir(self, tools_dir, staging_dir, gen_dir):
 432     staging_web_inf = os.path.join(staging_dir, 'WEB-INF')
 433     lib_dir = os.path.join(staging_web_inf, 'lib')
 434
 435     for jar_file in GetUserJspLibFiles(tools_dir):
 436       self._CopyOrLinkFile(
 437           jar_file, os.path.join(lib_dir, os.path.basename(jar_file)))
 438     for jar_file in GetSharedJspLibFiles(tools_dir):
 439       self._CopyOrLinkFile(
 440           jar_file, os.path.join(lib_dir, os.path.basename(jar_file)))
 441
 442     classes_dir = os.path.join(staging_web_inf, 'classes')
 443     generated_web_xml = os.path.join(staging_web_inf, 'generated_web.xml')
 444
 445     classpath = self._GetJspClasspath(tools_dir, classes_dir, gen_dir)
 446
 447     command_and_args = [
 448         self.java_command,
 449         '-classpath', classpath,
 450         _LOCAL_JSPC_CLASS,
 451         '-uriroot', staging_dir,
 452         '-p', 'org.apache.jsp',
 453         '-l',
 454         '-v',
 455         '-webinc', generated_web_xml,
 456         '-d', gen_dir,
 457         '-javaEncoding', self.options.compile_encoding,
 458     ]
 459
 460     status = subprocess.call(command_and_args)
 461     if status:
 462       raise CompileError(
 463           'Compilation of JSPs exited with status %d' % status)
 464
 465     self._CompileJavaFiles(classpath, staging_web_inf, gen_dir)
 466
 467
 468     self.web_xml = self._ReadWebXml(staging_dir)
 469
 470   def _CompileJavaFiles(self, classpath, web_inf, jsp_class_dir):
 471     """Compile all *.java files found under jsp_class_dir."""
 472     java_files = _FilesMatching(jsp_class_dir, lambda f: f.endswith('.java'))
 473     if not java_files:
 474       return
 475
 476     command_and_args = [
 477         self.javac_command,
 478         '-classpath', classpath,
 479         '-d', jsp_class_dir,
 480         '-encoding', self.options.compile_encoding,
 481     ] + java_files
 482
 483     status = subprocess.call(command_and_args)
 484     if status:
 485       raise CompileError(
 486           'Compilation of JSP-generated code exited with status %d' % status)
 487
 488     if self.options.jar_jsps:
 489       self._ZipJasperGeneratedFiles(web_inf, jsp_class_dir)
 490     else:
 491       web_inf_classes = os.path.join(web_inf, 'classes')
 492       self._MoveDirectoryContents(jsp_class_dir, web_inf_classes)
 493
 494     if self.options.delete_jsps:
 495       jsps = _FilesMatching(os.path.dirname(web_inf),
 496                             lambda f: f.endswith('.jsp'))
 497       for f in jsps:
 498         os.remove(f)
 499
 500     if self.options.do_jar_classes:
 501       self._ZipWebInfClassesFiles(web_inf)
 502
 503   @staticmethod
 504   def _ZipJasperGeneratedFiles(web_inf, jsp_class_dir):
 505     lib_dir = os.path.join(web_inf, 'lib')
 506     jarfile.Make(jsp_class_dir, lib_dir, _COMPILED_JSP_JAR_NAME_PREFIX,
 507                  maximum_size=_MAX_COMPILED_JSP_JAR_SIZE,
 508                  include_predicate=lambda name: not name.endswith('.java'))
 509
 510   @staticmethod
 511   def _ZipWebInfClassesFiles(web_inf):
 512
 513
 514     lib_dir = os.path.join(web_inf, 'lib')
 515     classes_dir = os.path.join(web_inf, 'classes')
 516     jarfile.Make(classes_dir, lib_dir, _CLASSES_JAR_NAME_PREFIX,
 517                  maximum_size=_MAX_COMPILED_JSP_JAR_SIZE)
 518     shutil.rmtree(classes_dir)
 519
 520     os.mkdir(classes_dir)
 521
 522   @staticmethod
 523   def _GetJspClasspath(tools_dir, classes_dir, gen_dir):
 524     """Builds the classpath for the JSP Compilation system call."""
 525     lib_dir = os.path.join(os.path.dirname(classes_dir), 'lib')
 526     elements = (
 527         GetImplLibs(tools_dir) + GetSharedLibFiles(tools_dir) +
 528         [classes_dir, gen_dir] +
 529         _FilesMatching(
 530             lib_dir, lambda f: f.endswith('.jar') or f.endswith('.zip')))
 531
 532     return (os.pathsep).join(elements)
 533
 534   @staticmethod
 535   def _MatchingFileExists(regex, dir_path):
 536     for _, _, files in os.walk(dir_path):
 537       for f in files:
 538         if re.search(regex, f):
 539           return True
 540     return False
 541
 542
 543 def GetImplLibs(tools_dir):
 544   return _GetLibsShallow(os.path.join(tools_dir, 'java', 'lib', 'impl'))
 545
 546
 547 def GetSharedLibFiles(tools_dir):
 548   return _GetLibsRecursive(os.path.join(tools_dir, 'java', 'lib', 'shared'))
 549
 550
 551 def GetUserJspLibFiles(tools_dir):
 552   return _GetLibsRecursive(
 553       os.path.join(tools_dir, 'java', 'lib', 'tools', 'jsp'))
 554
 555
 556 def GetSharedJspLibFiles(tools_dir):
 557   return _GetLibsRecursive(
 558       os.path.join(tools_dir, 'java', 'lib', 'shared', 'jsp'))
 559
 560
 561 def _GetLibsRecursive(dir_path):
 562   return _FilesMatching(dir_path, lambda f: f.endswith('.jar'))
 563
 564
 565 def _GetLibsShallow(dir_path):
 566   libs = []
 567   for f in os.listdir(dir_path):
 568     if os.path.isfile(os.path.join(dir_path, f)) and f.endswith('.jar'):
 569       libs.append(os.path.join(dir_path, f))
 570   return libs
 571
 572
 573 def _FilesMatching(root, predicate=lambda f: True):
 574   """Finds all files under the given root that match the given predicate.
 575
 576   Args:
 577     root: a string that is the absolute or relative path to a directory.
 578     predicate: a function that takes a file name (without a directory) and
 579       returns a truth value.
 580
 581   Returns:
 582     A list of strings that are the paths of every file under the given root
 583     that satisfies the given predicate. The paths are absolute if and only if
 584     the input root is absolute.
 585   """
 586   matches = []
 587   for path, _, files in os.walk(root):
 588     matches += [os.path.join(path, f) for f in files if predicate(f)]
 589   return matches
 590
 591
 592 def JavaHomeAndSuffix():
 593   """Find the directory that the JDK is installed in.
 594
 595   The JDK install directory is expected to have a bin directory that contains
 596   at a minimum the java and javac executables. If the environment variable
 597   JAVA_HOME is set then it must point to such a directory. Otherwise, we look
 598   for javac on the PATH and check that it is inside a JDK install directory.
 599
 600   Returns:
 601     A tuple where the first element is the JDK install directory and the second
 602     element is a suffix that must be appended to executables in that directory
 603     ('' on Unix-like systems, '.exe' on Windows).
 604
 605   Raises:
 606     RuntimeError: If JAVA_HOME is set but is not a JDK install directory, or
 607     otherwise if a JDK install directory cannot be found based on the PATH.
 608   """
 609   def ResultForJdkAt(path):
 610     """Return (path, suffix) if path is a JDK install directory, else None."""
 611     def IsExecutable(binary):
 612       return os.path.isfile(binary) and os.access(binary, os.X_OK)
 613
 614     def ResultFor(path):
 615       for suffix in ['', '.exe']:
 616         if all(IsExecutable(os.path.join(path, 'bin', binary + suffix))
 617                for binary in ['java', 'javac', 'jar']):
 618           return (path, suffix)
 619       return None
 620
 621     result = ResultFor(path)
 622     if not result:
 623
 624
 625       head, tail = os.path.split(path)
 626       if tail == 'jre':
 627         result = ResultFor(head)
 628     return result
 629
 630   java_home = os.getenv('JAVA_HOME')
 631   if java_home:
 632     result = ResultForJdkAt(java_home)
 633     if result:
 634       return result
 635     else:
 636       raise RuntimeError(
 637           'JAVA_HOME is set but does not reference a valid JDK: %s' % java_home)
 638   for path_dir in os.environ['PATH'].split(os.pathsep):
 639     maybe_root, last = os.path.split(path_dir)
 640     if last == 'bin':
 641       result = ResultForJdkAt(maybe_root)
 642       if result:
 643         return result
 644   raise RuntimeError('Did not find JDK in PATH and JAVA_HOME is not set')
 645
 646
 647 def _FindApiJars(lib_dir):
 648   """Find the appengine-api-*.jar and its version.
 649
 650   The version of an appengine-api-*.jar is the Specification-Version attribute
 651   in the jar's manifest section whose Name is 'com/google/appengine/api/'.
 652
 653   Args:
 654     lib_dir: the base directory under which jars are to be found.
 655
 656   Returns:
 657     A dict from string to string, mapping all found API jars to their
 658     corresponding versions.
 659
 660   Raises:
 661     IOError: if there was a problem reading the jars.
 662   """
 663   result = {}
 664   for jar_file in _FilesMatching(lib_dir, lambda f: f.endswith('.jar')):
 665     manifest = jarfile.ReadManifest(jar_file)
 666     if manifest:
 667       section = manifest.sections.get('com/google/appengine/api/')
 668       if section and 'Specification-Version' in section:
 669         result[jar_file] = section['Specification-Version']
 670   return result