python/lib/distutils/distutils/command/sdist.py

   1 """distutils.command.sdist
   2
   3 Implements the Distutils 'sdist' command (create a source distribution)."""
   4
   5 __revision__ = "$Id$"
   6
   7 import os
   8 import string
   9 import sys
  10 from glob import glob
  11 from warnings import warn
  12
  13 from distutils.core import Command
  14 from distutils import dir_util, dep_util, file_util, archive_util
  15 from distutils.text_file import TextFile
  16 from distutils.errors import (DistutilsPlatformError, DistutilsOptionError,
  17                               DistutilsTemplateError)
  18 from distutils.filelist import FileList
  19 from distutils import log
  20 from distutils.util import convert_path
  21
  22 def show_formats():
  23     """Print all possible values for the 'formats' option (used by
  24     the "--help-formats" command-line option).
  25     """
  26     from distutils.fancy_getopt import FancyGetopt
  27     from distutils.archive_util import ARCHIVE_FORMATS
  28     formats = []
  29     for format in ARCHIVE_FORMATS.keys():
  30         formats.append(("formats=" + format, None,
  31                         ARCHIVE_FORMATS[format][2]))
  32     formats.sort()
  33     FancyGetopt(formats).print_help(
  34         "List of available source distribution formats:")
  35
  36 class sdist(Command):
  37
  38     description = "create a source distribution (tarball, zip file, etc.)"
  39
  40     def checking_metadata(self):
  41         """Callable used for the check sub-command.
  42
  43         Placed here so user_options can view it"""
  44         return self.metadata_check
  45
  46     user_options = [
  47         ('template=', 't',
  48          "name of manifest template file [default: MANIFEST.in]"),
  49         ('manifest=', 'm',
  50          "name of manifest file [default: MANIFEST]"),
  51         ('use-defaults', None,
  52          "include the default file set in the manifest "
  53          "[default; disable with --no-defaults]"),
  54         ('no-defaults', None,
  55          "don't include the default file set"),
  56         ('prune', None,
  57          "specifically exclude files/directories that should not be "
  58          "distributed (build tree, RCS/CVS dirs, etc.) "
  59          "[default; disable with --no-prune]"),
  60         ('no-prune', None,
  61          "don't automatically exclude anything"),
  62         ('manifest-only', 'o',
  63          "just regenerate the manifest and then stop "
  64          "(implies --force-manifest)"),
  65         ('force-manifest', 'f',
  66          "forcibly regenerate the manifest and carry on as usual. "
  67          "Deprecated: now the manifest is always regenerated."),
  68         ('formats=', None,
  69          "formats for source distribution (comma-separated list)"),
  70         ('keep-temp', 'k',
  71          "keep the distribution tree around after creating " +
  72          "archive file(s)"),
  73         ('dist-dir=', 'd',
  74          "directory to put the source distribution archive(s) in "
  75          "[default: dist]"),
  76         ('metadata-check', None,
  77          "Ensure that all required elements of meta-data "
  78          "are supplied. Warn if any missing. [default]"),
  79         ('owner=', 'u',
  80          "Owner name used when creating a tar file [default: current user]"),
  81         ('group=', 'g',
  82          "Group name used when creating a tar file [default: current group]"),
  83         ]
  84
  85     boolean_options = ['use-defaults', 'prune',
  86                        'manifest-only', 'force-manifest',
  87                        'keep-temp', 'metadata-check']
  88
  89     help_options = [
  90         ('help-formats', None,
  91          "list available distribution formats", show_formats),
  92         ]
  93
  94     negative_opt = {'no-defaults': 'use-defaults',
  95                     'no-prune': 'prune' }
  96
  97     default_format = {'posix': 'gztar',
  98                       'nt': 'zip' }
  99
 100     sub_commands = [('check', checking_metadata)]
 101
 102     def initialize_options(self):
 103         # 'template' and 'manifest' are, respectively, the names of
 104         # the manifest template and manifest file.
 105         self.template = None
 106         self.manifest = None
 107
 108         # 'use_defaults': if true, we will include the default file set
 109         # in the manifest
 110         self.use_defaults = 1
 111         self.prune = 1
 112
 113         self.manifest_only = 0
 114         self.force_manifest = 0
 115
 116         self.formats = None
 117         self.keep_temp = 0
 118         self.dist_dir = None
 119
 120         self.archive_files = None
 121         self.metadata_check = 1
 122         self.owner = None
 123         self.group = None
 124
 125     def finalize_options(self):
 126         if self.manifest is None:
 127             self.manifest = "MANIFEST"
 128         if self.template is None:
 129             self.template = "MANIFEST.in"
 130
 131         self.ensure_string_list('formats')
 132         if self.formats is None:
 133             try:
 134                 self.formats = [self.default_format[os.name]]
 135             except KeyError:
 136                 raise DistutilsPlatformError, \
 137                       "don't know how to create source distributions " + \
 138                       "on platform %s" % os.name
 139
 140         bad_format = archive_util.check_archive_formats(self.formats)
 141         if bad_format:
 142             raise DistutilsOptionError, \
 143                   "unknown archive format '%s'" % bad_format
 144
 145         if self.dist_dir is None:
 146             self.dist_dir = "dist"
 147
 148     def run(self):
 149         # 'filelist' contains the list of files that will make up the
 150         # manifest
 151         self.filelist = FileList()
 152
 153         # Run sub commands
 154         for cmd_name in self.get_sub_commands():
 155             self.run_command(cmd_name)
 156
 157         # Do whatever it takes to get the list of files to process
 158         # (process the manifest template, read an existing manifest,
 159         # whatever).  File list is accumulated in 'self.filelist'.
 160         self.get_file_list()
 161
 162         # If user just wanted us to regenerate the manifest, stop now.
 163         if self.manifest_only:
 164             return
 165
 166         # Otherwise, go ahead and create the source distribution tarball,
 167         # or zipfile, or whatever.
 168         self.make_distribution()
 169
 170     def check_metadata(self):
 171         """Deprecated API."""
 172         warn("distutils.command.sdist.check_metadata is deprecated, \
 173               use the check command instead", PendingDeprecationWarning)
 174         check = self.distribution.get_command_obj('check')
 175         check.ensure_finalized()
 176         check.run()
 177
 178     def get_file_list(self):
 179         """Figure out the list of files to include in the source
 180         distribution, and put it in 'self.filelist'.  This might involve
 181         reading the manifest template (and writing the manifest), or just
 182         reading the manifest, or just using the default file set -- it all
 183         depends on the user's options.
 184         """
 185         # new behavior when using a template:
 186         # the file list is recalculated every time because
 187         # even if MANIFEST.in or setup.py are not changed
 188         # the user might have added some files in the tree that
 189         # need to be included.
 190         #
 191         #  This makes --force the default and only behavior with templates.
 192         template_exists = os.path.isfile(self.template)
 193         if not template_exists and self._manifest_is_not_generated():
 194             self.read_manifest()
 195             self.filelist.sort()
 196             self.filelist.remove_duplicates()
 197             return
 198
 199         if not template_exists:
 200             self.warn(("manifest template '%s' does not exist " +
 201                         "(using default file list)") %
 202                         self.template)
 203         self.filelist.findall()
 204
 205         if self.use_defaults:
 206             self.add_defaults()
 207
 208         if template_exists:
 209             self.read_template()
 210
 211         if self.prune:
 212             self.prune_file_list()
 213
 214         self.filelist.sort()
 215         self.filelist.remove_duplicates()
 216         self.write_manifest()
 217
 218     def add_defaults(self):
 219         """Add all the default files to self.filelist:
 220           - README or README.txt
 221           - setup.py
 222           - test/test*.py
 223           - all pure Python modules mentioned in setup script
 224           - all files pointed by package_data (build_py)
 225           - all files defined in data_files.
 226           - all files defined as scripts.
 227           - all C sources listed as part of extensions or C libraries
 228             in the setup script (doesn't catch C headers!)
 229         Warns if (README or README.txt) or setup.py are missing; everything
 230         else is optional.
 231         """
 232
 233         standards = [('README', 'README.txt'), self.distribution.script_name]
 234         for fn in standards:
 235             if isinstance(fn, tuple):
 236                 alts = fn
 237                 got_it = 0
 238                 for fn in alts:
 239                     if os.path.exists(fn):
 240                         got_it = 1
 241                         self.filelist.append(fn)
 242                         break
 243
 244                 if not got_it:
 245                     self.warn("standard file not found: should have one of " +
 246                               string.join(alts, ', '))
 247             else:
 248                 if os.path.exists(fn):
 249                     self.filelist.append(fn)
 250                 else:
 251                     self.warn("standard file '%s' not found" % fn)
 252
 253         optional = ['test/test*.py', 'setup.cfg']
 254         for pattern in optional:
 255             files = filter(os.path.isfile, glob(pattern))
 256             if files:
 257                 self.filelist.extend(files)
 258
 259         # build_py is used to get:
 260         #  - python modules
 261         #  - files defined in package_data
 262         build_py = self.get_finalized_command('build_py')
 263
 264         # getting python files
 265         if self.distribution.has_pure_modules():
 266             self.filelist.extend(build_py.get_source_files())
 267
 268         # getting package_data files
 269         # (computed in build_py.data_files by build_py.finalize_options)
 270         for pkg, src_dir, build_dir, filenames in build_py.data_files:
 271             for filename in filenames:
 272                 self.filelist.append(os.path.join(src_dir, filename))
 273
 274         # getting distribution.data_files
 275         if self.distribution.has_data_files():
 276             for item in self.distribution.data_files:
 277                 if isinstance(item, str): # plain file
 278                     item = convert_path(item)
 279                     if os.path.isfile(item):
 280                         self.filelist.append(item)
 281                 else:    # a (dirname, filenames) tuple
 282                     dirname, filenames = item
 283                     for f in filenames:
 284                         f = convert_path(f)
 285                         if os.path.isfile(f):
 286                             self.filelist.append(f)
 287
 288         if self.distribution.has_ext_modules():
 289             build_ext = self.get_finalized_command('build_ext')
 290             self.filelist.extend(build_ext.get_source_files())
 291
 292         if self.distribution.has_c_libraries():
 293             build_clib = self.get_finalized_command('build_clib')
 294             self.filelist.extend(build_clib.get_source_files())
 295
 296         if self.distribution.has_scripts():
 297             build_scripts = self.get_finalized_command('build_scripts')
 298             self.filelist.extend(build_scripts.get_source_files())
 299
 300     def read_template(self):
 301         """Read and parse manifest template file named by self.template.
 302
 303         (usually "MANIFEST.in") The parsing and processing is done by
 304         'self.filelist', which updates itself accordingly.
 305         """
 306         log.info("reading manifest template '%s'", self.template)
 307         template = TextFile(self.template,
 308                             strip_comments=1,
 309                             skip_blanks=1,
 310                             join_lines=1,
 311                             lstrip_ws=1,
 312                             rstrip_ws=1,
 313                             collapse_join=1)
 314
 315         try:
 316             while 1:
 317                 line = template.readline()
 318                 if line is None:            # end of file
 319                     break
 320
 321                 try:
 322                     self.filelist.process_template_line(line)
 323                 # the call above can raise a DistutilsTemplateError for
 324                 # malformed lines, or a ValueError from the lower-level
 325                 # convert_path function
 326                 except (DistutilsTemplateError, ValueError) as msg:
 327                     self.warn("%s, line %d: %s" % (template.filename,
 328                                                    template.current_line,
 329                                                    msg))
 330         finally:
 331             template.close()
 332
 333     def prune_file_list(self):
 334         """Prune off branches that might slip into the file list as created
 335         by 'read_template()', but really don't belong there:
 336           * the build tree (typically "build")
 337           * the release tree itself (only an issue if we ran "sdist"
 338             previously with --keep-temp, or it aborted)
 339           * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories
 340         """
 341         build = self.get_finalized_command('build')
 342         base_dir = self.distribution.get_fullname()
 343
 344         self.filelist.exclude_pattern(None, prefix=build.build_base)
 345         self.filelist.exclude_pattern(None, prefix=base_dir)
 346
 347         # pruning out vcs directories
 348         # both separators are used under win32
 349         if sys.platform == 'win32':
 350             seps = r'/|\\'
 351         else:
 352             seps = '/'
 353
 354         vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr',
 355                     '_darcs']
 356         vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps)
 357         self.filelist.exclude_pattern(vcs_ptrn, is_regex=1)
 358
 359     def write_manifest(self):
 360         """Write the file list in 'self.filelist' (presumably as filled in
 361         by 'add_defaults()' and 'read_template()') to the manifest file
 362         named by 'self.manifest'.
 363         """
 364         if self._manifest_is_not_generated():
 365             log.info("not writing to manually maintained "
 366                      "manifest file '%s'" % self.manifest)
 367             return
 368
 369         content = self.filelist.files[:]
 370         content.insert(0, '# file GENERATED by distutils, do NOT edit')
 371         self.execute(file_util.write_file, (self.manifest, content),
 372                      "writing manifest file '%s'" % self.manifest)
 373
 374     def _manifest_is_not_generated(self):
 375         # check for special comment used in 2.7.1 and higher
 376         if not os.path.isfile(self.manifest):
 377             return False
 378
 379         fp = open(self.manifest, 'rU')
 380         try:
 381             first_line = fp.readline()
 382         finally:
 383             fp.close()
 384         return first_line != '# file GENERATED by distutils, do NOT edit\n'
 385
 386     def read_manifest(self):
 387         """Read the manifest file (named by 'self.manifest') and use it to
 388         fill in 'self.filelist', the list of files to include in the source
 389         distribution.
 390         """
 391         log.info("reading manifest file '%s'", self.manifest)
 392         manifest = open(self.manifest)
 393         for line in manifest:
 394             # ignore comments and blank lines
 395             line = line.strip()
 396             if line.startswith('#') or not line:
 397                 continue
 398             self.filelist.append(line)
 399         manifest.close()
 400
 401     def make_release_tree(self, base_dir, files):
 402         """Create the directory tree that will become the source
 403         distribution archive.  All directories implied by the filenames in
 404         'files' are created under 'base_dir', and then we hard link or copy
 405         (if hard linking is unavailable) those files into place.
 406         Essentially, this duplicates the developer's source tree, but in a
 407         directory named after the distribution, containing only the files
 408         to be distributed.
 409         """
 410         # Create all the directories under 'base_dir' necessary to
 411         # put 'files' there; the 'mkpath()' is just so we don't die
 412         # if the manifest happens to be empty.
 413         self.mkpath(base_dir)
 414         dir_util.create_tree(base_dir, files, dry_run=self.dry_run)
 415
 416         # And walk over the list of files, either making a hard link (if
 417         # os.link exists) to each one that doesn't already exist in its
 418         # corresponding location under 'base_dir', or copying each file
 419         # that's out-of-date in 'base_dir'.  (Usually, all files will be
 420         # out-of-date, because by default we blow away 'base_dir' when
 421         # we're done making the distribution archives.)
 422
 423         if hasattr(os, 'link'):        # can make hard links on this system
 424             link = 'hard'
 425             msg = "making hard links in %s..." % base_dir
 426         else:                           # nope, have to copy
 427             link = None
 428             msg = "copying files to %s..." % base_dir
 429
 430         if not files:
 431             log.warn("no files to distribute -- empty manifest?")
 432         else:
 433             log.info(msg)
 434         for file in files:
 435             if not os.path.isfile(file):
 436                 log.warn("'%s' not a regular file -- skipping" % file)
 437             else:
 438                 dest = os.path.join(base_dir, file)
 439                 self.copy_file(file, dest, link=link)
 440
 441         self.distribution.metadata.write_pkg_info(base_dir)
 442
 443     def make_distribution(self):
 444         """Create the source distribution(s).  First, we create the release
 445         tree with 'make_release_tree()'; then, we create all required
 446         archive files (according to 'self.formats') from the release tree.
 447         Finally, we clean up by blowing away the release tree (unless
 448         'self.keep_temp' is true).  The list of archive files created is
 449         stored so it can be retrieved later by 'get_archive_files()'.
 450         """
 451         # Don't warn about missing meta-data here -- should be (and is!)
 452         # done elsewhere.
 453         base_dir = self.distribution.get_fullname()
 454         base_name = os.path.join(self.dist_dir, base_dir)
 455
 456         self.make_release_tree(base_dir, self.filelist.files)
 457         archive_files = []              # remember names of files we create
 458         # tar archive must be created last to avoid overwrite and remove
 459         if 'tar' in self.formats:
 460             self.formats.append(self.formats.pop(self.formats.index('tar')))
 461
 462         for fmt in self.formats:
 463             file = self.make_archive(base_name, fmt, base_dir=base_dir,
 464                                      owner=self.owner, group=self.group)
 465             archive_files.append(file)
 466             self.distribution.dist_files.append(('sdist', '', file))
 467
 468         self.archive_files = archive_files
 469
 470         if not self.keep_temp:
 471             dir_util.remove_tree(base_dir, dry_run=self.dry_run)
 472
 473     def get_archive_files(self):
 474         """Return the list of archive files created when the command
 475         was run, or None if the command hasn't run yet.
 476         """
 477         return self.archive_files