docutils/frontend.py

   1 # $Id$
   2 # Author: David Goodger <goodger@python.org>
   3 # Copyright: This module has been placed in the public domain.
   4
   5 """
   6 Command-line and common processing for Docutils front-end tools.
   7
   8 Exports the following classes:
   9
  10 * `OptionParser`: Standard Docutils command-line processing.
  11 * `Option`: Customized version of `optparse.Option`; validation support.
  12 * `Values`: Runtime settings; objects are simple structs
  13   (``object.attribute``).  Supports cumulative list settings (attributes).
  14 * `ConfigParser`: Standard Docutils config file processing.
  15
  16 Also exports the following functions:
  17
  18 * Option callbacks: `store_multiple`, `read_config_file`.
  19 * Setting validators: `validate_encoding`,
  20   `validate_encoding_error_handler`,
  21   `validate_encoding_and_error_handler`,
  22   `validate_boolean`, `validate_ternary`, `validate_threshold`,
  23   `validate_colon_separated_string_list`,
  24   `validate_comma_separated_string_list`,
  25   `validate_dependency_file`.
  26 * `make_paths_absolute`.
  27 * SettingSpec manipulation: `filter_settings_spec`.
  28 """
  29
  30 __docformat__ = 'reStructuredText'
  31
  32 import os
  33 import os.path
  34 import sys
  35 import warnings
  36 import ConfigParser as CP
  37 import codecs
  38 import optparse
  39 from optparse import SUPPRESS_HELP
  40 import docutils
  41 import docutils.utils
  42 import docutils.nodes
  43 from docutils.utils.error_reporting import (locale_encoding, SafeString,
  44                                             ErrorOutput, ErrorString)
  45
  46
  47 def store_multiple(option, opt, value, parser, *args, **kwargs):
  48     """
  49     Store multiple values in `parser.values`.  (Option callback.)
  50
  51     Store `None` for each attribute named in `args`, and store the value for
  52     each key (attribute name) in `kwargs`.
  53     """
  54     for attribute in args:
  55         setattr(parser.values, attribute, None)
  56     for key, value in kwargs.items():
  57         setattr(parser.values, key, value)
  58
  59 def read_config_file(option, opt, value, parser):
  60     """
  61     Read a configuration file during option processing.  (Option callback.)
  62     """
  63     try:
  64         new_settings = parser.get_config_file_settings(value)
  65     except ValueError, error:
  66         parser.error(error)
  67     parser.values.update(new_settings, parser)
  68
  69 def validate_encoding(setting, value, option_parser,
  70                       config_parser=None, config_section=None):
  71     try:
  72         codecs.lookup(value)
  73     except LookupError:
  74         raise (LookupError('setting "%s": unknown encoding: "%s"'
  75                            % (setting, value)),
  76                None, sys.exc_info()[2])
  77     return value
  78
  79 def validate_encoding_error_handler(setting, value, option_parser,
  80                                     config_parser=None, config_section=None):
  81     try:
  82         codecs.lookup_error(value)
  83     except LookupError:
  84         raise (LookupError(
  85             'unknown encoding error handler: "%s" (choices: '
  86             '"strict", "ignore", "replace", "backslashreplace", '
  87             '"xmlcharrefreplace", and possibly others; see documentation for '
  88             'the Python ``codecs`` module)' % value),
  89                None, sys.exc_info()[2])
  90     return value
  91
  92 def validate_encoding_and_error_handler(
  93     setting, value, option_parser, config_parser=None, config_section=None):
  94     """
  95     Side-effect: if an error handler is included in the value, it is inserted
  96     into the appropriate place as if it was a separate setting/option.
  97     """
  98     if ':' in value:
  99         encoding, handler = value.split(':')
 100         validate_encoding_error_handler(
 101             setting + '_error_handler', handler, option_parser,
 102             config_parser, config_section)
 103         if config_parser:
 104             config_parser.set(config_section, setting + '_error_handler',
 105                               handler)
 106         else:
 107             setattr(option_parser.values, setting + '_error_handler', handler)
 108     else:
 109         encoding = value
 110     validate_encoding(setting, encoding, option_parser,
 111                       config_parser, config_section)
 112     return encoding
 113
 114 def validate_boolean(setting, value, option_parser,
 115                      config_parser=None, config_section=None):
 116     """Check/normalize boolean settings:
 117          True:  '1', 'on', 'yes', 'true'
 118          False: '0', 'off', 'no','false', ''
 119     """
 120     if isinstance(value, bool):
 121         return value
 122     try:
 123         return option_parser.booleans[value.strip().lower()]
 124     except KeyError:
 125         raise (LookupError('unknown boolean value: "%s"' % value),
 126                None, sys.exc_info()[2])
 127
 128 def validate_ternary(setting, value, option_parser,
 129                      config_parser=None, config_section=None):
 130     """Check/normalize three-value settings:
 131          True:  '1', 'on', 'yes', 'true'
 132          False: '0', 'off', 'no','false', ''
 133          any other value: returned as-is.
 134     """
 135     if isinstance(value, bool) or value is None:
 136         return value
 137     try:
 138         return option_parser.booleans[value.strip().lower()]
 139     except KeyError:
 140         return value
 141
 142 def validate_nonnegative_int(setting, value, option_parser,
 143                              config_parser=None, config_section=None):
 144     value = int(value)
 145     if value < 0:
 146         raise ValueError('negative value; must be positive or zero')
 147     return value
 148
 149 def validate_threshold(setting, value, option_parser,
 150                        config_parser=None, config_section=None):
 151     try:
 152         return int(value)
 153     except ValueError:
 154         try:
 155             return option_parser.thresholds[value.lower()]
 156         except (KeyError, AttributeError):
 157             raise (LookupError('unknown threshold: %r.' % value),
 158                    None, sys.exc_info[2])
 159
 160 def validate_colon_separated_string_list(
 161     setting, value, option_parser, config_parser=None, config_section=None):
 162     if not isinstance(value, list):
 163         value = value.split(':')
 164     else:
 165         last = value.pop()
 166         value.extend(last.split(':'))
 167     return value
 168
 169 def validate_comma_separated_list(setting, value, option_parser,
 170                                     config_parser=None, config_section=None):
 171     """Check/normalize list arguments (split at "," and strip whitespace).
 172     """
 173     # `value` is already a ``list`` when  given as command line option
 174     # and "action" is "append" and ``unicode`` or ``str`` else.
 175     if not isinstance(value, list):
 176         value = [value]
 177     # this function is called for every option added to `value`
 178     # -> split the last item and append the result:
 179     last = value.pop()
 180     items = [i.strip(u' \t\n') for i in last.split(u',') if i.strip(u' \t\n')]
 181     value.extend(items)
 182     return value
 183
 184 def validate_url_trailing_slash(
 185     setting, value, option_parser, config_parser=None, config_section=None):
 186     if not value:
 187         return './'
 188     elif value.endswith('/'):
 189         return value
 190     else:
 191         return value + '/'
 192
 193 def validate_dependency_file(setting, value, option_parser,
 194                              config_parser=None, config_section=None):
 195     try:
 196         return docutils.utils.DependencyList(value)
 197     except IOError:
 198         return docutils.utils.DependencyList(None)
 199
 200 def validate_strip_class(setting, value, option_parser,
 201                          config_parser=None, config_section=None):
 202     # value is a comma separated string list:
 203     value = validate_comma_separated_list(setting, value, option_parser,
 204                                           config_parser, config_section)
 205     # validate list elements:
 206     for cls in value:
 207         normalized = docutils.nodes.make_id(cls)
 208         if cls != normalized:
 209             raise ValueError('Invalid class value %r (perhaps %r?)'
 210                              % (cls, normalized))
 211     return value
 212
 213 def validate_smartquotes_locales(setting, value, option_parser,
 214                          config_parser=None, config_section=None):
 215     """Check/normalize a comma separated list of smart quote definitions.
 216
 217     Return a list of (language-tag, quotes) string tuples."""
 218
 219     # value is a comma separated string list:
 220     value = validate_comma_separated_list(setting, value, option_parser,
 221                                           config_parser, config_section)
 222     # validate list elements
 223     lc_quotes = []
 224     for item in value:
 225         try:
 226             lang, quotes = item.split(':', 1)
 227         except AttributeError:
 228             # this function is called for every option added to `value`
 229             # -> ignore if already a tuple:
 230             lc_quotes.append(item)
 231             continue
 232         except ValueError:
 233             raise ValueError(u'Invalid value "%s".'
 234                              ' Format is "<language>:<quotes>".'
 235                              % item.encode('ascii', 'backslashreplace'))
 236         # parse colon separated string list:
 237         quotes = quotes.strip()
 238         multichar_quotes = quotes.split(':')
 239         if len(multichar_quotes) == 4:
 240             quotes = multichar_quotes
 241         elif len(quotes) != 4:
 242             raise ValueError('Invalid value "%s". Please specify 4 quotes\n'
 243                 '    (primary open/close; secondary open/close).'
 244                              % item.encode('ascii', 'backslashreplace'))
 245         lc_quotes.append((lang,quotes))
 246     return lc_quotes
 247
 248 def make_paths_absolute(pathdict, keys, base_path=None):
 249     """
 250     Interpret filesystem path settings relative to the `base_path` given.
 251
 252     Paths are values in `pathdict` whose keys are in `keys`.  Get `keys` from
 253     `OptionParser.relative_path_settings`.
 254     """
 255     if base_path is None:
 256         base_path = os.getcwdu() # type(base_path) == unicode
 257         # to allow combining non-ASCII cwd with unicode values in `pathdict`
 258     for key in keys:
 259         if key in pathdict:
 260             value = pathdict[key]
 261             if isinstance(value, list):
 262                 value = [make_one_path_absolute(base_path, path)
 263                          for path in value]
 264             elif value:
 265                 value = make_one_path_absolute(base_path, value)
 266             pathdict[key] = value
 267
 268 def make_one_path_absolute(base_path, path):
 269     return os.path.abspath(os.path.join(base_path, path))
 270
 271 def filter_settings_spec(settings_spec, *exclude, **replace):
 272     """Return a copy of `settings_spec` excluding/replacing some settings.
 273
 274     `settings_spec` is a tuple of configuration settings with a structure
 275     described for docutils.SettingsSpec.settings_spec.
 276
 277     Optional positional arguments are names of to-be-excluded settings.
 278     Keyword arguments are option specification replacements.
 279     (See the html4strict writer for an example.)
 280     """
 281     settings = list(settings_spec)
 282     # every third item is a sequence of option tuples
 283     for i in range(2, len(settings), 3):
 284         newopts = []
 285         for opt_spec in settings[i]:
 286             # opt_spec is ("<help>", [<option strings>], {<keyword args>})
 287             opt_name = [opt_string[2:].replace('-', '_')
 288                         for opt_string in opt_spec[1]
 289                             if opt_string.startswith('--')
 290                        ][0]
 291             if opt_name in exclude:
 292                 continue
 293             if opt_name in replace.keys():
 294                 newopts.append(replace[opt_name])
 295             else:
 296                 newopts.append(opt_spec)
 297         settings[i] = tuple(newopts)
 298     return tuple(settings)
 299
 300
 301 class Values(optparse.Values):
 302
 303     """
 304     Updates list attributes by extension rather than by replacement.
 305     Works in conjunction with the `OptionParser.lists` instance attribute.
 306     """
 307
 308     def __init__(self, *args, **kwargs):
 309         optparse.Values.__init__(self, *args, **kwargs)
 310         if (not hasattr(self, 'record_dependencies')
 311             or self.record_dependencies is None):
 312             # Set up dependency list, in case it is needed.
 313             self.record_dependencies = docutils.utils.DependencyList()
 314
 315     def update(self, other_dict, option_parser):
 316         if isinstance(other_dict, Values):
 317             other_dict = other_dict.__dict__
 318         other_dict = other_dict.copy()
 319         for setting in option_parser.lists.keys():
 320             if (hasattr(self, setting) and setting in other_dict):
 321                 value = getattr(self, setting)
 322                 if value:
 323                     value += other_dict[setting]
 324                     del other_dict[setting]
 325         self._update_loose(other_dict)
 326
 327     def copy(self):
 328         """Return a shallow copy of `self`."""
 329         return self.__class__(defaults=self.__dict__)
 330
 331
 332 class Option(optparse.Option):
 333
 334     ATTRS = optparse.Option.ATTRS + ['validator', 'overrides']
 335
 336     def process(self, opt, value, values, parser):
 337         """
 338         Call the validator function on applicable settings and
 339         evaluate the 'overrides' option.
 340         Extends `optparse.Option.process`.
 341         """
 342         result = optparse.Option.process(self, opt, value, values, parser)
 343         setting = self.dest
 344         if setting:
 345             if self.validator:
 346                 value = getattr(values, setting)
 347                 try:
 348                     new_value = self.validator(setting, value, parser)
 349                 except Exception, error:
 350                     raise (optparse.OptionValueError(
 351                         'Error in option "%s":\n    %s'
 352                         % (opt, ErrorString(error))),
 353                            None, sys.exc_info()[2])
 354                 setattr(values, setting, new_value)
 355             if self.overrides:
 356                 setattr(values, self.overrides, None)
 357         return result
 358
 359
 360 class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
 361
 362     """
 363     Parser for command-line and library use.  The `settings_spec`
 364     specification here and in other Docutils components are merged to build
 365     the set of command-line options and runtime settings for this process.
 366
 367     Common settings (defined below) and component-specific settings must not
 368     conflict.  Short options are reserved for common settings, and components
 369     are restrict to using long options.
 370     """
 371
 372     standard_config_files = [
 373         '/etc/docutils.conf',           # system-wide
 374         './docutils.conf',              # project-specific
 375         '~/.docutils']                  # user-specific
 376     """Docutils configuration files, using ConfigParser syntax.  Filenames
 377     will be tilde-expanded later.  Later files override earlier ones."""
 378
 379     threshold_choices = 'info 1 warning 2 error 3 severe 4 none 5'.split()
 380     """Possible inputs for for --report and --halt threshold values."""
 381
 382     thresholds = {'info': 1, 'warning': 2, 'error': 3, 'severe': 4, 'none': 5}
 383     """Lookup table for --report and --halt threshold values."""
 384
 385     booleans={'1': True, 'on': True, 'yes': True, 'true': True,
 386               '0': False, 'off': False, 'no': False, 'false': False, '': False}
 387     """Lookup table for boolean configuration file settings."""
 388
 389     default_error_encoding = getattr(sys.stderr, 'encoding',
 390                                      None) or locale_encoding or 'ascii'
 391
 392     default_error_encoding_error_handler = 'backslashreplace'
 393
 394     settings_spec = (
 395         'General Docutils Options',
 396         None,
 397         (('Specify the document title as metadata.',
 398           ['--title'], {}),
 399          ('Include a "Generated by Docutils" credit and link.',
 400           ['--generator', '-g'], {'action': 'store_true',
 401                                   'validator': validate_boolean}),
 402          ('Do not include a generator credit.',
 403           ['--no-generator'], {'action': 'store_false', 'dest': 'generator'}),
 404          ('Include the date at the end of the document (UTC).',
 405           ['--date', '-d'], {'action': 'store_const', 'const': '%Y-%m-%d',
 406                              'dest': 'datestamp'}),
 407          ('Include the time & date (UTC).',
 408           ['--time', '-t'], {'action': 'store_const',
 409                              'const': '%Y-%m-%d %H:%M UTC',
 410                              'dest': 'datestamp'}),
 411          ('Do not include a datestamp of any kind.',
 412           ['--no-datestamp'], {'action': 'store_const', 'const': None,
 413                                'dest': 'datestamp'}),
 414          ('Include a "View document source" link.',
 415           ['--source-link', '-s'], {'action': 'store_true',
 416                                     'validator': validate_boolean}),
 417          ('Use <URL> for a source link; implies --source-link.',
 418           ['--source-url'], {'metavar': '<URL>'}),
 419          ('Do not include a "View document source" link.',
 420           ['--no-source-link'],
 421           {'action': 'callback', 'callback': store_multiple,
 422            'callback_args': ('source_link', 'source_url')}),
 423          ('Link from section headers to TOC entries.  (default)',
 424           ['--toc-entry-backlinks'],
 425           {'dest': 'toc_backlinks', 'action': 'store_const', 'const': 'entry',
 426            'default': 'entry'}),
 427          ('Link from section headers to the top of the TOC.',
 428           ['--toc-top-backlinks'],
 429           {'dest': 'toc_backlinks', 'action': 'store_const', 'const': 'top'}),
 430          ('Disable backlinks to the table of contents.',
 431           ['--no-toc-backlinks'],
 432           {'dest': 'toc_backlinks', 'action': 'store_false'}),
 433          ('Link from footnotes/citations to references. (default)',
 434           ['--footnote-backlinks'],
 435           {'action': 'store_true', 'default': 1,
 436            'validator': validate_boolean}),
 437          ('Disable backlinks from footnotes and citations.',
 438           ['--no-footnote-backlinks'],
 439           {'dest': 'footnote_backlinks', 'action': 'store_false'}),
 440          ('Enable section numbering by Docutils.  (default)',
 441           ['--section-numbering'],
 442           {'action': 'store_true', 'dest': 'sectnum_xform',
 443            'default': 1, 'validator': validate_boolean}),
 444          ('Disable section numbering by Docutils.',
 445           ['--no-section-numbering'],
 446           {'action': 'store_false', 'dest': 'sectnum_xform'}),
 447          ('Remove comment elements from the document tree.',
 448           ['--strip-comments'],
 449           {'action': 'store_true', 'validator': validate_boolean}),
 450          ('Leave comment elements in the document tree. (default)',
 451           ['--leave-comments'],
 452           {'action': 'store_false', 'dest': 'strip_comments'}),
 453          ('Remove all elements with classes="<class>" from the document tree. '
 454           'Warning: potentially dangerous; use with caution. '
 455           '(Multiple-use option.)',
 456           ['--strip-elements-with-class'],
 457           {'action': 'append', 'dest': 'strip_elements_with_classes',
 458            'metavar': '<class>', 'validator': validate_strip_class}),
 459          ('Remove all classes="<class>" attributes from elements in the '
 460           'document tree. Warning: potentially dangerous; use with caution. '
 461           '(Multiple-use option.)',
 462           ['--strip-class'],
 463           {'action': 'append', 'dest': 'strip_classes',
 464            'metavar': '<class>', 'validator': validate_strip_class}),
 465          ('Report system messages at or higher than <level>: "info" or "1", '
 466           '"warning"/"2" (default), "error"/"3", "severe"/"4", "none"/"5"',
 467           ['--report', '-r'], {'choices': threshold_choices, 'default': 2,
 468                                'dest': 'report_level', 'metavar': '<level>',
 469                                'validator': validate_threshold}),
 470          ('Report all system messages.  (Same as "--report=1".)',
 471           ['--verbose', '-v'], {'action': 'store_const', 'const': 1,
 472                                 'dest': 'report_level'}),
 473          ('Report no system messages.  (Same as "--report=5".)',
 474           ['--quiet', '-q'], {'action': 'store_const', 'const': 5,
 475                               'dest': 'report_level'}),
 476          ('Halt execution at system messages at or above <level>.  '
 477           'Levels as in --report.  Default: 4 (severe).',
 478           ['--halt'], {'choices': threshold_choices, 'dest': 'halt_level',
 479                        'default': 4, 'metavar': '<level>',
 480                        'validator': validate_threshold}),
 481          ('Halt at the slightest problem.  Same as "--halt=info".',
 482           ['--strict'], {'action': 'store_const', 'const': 1,
 483                          'dest': 'halt_level'}),
 484          ('Enable a non-zero exit status for non-halting system messages at '
 485           'or above <level>.  Default: 5 (disabled).',
 486           ['--exit-status'], {'choices': threshold_choices,
 487                               'dest': 'exit_status_level',
 488                               'default': 5, 'metavar': '<level>',
 489                               'validator': validate_threshold}),
 490          ('Enable debug-level system messages and diagnostics.',
 491           ['--debug'], {'action': 'store_true', 'validator': validate_boolean}),
 492          ('Disable debug output.  (default)',
 493           ['--no-debug'], {'action': 'store_false', 'dest': 'debug'}),
 494          ('Send the output of system messages to <file>.',
 495           ['--warnings'], {'dest': 'warning_stream', 'metavar': '<file>'}),
 496          ('Enable Python tracebacks when Docutils is halted.',
 497           ['--traceback'], {'action': 'store_true', 'default': None,
 498                             'validator': validate_boolean}),
 499          ('Disable Python tracebacks.  (default)',
 500           ['--no-traceback'], {'dest': 'traceback', 'action': 'store_false'}),
 501          ('Specify the encoding and optionally the '
 502           'error handler of input text.  Default: <locale-dependent>:strict.',
 503           ['--input-encoding', '-i'],
 504           {'metavar': '<name[:handler]>',
 505            'validator': validate_encoding_and_error_handler}),
 506          ('Specify the error handler for undecodable characters.  '
 507           'Choices: "strict" (default), "ignore", and "replace".',
 508           ['--input-encoding-error-handler'],
 509           {'default': 'strict', 'validator': validate_encoding_error_handler}),
 510          ('Specify the text encoding and optionally the error handler for '
 511           'output.  Default: UTF-8:strict.',
 512           ['--output-encoding', '-o'],
 513           {'metavar': '<name[:handler]>', 'default': 'utf-8',
 514            'validator': validate_encoding_and_error_handler}),
 515          ('Specify error handler for unencodable output characters; '
 516           '"strict" (default), "ignore", "replace", '
 517           '"xmlcharrefreplace", "backslashreplace".',
 518           ['--output-encoding-error-handler'],
 519           {'default': 'strict', 'validator': validate_encoding_error_handler}),
 520          ('Specify text encoding and error handler for error output.  '
 521           'Default: %s:%s.'
 522           % (default_error_encoding, default_error_encoding_error_handler),
 523           ['--error-encoding', '-e'],
 524           {'metavar': '<name[:handler]>', 'default': default_error_encoding,
 525            'validator': validate_encoding_and_error_handler}),
 526          ('Specify the error handler for unencodable characters in '
 527           'error output.  Default: %s.'
 528           % default_error_encoding_error_handler,
 529           ['--error-encoding-error-handler'],
 530           {'default': default_error_encoding_error_handler,
 531            'validator': validate_encoding_error_handler}),
 532          ('Specify the language (as BCP 47 language tag).  Default: en.',
 533           ['--language', '-l'], {'dest': 'language_code', 'default': 'en',
 534                                  'metavar': '<name>'}),
 535          ('Write output file dependencies to <file>.',
 536           ['--record-dependencies'],
 537           {'metavar': '<file>', 'validator': validate_dependency_file,
 538            'default': None}),           # default set in Values class
 539          ('Read configuration settings from <file>, if it exists.',
 540           ['--config'], {'metavar': '<file>', 'type': 'string',
 541                          'action': 'callback', 'callback': read_config_file}),
 542          ("Show this program's version number and exit.",
 543           ['--version', '-V'], {'action': 'version'}),
 544          ('Show this help message and exit.',
 545           ['--help', '-h'], {'action': 'help'}),
 546          # Typically not useful for non-programmatical use:
 547          (SUPPRESS_HELP, ['--id-prefix'], {'default': ''}),
 548          (SUPPRESS_HELP, ['--auto-id-prefix'], {'default': 'id'}),
 549          # Hidden options, for development use only:
 550          (SUPPRESS_HELP, ['--dump-settings'], {'action': 'store_true'}),
 551          (SUPPRESS_HELP, ['--dump-internals'], {'action': 'store_true'}),
 552          (SUPPRESS_HELP, ['--dump-transforms'], {'action': 'store_true'}),
 553          (SUPPRESS_HELP, ['--dump-pseudo-xml'], {'action': 'store_true'}),
 554          (SUPPRESS_HELP, ['--expose-internal-attribute'],
 555           {'action': 'append', 'dest': 'expose_internals',
 556            'validator': validate_colon_separated_string_list}),
 557          (SUPPRESS_HELP, ['--strict-visitor'], {'action': 'store_true'}),
 558          ))
 559     """Runtime settings and command-line options common to all Docutils front
 560     ends.  Setting specs specific to individual Docutils components are also
 561     used (see `populate_from_components()`)."""
 562
 563     settings_defaults = {'_disable_config': None,
 564                          '_source': None,
 565                          '_destination': None,
 566                          '_config_files': None}
 567     """Defaults for settings that don't have command-line option equivalents."""
 568
 569     relative_path_settings = ('warning_stream',)
 570
 571     config_section = 'general'
 572
 573     version_template = ('%%prog (Docutils %s [%s], Python %s, on %s)'
 574                         % (docutils.__version__, docutils.__version_details__,
 575                            sys.version.split()[0], sys.platform))
 576     """Default version message."""
 577
 578     def __init__(self, components=(), defaults=None, read_config_files=None,
 579                  *args, **kwargs):
 580         """
 581         `components` is a list of Docutils components each containing a
 582         ``.settings_spec`` attribute.  `defaults` is a mapping of setting
 583         default overrides.
 584         """
 585
 586         self.lists = {}
 587         """Set of list-type settings."""
 588
 589         self.config_files = []
 590         """List of paths of applied configuration files."""
 591
 592         optparse.OptionParser.__init__(
 593             self, option_class=Option, add_help_option=None,
 594             formatter=optparse.TitledHelpFormatter(width=78),
 595             *args, **kwargs)
 596         if not self.version:
 597             self.version = self.version_template
 598         # Make an instance copy (it will be modified):
 599         self.relative_path_settings = list(self.relative_path_settings)
 600         self.components = (self,) + tuple(components)
 601         self.populate_from_components(self.components)
 602         self.set_defaults_from_dict(defaults or {})
 603         if read_config_files and not self.defaults['_disable_config']:
 604             try:
 605                 config_settings = self.get_standard_config_settings()
 606             except ValueError, error:
 607                 self.error(SafeString(error))
 608             self.set_defaults_from_dict(config_settings.__dict__)
 609
 610     def populate_from_components(self, components):
 611         """
 612         For each component, first populate from the `SettingsSpec.settings_spec`
 613         structure, then from the `SettingsSpec.settings_defaults` dictionary.
 614         After all components have been processed, check for and populate from
 615         each component's `SettingsSpec.settings_default_overrides` dictionary.
 616         """
 617         for component in components:
 618             if component is None:
 619                 continue
 620             settings_spec = component.settings_spec
 621             self.relative_path_settings.extend(
 622                 component.relative_path_settings)
 623             for i in range(0, len(settings_spec), 3):
 624                 title, description, option_spec = settings_spec[i:i+3]
 625                 if title:
 626                     group = optparse.OptionGroup(self, title, description)
 627                     self.add_option_group(group)
 628                 else:
 629                     group = self        # single options
 630                 for (help_text, option_strings, kwargs) in option_spec:
 631                     option = group.add_option(help=help_text, *option_strings,
 632                                               **kwargs)
 633                     if kwargs.get('action') == 'append':
 634                         self.lists[option.dest] = 1
 635                 if component.settings_defaults:
 636                     self.defaults.update(component.settings_defaults)
 637         for component in components:
 638             if component and component.settings_default_overrides:
 639                 self.defaults.update(component.settings_default_overrides)
 640
 641     def get_standard_config_files(self):
 642         """Return list of config files, from environment or standard."""
 643         try:
 644             config_files = os.environ['DOCUTILSCONFIG'].split(os.pathsep)
 645         except KeyError:
 646             config_files = self.standard_config_files
 647
 648         # If 'HOME' is not set, expandvars() requires the 'pwd' module which is
 649         # not available under certain environments, for example, within
 650         # mod_python.  The publisher ends up in here, and we need to publish
 651         # from within mod_python.  Therefore we need to avoid expanding when we
 652         # are in those environments.
 653         expand = os.path.expanduser
 654         if 'HOME' not in os.environ:
 655             try:
 656                 import pwd
 657             except ImportError:
 658                 expand = lambda x: x
 659         return [expand(f) for f in config_files if f.strip()]
 660
 661     def get_standard_config_settings(self):
 662         settings = Values()
 663         for filename in self.get_standard_config_files():
 664             settings.update(self.get_config_file_settings(filename), self)
 665         return settings
 666
 667     def get_config_file_settings(self, config_file):
 668         """Returns a dictionary containing appropriate config file settings."""
 669         parser = ConfigParser()
 670         parser.read(config_file, self)
 671         self.config_files.extend(parser._files)
 672         base_path = os.path.dirname(config_file)
 673         applied = {}
 674         settings = Values()
 675         for component in self.components:
 676             if not component:
 677                 continue
 678             for section in (tuple(component.config_section_dependencies or ())
 679                             + (component.config_section,)):
 680                 if section in applied:
 681                     continue
 682                 applied[section] = 1
 683                 settings.update(parser.get_section(section), self)
 684         make_paths_absolute(
 685             settings.__dict__, self.relative_path_settings, base_path)
 686         return settings.__dict__
 687
 688     def check_values(self, values, args):
 689         """Store positional arguments as runtime settings."""
 690         values._source, values._destination = self.check_args(args)
 691         make_paths_absolute(values.__dict__, self.relative_path_settings)
 692         values._config_files = self.config_files
 693         return values
 694
 695     def check_args(self, args):
 696         source = destination = None
 697         if args:
 698             source = args.pop(0)
 699             if source == '-':           # means stdin
 700                 source = None
 701         if args:
 702             destination = args.pop(0)
 703             if destination == '-':      # means stdout
 704                 destination = None
 705         if args:
 706             self.error('Maximum 2 arguments allowed.')
 707         if source and source == destination:
 708             self.error('Do not specify the same file for both source and '
 709                        'destination.  It will clobber the source file.')
 710         return source, destination
 711
 712     def set_defaults_from_dict(self, defaults):
 713         self.defaults.update(defaults)
 714
 715     def get_default_values(self):
 716         """Needed to get custom `Values` instances."""
 717         defaults = Values(self.defaults)
 718         defaults._config_files = self.config_files
 719         return defaults
 720
 721     def get_option_by_dest(self, dest):
 722         """
 723         Get an option by its dest.
 724
 725         If you're supplying a dest which is shared by several options,
 726         it is undefined which option of those is returned.
 727
 728         A KeyError is raised if there is no option with the supplied
 729         dest.
 730         """
 731         for group in self.option_groups + [self]:
 732             for option in group.option_list:
 733                 if option.dest == dest:
 734                     return option
 735         raise KeyError('No option with dest == %r.' % dest)
 736
 737
 738 class ConfigParser(CP.RawConfigParser):
 739
 740     old_settings = {
 741         'pep_stylesheet': ('pep_html writer', 'stylesheet'),
 742         'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'),
 743         'pep_template': ('pep_html writer', 'template')}
 744     """{old setting: (new section, new setting)} mapping, used by
 745     `handle_old_config`, to convert settings from the old [options] section."""
 746
 747     old_warning = """
 748 The "[option]" section is deprecated.  Support for old-format configuration
 749 files may be removed in a future Docutils release.  Please revise your
 750 configuration files.  See <http://docutils.sf.net/docs/user/config.html>,
 751 section "Old-Format Configuration Files".
 752 """
 753
 754     not_utf8_error = """\
 755 Unable to read configuration file "%s": content not encoded as UTF-8.
 756 Skipping "%s" configuration file.
 757 """
 758
 759     def __init__(self, *args, **kwargs):
 760         CP.RawConfigParser.__init__(self, *args, **kwargs)
 761
 762         self._files = []
 763         """List of paths of configuration files read."""
 764
 765         self._stderr = ErrorOutput()
 766         """Wrapper around sys.stderr catching en-/decoding errors"""
 767
 768     def read(self, filenames, option_parser):
 769         if type(filenames) in (str, unicode):
 770             filenames = [filenames]
 771         for filename in filenames:
 772             try:
 773                 # Config files must be UTF-8-encoded:
 774                 fp = codecs.open(filename, 'r', 'utf-8')
 775             except IOError:
 776                 continue
 777             try:
 778                 if sys.version_info < (3,2):
 779                     CP.RawConfigParser.readfp(self, fp, filename)
 780                 else:
 781                     CP.RawConfigParser.read_file(self, fp, filename)
 782             except UnicodeDecodeError:
 783                 self._stderr.write(self.not_utf8_error % (filename, filename))
 784                 fp.close()
 785                 continue
 786             fp.close()
 787             self._files.append(filename)
 788             if self.has_section('options'):
 789                 self.handle_old_config(filename)
 790             self.validate_settings(filename, option_parser)
 791
 792     def handle_old_config(self, filename):
 793         warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning,
 794                                filename, 0)
 795         options = self.get_section('options')
 796         if not self.has_section('general'):
 797             self.add_section('general')
 798         for key, value in options.items():
 799             if key in self.old_settings:
 800                 section, setting = self.old_settings[key]
 801                 if not self.has_section(section):
 802                     self.add_section(section)
 803             else:
 804                 section = 'general'
 805                 setting = key
 806             if not self.has_option(section, setting):
 807                 self.set(section, setting, value)
 808         self.remove_section('options')
 809
 810     def validate_settings(self, filename, option_parser):
 811         """
 812         Call the validator function and implement overrides on all applicable
 813         settings.
 814         """
 815         for section in self.sections():
 816             for setting in self.options(section):
 817                 try:
 818                     option = option_parser.get_option_by_dest(setting)
 819                 except KeyError:
 820                     continue
 821                 if option.validator:
 822                     value = self.get(section, setting)
 823                     try:
 824                         new_value = option.validator(
 825                             setting, value, option_parser,
 826                             config_parser=self, config_section=section)
 827                     except Exception, error:
 828                         raise (ValueError(
 829                             'Error in config file "%s", section "[%s]":\n'
 830                             '    %s\n'
 831                             '        %s = %s'
 832                             % (filename, section, ErrorString(error),
 833                                setting, value)), None, sys.exc_info()[2])
 834                     self.set(section, setting, new_value)
 835                 if option.overrides:
 836                     self.set(section, option.overrides, None)
 837
 838     def optionxform(self, optionstr):
 839         """
 840         Transform '-' to '_' so the cmdline form of option names can be used.
 841         """
 842         return optionstr.lower().replace('-', '_')
 843
 844     def get_section(self, section):
 845         """
 846         Return a given section as a dictionary (empty if the section
 847         doesn't exist).
 848         """
 849         section_dict = {}
 850         if self.has_section(section):
 851             for option in self.options(section):
 852                 section_dict[option] = self.get(section, option)
 853         return section_dict
 854
 855
 856 class ConfigDeprecationWarning(DeprecationWarning):
 857     """Warning for deprecated configuration file features."""