configobj.py

   1 # configobj.py
   2 # A config file reader/writer that supports nested sections in config files.
   3 # Copyright (C) 2005-2006 Michael Foord, Nicola Larosa
   4 # E-mail: fuzzyman AT voidspace DOT org DOT uk
   5 #         nico AT tekNico DOT net
   6
   7 # ConfigObj 4
   8 # http://www.voidspace.org.uk/python/configobj.html
   9
  10 # Released subject to the BSD License
  11 # Please see http://www.voidspace.org.uk/python/license.shtml
  12
  13 # Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
  14 # For information about bugfixes, updates and support, please join the
  15 # ConfigObj mailing list:
  16 # http://lists.sourceforge.net/lists/listinfo/configobj-develop
  17 # Comments, suggestions and bug reports welcome.
  18
  19 from __future__ import generators
  20
  21 import sys
  22 INTP_VER = sys.version_info[:2]
  23 if INTP_VER < (2, 2):
  24     raise RuntimeError("Python v.2.2 or later needed")
  25
  26 import os, re
  27 import compiler
  28 from types import StringTypes
  29 from warnings import warn
  30 from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
  31
  32 # A dictionary mapping BOM to
  33 # the encoding to decode with, and what to set the
  34 # encoding attribute to.
  35 BOMS = {
  36     BOM_UTF8: ('utf_8', None),
  37     BOM_UTF16_BE: ('utf16_be', 'utf_16'),
  38     BOM_UTF16_LE: ('utf16_le', 'utf_16'),
  39     BOM_UTF16: ('utf_16', 'utf_16'),
  40     }
  41 # All legal variants of the BOM codecs.
  42 # TODO: the list of aliases is not meant to be exhaustive, is there a
  43 #   better way ?
  44 BOM_LIST = {
  45     'utf_16': 'utf_16',
  46     'u16': 'utf_16',
  47     'utf16': 'utf_16',
  48     'utf-16': 'utf_16',
  49     'utf16_be': 'utf16_be',
  50     'utf_16_be': 'utf16_be',
  51     'utf-16be': 'utf16_be',
  52     'utf16_le': 'utf16_le',
  53     'utf_16_le': 'utf16_le',
  54     'utf-16le': 'utf16_le',
  55     'utf_8': 'utf_8',
  56     'u8': 'utf_8',
  57     'utf': 'utf_8',
  58     'utf8': 'utf_8',
  59     'utf-8': 'utf_8',
  60     }
  61
  62 # Map of encodings to the BOM to write.
  63 BOM_SET = {
  64     'utf_8': BOM_UTF8,
  65     'utf_16': BOM_UTF16,
  66     'utf16_be': BOM_UTF16_BE,
  67     'utf16_le': BOM_UTF16_LE,
  68     None: BOM_UTF8
  69     }
  70
  71 try:
  72     from validate import VdtMissingValue
  73 except ImportError:
  74     VdtMissingValue = None
  75
  76 try:
  77     enumerate
  78 except NameError:
  79     def enumerate(obj):
  80         """enumerate for Python 2.2."""
  81         i = -1
  82         for item in obj:
  83             i += 1
  84             yield i, item
  85
  86 try:
  87     True, False
  88 except NameError:
  89     True, False = 1, 0
  90
  91
  92 __version__ = '4.3.2'
  93
  94 __revision__ = '$Id: configobj.py 156 2006-01-31 14:57:08Z fuzzyman $'
  95
  96 __docformat__ = "restructuredtext en"
  97
  98 # NOTE: Does it make sense to have the following in __all__ ?
  99 # NOTE: DEFAULT_INDENT_TYPE, NUM_INDENT_SPACES, MAX_INTERPOL_DEPTH
 100 # NOTE: If used via ``from configobj import...``
 101 # NOTE: They are effectively read only
 102 __all__ = (
 103     '__version__',
 104     'DEFAULT_INDENT_TYPE',
 105     'NUM_INDENT_SPACES',
 106     'MAX_INTERPOL_DEPTH',
 107     'ConfigObjError',
 108     'NestingError',
 109     'ParseError',
 110     'DuplicateError',
 111     'ConfigspecError',
 112     'ConfigObj',
 113     'SimpleVal',
 114     'InterpolationError',
 115     'InterpolationDepthError',
 116     'MissingInterpolationOption',
 117     'RepeatSectionError',
 118     'UnreprError',
 119     'UnknownType',
 120     '__docformat__',
 121     'flatten_errors',
 122 )
 123
 124 DEFAULT_INDENT_TYPE = ' '
 125 NUM_INDENT_SPACES = 4
 126 MAX_INTERPOL_DEPTH = 10
 127
 128 OPTION_DEFAULTS = {
 129     'interpolation': True,
 130     'raise_errors': False,
 131     'list_values': True,
 132     'create_empty': False,
 133     'file_error': False,
 134     'configspec': None,
 135     'stringify': True,
 136     # option may be set to one of ('', ' ', '\t')
 137     'indent_type': None,
 138     'encoding': None,
 139     'default_encoding': None,
 140     'unrepr': False,
 141     'write_empty_values': False,
 142 }
 143
 144
 145 def getObj(s):
 146     s = "a=" + s
 147     p = compiler.parse(s)
 148     return p.getChildren()[1].getChildren()[0].getChildren()[1]
 149
 150 class UnknownType(Exception):
 151     pass
 152
 153 class Builder:
 154
 155     def build(self, o):
 156         m = getattr(self, 'build_' + o.__class__.__name__, None)
 157         if m is None:
 158             raise UnknownType(o.__class__.__name__)
 159         return m(o)
 160
 161     def build_List(self, o):
 162         return map(self.build, o.getChildren())
 163
 164     def build_Const(self, o):
 165         return o.value
 166
 167     def build_Dict(self, o):
 168         d = {}
 169         i = iter(map(self.build, o.getChildren()))
 170         for el in i:
 171             d[el] = i.next()
 172         return d
 173
 174     def build_Tuple(self, o):
 175         return tuple(self.build_List(o))
 176
 177     def build_Name(self, o):
 178         if o.name == 'None':
 179             return None
 180         if o.name == 'True':
 181             return True
 182         if o.name == 'False':
 183             return False
 184
 185         # An undefinted Name
 186         raise UnknownType('Undefined Name')
 187
 188     def build_Add(self, o):
 189         real, imag = map(self.build_Const, o.getChildren())
 190         try:
 191             real = float(real)
 192         except TypeError:
 193             raise UnknownType('Add')
 194         if not isinstance(imag, complex) or imag.real != 0.0:
 195             raise UnknownType('Add')
 196         return real+imag
 197
 198     def build_Getattr(self, o):
 199         parent = self.build(o.expr)
 200         return getattr(parent, o.attrname)
 201
 202     def build_UnarySub(self, o):
 203         return -self.build_Const(o.getChildren()[0])
 204
 205     def build_UnaryAdd(self, o):
 206         return self.build_Const(o.getChildren()[0])
 207
 208 def unrepr(s):
 209     if not s:
 210         return s
 211     return Builder().build(getObj(s))
 212
 213 def _splitlines(instring):
 214     """Split a string on lines, without losing line endings or truncating."""
 215
 216
 217 class ConfigObjError(SyntaxError):
 218     """
 219     This is the base class for all errors that ConfigObj raises.
 220     It is a subclass of SyntaxError.
 221     """
 222     def __init__(self, message='', line_number=None, line=''):
 223         self.line = line
 224         self.line_number = line_number
 225         self.message = message
 226         SyntaxError.__init__(self, message)
 227
 228 class NestingError(ConfigObjError):
 229     """
 230     This error indicates a level of nesting that doesn't match.
 231     """
 232
 233 class ParseError(ConfigObjError):
 234     """
 235     This error indicates that a line is badly written.
 236     It is neither a valid ``key = value`` line,
 237     nor a valid section marker line.
 238     """
 239
 240 class DuplicateError(ConfigObjError):
 241     """
 242     The keyword or section specified already exists.
 243     """
 244
 245 class ConfigspecError(ConfigObjError):
 246     """
 247     An error occured whilst parsing a configspec.
 248     """
 249
 250 class InterpolationError(ConfigObjError):
 251     """Base class for the two interpolation errors."""
 252
 253 class InterpolationDepthError(InterpolationError):
 254     """Maximum interpolation depth exceeded in string interpolation."""
 255
 256     def __init__(self, option):
 257         InterpolationError.__init__(
 258             self,
 259             'max interpolation depth exceeded in value "%s".' % option)
 260
 261 class RepeatSectionError(ConfigObjError):
 262     """
 263     This error indicates additional sections in a section with a
 264     ``__many__`` (repeated) section.
 265     """
 266
 267 class MissingInterpolationOption(InterpolationError):
 268     """A value specified for interpolation was missing."""
 269
 270     def __init__(self, option):
 271         InterpolationError.__init__(
 272             self,
 273             'missing option "%s" in interpolation.' % option)
 274
 275 class UnreprError(ConfigObjError):
 276     """An error parsing in unrepr mode."""
 277
 278
 279 class Section(dict):
 280     """
 281     A dictionary-like object that represents a section in a config file.
 282
 283     It does string interpolation if the 'interpolate' attribute
 284     of the 'main' object is set to True.
 285
 286     Interpolation is tried first from the 'DEFAULT' section of this object,
 287     next from the 'DEFAULT' section of the parent, lastly the main object.
 288
 289     A Section will behave like an ordered dictionary - following the
 290     order of the ``scalars`` and ``sections`` attributes.
 291     You can use this to change the order of members.
 292
 293     Iteration follows the order: scalars, then sections.
 294     """
 295
 296     _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
 297
 298     def __init__(self, parent, depth, main, indict=None, name=None):
 299         """
 300         * parent is the section above
 301         * depth is the depth level of this section
 302         * main is the main ConfigObj
 303         * indict is a dictionary to initialise the section with
 304         """
 305         if indict is None:
 306             indict = {}
 307         dict.__init__(self)
 308         # used for nesting level *and* interpolation
 309         self.parent = parent
 310         # used for the interpolation attribute
 311         self.main = main
 312         # level of nesting depth of this Section
 313         self.depth = depth
 314         # the sequence of scalar values in this Section
 315         self.scalars = []
 316         # the sequence of sections in this Section
 317         self.sections = []
 318         # purely for information
 319         self.name = name
 320         # for comments :-)
 321         self.comments = {}
 322         self.inline_comments = {}
 323         # for the configspec
 324         self.configspec = {}
 325         self._order = []
 326         self._configspec_comments = {}
 327         self._configspec_inline_comments = {}
 328         self._cs_section_comments = {}
 329         self._cs_section_inline_comments = {}
 330         # for defaults
 331         self.defaults = []
 332         #
 333         # we do this explicitly so that __setitem__ is used properly
 334         # (rather than just passing to ``dict.__init__``)
 335         for entry in indict:
 336             self[entry] = indict[entry]
 337
 338     def _interpolate(self, value):
 339         """Nicked from ConfigParser."""
 340         depth = MAX_INTERPOL_DEPTH
 341         # loop through this until it's done
 342         while depth:
 343             depth -= 1
 344             if value.find("%(") != -1:
 345                 value = self._KEYCRE.sub(self._interpolation_replace, value)
 346             else:
 347                 break
 348         else:
 349             raise InterpolationDepthError(value)
 350         return value
 351
 352     def _interpolation_replace(self, match):
 353         """ """
 354         s = match.group(1)
 355         if s is None:
 356             return match.group()
 357         else:
 358             # switch off interpolation before we try and fetch anything !
 359             self.main.interpolation = False
 360             # try the 'DEFAULT' member of *this section* first
 361             val = self.get('DEFAULT', {}).get(s)
 362             # try the 'DEFAULT' member of the *parent section* next
 363             if val is None:
 364                 val = self.parent.get('DEFAULT', {}).get(s)
 365             # last, try the 'DEFAULT' member of the *main section*
 366             if val is None:
 367                 val = self.main.get('DEFAULT', {}).get(s)
 368             self.main.interpolation = True
 369             if val is None:
 370                 raise MissingInterpolationOption(s)
 371             return val
 372
 373     def __getitem__(self, key):
 374         """Fetch the item and do string interpolation."""
 375         val = dict.__getitem__(self, key)
 376         if self.main.interpolation and isinstance(val, StringTypes):
 377             return self._interpolate(val)
 378         return val
 379
 380     def __setitem__(self, key, value, unrepr=False):
 381         """
 382         Correctly set a value.
 383
 384         Making dictionary values Section instances.
 385         (We have to special case 'Section' instances - which are also dicts)
 386
 387         Keys must be strings.
 388         Values need only be strings (or lists of strings) if
 389         ``main.stringify`` is set.
 390
 391         `unrepr`` must be set when setting a value to a dictionary, without
 392         creating a new sub-section.
 393         """
 394         if not isinstance(key, StringTypes):
 395             raise ValueError, 'The key "%s" is not a string.' % key
 396         # add the comment
 397         if not self.comments.has_key(key):
 398             self.comments[key] = []
 399             self.inline_comments[key] = ''
 400         # remove the entry from defaults
 401         if key in self.defaults:
 402             self.defaults.remove(key)
 403         #
 404         if isinstance(value, Section):
 405             if not self.has_key(key):
 406                 self.sections.append(key)
 407             dict.__setitem__(self, key, value)
 408         elif isinstance(value, dict) and not unrepr:
 409             # First create the new depth level,
 410             # then create the section
 411             if not self.has_key(key):
 412                 self.sections.append(key)
 413             new_depth = self.depth + 1
 414             dict.__setitem__(
 415                 self,
 416                 key,
 417                 Section(
 418                     self,
 419                     new_depth,
 420                     self.main,
 421                     indict=value,
 422                     name=key))
 423         else:
 424             if not self.has_key(key):
 425                 self.scalars.append(key)
 426             if not self.main.stringify:
 427                 if isinstance(value, StringTypes):
 428                     pass
 429                 elif isinstance(value, (list, tuple)):
 430                     for entry in value:
 431                         if not isinstance(entry, StringTypes):
 432                             raise TypeError, (
 433                                 'Value is not a string "%s".' % entry)
 434                 else:
 435                     raise TypeError, 'Value is not a string "%s".' % value
 436             dict.__setitem__(self, key, value)
 437
 438     def __delitem__(self, key):
 439         """Remove items from the sequence when deleting."""
 440         dict. __delitem__(self, key)
 441         if key in self.scalars:
 442             self.scalars.remove(key)
 443         else:
 444             self.sections.remove(key)
 445         del self.comments[key]
 446         del self.inline_comments[key]
 447
 448     def get(self, key, default=None):
 449         """A version of ``get`` that doesn't bypass string interpolation."""
 450         try:
 451             return self[key]
 452         except KeyError:
 453             return default
 454
 455     def update(self, indict):
 456         """
 457         A version of update that uses our ``__setitem__``.
 458         """
 459         for entry in indict:
 460             self[entry] = indict[entry]
 461
 462     def pop(self, key, *args):
 463         """ """
 464         val = dict.pop(self, key, *args)
 465         if key in self.scalars:
 466             del self.comments[key]
 467             del self.inline_comments[key]
 468             self.scalars.remove(key)
 469         elif key in self.sections:
 470             del self.comments[key]
 471             del self.inline_comments[key]
 472             self.sections.remove(key)
 473         if self.main.interpolation and isinstance(val, StringTypes):
 474             return self._interpolate(val)
 475         return val
 476
 477     def popitem(self):
 478         """Pops the first (key,val)"""
 479         sequence = (self.scalars + self.sections)
 480         if not sequence:
 481             raise KeyError, ": 'popitem(): dictionary is empty'"
 482         key = sequence[0]
 483         val =  self[key]
 484         del self[key]
 485         return key, val
 486
 487     def clear(self):
 488         """
 489         A version of clear that also affects scalars/sections
 490         Also clears comments and configspec.
 491
 492         Leaves other attributes alone :
 493             depth/main/parent are not affected
 494         """
 495         dict.clear(self)
 496         self.scalars = []
 497         self.sections = []
 498         self.comments = {}
 499         self.inline_comments = {}
 500         self.configspec = {}
 501
 502     def setdefault(self, key, default=None):
 503         """A version of setdefault that sets sequence if appropriate."""
 504         try:
 505             return self[key]
 506         except KeyError:
 507             self[key] = default
 508             return self[key]
 509
 510     def items(self):
 511         """ """
 512         return zip((self.scalars + self.sections), self.values())
 513
 514     def keys(self):
 515         """ """
 516         return (self.scalars + self.sections)
 517
 518     def values(self):
 519         """ """
 520         return [self[key] for key in (self.scalars + self.sections)]
 521
 522     def iteritems(self):
 523         """ """
 524         return iter(self.items())
 525
 526     def iterkeys(self):
 527         """ """
 528         return iter((self.scalars + self.sections))
 529
 530     __iter__ = iterkeys
 531
 532     def itervalues(self):
 533         """ """
 534         return iter(self.values())
 535
 536     def __repr__(self):
 537         return '{%s}' % ', '.join([('%s: %s' % (repr(key), repr(self[key])))
 538             for key in (self.scalars + self.sections)])
 539
 540     __str__ = __repr__
 541
 542     # Extra methods - not in a normal dictionary
 543
 544     def dict(self):
 545         """
 546         Return a deepcopy of self as a dictionary.
 547
 548         All members that are ``Section`` instances are recursively turned to
 549         ordinary dictionaries - by calling their ``dict`` method.
 550
 551         >>> n = a.dict()
 552         >>> n == a
 553         1
 554         >>> n is a
 555         0
 556         """
 557         newdict = {}
 558         for entry in self:
 559             this_entry = self[entry]
 560             if isinstance(this_entry, Section):
 561                 this_entry = this_entry.dict()
 562             elif isinstance(this_entry, list):
 563                 # create a copy rather than a reference
 564                 this_entry = list(this_entry)
 565             elif isinstance(this_entry, tuple):
 566                 # create a copy rather than a reference
 567                 this_entry = tuple(this_entry)
 568             newdict[entry] = this_entry
 569         return newdict
 570
 571     def merge(self, indict):
 572         """
 573         A recursive update - useful for merging config files.
 574
 575         >>> a = '''[section1]
 576         ...     option1 = True
 577         ...     [[subsection]]
 578         ...     more_options = False
 579         ...     # end of file'''.splitlines()
 580         >>> b = '''# File is user.ini
 581         ...     [section1]
 582         ...     option1 = False
 583         ...     # end of file'''.splitlines()
 584         >>> c1 = ConfigObj(b)
 585         >>> c2 = ConfigObj(a)
 586         >>> c2.merge(c1)
 587         >>> c2
 588         {'section1': {'option1': 'False', 'subsection': {'more_options': 'False'}}}
 589         """
 590         for key, val in indict.items():
 591             if (key in self and isinstance(self[key], dict) and
 592                                 isinstance(val, dict)):
 593                 self[key].merge(val)
 594             else:
 595                 self[key] = val
 596
 597     def rename(self, oldkey, newkey):
 598         """
 599         Change a keyname to another, without changing position in sequence.
 600
 601         Implemented so that transformations can be made on keys,
 602         as well as on values. (used by encode and decode)
 603
 604         Also renames comments.
 605         """
 606         if oldkey in self.scalars:
 607             the_list = self.scalars
 608         elif oldkey in self.sections:
 609             the_list = self.sections
 610         else:
 611             raise KeyError, 'Key "%s" not found.' % oldkey
 612         pos = the_list.index(oldkey)
 613         #
 614         val = self[oldkey]
 615         dict.__delitem__(self, oldkey)
 616         dict.__setitem__(self, newkey, val)
 617         the_list.remove(oldkey)
 618         the_list.insert(pos, newkey)
 619         comm = self.comments[oldkey]
 620         inline_comment = self.inline_comments[oldkey]
 621         del self.comments[oldkey]
 622         del self.inline_comments[oldkey]
 623         self.comments[newkey] = comm
 624         self.inline_comments[newkey] = inline_comment
 625
 626     def walk(self, function, raise_errors=True,
 627             call_on_sections=False, **keywargs):
 628         """
 629         Walk every member and call a function on the keyword and value.
 630
 631         Return a dictionary of the return values
 632
 633         If the function raises an exception, raise the errror
 634         unless ``raise_errors=False``, in which case set the return value to
 635         ``False``.
 636
 637         Any unrecognised keyword arguments you pass to walk, will be pased on
 638         to the function you pass in.
 639
 640         Note: if ``call_on_sections`` is ``True`` then - on encountering a
 641         subsection, *first* the function is called for the *whole* subsection,
 642         and then recurses into it's members. This means your function must be
 643         able to handle strings, dictionaries and lists. This allows you
 644         to change the key of subsections as well as for ordinary members. The
 645         return value when called on the whole subsection has to be discarded.
 646
 647         See  the encode and decode methods for examples, including functions.
 648
 649         .. caution::
 650
 651             You can use ``walk`` to transform the names of members of a section
 652             but you mustn't add or delete members.
 653
 654         >>> config = '''[XXXXsection]
 655         ... XXXXkey = XXXXvalue'''.splitlines()
 656         >>> cfg = ConfigObj(config)
 657         >>> cfg
 658         {'XXXXsection': {'XXXXkey': 'XXXXvalue'}}
 659         >>> def transform(section, key):
 660         ...     val = section[key]
 661         ...     newkey = key.replace('XXXX', 'CLIENT1')
 662         ...     section.rename(key, newkey)
 663         ...     if isinstance(val, (tuple, list, dict)):
 664         ...         pass
 665         ...     else:
 666         ...         val = val.replace('XXXX', 'CLIENT1')
 667         ...         section[newkey] = val
 668         >>> cfg.walk(transform, call_on_sections=True)
 669         {'CLIENT1section': {'CLIENT1key': None}}
 670         >>> cfg
 671         {'CLIENT1section': {'CLIENT1key': 'CLIENT1value'}}
 672         """
 673         out = {}
 674         # scalars first
 675         for i in range(len(self.scalars)):
 676             entry = self.scalars[i]
 677             try:
 678                 val = function(self, entry, **keywargs)
 679                 # bound again in case name has changed
 680                 entry = self.scalars[i]
 681                 out[entry] = val
 682             except Exception:
 683                 if raise_errors:
 684                     raise
 685                 else:
 686                     entry = self.scalars[i]
 687                     out[entry] = False
 688         # then sections
 689         for i in range(len(self.sections)):
 690             entry = self.sections[i]
 691             if call_on_sections:
 692                 try:
 693                     function(self, entry, **keywargs)
 694                 except Exception:
 695                     if raise_errors:
 696                         raise
 697                     else:
 698                         entry = self.sections[i]
 699                         out[entry] = False
 700                 # bound again in case name has changed
 701                 entry = self.sections[i]
 702             # previous result is discarded
 703             out[entry] = self[entry].walk(
 704                 function,
 705                 raise_errors=raise_errors,
 706                 call_on_sections=call_on_sections,
 707                 **keywargs)
 708         return out
 709
 710     def decode(self, encoding):
 711         """
 712         Decode all strings and values to unicode, using the specified encoding.
 713
 714         Works with subsections and list values.
 715
 716         Uses the ``walk`` method.
 717
 718         Testing ``encode`` and ``decode``.
 719         >>> m = ConfigObj(a)
 720         >>> m.decode('ascii')
 721         >>> def testuni(val):
 722         ...     for entry in val:
 723         ...         if not isinstance(entry, unicode):
 724         ...             print >> sys.stderr, type(entry)
 725         ...             raise AssertionError, 'decode failed.'
 726         ...         if isinstance(val[entry], dict):
 727         ...             testuni(val[entry])
 728         ...         elif not isinstance(val[entry], unicode):
 729         ...             raise AssertionError, 'decode failed.'
 730         >>> testuni(m)
 731         >>> m.encode('ascii')
 732         >>> a == m
 733         1
 734         """
 735         warn('use of ``decode`` is deprecated.', DeprecationWarning)
 736         def decode(section, key, encoding=encoding, warn=True):
 737             """ """
 738             val = section[key]
 739             if isinstance(val, (list, tuple)):
 740                 newval = []
 741                 for entry in val:
 742                     newval.append(entry.decode(encoding))
 743             elif isinstance(val, dict):
 744                 newval = val
 745             else:
 746                 newval = val.decode(encoding)
 747             newkey = key.decode(encoding)
 748             section.rename(key, newkey)
 749             section[newkey] = newval
 750         # using ``call_on_sections`` allows us to modify section names
 751         self.walk(decode, call_on_sections=True)
 752
 753     def encode(self, encoding):
 754         """
 755         Encode all strings and values from unicode,
 756         using the specified encoding.
 757
 758         Works with subsections and list values.
 759         Uses the ``walk`` method.
 760         """
 761         warn('use of ``encode`` is deprecated.', DeprecationWarning)
 762         def encode(section, key, encoding=encoding):
 763             """ """
 764             val = section[key]
 765             if isinstance(val, (list, tuple)):
 766                 newval = []
 767                 for entry in val:
 768                     newval.append(entry.encode(encoding))
 769             elif isinstance(val, dict):
 770                 newval = val
 771             else:
 772                 newval = val.encode(encoding)
 773             newkey = key.encode(encoding)
 774             section.rename(key, newkey)
 775             section[newkey] = newval
 776         self.walk(encode, call_on_sections=True)
 777
 778     def istrue(self, key):
 779         """A deprecated version of ``as_bool``."""
 780         warn('use of ``istrue`` is deprecated. Use ``as_bool`` method '
 781                 'instead.', DeprecationWarning)
 782         return self.as_bool(key)
 783
 784     def as_bool(self, key):
 785         """
 786         Accepts a key as input. The corresponding value must be a string or
 787         the objects (``True`` or 1) or (``False`` or 0). We allow 0 and 1 to
 788         retain compatibility with Python 2.2.
 789
 790         If the string is one of  ``True``, ``On``, ``Yes``, or ``1`` it returns
 791         ``True``.
 792
 793         If the string is one of  ``False``, ``Off``, ``No``, or ``0`` it returns
 794         ``False``.
 795
 796         ``as_bool`` is not case sensitive.
 797
 798         Any other input will raise a ``ValueError``.
 799
 800         >>> a = ConfigObj()
 801         >>> a['a'] = 'fish'
 802         >>> a.as_bool('a')
 803         Traceback (most recent call last):
 804         ValueError: Value "fish" is neither True nor False
 805         >>> a['b'] = 'True'
 806         >>> a.as_bool('b')
 807         1
 808         >>> a['b'] = 'off'
 809         >>> a.as_bool('b')
 810         0
 811         """
 812         val = self[key]
 813         if val == True:
 814             return True
 815         elif val == False:
 816             return False
 817         else:
 818             try:
 819                 if not isinstance(val, StringTypes):
 820                     raise KeyError
 821                 else:
 822                     return self.main._bools[val.lower()]
 823             except KeyError:
 824                 raise ValueError('Value "%s" is neither True nor False' % val)
 825
 826     def as_int(self, key):
 827         """
 828         A convenience method which coerces the specified value to an integer.
 829
 830         If the value is an invalid literal for ``int``, a ``ValueError`` will
 831         be raised.
 832
 833         >>> a = ConfigObj()
 834         >>> a['a'] = 'fish'
 835         >>> a.as_int('a')
 836         Traceback (most recent call last):
 837         ValueError: invalid literal for int(): fish
 838         >>> a['b'] = '1'
 839         >>> a.as_int('b')
 840         1
 841         >>> a['b'] = '3.2'
 842         >>> a.as_int('b')
 843         Traceback (most recent call last):
 844         ValueError: invalid literal for int(): 3.2
 845         """
 846         return int(self[key])
 847
 848     def as_float(self, key):
 849         """
 850         A convenience method which coerces the specified value to a float.
 851
 852         If the value is an invalid literal for ``float``, a ``ValueError`` will
 853         be raised.
 854
 855         >>> a = ConfigObj()
 856         >>> a['a'] = 'fish'
 857         >>> a.as_float('a')
 858         Traceback (most recent call last):
 859         ValueError: invalid literal for float(): fish
 860         >>> a['b'] = '1'
 861         >>> a.as_float('b')
 862         1.0
 863         >>> a['b'] = '3.2'
 864         >>> a.as_float('b')
 865         3.2000000000000002
 866         """
 867         return float(self[key])
 868
 869
 870 class ConfigObj(Section):
 871     """An object to read, create, and write config files."""
 872
 873     _keyword = re.compile(r'''^ # line start
 874         (\s*)                   # indentation
 875         (                       # keyword
 876             (?:".*?")|          # double quotes
 877             (?:'.*?')|          # single quotes
 878             (?:[^'"=].*?)       # no quotes
 879         )
 880         \s*=\s*                 # divider
 881         (.*)                    # value (including list values and comments)
 882         $   # line end
 883         ''',
 884         re.VERBOSE)
 885
 886     _sectionmarker = re.compile(r'''^
 887         (\s*)                     # 1: indentation
 888         ((?:\[\s*)+)              # 2: section marker open
 889         (                         # 3: section name open
 890             (?:"\s*\S.*?\s*")|    # at least one non-space with double quotes
 891             (?:'\s*\S.*?\s*')|    # at least one non-space with single quotes
 892             (?:[^'"\s].*?)        # at least one non-space unquoted
 893         )                         # section name close
 894         ((?:\s*\])+)              # 4: section marker close
 895         \s*(\#.*)?                # 5: optional comment
 896         $''',
 897         re.VERBOSE)
 898
 899     # this regexp pulls list values out as a single string
 900     # or single values and comments
 901     # FIXME: this regex adds a '' to the end of comma terminated lists
 902     #   workaround in ``_handle_value``
 903     _valueexp = re.compile(r'''^
 904         (?:
 905             (?:
 906                 (
 907                     (?:
 908                         (?:
 909                             (?:".*?")|              # double quotes
 910                             (?:'.*?')|              # single quotes
 911                             (?:[^'",\#][^,\#]*?)    # unquoted
 912                         )
 913                         \s*,\s*                     # comma
 914                     )*      # match all list items ending in a comma (if any)
 915                 )
 916                 (
 917                     (?:".*?")|                      # double quotes
 918                     (?:'.*?')|                      # single quotes
 919                     (?:[^'",\#\s][^,]*?)|           # unquoted
 920                     (?:(?<!,))                      # Empty value
 921                 )?          # last item in a list - or string value
 922             )|
 923             (,)             # alternatively a single comma - empty list
 924         )
 925         \s*(\#.*)?          # optional comment
 926         $''',
 927         re.VERBOSE)
 928
 929     # use findall to get the members of a list value
 930     _listvalueexp = re.compile(r'''
 931         (
 932             (?:".*?")|          # double quotes
 933             (?:'.*?')|          # single quotes
 934             (?:[^'",\#].*?)       # unquoted
 935         )
 936         \s*,\s*                 # comma
 937         ''',
 938         re.VERBOSE)
 939
 940     # this regexp is used for the value
 941     # when lists are switched off
 942     _nolistvalue = re.compile(r'''^
 943         (
 944             (?:".*?")|          # double quotes
 945             (?:'.*?')|          # single quotes
 946             (?:[^'"\#].*?)|     # unquoted
 947             (?:)                # Empty value
 948         )
 949         \s*(\#.*)?              # optional comment
 950         $''',
 951         re.VERBOSE)
 952
 953     # regexes for finding triple quoted values on one line
 954     _single_line_single = re.compile(r"^'''(.*?)'''\s*(#.*)?$")
 955     _single_line_double = re.compile(r'^"""(.*?)"""\s*(#.*)?$')
 956     _multi_line_single = re.compile(r"^(.*?)'''\s*(#.*)?$")
 957     _multi_line_double = re.compile(r'^(.*?)"""\s*(#.*)?$')
 958
 959     _triple_quote = {
 960         "'''": (_single_line_single, _multi_line_single),
 961         '"""': (_single_line_double, _multi_line_double),
 962     }
 963
 964     # Used by the ``istrue`` Section method
 965     _bools = {
 966         'yes': True, 'no': False,
 967         'on': True, 'off': False,
 968         '1': True, '0': False,
 969         'true': True, 'false': False,
 970         }
 971
 972     def __init__(self, infile=None, options=None, **kwargs):
 973         """
 974         Parse or create a config file object.
 975
 976         ``ConfigObj(infile=None, options=None, **kwargs)``
 977         """
 978         if infile is None:
 979             infile = []
 980         if options is None:
 981             options = {}
 982         else:
 983             options = dict(options)
 984         # keyword arguments take precedence over an options dictionary
 985         options.update(kwargs)
 986         # init the superclass
 987         Section.__init__(self, self, 0, self)
 988         #
 989         defaults = OPTION_DEFAULTS.copy()
 990         for entry in options.keys():
 991             if entry not in defaults.keys():
 992                 raise TypeError, 'Unrecognised option "%s".' % entry
 993         # TODO: check the values too.
 994         #
 995         # Add any explicit options to the defaults
 996         defaults.update(options)
 997         #
 998         # initialise a few variables
 999         self.filename = None
1000         self._errors = []
1001         self.raise_errors = defaults['raise_errors']
1002         self.interpolation = defaults['interpolation']
1003         self.list_values = defaults['list_values']
1004         self.create_empty = defaults['create_empty']
1005         self.file_error = defaults['file_error']
1006         self.stringify = defaults['stringify']
1007         self.indent_type = defaults['indent_type']
1008         self.encoding = defaults['encoding']
1009         self.default_encoding = defaults['default_encoding']
1010         self.BOM = False
1011         self.newlines = None
1012         self.write_empty_values = defaults['write_empty_values']
1013         self.unrepr = defaults['unrepr']
1014         #
1015         self.initial_comment = []
1016         self.final_comment = []
1017         #
1018         self._terminated = False
1019         #
1020         if isinstance(infile, StringTypes):
1021             self.filename = infile
1022             if os.path.isfile(infile):
1023                 infile = open(infile).read() or []
1024             elif self.file_error:
1025                 # raise an error if the file doesn't exist
1026                 raise IOError, 'Config file not found: "%s".' % self.filename
1027             else:
1028                 # file doesn't already exist
1029                 if self.create_empty:
1030                     # this is a good test that the filename specified
1031                     # isn't impossible - like on a non existent device
1032                     h = open(infile, 'w')
1033                     h.write('')
1034                     h.close()
1035                 infile = []
1036         elif isinstance(infile, (list, tuple)):
1037             infile = list(infile)
1038         elif isinstance(infile, dict):
1039             # initialise self
1040             # the Section class handles creating subsections
1041             if isinstance(infile, ConfigObj):
1042                 # get a copy of our ConfigObj
1043                 infile = infile.dict()
1044             for entry in infile:
1045                 self[entry] = infile[entry]
1046             del self._errors
1047             if defaults['configspec'] is not None:
1048                 self._handle_configspec(defaults['configspec'])
1049             else:
1050                 self.configspec = None
1051             return
1052         elif hasattr(infile, 'read'):
1053             # This supports file like objects
1054             infile = infile.read() or []
1055             # needs splitting into lines - but needs doing *after* decoding
1056             # in case it's not an 8 bit encoding
1057         else:
1058             raise TypeError, ('infile must be a filename,'
1059                 ' file like object, or list of lines.')
1060         #
1061         if infile:
1062             # don't do it for the empty ConfigObj
1063             infile = self._handle_bom(infile)
1064             # infile is now *always* a list
1065             #
1066             # Set the newlines attribute (first line ending it finds)
1067             # and strip trailing '\n' or '\r' from lines
1068             for line in infile:
1069                 if (not line) or (line[-1] not in '\r\n'):
1070                     continue
1071                 for end in ('\r\n', '\n', '\r'):
1072                     if line.endswith(end):
1073                         self.newlines = end
1074                         break
1075                 break
1076             if infile[-1] and infile[-1] in '\r\n':
1077                 self._terminated = True
1078             infile = [line.rstrip('\r\n') for line in infile]
1079         #
1080         self._parse(infile)
1081         # if we had any errors, now is the time to raise them
1082         if self._errors:
1083             info = "at line %s." % self._errors[0].line_number
1084             if len(self._errors) > 1:
1085                 msg = ("Parsing failed with several errors.\nFirst error %s" %
1086                     info)
1087                 error = ConfigObjError(msg)
1088             else:
1089                 error = self._errors[0]
1090             # set the errors attribute; it's a list of tuples:
1091             # (error_type, message, line_number)
1092             error.errors = self._errors
1093             # set the config attribute
1094             error.config = self
1095             raise error
1096         # delete private attributes
1097         del self._errors
1098         #
1099         if defaults['configspec'] is None:
1100             self.configspec = None
1101         else:
1102             self._handle_configspec(defaults['configspec'])
1103
1104     def __repr__(self):
1105         return 'ConfigObj({%s})' % ', '.join(
1106             [('%s: %s' % (repr(key), repr(self[key]))) for key in
1107             (self.scalars + self.sections)])
1108
1109     def _handle_bom(self, infile):
1110         """
1111         Handle any BOM, and decode if necessary.
1112
1113         If an encoding is specified, that *must* be used - but the BOM should
1114         still be removed (and the BOM attribute set).
1115
1116         (If the encoding is wrongly specified, then a BOM for an alternative
1117         encoding won't be discovered or removed.)
1118
1119         If an encoding is not specified, UTF8 or UTF16 BOM will be detected and
1120         removed. The BOM attribute will be set. UTF16 will be decoded to
1121         unicode.
1122
1123         NOTE: This method must not be called with an empty ``infile``.
1124
1125         Specifying the *wrong* encoding is likely to cause a
1126         ``UnicodeDecodeError``.
1127
1128         ``infile`` must always be returned as a list of lines, but may be
1129         passed in as a single string.
1130         """
1131         if ((self.encoding is not None) and
1132             (self.encoding.lower() not in BOM_LIST)):
1133             # No need to check for a BOM
1134             # the encoding specified doesn't have one
1135             # just decode
1136             return self._decode(infile, self.encoding)
1137         #
1138         if isinstance(infile, (list, tuple)):
1139             line = infile[0]
1140         else:
1141             line = infile
1142         if self.encoding is not None:
1143             # encoding explicitly supplied
1144             # And it could have an associated BOM
1145             # TODO: if encoding is just UTF16 - we ought to check for both
1146             # TODO: big endian and little endian versions.
1147             enc = BOM_LIST[self.encoding.lower()]
1148             if enc == 'utf_16':
1149                 # For UTF16 we try big endian and little endian
1150                 for BOM, (encoding, final_encoding) in BOMS.items():
1151                     if not final_encoding:
1152                         # skip UTF8
1153                         continue
1154                     if infile.startswith(BOM):
1155                         ### BOM discovered
1156                         ##self.BOM = True
1157                         # Don't need to remove BOM
1158                         return self._decode(infile, encoding)
1159                 #
1160                 # If we get this far, will *probably* raise a DecodeError
1161                 # As it doesn't appear to start with a BOM
1162                 return self._decode(infile, self.encoding)
1163             #
1164             # Must be UTF8
1165             BOM = BOM_SET[enc]
1166             if not line.startswith(BOM):
1167                 return self._decode(infile, self.encoding)
1168             #
1169             newline = line[len(BOM):]
1170             #
1171             # BOM removed
1172             if isinstance(infile, (list, tuple)):
1173                 infile[0] = newline
1174             else:
1175                 infile = newline
1176             self.BOM = True
1177             return self._decode(infile, self.encoding)
1178         #
1179         # No encoding specified - so we need to check for UTF8/UTF16
1180         for BOM, (encoding, final_encoding) in BOMS.items():
1181             if not line.startswith(BOM):
1182                 continue
1183             else:
1184                 # BOM discovered
1185                 self.encoding = final_encoding
1186                 if not final_encoding:
1187                     self.BOM = True
1188                     # UTF8
1189                     # remove BOM
1190                     newline = line[len(BOM):]
1191                     if isinstance(infile, (list, tuple)):
1192                         infile[0] = newline
1193                     else:
1194                         infile = newline
1195                     # UTF8 - don't decode
1196                     if isinstance(infile, StringTypes):
1197                         return infile.splitlines(True)
1198                     else:
1199                         return infile
1200                 # UTF16 - have to decode
1201                 return self._decode(infile, encoding)
1202         #
1203         # No BOM discovered and no encoding specified, just return
1204         if isinstance(infile, StringTypes):
1205             # infile read from a file will be a single string
1206             return infile.splitlines(True)
1207         else:
1208             return infile
1209
1210     def _a_to_u(self, string):
1211         """Decode ascii strings to unicode if a self.encoding is specified."""
1212         if not self.encoding:
1213             return string
1214         else:
1215             return string.decode('ascii')
1216
1217     def _decode(self, infile, encoding):
1218         """
1219         Decode infile to unicode. Using the specified encoding.
1220
1221         if is a string, it also needs converting to a list.
1222         """
1223         if isinstance(infile, StringTypes):
1224             # can't be unicode
1225             # NOTE: Could raise a ``UnicodeDecodeError``
1226             return infile.decode(encoding).splitlines(True)
1227         for i, line in enumerate(infile):
1228             if not isinstance(line, unicode):
1229                 # NOTE: The isinstance test here handles mixed lists of unicode/string
1230                 # NOTE: But the decode will break on any non-string values
1231                 # NOTE: Or could raise a ``UnicodeDecodeError``
1232                 infile[i] = line.decode(encoding)
1233         return infile
1234
1235     def _decode_element(self, line):
1236         """Decode element to unicode if necessary."""
1237         if not self.encoding:
1238             return line
1239         if isinstance(line, str) and self.default_encoding:
1240             return line.decode(self.default_encoding)
1241         return line
1242
1243     def _str(self, value):
1244         """
1245         Used by ``stringify`` within validate, to turn non-string values
1246         into strings.
1247         """
1248         if not isinstance(value, StringTypes):
1249             return str(value)
1250         else:
1251             return value
1252
1253     def _parse(self, infile):
1254         """Actually parse the config file."""
1255         temp_list_values = self.list_values
1256         if self.unrepr:
1257             self.list_values = False
1258         comment_list = []
1259         done_start = False
1260         this_section = self
1261         maxline = len(infile) - 1
1262         cur_index = -1
1263         reset_comment = False
1264         while cur_index < maxline:
1265             if reset_comment:
1266                 comment_list = []
1267             cur_index += 1
1268             line = infile[cur_index]
1269             sline = line.strip()
1270             # do we have anything on the line ?
1271             if not sline or sline.startswith('#'):
1272                 reset_comment = False
1273                 comment_list.append(line)
1274                 continue
1275             if not done_start:
1276                 # preserve initial comment
1277                 self.initial_comment = comment_list
1278                 comment_list = []
1279                 done_start = True
1280             reset_comment = True
1281             # first we check if it's a section marker
1282             mat = self._sectionmarker.match(line)
1283             if mat is not None:
1284                 # is a section line
1285                 (indent, sect_open, sect_name, sect_close, comment) = (
1286                     mat.groups())
1287                 if indent and (self.indent_type is None):
1288                     self.indent_type = indent[0]
1289                 cur_depth = sect_open.count('[')
1290                 if cur_depth != sect_close.count(']'):
1291                     self._handle_error(
1292                         "Cannot compute the section depth at line %s.",
1293                         NestingError, infile, cur_index)
1294                     continue
1295                 #
1296                 if cur_depth < this_section.depth:
1297                     # the new section is dropping back to a previous level
1298                     try:
1299                         parent = self._match_depth(
1300                             this_section,
1301                             cur_depth).parent
1302                     except SyntaxError:
1303                         self._handle_error(
1304                             "Cannot compute nesting level at line %s.",
1305                             NestingError, infile, cur_index)
1306                         continue
1307                 elif cur_depth == this_section.depth:
1308                     # the new section is a sibling of the current section
1309                     parent = this_section.parent
1310                 elif cur_depth == this_section.depth + 1:
1311                     # the new section is a child the current section
1312                     parent = this_section
1313                 else:
1314                     self._handle_error(
1315                         "Section too nested at line %s.",
1316                         NestingError, infile, cur_index)
1317                 #
1318                 sect_name = self._unquote(sect_name)
1319                 if parent.has_key(sect_name):
1320                     self._handle_error(
1321                         'Duplicate section name at line %s.',
1322                         DuplicateError, infile, cur_index)
1323                     continue
1324                 # create the new section
1325                 this_section = Section(
1326                     parent,
1327                     cur_depth,
1328                     self,
1329                     name=sect_name)
1330                 parent[sect_name] = this_section
1331                 parent.inline_comments[sect_name] = comment
1332                 parent.comments[sect_name] = comment_list
1333                 continue
1334             #
1335             # it's not a section marker,
1336             # so it should be a valid ``key = value`` line
1337             mat = self._keyword.match(line)
1338             if mat is None:
1339                 # it neither matched as a keyword
1340                 # or a section marker
1341                 self._handle_error(
1342                     'Invalid line at line "%s".',
1343                     ParseError, infile, cur_index)
1344             else:
1345                 # is a keyword value
1346                 # value will include any inline comment
1347                 (indent, key, value) = mat.groups()
1348                 if indent and (self.indent_type is None):
1349                     self.indent_type = indent[0]
1350                 # check for a multiline value
1351                 if value[:3] in ['"""', "'''"]:
1352                     try:
1353                         (value, comment, cur_index) = self._multiline(
1354                             value, infile, cur_index, maxline)
1355                     except SyntaxError:
1356                         self._handle_error(
1357                             'Parse error in value at line %s.',
1358                             ParseError, infile, cur_index)
1359                         continue
1360                     else:
1361                         if self.unrepr:
1362                             comment = ''
1363                             try:
1364                                 value = unrepr(value)
1365                             except Exception, e:
1366                                 if type(e) == UnknownType:
1367                                     msg = 'Unknown name or type in value at line %s.'
1368                                 else:
1369                                     msg = 'Parse error in value at line %s.'
1370                                 self._handle_error(msg, UnreprError, infile,
1371                                     cur_index)
1372                                 continue
1373                 else:
1374                     if self.unrepr:
1375                         comment = ''
1376                         try:
1377                             value = unrepr(value)
1378                         except Exception, e:
1379                             if isinstance(e, UnknownType):
1380                                 msg = 'Unknown name or type in value at line %s.'
1381                             else:
1382                                 msg = 'Parse error in value at line %s.'
1383                             self._handle_error(msg, UnreprError, infile,
1384                                 cur_index)
1385                             continue
1386                     else:
1387                         # extract comment and lists
1388                         try:
1389                             (value, comment) = self._handle_value(value)
1390                         except SyntaxError:
1391                             self._handle_error(
1392                                 'Parse error in value at line %s.',
1393                                 ParseError, infile, cur_index)
1394                             continue
1395                 #
1396                 key = self._unquote(key)
1397                 if this_section.has_key(key):
1398                     self._handle_error(
1399                         'Duplicate keyword name at line %s.',
1400                         DuplicateError, infile, cur_index)
1401                     continue
1402                 # add the key.
1403                 # we set unrepr because if we have got this far we will never
1404                 # be creating a new section
1405                 this_section.__setitem__(key, value, unrepr=True)
1406                 this_section.inline_comments[key] = comment
1407                 this_section.comments[key] = comment_list
1408                 continue
1409         #
1410         if self.indent_type is None:
1411             # no indentation used, set the type accordingly
1412             self.indent_type = ''
1413         #
1414         if self._terminated:
1415             comment_list.append('')
1416         # preserve the final comment
1417         if not self and not self.initial_comment:
1418             self.initial_comment = comment_list
1419         elif not reset_comment:
1420             self.final_comment = comment_list
1421         self.list_values = temp_list_values
1422
1423     def _match_depth(self, sect, depth):
1424         """
1425         Given a section and a depth level, walk back through the sections
1426         parents to see if the depth level matches a previous section.
1427
1428         Return a reference to the right section,
1429         or raise a SyntaxError.
1430         """
1431         while depth < sect.depth:
1432             if sect is sect.parent:
1433                 # we've reached the top level already
1434                 raise SyntaxError
1435             sect = sect.parent
1436         if sect.depth == depth:
1437             return sect
1438         # shouldn't get here
1439         raise SyntaxError
1440
1441     def _handle_error(self, text, ErrorClass, infile, cur_index):
1442         """
1443         Handle an error according to the error settings.
1444
1445         Either raise the error or store it.
1446         The error will have occured at ``cur_index``
1447         """
1448         line = infile[cur_index]
1449         cur_index += 1
1450         message = text % cur_index
1451         error = ErrorClass(message, cur_index, line)
1452         if self.raise_errors:
1453             # raise the error - parsing stops here
1454             raise error
1455         # store the error
1456         # reraise when parsing has finished
1457         self._errors.append(error)
1458
1459     def _unquote(self, value):
1460         """Return an unquoted version of a value"""
1461         if (value[0] == value[-1]) and (value[0] in ('"', "'")):
1462             value = value[1:-1]
1463         return value
1464
1465     def _quote(self, value, multiline=True):
1466         """
1467         Return a safely quoted version of a value.
1468
1469         Raise a ConfigObjError if the value cannot be safely quoted.
1470         If multiline is ``True`` (default) then use triple quotes
1471         if necessary.
1472
1473         Don't quote values that don't need it.
1474         Recursively quote members of a list and return a comma joined list.
1475         Multiline is ``False`` for lists.
1476         Obey list syntax for empty and single member lists.
1477
1478         If ``list_values=False`` then the value is only quoted if it contains
1479         a ``\n`` (is multiline).
1480
1481         If ``write_empty_values`` is set, and the value is an empty string, it
1482         won't be quoted.
1483         """
1484         if multiline and self.write_empty_values and value == '':
1485             # Only if multiline is set, so that it is used for values not
1486             # keys, and not values that are part of a list
1487             return ''
1488         if multiline and isinstance(value, (list, tuple)):
1489             if not value:
1490                 return ','
1491             elif len(value) == 1:
1492                 return self._quote(value[0], multiline=False) + ','
1493             return ', '.join([self._quote(val, multiline=False)
1494                 for val in value])
1495         if not isinstance(value, StringTypes):
1496             if self.stringify:
1497                 value = str(value)
1498             else:
1499                 raise TypeError, 'Value "%s" is not a string.' % value
1500         squot = "'%s'"
1501         dquot = '"%s"'
1502         noquot = "%s"
1503         wspace_plus = ' \r\t\n\v\t\'"'
1504         tsquot = '"""%s"""'
1505         tdquot = "'''%s'''"
1506         if not value:
1507             return '""'
1508         if (not self.list_values and '\n' not in value) or not (multiline and
1509                 ((("'" in value) and ('"' in value)) or ('\n' in value))):
1510             if not self.list_values:
1511                 # we don't quote if ``list_values=False``
1512                 quot = noquot
1513             # for normal values either single or double quotes will do
1514             elif '\n' in value:
1515                 # will only happen if multiline is off - e.g. '\n' in key
1516                 raise ConfigObjError, ('Value "%s" cannot be safely quoted.' %
1517                     value)
1518             elif ((value[0] not in wspace_plus) and
1519                     (value[-1] not in wspace_plus) and
1520                     (',' not in value)):
1521                 quot = noquot
1522             else:
1523                 if ("'" in value) and ('"' in value):
1524                     raise ConfigObjError, (
1525                         'Value "%s" cannot be safely quoted.' % value)
1526                 elif '"' in value:
1527                     quot = squot
1528                 else:
1529                     quot = dquot
1530         else:
1531             # if value has '\n' or "'" *and* '"', it will need triple quotes
1532             if (value.find('"""') != -1) and (value.find("'''") != -1):
1533                 raise ConfigObjError, (
1534                     'Value "%s" cannot be safely quoted.' % value)
1535             if value.find('"""') == -1:
1536                 quot = tdquot
1537             else:
1538                 quot = tsquot
1539         return quot % value
1540
1541     def _handle_value(self, value):
1542         """
1543         Given a value string, unquote, remove comment,
1544         handle lists. (including empty and single member lists)
1545         """
1546         # do we look for lists in values ?
1547         if not self.list_values:
1548             mat = self._nolistvalue.match(value)
1549             if mat is None:
1550                 raise SyntaxError
1551             # NOTE: we don't unquote here
1552             return mat.groups()
1553         #
1554         mat = self._valueexp.match(value)
1555         if mat is None:
1556             # the value is badly constructed, probably badly quoted,
1557             # or an invalid list
1558             raise SyntaxError
1559         (list_values, single, empty_list, comment) = mat.groups()
1560         if (list_values == '') and (single is None):
1561             # change this if you want to accept empty values
1562             raise SyntaxError
1563         # NOTE: note there is no error handling from here if the regex
1564         # is wrong: then incorrect values will slip through
1565         if empty_list is not None:
1566             # the single comma - meaning an empty list
1567             return ([], comment)
1568         if single is not None:
1569             # handle empty values
1570             if list_values and not single:
1571                 # FIXME: the '' is a workaround because our regex now matches
1572                 #   '' at the end of a list if it has a trailing comma
1573                 single = None
1574             else:
1575                 single = single or '""'
1576                 single = self._unquote(single)
1577         if list_values == '':
1578             # not a list value
1579             return (single, comment)
1580         the_list = self._listvalueexp.findall(list_values)
1581         the_list = [self._unquote(val) for val in the_list]
1582         if single is not None:
1583             the_list += [single]
1584         return (the_list, comment)
1585
1586     def _multiline(self, value, infile, cur_index, maxline):
1587         """Extract the value, where we are in a multiline situation."""
1588         quot = value[:3]
1589         newvalue = value[3:]
1590         single_line = self._triple_quote[quot][0]
1591         multi_line = self._triple_quote[quot][1]
1592         mat = single_line.match(value)
1593         if mat is not None:
1594             retval = list(mat.groups())
1595             retval.append(cur_index)
1596             return retval
1597         elif newvalue.find(quot) != -1:
1598             # somehow the triple quote is missing
1599             raise SyntaxError
1600         #
1601         while cur_index < maxline:
1602             cur_index += 1
1603             newvalue += '\n'
1604             line = infile[cur_index]
1605             if line.find(quot) == -1:
1606                 newvalue += line
1607             else:
1608                 # end of multiline, process it
1609                 break
1610         else:
1611             # we've got to the end of the config, oops...
1612             raise SyntaxError
1613         mat = multi_line.match(line)
1614         if mat is None:
1615             # a badly formed line
1616             raise SyntaxError
1617         (value, comment) = mat.groups()
1618         return (newvalue + value, comment, cur_index)
1619
1620     def _handle_configspec(self, configspec):
1621         """Parse the configspec."""
1622         # FIXME: Should we check that the configspec was created with the
1623         #   correct settings ? (i.e. ``list_values=False``)
1624         if not isinstance(configspec, ConfigObj):
1625             try:
1626                 configspec = ConfigObj(
1627                     configspec,
1628                     raise_errors=True,
1629                     file_error=True,
1630                     list_values=False)
1631             except ConfigObjError, e:
1632                 # FIXME: Should these errors have a reference
1633                 # to the already parsed ConfigObj ?
1634                 raise ConfigspecError('Parsing configspec failed: %s' % e)
1635             except IOError, e:
1636                 raise IOError('Reading configspec failed: %s' % e)
1637         self._set_configspec_value(configspec, self)
1638
1639     def _set_configspec_value(self, configspec, section):
1640         """Used to recursively set configspec values."""
1641         if '__many__' in configspec.sections:
1642             section.configspec['__many__'] = configspec['__many__']
1643             if len(configspec.sections) > 1:
1644                 # FIXME: can we supply any useful information here ?
1645                 raise RepeatSectionError
1646         if hasattr(configspec, 'initial_comment'):
1647             section._configspec_initial_comment = configspec.initial_comment
1648             section._configspec_final_comment = configspec.final_comment
1649             section._configspec_encoding = configspec.encoding
1650             section._configspec_BOM = configspec.BOM
1651             section._configspec_newlines = configspec.newlines
1652             section._configspec_indent_type = configspec.indent_type
1653         for entry in configspec.scalars:
1654             section._configspec_comments[entry] = configspec.comments[entry]
1655             section._configspec_inline_comments[entry] = (
1656                 configspec.inline_comments[entry])
1657             section.configspec[entry] = configspec[entry]
1658             section._order.append(entry)
1659         for entry in configspec.sections:
1660             if entry == '__many__':
1661                 continue
1662             section._cs_section_comments[entry] = configspec.comments[entry]
1663             section._cs_section_inline_comments[entry] = (
1664                 configspec.inline_comments[entry])
1665             if not section.has_key(entry):
1666                 section[entry] = {}
1667             self._set_configspec_value(configspec[entry], section[entry])
1668
1669     def _handle_repeat(self, section, configspec):
1670         """Dynamically assign configspec for repeated section."""
1671         try:
1672             section_keys = configspec.sections
1673             scalar_keys = configspec.scalars
1674         except AttributeError:
1675             section_keys = [entry for entry in configspec
1676                                 if isinstance(configspec[entry], dict)]
1677             scalar_keys = [entry for entry in configspec
1678                                 if not isinstance(configspec[entry], dict)]
1679         if '__many__' in section_keys and len(section_keys) > 1:
1680             # FIXME: can we supply any useful information here ?
1681             raise RepeatSectionError
1682         scalars = {}
1683         sections = {}
1684         for entry in scalar_keys:
1685             val = configspec[entry]
1686             scalars[entry] = val
1687         for entry in section_keys:
1688             val = configspec[entry]
1689             if entry == '__many__':
1690                 scalars[entry] = val
1691                 continue
1692             sections[entry] = val
1693         #
1694         section.configspec = scalars
1695         for entry in sections:
1696             if not section.has_key(entry):
1697                 section[entry] = {}
1698             self._handle_repeat(section[entry], sections[entry])
1699
1700     def _write_line(self, indent_string, entry, this_entry, comment):
1701         """Write an individual line, for the write method"""
1702         # NOTE: the calls to self._quote here handles non-StringType values.
1703         if not self.unrepr:
1704             val = self._decode_element(self._quote(this_entry))
1705         else:
1706             val = repr(this_entry)
1707         return '%s%s%s%s%s' % (
1708             indent_string,
1709             self._decode_element(self._quote(entry, multiline=False)),
1710             self._a_to_u(' = '),
1711             val,
1712             self._decode_element(comment))
1713
1714     def _write_marker(self, indent_string, depth, entry, comment):
1715         """Write a section marker line"""
1716         return '%s%s%s%s%s' % (
1717             indent_string,
1718             self._a_to_u('[' * depth),
1719             self._quote(self._decode_element(entry), multiline=False),
1720             self._a_to_u(']' * depth),
1721             self._decode_element(comment))
1722
1723     def _handle_comment(self, comment):
1724         """Deal with a comment."""
1725         if not comment:
1726             return ''
1727         if self.indent_type == '\t':
1728             start = self._a_to_u('\t')
1729         else:
1730             start = self._a_to_u(' ' * NUM_INDENT_SPACES)
1731         if not comment.startswith('#'):
1732             start += _a_to_u('# ')
1733         return (start + comment)
1734
1735     def _compute_indent_string(self, depth):
1736         """
1737         Compute the indent string, according to current indent_type and depth
1738         """
1739         if self.indent_type == '':
1740             # no indentation at all
1741             return ''
1742         if self.indent_type == '\t':
1743             return '\t' * depth
1744         if self.indent_type == ' ':
1745             return ' ' * NUM_INDENT_SPACES * depth
1746         raise SyntaxError
1747
1748     # Public methods
1749
1750     def write(self, outfile=None, section=None):
1751         """
1752         Write the current ConfigObj as a file
1753
1754         tekNico: FIXME: use StringIO instead of real files
1755
1756         >>> filename = a.filename
1757         >>> a.filename = 'test.ini'
1758         >>> a.write()
1759         >>> a.filename = filename
1760         >>> a == ConfigObj('test.ini', raise_errors=True)
1761         1
1762         """
1763         if self.indent_type is None:
1764             # this can be true if initialised from a dictionary
1765             self.indent_type = DEFAULT_INDENT_TYPE
1766         #
1767         out = []
1768         cs = self._a_to_u('#')
1769         csp = self._a_to_u('# ')
1770         if section is None:
1771             int_val = self.interpolation
1772             self.interpolation = False
1773             section = self
1774             for line in self.initial_comment:
1775                 line = self._decode_element(line)
1776                 stripped_line = line.strip()
1777                 if stripped_line and not stripped_line.startswith(cs):
1778                     line = csp + line
1779                 out.append(line)
1780         #
1781         indent_string = self._a_to_u(
1782             self._compute_indent_string(section.depth))
1783         for entry in (section.scalars + section.sections):
1784             if entry in section.defaults:
1785                 # don't write out default values
1786                 continue
1787             for comment_line in section.comments[entry]:
1788                 comment_line = self._decode_element(comment_line.lstrip())
1789                 if comment_line and not comment_line.startswith(cs):
1790                     comment_line = csp + comment_line
1791                 out.append(indent_string + comment_line)
1792             this_entry = section[entry]
1793             comment = self._handle_comment(section.inline_comments[entry])
1794             #
1795             if isinstance(this_entry, dict):
1796                 # a section
1797                 out.append(self._write_marker(
1798                     indent_string,
1799                     this_entry.depth,
1800                     entry,
1801                     comment))
1802                 out.extend(self.write(section=this_entry))
1803             else:
1804                 out.append(self._write_line(
1805                     indent_string,
1806                     entry,
1807                     this_entry,
1808                     comment))
1809         #
1810         if section is self:
1811             for line in self.final_comment:
1812                 line = self._decode_element(line)
1813                 stripped_line = line.strip()
1814                 if stripped_line and not stripped_line.startswith(cs):
1815                     line = csp + line
1816                 out.append(line)
1817             self.interpolation = int_val
1818         #
1819         if section is not self:
1820             return out
1821         #
1822         if (self.filename is None) and (outfile is None):
1823             # output a list of lines
1824             # might need to encode
1825             # NOTE: This will *screw* UTF16, each line will start with the BOM
1826             if self.encoding:
1827                 out = [l.encode(self.encoding) for l in out]
1828             if (self.BOM and ((self.encoding is None) or
1829                 (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
1830                 # Add the UTF8 BOM
1831                 if not out:
1832                     out.append('')
1833                 out[0] = BOM_UTF8 + out[0]
1834             return out
1835         #
1836         # Turn the list to a string, joined with correct newlines
1837         output = (self._a_to_u(self.newlines or os.linesep)
1838             ).join(out)
1839         if self.encoding:
1840             output = output.encode(self.encoding)
1841         if (self.BOM and ((self.encoding is None) or
1842             (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
1843             # Add the UTF8 BOM
1844             output = BOM_UTF8 + output
1845         if outfile is not None:
1846             outfile.write(output)
1847         else:
1848             h = open(self.filename, 'wb')
1849             h.write(output)
1850             h.close()
1851
1852     def validate(self, validator, preserve_errors=False, copy=False,
1853         section=None):
1854         """
1855         Test the ConfigObj against a configspec.
1856
1857         It uses the ``validator`` object from *validate.py*.
1858
1859         To run ``validate`` on the current ConfigObj, call: ::
1860
1861             test = config.validate(validator)
1862
1863         (Normally having previously passed in the configspec when the ConfigObj
1864         was created - you can dynamically assign a dictionary of checks to the
1865         ``configspec`` attribute of a section though).
1866
1867         It returns ``True`` if everything passes, or a dictionary of
1868         pass/fails (True/False). If every member of a subsection passes, it
1869         will just have the value ``True``. (It also returns ``False`` if all
1870         members fail).
1871
1872         In addition, it converts the values from strings to their native
1873         types if their checks pass (and ``stringify`` is set).
1874
1875         If ``preserve_errors`` is ``True`` (``False`` is default) then instead
1876         of a marking a fail with a ``False``, it will preserve the actual
1877         exception object. This can contain info about the reason for failure.
1878         For example the ``VdtValueTooSmallError`` indeicates that the value
1879         supplied was too small. If a value (or section) is missing it will
1880         still be marked as ``False``.
1881
1882         You must have the validate module to use ``preserve_errors=True``.
1883
1884         You can then use the ``flatten_errors`` function to turn your nested
1885         results dictionary into a flattened list of failures - useful for
1886         displaying meaningful error messages.
1887         """
1888         if section is None:
1889             if self.configspec is None:
1890                 raise ValueError, 'No configspec supplied.'
1891             if preserve_errors:
1892                 if VdtMissingValue is None:
1893                     raise ImportError('Missing validate module.')
1894             section = self
1895         #
1896         spec_section = section.configspec
1897         if copy and hasattr(section, '_configspec_initial_comment'):
1898             section.initial_comment = section._configspec_initial_comment
1899             section.final_comment = section._configspec_final_comment
1900             section.encoding = section._configspec_encoding
1901             section.BOM = section._configspec_BOM
1902             section.newlines = section._configspec_newlines
1903             section.indent_type = section._configspec_indent_type
1904         if '__many__' in section.configspec:
1905             many = spec_section['__many__']
1906             # dynamically assign the configspecs
1907             # for the sections below
1908             for entry in section.sections:
1909                 self._handle_repeat(section[entry], many)
1910         #
1911         out = {}
1912         ret_true = True
1913         ret_false = True
1914         order = [k for k in section._order if k in spec_section]
1915         order += [k for k in spec_section if k not in order]
1916         for entry in order:
1917             if entry == '__many__':
1918                 continue
1919             if (not entry in section.scalars) or (entry in section.defaults):
1920                 # missing entries
1921                 # or entries from defaults
1922                 missing = True
1923                 val = None
1924                 if copy and not entry in section.scalars:
1925                     # copy comments
1926                     section.comments[entry] = (
1927                         section._configspec_comments.get(entry, []))
1928                     section.inline_comments[entry] = (
1929                         section._configspec_inline_comments.get(entry, ''))
1930                 #
1931             else:
1932                 missing = False
1933                 val = section[entry]
1934             try:
1935                 check = validator.check(spec_section[entry],
1936                                         val,
1937                                         missing=missing
1938                                         )
1939             except validator.baseErrorClass, e:
1940                 if not preserve_errors or isinstance(e, VdtMissingValue):
1941                     out[entry] = False
1942                 else:
1943                     # preserve the error
1944                     out[entry] = e
1945                     ret_false = False
1946                 ret_true = False
1947             else:
1948                 ret_false = False
1949                 out[entry] = True
1950                 if self.stringify or missing:
1951                     # if we are doing type conversion
1952                     # or the value is a supplied default
1953                     if not self.stringify:
1954                         if isinstance(check, (list, tuple)):
1955                             # preserve lists
1956                             check = [self._str(item) for item in check]
1957                         elif missing and check is None:
1958                             # convert the None from a default to a ''
1959                             check = ''
1960                         else:
1961                             check = self._str(check)
1962                     if (check != val) or missing:
1963                         section[entry] = check
1964                 if not copy and missing and entry not in section.defaults:
1965                     section.defaults.append(entry)
1966         #
1967         # Missing sections will have been created as empty ones when the
1968         # configspec was read.
1969         for entry in section.sections:
1970             # FIXME: this means DEFAULT is not copied in copy mode
1971             if section is self and entry == 'DEFAULT':
1972                 continue
1973             if copy:
1974                 section.comments[entry] = section._cs_section_comments[entry]
1975                 section.inline_comments[entry] = (
1976                     section._cs_section_inline_comments[entry])
1977             check = self.validate(validator, preserve_errors=preserve_errors,
1978                 copy=copy, section=section[entry])
1979             out[entry] = check
1980             if check == False:
1981                 ret_true = False
1982             elif check == True:
1983                 ret_false = False
1984             else:
1985                 ret_true = False
1986                 ret_false = False
1987         #
1988         if ret_true:
1989             return True
1990         elif ret_false:
1991             return False
1992         else:
1993             return out
1994
1995 class SimpleVal(object):
1996     """
1997     A simple validator.
1998     Can be used to check that all members expected are present.
1999
2000     To use it, provide a configspec with all your members in (the value given
2001     will be ignored). Pass an instance of ``SimpleVal`` to the ``validate``
2002     method of your ``ConfigObj``. ``validate`` will return ``True`` if all
2003     members are present, or a dictionary with True/False meaning
2004     present/missing. (Whole missing sections will be replaced with ``False``)
2005     """
2006
2007     def __init__(self):
2008         self.baseErrorClass = ConfigObjError
2009
2010     def check(self, check, member, missing=False):
2011         """A dummy check method, always returns the value unchanged."""
2012         if missing:
2013             raise self.baseErrorClass
2014         return member
2015
2016 # Check / processing functions for options
2017 def flatten_errors(cfg, res, levels=None, results=None):
2018     """
2019     An example function that will turn a nested dictionary of results
2020     (as returned by ``ConfigObj.validate``) into a flat list.
2021
2022     ``cfg`` is the ConfigObj instance being checked, ``res`` is the results
2023     dictionary returned by ``validate``.
2024
2025     (This is a recursive function, so you shouldn't use the ``levels`` or
2026     ``results`` arguments - they are used by the function.
2027
2028     Returns a list of keys that failed. Each member of the list is a tuple :
2029     ::
2030
2031         ([list of sections...], key, result)
2032
2033     If ``validate`` was called with ``preserve_errors=False`` (the default)
2034     then ``result`` will always be ``False``.
2035
2036     *list of sections* is a flattened list of sections that the key was found
2037     in.
2038
2039     If the section was missing then key will be ``None``.
2040
2041     If the value (or section) was missing then ``result`` will be ``False``.
2042
2043     If ``validate`` was called with ``preserve_errors=True`` and a value
2044     was present, but failed the check, then ``result`` will be the exception
2045     object returned. You can use this as a string that describes the failure.
2046
2047     For example *The value "3" is of the wrong type*.
2048
2049     >>> import validate
2050     >>> vtor = validate.Validator()
2051     >>> my_ini = '''
2052     ...     option1 = True
2053     ...     [section1]
2054     ...     option1 = True
2055     ...     [section2]
2056     ...     another_option = Probably
2057     ...     [section3]
2058     ...     another_option = True
2059     ...     [[section3b]]
2060     ...     value = 3
2061     ...     value2 = a
2062     ...     value3 = 11
2063     ...     '''
2064     >>> my_cfg = '''
2065     ...     option1 = boolean()
2066     ...     option2 = boolean()
2067     ...     option3 = boolean(default=Bad_value)
2068     ...     [section1]
2069     ...     option1 = boolean()
2070     ...     option2 = boolean()
2071     ...     option3 = boolean(default=Bad_value)
2072     ...     [section2]
2073     ...     another_option = boolean()
2074     ...     [section3]
2075     ...     another_option = boolean()
2076     ...     [[section3b]]
2077     ...     value = integer
2078     ...     value2 = integer
2079     ...     value3 = integer(0, 10)
2080     ...         [[[section3b-sub]]]
2081     ...         value = string
2082     ...     [section4]
2083     ...     another_option = boolean()
2084     ...     '''
2085     >>> cs = my_cfg.split('\\n')
2086     >>> ini = my_ini.split('\\n')
2087     >>> cfg = ConfigObj(ini, configspec=cs)
2088     >>> res = cfg.validate(vtor, preserve_errors=True)
2089     >>> errors = []
2090     >>> for entry in flatten_errors(cfg, res):
2091     ...     section_list, key, error = entry
2092     ...     section_list.insert(0, '[root]')
2093     ...     if key is not None:
2094     ...        section_list.append(key)
2095     ...     else:
2096     ...         section_list.append('[missing]')
2097     ...     section_string = ', '.join(section_list)
2098     ...     errors.append((section_string, ' = ', error))
2099     >>> errors.sort()
2100     >>> for entry in errors:
2101     ...     print entry[0], entry[1], (entry[2] or 0)
2102     [root], option2  =  0
2103     [root], option3  =  the value "Bad_value" is of the wrong type.
2104     [root], section1, option2  =  0
2105     [root], section1, option3  =  the value "Bad_value" is of the wrong type.
2106     [root], section2, another_option  =  the value "Probably" is of the wrong type.
2107     [root], section3, section3b, section3b-sub, [missing]  =  0
2108     [root], section3, section3b, value2  =  the value "a" is of the wrong type.
2109     [root], section3, section3b, value3  =  the value "11" is too big.
2110     [root], section4, [missing]  =  0
2111     """
2112     if levels is None:
2113         # first time called
2114         levels = []
2115         results = []
2116     if res is True:
2117         return results
2118     if res is False:
2119         results.append((levels[:], None, False))
2120         if levels:
2121             levels.pop()
2122         return results
2123     for (key, val) in res.items():
2124         if val == True:
2125             continue
2126         if isinstance(cfg.get(key), dict):
2127             # Go down one level
2128             levels.append(key)
2129             flatten_errors(cfg[key], val, levels, results)
2130             continue
2131         results.append((levels[:], key, val))
2132     #
2133     # Go up one level
2134     if levels:
2135         levels.pop()
2136     #
2137     return results
2138
2139 """*A programming language is a medium of expression.* - Paul Graham"""