Fix timestamp constructing and representing (close #25).
[pyyaml/python3.git] / lib / yaml / constructor.py
blobd1d4f6ada599f3b538f20a3abdc01d4bc12fe7bc
2 __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
3 'ConstructorError']
5 from error import *
6 from nodes import *
8 import datetime
10 try:
11 set
12 except NameError:
13 from sets import Set as set
15 import binascii, re, sys
17 class ConstructorError(MarkedYAMLError):
18 pass
20 class BaseConstructor(object):
22 yaml_constructors = {}
23 yaml_multi_constructors = {}
25 def __init__(self):
26 self.constructed_objects = {}
27 self.recursive_objects = {}
28 self.state_generators = []
29 self.deep_construct = False
31 def check_data(self):
32 # If there are more documents available?
33 return self.check_node()
35 def get_data(self):
36 # Construct and return the next document.
37 if self.check_node():
38 return self.construct_document(self.get_node())
40 def g(): yield None
41 generator_type = type(g())
42 del g
44 def construct_document(self, node):
45 data = self.construct_object(node)
46 while self.state_generators:
47 state_generators = self.state_generators
48 self.state_generators = []
49 for generator in state_generators:
50 for dummy in generator:
51 pass
52 self.constructed_objects = {}
53 self.recursive_objects = {}
54 self.deep_construct = False
55 return data
57 def construct_object(self, node, deep=False):
58 if deep:
59 old_deep = self.deep_construct
60 self.deep_construct = True
61 if node in self.constructed_objects:
62 return self.constructed_objects[node]
63 if node in self.recursive_objects:
64 raise ConstructorError(None, None,
65 "found unconstructable recursive node", node.start_mark)
66 self.recursive_objects[node] = None
67 constructor = None
68 state_constructor = None
69 tag_suffix = None
70 if node.tag in self.yaml_constructors:
71 constructor = self.yaml_constructors[node.tag]
72 else:
73 for tag_prefix in self.yaml_multi_constructors:
74 if node.tag.startswith(tag_prefix):
75 tag_suffix = node.tag[len(tag_prefix):]
76 constructor = self.yaml_multi_constructors[tag_prefix]
77 break
78 else:
79 if None in self.yaml_multi_constructors:
80 tag_suffix = node.tag
81 constructor = self.yaml_multi_constructors[None]
82 elif None in self.yaml_constructors:
83 constructor = self.yaml_constructors[None]
84 elif isinstance(node, ScalarNode):
85 constructor = self.__class__.construct_scalar
86 elif isinstance(node, SequenceNode):
87 constructor = self.__class__.construct_sequence
88 elif isinstance(node, MappingNode):
89 constructor = self.__class__.construct_mapping
90 if tag_suffix is None:
91 data = constructor(self, node)
92 else:
93 data = constructor(self, tag_suffix, node)
94 if isinstance(data, self.generator_type):
95 generator = data
96 data = generator.next()
97 if self.deep_construct:
98 for dummy in generator:
99 pass
100 else:
101 self.state_generators.append(generator)
102 self.constructed_objects[node] = data
103 del self.recursive_objects[node]
104 if deep:
105 self.deep_construct = old_deep
106 return data
108 def construct_scalar(self, node):
109 if not isinstance(node, ScalarNode):
110 raise ConstructorError(None, None,
111 "expected a scalar node, but found %s" % node.id,
112 node.start_mark)
113 return node.value
115 def construct_sequence(self, node, deep=False):
116 if not isinstance(node, SequenceNode):
117 raise ConstructorError(None, None,
118 "expected a sequence node, but found %s" % node.id,
119 node.start_mark)
120 return [self.construct_object(child, deep=deep)
121 for child in node.value]
123 def construct_mapping(self, node, deep=False):
124 if not isinstance(node, MappingNode):
125 raise ConstructorError(None, None,
126 "expected a mapping node, but found %s" % node.id,
127 node.start_mark)
128 mapping = {}
129 for key_node, value_node in node.value:
130 key = self.construct_object(key_node, deep=deep)
131 try:
132 hash(key)
133 except TypeError, exc:
134 raise ConstructorError("while constructing a mapping", node.start_mark,
135 "found unacceptable key (%s)" % exc, key_node.start_mark)
136 value = self.construct_object(value_node, deep=deep)
137 mapping[key] = value
138 return mapping
140 def construct_pairs(self, node, deep=False):
141 if not isinstance(node, MappingNode):
142 raise ConstructorError(None, None,
143 "expected a mapping node, but found %s" % node.id,
144 node.start_mark)
145 pairs = []
146 for key_node, value_node in node.value:
147 key = self.construct_object(key_node, deep=deep)
148 value = self.construct_object(value_node, deep=deep)
149 pairs.append((key, value))
150 return pairs
152 def add_constructor(cls, tag, constructor):
153 if not 'yaml_constructors' in cls.__dict__:
154 cls.yaml_constructors = cls.yaml_constructors.copy()
155 cls.yaml_constructors[tag] = constructor
156 add_constructor = classmethod(add_constructor)
158 def add_multi_constructor(cls, tag_prefix, multi_constructor):
159 if not 'yaml_multi_constructors' in cls.__dict__:
160 cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
161 cls.yaml_multi_constructors[tag_prefix] = multi_constructor
162 add_multi_constructor = classmethod(add_multi_constructor)
164 class SafeConstructor(BaseConstructor):
166 def construct_scalar(self, node):
167 if isinstance(node, MappingNode):
168 for key_node, value_node in node.value:
169 if key_node.tag == u'tag:yaml.org,2002:value':
170 return self.construct_scalar(value_node)
171 return BaseConstructor.construct_scalar(self, node)
173 def flatten_mapping(self, node):
174 merge = []
175 index = 0
176 while index < len(node.value):
177 key_node, value_node = node.value[index]
178 if key_node.tag == u'tag:yaml.org,2002:merge':
179 del node.value[index]
180 if isinstance(value_node, MappingNode):
181 self.flatten_mapping(value_node)
182 merge.extend(value_node.value)
183 elif isinstance(value_node, SequenceNode):
184 submerge = []
185 for subnode in value_node.value:
186 if not isinstance(subnode, MappingNode):
187 raise ConstructorError("while constructing a mapping",
188 node.start_mark,
189 "expected a mapping for merging, but found %s"
190 % subnode.id, subnode.start_mark)
191 self.flatten_mapping(subnode)
192 submerge.append(subnode.value)
193 submerge.reverse()
194 for value in submerge:
195 merge.extend(value)
196 else:
197 raise ConstructorError("while constructing a mapping", node.start_mark,
198 "expected a mapping or list of mappings for merging, but found %s"
199 % value_node.id, value_node.start_mark)
200 elif key_node.tag == u'tag:yaml.org,2002:value':
201 key_node.tag = u'tag:yaml.org,2002:str'
202 index += 1
203 else:
204 index += 1
205 if merge:
206 node.value = merge + node.value
208 def construct_mapping(self, node, deep=False):
209 if isinstance(node, MappingNode):
210 self.flatten_mapping(node)
211 return BaseConstructor.construct_mapping(self, node, deep=deep)
213 def construct_yaml_null(self, node):
214 self.construct_scalar(node)
215 return None
217 bool_values = {
218 u'yes': True,
219 u'no': False,
220 u'true': True,
221 u'false': False,
222 u'on': True,
223 u'off': False,
226 def construct_yaml_bool(self, node):
227 value = self.construct_scalar(node)
228 return self.bool_values[value.lower()]
230 def construct_yaml_int(self, node):
231 value = str(self.construct_scalar(node))
232 value = value.replace('_', '')
233 sign = +1
234 if value[0] == '-':
235 sign = -1
236 if value[0] in '+-':
237 value = value[1:]
238 if value == '0':
239 return 0
240 elif value.startswith('0b'):
241 return sign*int(value[2:], 2)
242 elif value.startswith('0x'):
243 return sign*int(value[2:], 16)
244 elif value[0] == '0':
245 return sign*int(value, 8)
246 elif ':' in value:
247 digits = [int(part) for part in value.split(':')]
248 digits.reverse()
249 base = 1
250 value = 0
251 for digit in digits:
252 value += digit*base
253 base *= 60
254 return sign*value
255 else:
256 return sign*int(value)
258 inf_value = 1e300
259 while inf_value != inf_value*inf_value:
260 inf_value *= inf_value
261 nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
263 def construct_yaml_float(self, node):
264 value = str(self.construct_scalar(node))
265 value = value.replace('_', '').lower()
266 sign = +1
267 if value[0] == '-':
268 sign = -1
269 if value[0] in '+-':
270 value = value[1:]
271 if value == '.inf':
272 return sign*self.inf_value
273 elif value == '.nan':
274 return self.nan_value
275 elif ':' in value:
276 digits = [float(part) for part in value.split(':')]
277 digits.reverse()
278 base = 1
279 value = 0.0
280 for digit in digits:
281 value += digit*base
282 base *= 60
283 return sign*value
284 else:
285 return sign*float(value)
287 def construct_yaml_binary(self, node):
288 value = self.construct_scalar(node)
289 try:
290 return str(value).decode('base64')
291 except (binascii.Error, UnicodeEncodeError), exc:
292 raise ConstructorError(None, None,
293 "failed to decode base64 data: %s" % exc, node.start_mark)
295 timestamp_regexp = re.compile(
296 ur'''^(?P<year>[0-9][0-9][0-9][0-9])
297 -(?P<month>[0-9][0-9]?)
298 -(?P<day>[0-9][0-9]?)
299 (?:(?:[Tt]|[ \t]+)
300 (?P<hour>[0-9][0-9]?)
301 :(?P<minute>[0-9][0-9])
302 :(?P<second>[0-9][0-9])
303 (?:(?P<fraction>\.[0-9]*))?
304 (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
305 (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
307 def construct_yaml_timestamp(self, node):
308 value = self.construct_scalar(node)
309 match = self.timestamp_regexp.match(node.value)
310 values = match.groupdict()
311 year = int(values['year'])
312 month = int(values['month'])
313 day = int(values['day'])
314 if not values['hour']:
315 return datetime.date(year, month, day)
316 hour = int(values['hour'])
317 minute = int(values['minute'])
318 second = int(values['second'])
319 fraction = 0
320 if values['fraction']:
321 fraction = int(float(values['fraction'])*1000000)
322 delta = None
323 if values['tz_sign']:
324 tz_hour = int(values['tz_hour'])
325 tz_minute = int(values['tz_minute'] or 0)
326 delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
327 if values['tz_sign'] == '-':
328 delta = -delta
329 data = datetime.datetime(year, month, day, hour, minute, second, fraction)
330 if delta:
331 data -= delta
332 return data
334 def construct_yaml_omap(self, node):
335 # Note: we do not check for duplicate keys, because it's too
336 # CPU-expensive.
337 omap = []
338 yield omap
339 if not isinstance(node, SequenceNode):
340 raise ConstructorError("while constructing an ordered map", node.start_mark,
341 "expected a sequence, but found %s" % node.id, node.start_mark)
342 for subnode in node.value:
343 if not isinstance(subnode, MappingNode):
344 raise ConstructorError("while constructing an ordered map", node.start_mark,
345 "expected a mapping of length 1, but found %s" % subnode.id,
346 subnode.start_mark)
347 if len(subnode.value) != 1:
348 raise ConstructorError("while constructing an ordered map", node.start_mark,
349 "expected a single mapping item, but found %d items" % len(subnode.value),
350 subnode.start_mark)
351 key_node, value_node = subnode.value[0]
352 key = self.construct_object(key_node)
353 value = self.construct_object(value_node)
354 omap.append((key, value))
356 def construct_yaml_pairs(self, node):
357 # Note: the same code as `construct_yaml_omap`.
358 pairs = []
359 yield pairs
360 if not isinstance(node, SequenceNode):
361 raise ConstructorError("while constructing pairs", node.start_mark,
362 "expected a sequence, but found %s" % node.id, node.start_mark)
363 for subnode in node.value:
364 if not isinstance(subnode, MappingNode):
365 raise ConstructorError("while constructing pairs", node.start_mark,
366 "expected a mapping of length 1, but found %s" % subnode.id,
367 subnode.start_mark)
368 if len(subnode.value) != 1:
369 raise ConstructorError("while constructing pairs", node.start_mark,
370 "expected a single mapping item, but found %d items" % len(subnode.value),
371 subnode.start_mark)
372 key_node, value_node = subnode.value[0]
373 key = self.construct_object(key_node)
374 value = self.construct_object(value_node)
375 pairs.append((key, value))
377 def construct_yaml_set(self, node):
378 data = set()
379 yield data
380 value = self.construct_mapping(node)
381 data.update(value)
383 def construct_yaml_str(self, node):
384 value = self.construct_scalar(node)
385 try:
386 return str(value)
387 except UnicodeEncodeError:
388 return value
390 def construct_yaml_seq(self, node):
391 data = []
392 yield data
393 data.extend(self.construct_sequence(node))
395 def construct_yaml_map(self, node):
396 data = {}
397 yield data
398 value = self.construct_mapping(node)
399 data.update(value)
401 def construct_yaml_object(self, node, cls):
402 data = cls.__new__(cls)
403 yield data
404 if hasattr(data, '__setstate__'):
405 state = self.construct_mapping(node, deep=True)
406 data.__setstate__(state)
407 else:
408 state = self.construct_mapping(node)
409 data.__dict__.update(state)
411 def construct_undefined(self, node):
412 raise ConstructorError(None, None,
413 "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'),
414 node.start_mark)
416 SafeConstructor.add_constructor(
417 u'tag:yaml.org,2002:null',
418 SafeConstructor.construct_yaml_null)
420 SafeConstructor.add_constructor(
421 u'tag:yaml.org,2002:bool',
422 SafeConstructor.construct_yaml_bool)
424 SafeConstructor.add_constructor(
425 u'tag:yaml.org,2002:int',
426 SafeConstructor.construct_yaml_int)
428 SafeConstructor.add_constructor(
429 u'tag:yaml.org,2002:float',
430 SafeConstructor.construct_yaml_float)
432 SafeConstructor.add_constructor(
433 u'tag:yaml.org,2002:binary',
434 SafeConstructor.construct_yaml_binary)
436 SafeConstructor.add_constructor(
437 u'tag:yaml.org,2002:timestamp',
438 SafeConstructor.construct_yaml_timestamp)
440 SafeConstructor.add_constructor(
441 u'tag:yaml.org,2002:omap',
442 SafeConstructor.construct_yaml_omap)
444 SafeConstructor.add_constructor(
445 u'tag:yaml.org,2002:pairs',
446 SafeConstructor.construct_yaml_pairs)
448 SafeConstructor.add_constructor(
449 u'tag:yaml.org,2002:set',
450 SafeConstructor.construct_yaml_set)
452 SafeConstructor.add_constructor(
453 u'tag:yaml.org,2002:str',
454 SafeConstructor.construct_yaml_str)
456 SafeConstructor.add_constructor(
457 u'tag:yaml.org,2002:seq',
458 SafeConstructor.construct_yaml_seq)
460 SafeConstructor.add_constructor(
461 u'tag:yaml.org,2002:map',
462 SafeConstructor.construct_yaml_map)
464 SafeConstructor.add_constructor(None,
465 SafeConstructor.construct_undefined)
467 class Constructor(SafeConstructor):
469 def construct_python_str(self, node):
470 return self.construct_scalar(node).encode('utf-8')
472 def construct_python_unicode(self, node):
473 return self.construct_scalar(node)
475 def construct_python_long(self, node):
476 return long(self.construct_yaml_int(node))
478 def construct_python_complex(self, node):
479 return complex(self.construct_scalar(node))
481 def construct_python_tuple(self, node):
482 return tuple(self.construct_sequence(node))
484 def find_python_module(self, name, mark):
485 if not name:
486 raise ConstructorError("while constructing a Python module", mark,
487 "expected non-empty name appended to the tag", mark)
488 try:
489 __import__(name)
490 except ImportError, exc:
491 raise ConstructorError("while constructing a Python module", mark,
492 "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark)
493 return sys.modules[name]
495 def find_python_name(self, name, mark):
496 if not name:
497 raise ConstructorError("while constructing a Python object", mark,
498 "expected non-empty name appended to the tag", mark)
499 if u'.' in name:
500 # Python 2.4 only
501 #module_name, object_name = name.rsplit('.', 1)
502 items = name.split('.')
503 object_name = items.pop()
504 module_name = '.'.join(items)
505 else:
506 module_name = '__builtin__'
507 object_name = name
508 try:
509 __import__(module_name)
510 except ImportError, exc:
511 raise ConstructorError("while constructing a Python object", mark,
512 "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark)
513 module = sys.modules[module_name]
514 if not hasattr(module, object_name):
515 raise ConstructorError("while constructing a Python object", mark,
516 "cannot find %r in the module %r" % (object_name.encode('utf-8'),
517 module.__name__), mark)
518 return getattr(module, object_name)
520 def construct_python_name(self, suffix, node):
521 value = self.construct_scalar(node)
522 if value:
523 raise ConstructorError("while constructing a Python name", node.start_mark,
524 "expected the empty value, but found %r" % value.encode('utf-8'),
525 node.start_mark)
526 return self.find_python_name(suffix, node.start_mark)
528 def construct_python_module(self, suffix, node):
529 value = self.construct_scalar(node)
530 if value:
531 raise ConstructorError("while constructing a Python module", node.start_mark,
532 "expected the empty value, but found %r" % value.encode('utf-8'),
533 node.start_mark)
534 return self.find_python_module(suffix, node.start_mark)
536 class classobj: pass
538 def make_python_instance(self, suffix, node,
539 args=None, kwds=None, newobj=False):
540 if not args:
541 args = []
542 if not kwds:
543 kwds = {}
544 cls = self.find_python_name(suffix, node.start_mark)
545 if newobj and isinstance(cls, type(self.classobj)) \
546 and not args and not kwds:
547 instance = self.classobj()
548 instance.__class__ = cls
549 return instance
550 elif newobj and isinstance(cls, type):
551 return cls.__new__(cls, *args, **kwds)
552 else:
553 return cls(*args, **kwds)
555 def set_python_instance_state(self, instance, state):
556 if hasattr(instance, '__setstate__'):
557 instance.__setstate__(state)
558 else:
559 slotstate = {}
560 if isinstance(state, tuple) and len(state) == 2:
561 state, slotstate = state
562 if hasattr(instance, '__dict__'):
563 instance.__dict__.update(state)
564 elif state:
565 slotstate.update(state)
566 for key, value in slotstate.items():
567 setattr(object, key, value)
569 def construct_python_object(self, suffix, node):
570 # Format:
571 # !!python/object:module.name { ... state ... }
572 instance = self.make_python_instance(suffix, node, newobj=True)
573 yield instance
574 deep = hasattr(instance, '__setstate__')
575 state = self.construct_mapping(node, deep=deep)
576 self.set_python_instance_state(instance, state)
578 def construct_python_object_apply(self, suffix, node, newobj=False):
579 # Format:
580 # !!python/object/apply # (or !!python/object/new)
581 # args: [ ... arguments ... ]
582 # kwds: { ... keywords ... }
583 # state: ... state ...
584 # listitems: [ ... listitems ... ]
585 # dictitems: { ... dictitems ... }
586 # or short format:
587 # !!python/object/apply [ ... arguments ... ]
588 # The difference between !!python/object/apply and !!python/object/new
589 # is how an object is created, check make_python_instance for details.
590 if isinstance(node, SequenceNode):
591 args = self.construct_sequence(node, deep=True)
592 kwds = {}
593 state = {}
594 listitems = []
595 dictitems = {}
596 else:
597 value = self.construct_mapping(node, deep=True)
598 args = value.get('args', [])
599 kwds = value.get('kwds', {})
600 state = value.get('state', {})
601 listitems = value.get('listitems', [])
602 dictitems = value.get('dictitems', {})
603 instance = self.make_python_instance(suffix, node, args, kwds, newobj)
604 if state:
605 self.set_python_instance_state(instance, state)
606 if listitems:
607 instance.extend(listitems)
608 if dictitems:
609 for key in dictitems:
610 instance[key] = dictitems[key]
611 return instance
613 def construct_python_object_new(self, suffix, node):
614 return self.construct_python_object_apply(suffix, node, newobj=True)
616 Constructor.add_constructor(
617 u'tag:yaml.org,2002:python/none',
618 Constructor.construct_yaml_null)
620 Constructor.add_constructor(
621 u'tag:yaml.org,2002:python/bool',
622 Constructor.construct_yaml_bool)
624 Constructor.add_constructor(
625 u'tag:yaml.org,2002:python/str',
626 Constructor.construct_python_str)
628 Constructor.add_constructor(
629 u'tag:yaml.org,2002:python/unicode',
630 Constructor.construct_python_unicode)
632 Constructor.add_constructor(
633 u'tag:yaml.org,2002:python/int',
634 Constructor.construct_yaml_int)
636 Constructor.add_constructor(
637 u'tag:yaml.org,2002:python/long',
638 Constructor.construct_python_long)
640 Constructor.add_constructor(
641 u'tag:yaml.org,2002:python/float',
642 Constructor.construct_yaml_float)
644 Constructor.add_constructor(
645 u'tag:yaml.org,2002:python/complex',
646 Constructor.construct_python_complex)
648 Constructor.add_constructor(
649 u'tag:yaml.org,2002:python/list',
650 Constructor.construct_yaml_seq)
652 Constructor.add_constructor(
653 u'tag:yaml.org,2002:python/tuple',
654 Constructor.construct_python_tuple)
656 Constructor.add_constructor(
657 u'tag:yaml.org,2002:python/dict',
658 Constructor.construct_yaml_map)
660 Constructor.add_multi_constructor(
661 u'tag:yaml.org,2002:python/name:',
662 Constructor.construct_python_name)
664 Constructor.add_multi_constructor(
665 u'tag:yaml.org,2002:python/module:',
666 Constructor.construct_python_module)
668 Constructor.add_multi_constructor(
669 u'tag:yaml.org,2002:python/object:',
670 Constructor.construct_python_object)
672 Constructor.add_multi_constructor(
673 u'tag:yaml.org,2002:python/object/apply:',
674 Constructor.construct_python_object_apply)
676 Constructor.add_multi_constructor(
677 u'tag:yaml.org,2002:python/object/new:',
678 Constructor.construct_python_object_new)