3 # Migration Stream Analyzer
5 # Copyright (c) 2015 Alexander Graf <agraf@suse.de>
7 # This library is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU Lesser General Public
9 # License as published by the Free Software Foundation; either
10 # version 2.1 of the License, or (at your option) any later version.
12 # This library is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 # Lesser General Public License for more details.
17 # You should have received a copy of the GNU Lesser General Public
18 # License along with this library; if not, see <http://www.gnu.org/licenses/>.
35 class MigrationFile(object):
36 def __init__(self
, filename
):
37 self
.filename
= filename
38 self
.file = open(self
.filename
, "rb")
41 return int.from_bytes(self
.file.read(8), byteorder
='big', signed
=False)
44 return int.from_bytes(self
.file.read(4), byteorder
='big', signed
=False)
47 return int.from_bytes(self
.file.read(2), byteorder
='big', signed
=False)
50 return int.from_bytes(self
.file.read(1), byteorder
='big', signed
=True)
52 def readstr(self
, len = None):
53 return self
.readvar(len).decode('utf-8')
55 def readvar(self
, size
= None):
60 value
= self
.file.read(size
)
61 if len(value
) != size
:
62 raise Exception("Unexpected end of %s at 0x%x" % (self
.filename
, self
.file.tell()))
66 return self
.file.tell()
68 # The VMSD description is at the end of the file, after EOF. Look for
69 # the last NULL byte, then for the beginning brace of JSON.
70 def read_migration_debug_json(self
):
71 QEMU_VM_VMDESCRIPTION
= 0x06
73 # Remember the offset in the file when we started
74 entrypos
= self
.file.tell()
77 self
.file.seek(0, os
.SEEK_END
)
78 endpos
= self
.file.tell()
79 self
.file.seek(max(-endpos
, -10 * 1024 * 1024), os
.SEEK_END
)
80 datapos
= self
.file.tell()
81 data
= self
.file.read()
82 # The full file read closed the file as well, reopen it
83 self
.file = open(self
.filename
, "rb")
85 # Find the last NULL byte, then the first brace after that. This should
86 # be the beginning of our JSON data.
87 nulpos
= data
.rfind(b
'\0')
88 jsonpos
= data
.find(b
'{', nulpos
)
90 # Check backwards from there and see whether we guessed right
91 self
.file.seek(datapos
+ jsonpos
- 5, 0)
92 if self
.read8() != QEMU_VM_VMDESCRIPTION
:
93 raise Exception("No Debug Migration device found")
95 jsonlen
= self
.read32()
97 # Seek back to where we were at the beginning
98 self
.file.seek(entrypos
, 0)
100 # explicit decode() needed for Python 3.5 compatibility
101 return data
[jsonpos
:jsonpos
+ jsonlen
].decode("utf-8")
106 class RamSection(object):
107 RAM_SAVE_FLAG_COMPRESS
= 0x02
108 RAM_SAVE_FLAG_MEM_SIZE
= 0x04
109 RAM_SAVE_FLAG_PAGE
= 0x08
110 RAM_SAVE_FLAG_EOS
= 0x10
111 RAM_SAVE_FLAG_CONTINUE
= 0x20
112 RAM_SAVE_FLAG_XBZRLE
= 0x40
113 RAM_SAVE_FLAG_HOOK
= 0x80
114 RAM_SAVE_FLAG_COMPRESS_PAGE
= 0x100
115 RAM_SAVE_FLAG_MULTIFD_FLUSH
= 0x200
117 def __init__(self
, file, version_id
, ramargs
, section_key
):
119 raise Exception("Unknown RAM version %d" % version_id
)
122 self
.section_key
= section_key
123 self
.TARGET_PAGE_SIZE
= ramargs
['page_size']
124 self
.dump_memory
= ramargs
['dump_memory']
125 self
.write_memory
= ramargs
['write_memory']
126 self
.ignore_shared
= ramargs
['ignore_shared']
127 self
.sizeinfo
= collections
.OrderedDict()
128 self
.data
= collections
.OrderedDict()
129 self
.data
['section sizes'] = self
.sizeinfo
131 if self
.write_memory
:
134 self
.memory
= collections
.OrderedDict()
135 self
.data
['memory'] = self
.memory
138 return self
.data
.__repr
__()
141 return self
.data
.__str
__()
147 # Read all RAM sections
149 addr
= self
.file.read64()
150 flags
= addr
& (self
.TARGET_PAGE_SIZE
- 1)
151 addr
&= ~
(self
.TARGET_PAGE_SIZE
- 1)
153 if flags
& self
.RAM_SAVE_FLAG_MEM_SIZE
:
155 namelen
= self
.file.read8()
156 # We assume that no RAM chunk is big enough to ever
157 # hit the first byte of the address, so when we see
158 # a zero here we know it has to be an address, not the
159 # length of the next block.
161 self
.file.file.seek(-1, 1)
163 self
.name
= self
.file.readstr(len = namelen
)
164 len = self
.file.read64()
165 self
.sizeinfo
[self
.name
] = '0x%016x' % len
166 if self
.write_memory
:
168 mkdir_p('./' + os
.path
.dirname(self
.name
))
169 f
= open('./' + self
.name
, "wb")
172 self
.files
[self
.name
] = f
173 if self
.ignore_shared
:
174 mr_addr
= self
.file.read64()
175 flags
&= ~self
.RAM_SAVE_FLAG_MEM_SIZE
177 if flags
& self
.RAM_SAVE_FLAG_COMPRESS
:
178 if flags
& self
.RAM_SAVE_FLAG_CONTINUE
:
179 flags
&= ~self
.RAM_SAVE_FLAG_CONTINUE
181 self
.name
= self
.file.readstr()
182 fill_char
= self
.file.read8()
183 # The page in question is filled with fill_char now
184 if self
.write_memory
and fill_char
!= 0:
185 self
.files
[self
.name
].seek(addr
, os
.SEEK_SET
)
186 self
.files
[self
.name
].write(chr(fill_char
) * self
.TARGET_PAGE_SIZE
)
188 self
.memory
['%s (0x%016x)' % (self
.name
, addr
)] = 'Filled with 0x%02x' % fill_char
189 flags
&= ~self
.RAM_SAVE_FLAG_COMPRESS
190 elif flags
& self
.RAM_SAVE_FLAG_PAGE
:
191 if flags
& self
.RAM_SAVE_FLAG_CONTINUE
:
192 flags
&= ~self
.RAM_SAVE_FLAG_CONTINUE
194 self
.name
= self
.file.readstr()
196 if self
.write_memory
or self
.dump_memory
:
197 data
= self
.file.readvar(size
= self
.TARGET_PAGE_SIZE
)
198 else: # Just skip RAM data
199 self
.file.file.seek(self
.TARGET_PAGE_SIZE
, 1)
201 if self
.write_memory
:
202 self
.files
[self
.name
].seek(addr
, os
.SEEK_SET
)
203 self
.files
[self
.name
].write(data
)
205 hexdata
= " ".join("{0:02x}".format(ord(c
)) for c
in data
)
206 self
.memory
['%s (0x%016x)' % (self
.name
, addr
)] = hexdata
208 flags
&= ~self
.RAM_SAVE_FLAG_PAGE
209 elif flags
& self
.RAM_SAVE_FLAG_XBZRLE
:
210 raise Exception("XBZRLE RAM compression is not supported yet")
211 elif flags
& self
.RAM_SAVE_FLAG_HOOK
:
212 raise Exception("RAM hooks don't make sense with files")
213 if flags
& self
.RAM_SAVE_FLAG_MULTIFD_FLUSH
:
217 if flags
& self
.RAM_SAVE_FLAG_EOS
:
221 raise Exception("Unknown RAM flags: %x" % flags
)
224 if self
.write_memory
:
225 for key
in self
.files
:
226 self
.files
[key
].close()
229 class HTABSection(object):
230 HASH_PTE_SIZE_64
= 16
232 def __init__(self
, file, version_id
, device
, section_key
):
234 raise Exception("Unknown HTAB version %d" % version_id
)
237 self
.section_key
= section_key
241 header
= self
.file.read32()
248 # First section, just the hash shift
251 # Read until end marker
253 index
= self
.file.read32()
254 n_valid
= self
.file.read16()
255 n_invalid
= self
.file.read16()
257 if index
== 0 and n_valid
== 0 and n_invalid
== 0:
260 self
.file.readvar(n_valid
* self
.HASH_PTE_SIZE_64
)
266 class ConfigurationSection(object):
267 def __init__(self
, file, desc
):
272 def parse_capabilities(self
, vmsd_caps
):
276 ncaps
= vmsd_caps
.data
['caps_count'].data
277 self
.caps
= vmsd_caps
.data
['capabilities']
279 if type(self
.caps
) != list:
280 self
.caps
= [self
.caps
]
282 if len(self
.caps
) != ncaps
:
283 raise Exception("Number of capabilities doesn't match "
286 def has_capability(self
, cap
):
287 return any([str(c
) == cap
for c
in self
.caps
])
291 version_id
= self
.desc
['version']
292 section
= VMSDSection(self
.file, version_id
, self
.desc
,
295 self
.parse_capabilities(
296 section
.data
.get("configuration/capabilities"))
298 # backward compatibility for older streams that don't have
299 # the configuration section in the json
300 name_len
= self
.file.read32()
301 name
= self
.file.readstr(len = name_len
)
303 class VMSDFieldGeneric(object):
304 def __init__(self
, desc
, file):
310 return str(self
.__str
__())
313 return " ".join("{0:02x}".format(c
) for c
in self
.data
)
316 return self
.__str
__()
319 size
= int(self
.desc
['size'])
320 self
.data
= self
.file.readvar(size
)
323 class VMSDFieldCap(object):
324 def __init__(self
, desc
, file):
336 len = self
.file.read8()
337 self
.data
= self
.file.readstr(len)
340 class VMSDFieldInt(VMSDFieldGeneric
):
341 def __init__(self
, desc
, file):
342 super(VMSDFieldInt
, self
).__init
__(desc
, file)
343 self
.size
= int(desc
['size'])
344 self
.format
= '0x%%0%dx' % (self
.size
* 2)
345 self
.sdtype
= '>i%d' % self
.size
346 self
.udtype
= '>u%d' % self
.size
350 return ('%s (%d)' % ((self
.format
% self
.udata
), self
.data
))
352 return self
.format
% self
.data
355 return self
.__repr
__()
358 return self
.__str
__()
361 super(VMSDFieldInt
, self
).read()
362 self
.sdata
= int.from_bytes(self
.data
, byteorder
='big', signed
=True)
363 self
.udata
= int.from_bytes(self
.data
, byteorder
='big', signed
=False)
364 self
.data
= self
.sdata
367 class VMSDFieldUInt(VMSDFieldInt
):
368 def __init__(self
, desc
, file):
369 super(VMSDFieldUInt
, self
).__init
__(desc
, file)
372 super(VMSDFieldUInt
, self
).read()
373 self
.data
= self
.udata
376 class VMSDFieldIntLE(VMSDFieldInt
):
377 def __init__(self
, desc
, file):
378 super(VMSDFieldIntLE
, self
).__init
__(desc
, file)
379 self
.dtype
= '<i%d' % self
.size
381 class VMSDFieldBool(VMSDFieldGeneric
):
382 def __init__(self
, desc
, file):
383 super(VMSDFieldBool
, self
).__init
__(desc
, file)
386 return self
.data
.__repr
__()
389 return self
.data
.__str
__()
395 super(VMSDFieldBool
, self
).read()
396 if self
.data
[0] == 0:
402 class VMSDFieldStruct(VMSDFieldGeneric
):
403 QEMU_VM_SUBSECTION
= 0x05
405 def __init__(self
, desc
, file):
406 super(VMSDFieldStruct
, self
).__init
__(desc
, file)
407 self
.data
= collections
.OrderedDict()
409 # When we see compressed array elements, unfold them here
411 for field
in self
.desc
['struct']['fields']:
412 if not 'array_len' in field
:
413 new_fields
.append(field
)
415 array_len
= field
.pop('array_len')
417 new_fields
.append(field
)
418 for i
in range(1, array_len
):
423 self
.desc
['struct']['fields'] = new_fields
426 return self
.data
.__repr
__()
429 return self
.data
.__str
__()
432 for field
in self
.desc
['struct']['fields']:
434 reader
= vmsd_field_readers
[field
['type']]
436 reader
= VMSDFieldGeneric
438 field
['data'] = reader(field
, self
.file)
442 if field
['name'] not in self
.data
:
443 self
.data
[field
['name']] = []
444 a
= self
.data
[field
['name']]
445 if len(a
) != int(field
['index']):
446 raise Exception("internal index of data field unmatched (%d/%d)" % (len(a
), int(field
['index'])))
447 a
.append(field
['data'])
449 self
.data
[field
['name']] = field
['data']
451 if 'subsections' in self
.desc
['struct']:
452 for subsection
in self
.desc
['struct']['subsections']:
453 if self
.file.read8() != self
.QEMU_VM_SUBSECTION
:
454 raise Exception("Subsection %s not found at offset %x" % ( subsection
['vmsd_name'], self
.file.tell()))
455 name
= self
.file.readstr()
456 version_id
= self
.file.read32()
457 self
.data
[name
] = VMSDSection(self
.file, version_id
, subsection
, (name
, 0))
458 self
.data
[name
].read()
460 def getDictItem(self
, value
):
461 # Strings would fall into the array category, treat
463 if value
.__class
__ is ''.__class
__:
467 return self
.getDictOrderedDict(value
)
470 return self
.getDictArray(value
)
473 return value
.getDict()
477 def getDictArray(self
, array
):
480 r
.append(self
.getDictItem(value
))
483 def getDictOrderedDict(self
, dict):
484 r
= collections
.OrderedDict()
485 for (key
, value
) in dict.items():
486 r
[key
] = self
.getDictItem(value
)
490 return self
.getDictOrderedDict(self
.data
)
492 vmsd_field_readers
= {
493 "bool" : VMSDFieldBool
,
494 "int8" : VMSDFieldInt
,
495 "int16" : VMSDFieldInt
,
496 "int32" : VMSDFieldInt
,
497 "int32 equal" : VMSDFieldInt
,
498 "int32 le" : VMSDFieldIntLE
,
499 "int64" : VMSDFieldInt
,
500 "uint8" : VMSDFieldUInt
,
501 "uint16" : VMSDFieldUInt
,
502 "uint32" : VMSDFieldUInt
,
503 "uint32 equal" : VMSDFieldUInt
,
504 "uint64" : VMSDFieldUInt
,
505 "int64 equal" : VMSDFieldInt
,
506 "uint8 equal" : VMSDFieldInt
,
507 "uint16 equal" : VMSDFieldInt
,
508 "float64" : VMSDFieldGeneric
,
509 "timer" : VMSDFieldGeneric
,
510 "buffer" : VMSDFieldGeneric
,
511 "unused_buffer" : VMSDFieldGeneric
,
512 "bitmap" : VMSDFieldGeneric
,
513 "struct" : VMSDFieldStruct
,
514 "capability": VMSDFieldCap
,
515 "unknown" : VMSDFieldGeneric
,
518 class VMSDSection(VMSDFieldStruct
):
519 def __init__(self
, file, version_id
, device
, section_key
):
523 self
.section_key
= section_key
525 if 'vmsd_name' in device
:
526 self
.vmsd_name
= device
['vmsd_name']
528 # A section really is nothing but a FieldStruct :)
529 super(VMSDSection
, self
).__init
__({ 'struct' : desc
}, file)
531 ###############################################################################
533 class MigrationDump(object):
534 QEMU_VM_FILE_MAGIC
= 0x5145564d
535 QEMU_VM_FILE_VERSION
= 0x00000003
537 QEMU_VM_SECTION_START
= 0x01
538 QEMU_VM_SECTION_PART
= 0x02
539 QEMU_VM_SECTION_END
= 0x03
540 QEMU_VM_SECTION_FULL
= 0x04
541 QEMU_VM_SUBSECTION
= 0x05
542 QEMU_VM_VMDESCRIPTION
= 0x06
543 QEMU_VM_CONFIGURATION
= 0x07
544 QEMU_VM_SECTION_FOOTER
= 0x7e
546 def __init__(self
, filename
):
547 self
.section_classes
= { ( 'ram', 0 ) : [ RamSection
, None ],
548 ( 'spapr/htab', 0) : ( HTABSection
, None ) }
549 self
.filename
= filename
550 self
.vmsd_desc
= None
552 def read(self
, desc_only
= False, dump_memory
= False, write_memory
= False):
553 # Read in the whole file
554 file = MigrationFile(self
.filename
)
558 if data
!= self
.QEMU_VM_FILE_MAGIC
:
559 raise Exception("Invalid file magic %x" % data
)
561 # Version (has to be v3)
563 if data
!= self
.QEMU_VM_FILE_VERSION
:
564 raise Exception("Invalid version number %d" % data
)
566 self
.load_vmsd_json(file)
569 self
.sections
= collections
.OrderedDict()
575 ramargs
['page_size'] = self
.vmsd_desc
['page_size']
576 ramargs
['dump_memory'] = dump_memory
577 ramargs
['write_memory'] = write_memory
578 ramargs
['ignore_shared'] = False
579 self
.section_classes
[('ram',0)][1] = ramargs
582 section_type
= file.read8()
583 if section_type
== self
.QEMU_VM_EOF
:
585 elif section_type
== self
.QEMU_VM_CONFIGURATION
:
586 config_desc
= self
.vmsd_desc
.get('configuration')
587 section
= ConfigurationSection(file, config_desc
)
589 ramargs
['ignore_shared'] = section
.has_capability('x-ignore-shared')
590 elif section_type
== self
.QEMU_VM_SECTION_START
or section_type
== self
.QEMU_VM_SECTION_FULL
:
591 section_id
= file.read32()
592 name
= file.readstr()
593 instance_id
= file.read32()
594 version_id
= file.read32()
595 section_key
= (name
, instance_id
)
596 classdesc
= self
.section_classes
[section_key
]
597 section
= classdesc
[0](file, version_id
, classdesc
[1], section_key
)
598 self
.sections
[section_id
] = section
600 elif section_type
== self
.QEMU_VM_SECTION_PART
or section_type
== self
.QEMU_VM_SECTION_END
:
601 section_id
= file.read32()
602 self
.sections
[section_id
].read()
603 elif section_type
== self
.QEMU_VM_SECTION_FOOTER
:
604 read_section_id
= file.read32()
605 if read_section_id
!= section_id
:
606 raise Exception("Mismatched section footer: %x vs %x" % (read_section_id
, section_id
))
608 raise Exception("Unknown section type: %d" % section_type
)
611 def load_vmsd_json(self
, file):
612 vmsd_json
= file.read_migration_debug_json()
613 self
.vmsd_desc
= json
.loads(vmsd_json
, object_pairs_hook
=collections
.OrderedDict
)
614 for device
in self
.vmsd_desc
['devices']:
615 key
= (device
['name'], device
['instance_id'])
616 value
= ( VMSDSection
, device
)
617 self
.section_classes
[key
] = value
620 r
= collections
.OrderedDict()
621 for (key
, value
) in self
.sections
.items():
622 key
= "%s (%d)" % ( value
.section_key
[0], key
)
623 r
[key
] = value
.getDict()
626 ###############################################################################
628 class JSONEncoder(json
.JSONEncoder
):
629 def default(self
, o
):
630 if isinstance(o
, VMSDFieldGeneric
):
632 return json
.JSONEncoder
.default(self
, o
)
634 parser
= argparse
.ArgumentParser()
635 parser
.add_argument("-f", "--file", help='migration dump to read from', required
=True)
636 parser
.add_argument("-m", "--memory", help='dump RAM contents as well', action
='store_true')
637 parser
.add_argument("-d", "--dump", help='what to dump ("state" or "desc")', default
='state')
638 parser
.add_argument("-x", "--extract", help='extract contents into individual files', action
='store_true')
639 args
= parser
.parse_args()
641 jsonenc
= JSONEncoder(indent
=4, separators
=(',', ': '))
644 dump
= MigrationDump(args
.file)
646 dump
.read(desc_only
= True)
648 f
= open("desc.json", "w")
650 f
.write(jsonenc
.encode(dump
.vmsd_desc
))
653 dump
.read(write_memory
= True)
654 dict = dump
.getDict()
656 f
= open("state.json", "w")
658 f
.write(jsonenc
.encode(dict))
660 elif args
.dump
== "state":
661 dump
= MigrationDump(args
.file)
662 dump
.read(dump_memory
= args
.memory
)
663 dict = dump
.getDict()
664 print(jsonenc
.encode(dict))
665 elif args
.dump
== "desc":
666 dump
= MigrationDump(args
.file)
667 dump
.read(desc_only
= True)
668 print(jsonenc
.encode(dump
.vmsd_desc
))
670 raise Exception("Please specify either -x, -d state or -d desc")