3 # Migration Stream Analyzer
5 # Copyright (c) 2015 Alexander Graf <agraf@suse.de>
7 # This library is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU Lesser General Public
9 # License as published by the Free Software Foundation; either
10 # version 2.1 of the License, or (at your option) any later version.
12 # This library is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 # Lesser General Public License for more details.
17 # You should have received a copy of the GNU Lesser General Public
18 # License along with this library; if not, see <http://www.gnu.org/licenses/>.
35 class MigrationFile(object):
36 def __init__(self
, filename
):
37 self
.filename
= filename
38 self
.file = open(self
.filename
, "rb")
41 return int.from_bytes(self
.file.read(8), byteorder
='big', signed
=False)
44 return int.from_bytes(self
.file.read(4), byteorder
='big', signed
=False)
47 return int.from_bytes(self
.file.read(2), byteorder
='big', signed
=False)
50 return int.from_bytes(self
.file.read(1), byteorder
='big', signed
=True)
52 def readstr(self
, len = None):
53 return self
.readvar(len).decode('utf-8')
55 def readvar(self
, size
= None):
60 value
= self
.file.read(size
)
61 if len(value
) != size
:
62 raise Exception("Unexpected end of %s at 0x%x" % (self
.filename
, self
.file.tell()))
66 return self
.file.tell()
68 # The VMSD description is at the end of the file, after EOF. Look for
69 # the last NULL byte, then for the beginning brace of JSON.
70 def read_migration_debug_json(self
):
71 QEMU_VM_VMDESCRIPTION
= 0x06
73 # Remember the offset in the file when we started
74 entrypos
= self
.file.tell()
77 self
.file.seek(0, os
.SEEK_END
)
78 endpos
= self
.file.tell()
79 self
.file.seek(max(-endpos
, -10 * 1024 * 1024), os
.SEEK_END
)
80 datapos
= self
.file.tell()
81 data
= self
.file.read()
82 # The full file read closed the file as well, reopen it
83 self
.file = open(self
.filename
, "rb")
85 # Find the last NULL byte, then the first brace after that. This should
86 # be the beginning of our JSON data.
87 nulpos
= data
.rfind(b
'\0')
88 jsonpos
= data
.find(b
'{', nulpos
)
90 # Check backwards from there and see whether we guessed right
91 self
.file.seek(datapos
+ jsonpos
- 5, 0)
92 if self
.read8() != QEMU_VM_VMDESCRIPTION
:
93 raise Exception("No Debug Migration device found")
95 jsonlen
= self
.read32()
97 # Seek back to where we were at the beginning
98 self
.file.seek(entrypos
, 0)
100 # explicit decode() needed for Python 3.5 compatibility
101 return data
[jsonpos
:jsonpos
+ jsonlen
].decode("utf-8")
106 class RamSection(object):
107 RAM_SAVE_FLAG_COMPRESS
= 0x02
108 RAM_SAVE_FLAG_MEM_SIZE
= 0x04
109 RAM_SAVE_FLAG_PAGE
= 0x08
110 RAM_SAVE_FLAG_EOS
= 0x10
111 RAM_SAVE_FLAG_CONTINUE
= 0x20
112 RAM_SAVE_FLAG_XBZRLE
= 0x40
113 RAM_SAVE_FLAG_HOOK
= 0x80
114 RAM_SAVE_FLAG_COMPRESS_PAGE
= 0x100
115 RAM_SAVE_FLAG_MULTIFD_FLUSH
= 0x200
117 def __init__(self
, file, version_id
, ramargs
, section_key
):
119 raise Exception("Unknown RAM version %d" % version_id
)
122 self
.section_key
= section_key
123 self
.TARGET_PAGE_SIZE
= ramargs
['page_size']
124 self
.dump_memory
= ramargs
['dump_memory']
125 self
.write_memory
= ramargs
['write_memory']
126 self
.ignore_shared
= ramargs
['ignore_shared']
127 self
.sizeinfo
= collections
.OrderedDict()
128 self
.data
= collections
.OrderedDict()
129 self
.data
['section sizes'] = self
.sizeinfo
131 if self
.write_memory
:
134 self
.memory
= collections
.OrderedDict()
135 self
.data
['memory'] = self
.memory
138 return self
.data
.__repr
__()
141 return self
.data
.__str
__()
147 # Read all RAM sections
149 addr
= self
.file.read64()
150 flags
= addr
& (self
.TARGET_PAGE_SIZE
- 1)
151 addr
&= ~
(self
.TARGET_PAGE_SIZE
- 1)
153 if flags
& self
.RAM_SAVE_FLAG_MEM_SIZE
:
155 namelen
= self
.file.read8()
156 # We assume that no RAM chunk is big enough to ever
157 # hit the first byte of the address, so when we see
158 # a zero here we know it has to be an address, not the
159 # length of the next block.
161 self
.file.file.seek(-1, 1)
163 self
.name
= self
.file.readstr(len = namelen
)
164 len = self
.file.read64()
165 self
.sizeinfo
[self
.name
] = '0x%016x' % len
166 if self
.write_memory
:
168 mkdir_p('./' + os
.path
.dirname(self
.name
))
169 f
= open('./' + self
.name
, "wb")
172 self
.files
[self
.name
] = f
173 if self
.ignore_shared
:
174 mr_addr
= self
.file.read64()
175 flags
&= ~self
.RAM_SAVE_FLAG_MEM_SIZE
177 if flags
& self
.RAM_SAVE_FLAG_COMPRESS
:
178 if flags
& self
.RAM_SAVE_FLAG_CONTINUE
:
179 flags
&= ~self
.RAM_SAVE_FLAG_CONTINUE
181 self
.name
= self
.file.readstr()
182 fill_char
= self
.file.read8()
183 # The page in question is filled with fill_char now
184 if self
.write_memory
and fill_char
!= 0:
185 self
.files
[self
.name
].seek(addr
, os
.SEEK_SET
)
186 self
.files
[self
.name
].write(chr(fill_char
) * self
.TARGET_PAGE_SIZE
)
188 self
.memory
['%s (0x%016x)' % (self
.name
, addr
)] = 'Filled with 0x%02x' % fill_char
189 flags
&= ~self
.RAM_SAVE_FLAG_COMPRESS
190 elif flags
& self
.RAM_SAVE_FLAG_PAGE
:
191 if flags
& self
.RAM_SAVE_FLAG_CONTINUE
:
192 flags
&= ~self
.RAM_SAVE_FLAG_CONTINUE
194 self
.name
= self
.file.readstr()
196 if self
.write_memory
or self
.dump_memory
:
197 data
= self
.file.readvar(size
= self
.TARGET_PAGE_SIZE
)
198 else: # Just skip RAM data
199 self
.file.file.seek(self
.TARGET_PAGE_SIZE
, 1)
201 if self
.write_memory
:
202 self
.files
[self
.name
].seek(addr
, os
.SEEK_SET
)
203 self
.files
[self
.name
].write(data
)
205 hexdata
= " ".join("{0:02x}".format(ord(c
)) for c
in data
)
206 self
.memory
['%s (0x%016x)' % (self
.name
, addr
)] = hexdata
208 flags
&= ~self
.RAM_SAVE_FLAG_PAGE
209 elif flags
& self
.RAM_SAVE_FLAG_XBZRLE
:
210 raise Exception("XBZRLE RAM compression is not supported yet")
211 elif flags
& self
.RAM_SAVE_FLAG_HOOK
:
212 raise Exception("RAM hooks don't make sense with files")
213 if flags
& self
.RAM_SAVE_FLAG_MULTIFD_FLUSH
:
217 if flags
& self
.RAM_SAVE_FLAG_EOS
:
221 raise Exception("Unknown RAM flags: %x" % flags
)
224 if self
.write_memory
:
225 for key
in self
.files
:
226 self
.files
[key
].close()
229 class HTABSection(object):
230 HASH_PTE_SIZE_64
= 16
232 def __init__(self
, file, version_id
, device
, section_key
):
234 raise Exception("Unknown HTAB version %d" % version_id
)
237 self
.section_key
= section_key
241 header
= self
.file.read32()
248 # First section, just the hash shift
251 # Read until end marker
253 index
= self
.file.read32()
254 n_valid
= self
.file.read16()
255 n_invalid
= self
.file.read16()
257 if index
== 0 and n_valid
== 0 and n_invalid
== 0:
260 self
.file.readvar(n_valid
* self
.HASH_PTE_SIZE_64
)
266 class S390StorageAttributes(object):
267 STATTR_FLAG_EOS
= 0x01
268 STATTR_FLAG_MORE
= 0x02
269 STATTR_FLAG_ERROR
= 0x04
270 STATTR_FLAG_DONE
= 0x08
272 def __init__(self
, file, version_id
, device
, section_key
):
274 raise Exception("Unknown storage_attributes version %d" % version_id
)
277 self
.section_key
= section_key
281 addr_flags
= self
.file.read64()
282 flags
= addr_flags
& 0xfff
283 if (flags
& (self
.STATTR_FLAG_DONE | self
.STATTR_FLAG_EOS
)):
285 if (flags
& self
.STATTR_FLAG_ERROR
):
286 raise Exception("Error in migration stream")
287 count
= self
.file.read64()
288 self
.file.readvar(count
)
294 class ConfigurationSection(object):
295 def __init__(self
, file, desc
):
300 def parse_capabilities(self
, vmsd_caps
):
304 ncaps
= vmsd_caps
.data
['caps_count'].data
305 self
.caps
= vmsd_caps
.data
['capabilities']
307 if type(self
.caps
) != list:
308 self
.caps
= [self
.caps
]
310 if len(self
.caps
) != ncaps
:
311 raise Exception("Number of capabilities doesn't match "
314 def has_capability(self
, cap
):
315 return any([str(c
) == cap
for c
in self
.caps
])
319 version_id
= self
.desc
['version']
320 section
= VMSDSection(self
.file, version_id
, self
.desc
,
323 self
.parse_capabilities(
324 section
.data
.get("configuration/capabilities"))
326 # backward compatibility for older streams that don't have
327 # the configuration section in the json
328 name_len
= self
.file.read32()
329 name
= self
.file.readstr(len = name_len
)
331 class VMSDFieldGeneric(object):
332 def __init__(self
, desc
, file):
338 return str(self
.__str
__())
341 return " ".join("{0:02x}".format(c
) for c
in self
.data
)
344 return self
.__str
__()
347 size
= int(self
.desc
['size'])
348 self
.data
= self
.file.readvar(size
)
351 class VMSDFieldCap(object):
352 def __init__(self
, desc
, file):
364 len = self
.file.read8()
365 self
.data
= self
.file.readstr(len)
368 class VMSDFieldInt(VMSDFieldGeneric
):
369 def __init__(self
, desc
, file):
370 super(VMSDFieldInt
, self
).__init
__(desc
, file)
371 self
.size
= int(desc
['size'])
372 self
.format
= '0x%%0%dx' % (self
.size
* 2)
373 self
.sdtype
= '>i%d' % self
.size
374 self
.udtype
= '>u%d' % self
.size
378 return ('%s (%d)' % ((self
.format
% self
.udata
), self
.data
))
380 return self
.format
% self
.data
383 return self
.__repr
__()
386 return self
.__str
__()
389 super(VMSDFieldInt
, self
).read()
390 self
.sdata
= int.from_bytes(self
.data
, byteorder
='big', signed
=True)
391 self
.udata
= int.from_bytes(self
.data
, byteorder
='big', signed
=False)
392 self
.data
= self
.sdata
395 class VMSDFieldUInt(VMSDFieldInt
):
396 def __init__(self
, desc
, file):
397 super(VMSDFieldUInt
, self
).__init
__(desc
, file)
400 super(VMSDFieldUInt
, self
).read()
401 self
.data
= self
.udata
404 class VMSDFieldIntLE(VMSDFieldInt
):
405 def __init__(self
, desc
, file):
406 super(VMSDFieldIntLE
, self
).__init
__(desc
, file)
407 self
.dtype
= '<i%d' % self
.size
409 class VMSDFieldBool(VMSDFieldGeneric
):
410 def __init__(self
, desc
, file):
411 super(VMSDFieldBool
, self
).__init
__(desc
, file)
414 return self
.data
.__repr
__()
417 return self
.data
.__str
__()
423 super(VMSDFieldBool
, self
).read()
424 if self
.data
[0] == 0:
430 class VMSDFieldStruct(VMSDFieldGeneric
):
431 QEMU_VM_SUBSECTION
= 0x05
433 def __init__(self
, desc
, file):
434 super(VMSDFieldStruct
, self
).__init
__(desc
, file)
435 self
.data
= collections
.OrderedDict()
437 # When we see compressed array elements, unfold them here
439 for field
in self
.desc
['struct']['fields']:
440 if not 'array_len' in field
:
441 new_fields
.append(field
)
443 array_len
= field
.pop('array_len')
445 new_fields
.append(field
)
446 for i
in range(1, array_len
):
451 self
.desc
['struct']['fields'] = new_fields
454 return self
.data
.__repr
__()
457 return self
.data
.__str
__()
460 for field
in self
.desc
['struct']['fields']:
462 reader
= vmsd_field_readers
[field
['type']]
464 reader
= VMSDFieldGeneric
466 field
['data'] = reader(field
, self
.file)
470 if field
['name'] not in self
.data
:
471 self
.data
[field
['name']] = []
472 a
= self
.data
[field
['name']]
473 if len(a
) != int(field
['index']):
474 raise Exception("internal index of data field unmatched (%d/%d)" % (len(a
), int(field
['index'])))
475 a
.append(field
['data'])
477 self
.data
[field
['name']] = field
['data']
479 if 'subsections' in self
.desc
['struct']:
480 for subsection
in self
.desc
['struct']['subsections']:
481 if self
.file.read8() != self
.QEMU_VM_SUBSECTION
:
482 raise Exception("Subsection %s not found at offset %x" % ( subsection
['vmsd_name'], self
.file.tell()))
483 name
= self
.file.readstr()
484 version_id
= self
.file.read32()
485 self
.data
[name
] = VMSDSection(self
.file, version_id
, subsection
, (name
, 0))
486 self
.data
[name
].read()
488 def getDictItem(self
, value
):
489 # Strings would fall into the array category, treat
491 if value
.__class
__ is ''.__class
__:
495 return self
.getDictOrderedDict(value
)
498 return self
.getDictArray(value
)
501 return value
.getDict()
505 def getDictArray(self
, array
):
508 r
.append(self
.getDictItem(value
))
511 def getDictOrderedDict(self
, dict):
512 r
= collections
.OrderedDict()
513 for (key
, value
) in dict.items():
514 r
[key
] = self
.getDictItem(value
)
518 return self
.getDictOrderedDict(self
.data
)
520 vmsd_field_readers
= {
521 "bool" : VMSDFieldBool
,
522 "int8" : VMSDFieldInt
,
523 "int16" : VMSDFieldInt
,
524 "int32" : VMSDFieldInt
,
525 "int32 equal" : VMSDFieldInt
,
526 "int32 le" : VMSDFieldIntLE
,
527 "int64" : VMSDFieldInt
,
528 "uint8" : VMSDFieldUInt
,
529 "uint16" : VMSDFieldUInt
,
530 "uint32" : VMSDFieldUInt
,
531 "uint32 equal" : VMSDFieldUInt
,
532 "uint64" : VMSDFieldUInt
,
533 "int64 equal" : VMSDFieldInt
,
534 "uint8 equal" : VMSDFieldInt
,
535 "uint16 equal" : VMSDFieldInt
,
536 "float64" : VMSDFieldGeneric
,
537 "timer" : VMSDFieldGeneric
,
538 "buffer" : VMSDFieldGeneric
,
539 "unused_buffer" : VMSDFieldGeneric
,
540 "bitmap" : VMSDFieldGeneric
,
541 "struct" : VMSDFieldStruct
,
542 "capability": VMSDFieldCap
,
543 "unknown" : VMSDFieldGeneric
,
546 class VMSDSection(VMSDFieldStruct
):
547 def __init__(self
, file, version_id
, device
, section_key
):
551 self
.section_key
= section_key
553 if 'vmsd_name' in device
:
554 self
.vmsd_name
= device
['vmsd_name']
556 # A section really is nothing but a FieldStruct :)
557 super(VMSDSection
, self
).__init
__({ 'struct' : desc
}, file)
559 ###############################################################################
561 class MigrationDump(object):
562 QEMU_VM_FILE_MAGIC
= 0x5145564d
563 QEMU_VM_FILE_VERSION
= 0x00000003
565 QEMU_VM_SECTION_START
= 0x01
566 QEMU_VM_SECTION_PART
= 0x02
567 QEMU_VM_SECTION_END
= 0x03
568 QEMU_VM_SECTION_FULL
= 0x04
569 QEMU_VM_SUBSECTION
= 0x05
570 QEMU_VM_VMDESCRIPTION
= 0x06
571 QEMU_VM_CONFIGURATION
= 0x07
572 QEMU_VM_SECTION_FOOTER
= 0x7e
574 def __init__(self
, filename
):
575 self
.section_classes
= {
576 ( 'ram', 0 ) : [ RamSection
, None ],
577 ( 's390-storage_attributes', 0 ) : [ S390StorageAttributes
, None],
578 ( 'spapr/htab', 0) : ( HTABSection
, None )
580 self
.filename
= filename
581 self
.vmsd_desc
= None
583 def read(self
, desc_only
= False, dump_memory
= False, write_memory
= False):
584 # Read in the whole file
585 file = MigrationFile(self
.filename
)
589 if data
!= self
.QEMU_VM_FILE_MAGIC
:
590 raise Exception("Invalid file magic %x" % data
)
592 # Version (has to be v3)
594 if data
!= self
.QEMU_VM_FILE_VERSION
:
595 raise Exception("Invalid version number %d" % data
)
597 self
.load_vmsd_json(file)
600 self
.sections
= collections
.OrderedDict()
606 ramargs
['page_size'] = self
.vmsd_desc
['page_size']
607 ramargs
['dump_memory'] = dump_memory
608 ramargs
['write_memory'] = write_memory
609 ramargs
['ignore_shared'] = False
610 self
.section_classes
[('ram',0)][1] = ramargs
613 section_type
= file.read8()
614 if section_type
== self
.QEMU_VM_EOF
:
616 elif section_type
== self
.QEMU_VM_CONFIGURATION
:
617 config_desc
= self
.vmsd_desc
.get('configuration')
618 section
= ConfigurationSection(file, config_desc
)
620 ramargs
['ignore_shared'] = section
.has_capability('x-ignore-shared')
621 elif section_type
== self
.QEMU_VM_SECTION_START
or section_type
== self
.QEMU_VM_SECTION_FULL
:
622 section_id
= file.read32()
623 name
= file.readstr()
624 instance_id
= file.read32()
625 version_id
= file.read32()
626 section_key
= (name
, instance_id
)
627 classdesc
= self
.section_classes
[section_key
]
628 section
= classdesc
[0](file, version_id
, classdesc
[1], section_key
)
629 self
.sections
[section_id
] = section
631 elif section_type
== self
.QEMU_VM_SECTION_PART
or section_type
== self
.QEMU_VM_SECTION_END
:
632 section_id
= file.read32()
633 self
.sections
[section_id
].read()
634 elif section_type
== self
.QEMU_VM_SECTION_FOOTER
:
635 read_section_id
= file.read32()
636 if read_section_id
!= section_id
:
637 raise Exception("Mismatched section footer: %x vs %x" % (read_section_id
, section_id
))
639 raise Exception("Unknown section type: %d" % section_type
)
642 def load_vmsd_json(self
, file):
643 vmsd_json
= file.read_migration_debug_json()
644 self
.vmsd_desc
= json
.loads(vmsd_json
, object_pairs_hook
=collections
.OrderedDict
)
645 for device
in self
.vmsd_desc
['devices']:
646 key
= (device
['name'], device
['instance_id'])
647 value
= ( VMSDSection
, device
)
648 self
.section_classes
[key
] = value
651 r
= collections
.OrderedDict()
652 for (key
, value
) in self
.sections
.items():
653 key
= "%s (%d)" % ( value
.section_key
[0], key
)
654 r
[key
] = value
.getDict()
657 ###############################################################################
659 class JSONEncoder(json
.JSONEncoder
):
660 def default(self
, o
):
661 if isinstance(o
, VMSDFieldGeneric
):
663 return json
.JSONEncoder
.default(self
, o
)
665 parser
= argparse
.ArgumentParser()
666 parser
.add_argument("-f", "--file", help='migration dump to read from', required
=True)
667 parser
.add_argument("-m", "--memory", help='dump RAM contents as well', action
='store_true')
668 parser
.add_argument("-d", "--dump", help='what to dump ("state" or "desc")', default
='state')
669 parser
.add_argument("-x", "--extract", help='extract contents into individual files', action
='store_true')
670 args
= parser
.parse_args()
672 jsonenc
= JSONEncoder(indent
=4, separators
=(',', ': '))
675 dump
= MigrationDump(args
.file)
677 dump
.read(desc_only
= True)
679 f
= open("desc.json", "w")
681 f
.write(jsonenc
.encode(dump
.vmsd_desc
))
684 dump
.read(write_memory
= True)
685 dict = dump
.getDict()
687 f
= open("state.json", "w")
689 f
.write(jsonenc
.encode(dict))
691 elif args
.dump
== "state":
692 dump
= MigrationDump(args
.file)
693 dump
.read(dump_memory
= args
.memory
)
694 dict = dump
.getDict()
695 print(jsonenc
.encode(dict))
696 elif args
.dump
== "desc":
697 dump
= MigrationDump(args
.file)
698 dump
.read(desc_only
= True)
699 print(jsonenc
.encode(dump
.vmsd_desc
))
701 raise Exception("Please specify either -x, -d state or -d desc")