3 # Script to compare machine type compatible properties (include/hw/boards.h).
4 # compat_props are applied to the driver during initialization to change
5 # default values, for instance, to maintain compatibility.
6 # This script constructs table with machines and values of their compat_props
7 # to compare and to find places for improvements or places with bugs. If
8 # during the comparison, some machine type doesn't have a property (it is in
9 # the comparison table because another machine type has it), then the
10 # appropriate method will be used to obtain the default value of this driver
11 # property via qmp command (e.g. query-cpu-model-expansion for x86_64-cpu).
12 # These methods are defined below in qemu_property_methods.
14 # Copyright (c) Yandex Technologies LLC, 2023
16 # This program is free software; you can redistribute it and/or modify
17 # it under the terms of the GNU General Public License as published by
18 # the Free Software Foundation; either version 2 of the License, or
19 # (at your option) any later version.
21 # This program is distributed in the hope that it will be useful,
22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 # GNU General Public License for more details.
26 # You should have received a copy of the GNU General Public License
27 # along with this program; if not, see <http://www.gnu.org/licenses/>.
31 from argparse
import ArgumentParser
, RawTextHelpFormatter
, Namespace
33 from contextlib
import ExitStack
34 from typing
import Optional
, List
, Dict
, Generator
, Tuple
, Union
, Any
, Set
37 qemu_dir
= path
.abspath(path
.dirname(path
.dirname(__file__
)))
38 sys
.path
.append(path
.join(qemu_dir
, 'python'))
39 from qemu
.machine
import QEMUMachine
40 except ModuleNotFoundError
as exc
:
41 print(f
"Module '{exc.name}' not found.")
42 print("Try export PYTHONPATH=top-qemu-dir/python or run from top-qemu-dir")
46 default_qemu_args
= '-enable-kvm -machine none'
47 default_qemu_binary
= 'build/qemu-system-x86_64'
50 # Methods for gettig the right values of drivers properties
52 # Use these methods as a 'whitelist' and add entries only if necessary. It's
53 # important to be stable and predictable in analysis and tests.
55 # * Class must be inherited from 'QEMUObject' and used in new_driver()
56 # * Class has to implement get_prop method in order to get values
57 # * Specialization always wins (with the given classes for 'device' and
58 # 'x86_64-cpu', method of 'x86_64-cpu' will be used for '486-x86_64-cpu')
61 def __init__(self
, vm
: QEMUMachine
, name
: str, abstract
: bool) -> None:
64 self
.abstract
= abstract
65 self
.parent
: Optional
[Driver
] = None
66 self
.property_getter
: Optional
[Driver
] = None
68 def get_prop(self
, driver
: str, prop
: str) -> str:
69 if self
.property_getter
:
70 return self
.property_getter
.get_prop(driver
, prop
)
72 return 'Unavailable method'
74 def is_child_of(self
, parent
: 'Driver') -> bool:
75 """Checks whether self is (recursive) child of @parent"""
76 cur_parent
= self
.parent
78 if cur_parent
is parent
:
80 cur_parent
= cur_parent
.parent
84 def set_implementations(self
, implementations
: List
['Driver']) -> None:
85 self
.implementations
= implementations
88 class QEMUObject(Driver
):
89 def __init__(self
, vm
: QEMUMachine
, name
: str) -> None:
90 super().__init
__(vm
, name
, True)
92 def set_implementations(self
, implementations
: List
[Driver
]) -> None:
93 self
.implementations
= implementations
95 # each implementation of the abstract driver has to use property getter
96 # of this abstract driver unless it has specialization. (e.g. having
97 # 'device' and 'x86_64-cpu', property getter of 'x86_64-cpu' will be
98 # used for '486-x86_64-cpu')
99 for impl
in implementations
:
100 if not impl
.property_getter
or\
101 self
.is_child_of(impl
.property_getter
):
102 impl
.property_getter
= self
105 class QEMUDevice(QEMUObject
):
106 def __init__(self
, vm
: QEMUMachine
) -> None:
107 super().__init
__(vm
, 'device')
108 self
.cached
: Dict
[str, List
[Dict
[str, Any
]]] = {}
110 def get_prop(self
, driver
: str, prop_name
: str) -> str:
111 if driver
not in self
.cached
:
112 self
.cached
[driver
] = self
.vm
.cmd('device-list-properties',
114 for prop
in self
.cached
[driver
]:
115 if prop
['name'] == prop_name
:
116 return str(prop
.get('default-value', 'No default value'))
118 return 'Unknown property'
121 class QEMUx86CPU(QEMUObject
):
122 def __init__(self
, vm
: QEMUMachine
) -> None:
123 super().__init
__(vm
, 'x86_64-cpu')
124 self
.cached
: Dict
[str, Dict
[str, Any
]] = {}
126 def get_prop(self
, driver
: str, prop_name
: str) -> str:
127 if not driver
.endswith('-x86_64-cpu'):
128 return 'Wrong x86_64-cpu name'
130 # crop last 11 chars '-x86_64-cpu'
132 if name
not in self
.cached
:
133 self
.cached
[name
] = self
.vm
.cmd(
134 'query-cpu-model-expansion', type='full',
135 model
={'name': name
})['model']['props']
136 return str(self
.cached
[name
].get(prop_name
, 'Unknown property'))
139 # Now it's stub, because all memory_backend types don't have default values
140 # but this behaviour can be changed
141 class QEMUMemoryBackend(QEMUObject
):
142 def __init__(self
, vm
: QEMUMachine
) -> None:
143 super().__init
__(vm
, 'memory-backend')
144 self
.cached
: Dict
[str, List
[Dict
[str, Any
]]] = {}
146 def get_prop(self
, driver
: str, prop_name
: str) -> str:
147 if driver
not in self
.cached
:
148 self
.cached
[driver
] = self
.vm
.cmd('qom-list-properties',
150 for prop
in self
.cached
[driver
]:
151 if prop
['name'] == prop_name
:
152 return str(prop
.get('default-value', 'No default value'))
154 return 'Unknown property'
157 def new_driver(vm
: QEMUMachine
, name
: str, is_abstr
: bool) -> Driver
:
159 return QEMUObject(vm
, 'object')
160 elif name
== 'device':
161 return QEMUDevice(vm
)
162 elif name
== 'x86_64-cpu':
163 return QEMUx86CPU(vm
)
164 elif name
== 'memory-backend':
165 return QEMUMemoryBackend(vm
)
167 return Driver(vm
, name
, is_abstr
)
168 # End of methods definition
171 class VMPropertyGetter
:
172 """It implements the relationship between drivers and how to get their
174 def __init__(self
, vm
: QEMUMachine
) -> None:
175 self
.drivers
: Dict
[str, Driver
] = {}
177 qom_all_types
= vm
.cmd('qom-list-types', abstract
=True)
178 self
.drivers
= {t
['name']: new_driver(vm
, t
['name'],
179 t
.get('abstract', False))
180 for t
in qom_all_types
}
182 for t
in qom_all_types
:
183 drv
= self
.drivers
[t
['name']]
185 drv
.parent
= self
.drivers
[t
['parent']]
187 for drv
in self
.drivers
.values():
188 imps
= vm
.cmd('qom-list-types', implements
=drv
.name
)
189 # only implementations inherit property getter
190 drv
.set_implementations([self
.drivers
[imp
['name']]
193 def get_prop(self
, driver
: str, prop
: str) -> str:
194 # wrong driver name or disabled in config driver
196 drv
= self
.drivers
[driver
]
198 return 'Unavailable driver'
200 assert not drv
.abstract
202 return drv
.get_prop(driver
, prop
)
204 def get_implementations(self
, driver
: str) -> List
[str]:
205 return [impl
.name
for impl
in self
.drivers
[driver
].implementations
]
209 """A short QEMU machine type description. It contains only processed
210 compat_props (properties of abstract classes are applied to its
213 # raw_mt_dict - dict produced by `query-machines`
214 def __init__(self
, raw_mt_dict
: Dict
[str, Any
],
215 qemu_drivers
: VMPropertyGetter
) -> None:
216 self
.name
= raw_mt_dict
['name']
217 self
.compat_props
: Dict
[str, Any
] = {}
218 # properties are applied sequentially and can rewrite values like in
219 # QEMU. Also it has to resolve class relationships to apply appropriate
220 # values from abstract class to all implementations
221 for prop
in raw_mt_dict
['compat-props']:
222 driver
= prop
['qom-type']
224 # implementation adds only itself, abstract class adds
225 # lementation (abstract classes are uninterestiong)
226 impls
= qemu_drivers
.get_implementations(driver
)
228 if impl
not in self
.compat_props
:
229 self
.compat_props
[impl
] = {}
230 self
.compat_props
[impl
][prop
['property']] = prop
['value']
232 # QEMU doesn't know this driver thus it has to be saved
233 if driver
not in self
.compat_props
:
234 self
.compat_props
[driver
] = {}
235 self
.compat_props
[driver
][prop
['property']] = prop
['value']
238 class Configuration():
239 """Class contains all necessary components to generate table and is used
240 to compare different binaries"""
241 def __init__(self
, vm
: QEMUMachine
,
242 req_mt
: List
[str], all_mt
: bool) -> None:
244 self
._binary
= vm
.binary
245 self
._qemu
_args
= args
.qemu_args
.split(' ')
247 self
._qemu
_drivers
= VMPropertyGetter(vm
)
248 self
.req_mt
= get_req_mt(self
._qemu
_drivers
, vm
, req_mt
, all_mt
)
250 def get_implementations(self
, driver_name
: str) -> List
[str]:
251 return self
._qemu
_drivers
.get_implementations(driver_name
)
253 def get_table(self
, req_props
: List
[Tuple
[str, str]]) -> pd
.DataFrame
:
254 table
: List
[pd
.DataFrame
] = []
255 for mt
in self
.req_mt
:
256 name
= f
'{self._binary}\n{mt.name}'
258 for driver
, prop
in req_props
:
260 # values from QEMU machine type definitions
261 column
.append(mt
.compat_props
[driver
][prop
])
263 # values from QEMU type definitions
264 column
.append(self
._qemu
_drivers
.get_prop(driver
, prop
))
265 table
.append(pd
.DataFrame({name
: column
}))
267 return pd
.concat(table
, axis
=1)
270 script_desc
= """Script to compare machine types (their compat_props).
273 * save info about all machines: ./scripts/compare-machine-types.py --all \
274 --format csv --raw > table.csv
275 * compare machines: ./scripts/compare-machine-types.py --mt pc-q35-2.12 \
277 * compare binaries and machines: ./scripts/compare-machine-types.py \
278 --mt pc-q35-6.2 pc-q35-7.0 --qemu-binary build/qemu-system-x86_64 \
280 ╒════════════╤══════════════════════════╤════════════════════════════\
281 ╤════════════════════════════╤══════════════════╤══════════════════╕
282 │ Driver │ Property │ build/qemu-system-x86_64 \
283 │ build/qemu-system-x86_64 │ build/qemu-exp │ build/qemu-exp │
285 │ pc-q35-7.0 │ pc-q35-6.2 │ pc-q35-7.0 │
286 ╞════════════╪══════════════════════════╪════════════════════════════\
287 ╪════════════════════════════╪══════════════════╪══════════════════╡
288 │ PIIX4_PM │ x-not-migrate-acpi-index │ True \
289 │ False │ False │ False │
290 ├────────────┼──────────────────────────┼────────────────────────────\
291 ┼────────────────────────────┼──────────────────┼──────────────────┤
292 │ virtio-mem │ unplugged-inaccessible │ False \
293 │ auto │ False │ auto │
294 ╘════════════╧══════════════════════════╧════════════════════════════\
295 ╧════════════════════════════╧══════════════════╧══════════════════╛
297 If a property from QEMU machine defintion applies to an abstract class (e.g. \
298 x86_64-cpu) this script will compare all implementations of this class.
300 "Unavailable method" - means that this script doesn't know how to get \
301 default values of the driver. To add method use the construction described \
302 at the top of the script.
303 "Unavailable driver" - means that this script doesn't know this driver. \
304 For instance, this can happen if you configure QEMU without this device or \
305 if machine type definition has error.
306 "No default value" - means that the appropriate method can't get the default \
307 value and most likely that this property doesn't have it.
308 "Unknown property" - means that the appropriate method can't find property \
312 def parse_args() -> Namespace
:
313 parser
= ArgumentParser(formatter_class
=RawTextHelpFormatter
,
314 description
=script_desc
)
315 parser
.add_argument('--format', choices
=['human-readable', 'json', 'csv'],
316 default
='human-readable',
317 help='returns table in json format')
318 parser
.add_argument('--raw', action
='store_true',
319 help='prints ALL defined properties without value '
320 'transformation. By default, only rows '
321 'with different values will be printed and '
322 'values will be transformed(e.g. "on" -> True)')
323 parser
.add_argument('--qemu-args', default
=default_qemu_args
,
324 help='command line to start qemu. '
325 f
'Default: "{default_qemu_args}"')
326 parser
.add_argument('--qemu-binary', nargs
="*", type=str,
327 default
=[default_qemu_binary
],
328 help='list of qemu binaries that will be compared. '
329 f
'Deafult: {default_qemu_binary}')
331 mt_args_group
= parser
.add_mutually_exclusive_group()
332 mt_args_group
.add_argument('--all', action
='store_true',
333 help='prints all available machine types (list '
334 'of machine types will be ignored)')
335 mt_args_group
.add_argument('--mt', nargs
="*", type=str,
336 help='list of Machine Types '
337 'that will be compared')
339 return parser
.parse_args()
342 def mt_comp(mt
: Machine
) -> Tuple
[str, int, int, int]:
343 """Function to compare and sort machine by names.
344 It returns socket_name, major version, minor version, revision"""
345 # none, microvm, x-remote and etc.
346 if '-' not in mt
.name
or '.' not in mt
.name
:
347 return mt
.name
, 0, 0, 0
349 socket
, ver
= mt
.name
.rsplit('-', 1)
350 ver_list
= list(map(int, ver
.split('.', 2)))
351 ver_list
+= [0] * (3 - len(ver_list
))
352 return socket
, ver_list
[0], ver_list
[1], ver_list
[2]
355 def get_mt_definitions(qemu_drivers
: VMPropertyGetter
,
356 vm
: QEMUMachine
) -> List
[Machine
]:
357 """Constructs list of machine definitions (primarily compat_props) via
359 raw_mt_defs
= vm
.cmd('query-machines', compat_props
=True)
361 for raw_mt
in raw_mt_defs
:
362 mt_defs
.append(Machine(raw_mt
, qemu_drivers
))
364 mt_defs
.sort(key
=mt_comp
)
368 def get_req_mt(qemu_drivers
: VMPropertyGetter
, vm
: QEMUMachine
,
369 req_mt
: Optional
[List
[str]], all_mt
: bool) -> List
[Machine
]:
370 """Returns list of requested by user machines"""
371 mt_defs
= get_mt_definitions(qemu_drivers
, vm
)
376 print('Enter machine types for comparision')
381 if mt
.name
in req_mt
:
382 matched_mt
.append(mt
)
387 def get_affected_props(configs
: List
[Configuration
]) -> Generator
[Tuple
[str,
390 """Helps to go through all affected in machine definitions drivers
392 driver_props
: Dict
[str, Set
[Any
]] = {}
393 for config
in configs
:
394 for mt
in config
.req_mt
:
395 compat_props
= mt
.compat_props
396 for driver
, prop
in compat_props
.items():
397 if driver
not in driver_props
:
398 driver_props
[driver
] = set()
399 driver_props
[driver
].update(prop
.keys())
401 for driver
, props
in sorted(driver_props
.items()):
402 for prop
in sorted(props
):
406 def transform_value(value
: str) -> Union
[str, bool]:
407 true_list
= ['true', 'on']
408 false_list
= ['false', 'off']
415 if out
in false_list
:
421 def simplify_table(table
: pd
.DataFrame
) -> pd
.DataFrame
:
422 """transforms values to make it easier to compare it and drops rows
423 with the same values for all columns"""
425 table
= table
.map(transform_value
)
427 return table
[~table
.iloc
[:, 3:].eq(table
.iloc
[:, 2], axis
=0).all(axis
=1)]
430 # constructs table in the format:
432 # Driver | Property | binary1 | binary1 | ...
433 # | | machine1 | machine2 | ...
434 # ------------------------------------------------------ ...
435 # driver1 | property1 | value1 | value2 | ...
436 # driver1 | property2 | value3 | value4 | ...
437 # driver2 | property3 | value5 | value6 | ...
438 # ... | ... | ... | ... | ...
440 def fill_prop_table(configs
: List
[Configuration
],
441 is_raw
: bool) -> pd
.DataFrame
:
442 req_props
= list(get_affected_props(configs
))
444 print('No drivers to compare. Check machine names')
447 driver_col
, prop_col
= tuple(zip(*req_props
))
448 table
= [pd
.DataFrame({'Driver': driver_col
}),
449 pd
.DataFrame({'Property': prop_col
})]
451 table
.extend([config
.get_table(req_props
) for config
in configs
])
453 df_table
= pd
.concat(table
, axis
=1)
458 return simplify_table(df_table
)
461 def print_table(table
: pd
.DataFrame
, table_format
: str) -> None:
462 if table_format
== 'json':
463 print(comp_table
.to_json())
464 elif table_format
== 'csv':
465 print(comp_table
.to_csv())
467 print(comp_table
.to_markdown(index
=False, stralign
='center',
468 colalign
=('center',), headers
='keys',
469 tablefmt
='fancy_grid',
470 disable_numparse
=True))
473 if __name__
== '__main__':
475 with
ExitStack() as stack
:
476 vms
= [stack
.enter_context(QEMUMachine(binary
=binary
, qmp_timer
=15,
477 args
=args
.qemu_args
.split(' '))) for binary
in args
.qemu_binary
]
482 configurations
.append(Configuration(vm
, args
.mt
, args
.all
))
484 comp_table
= fill_prop_table(configurations
, args
.raw
)
485 if not comp_table
.empty
:
486 print_table(comp_table
, args
.format
)