2 * QEMU PowerPC pSeries Logical Partition NUMA associativity handling
4 * Copyright IBM Corp. 2020
7 * Daniel Henrique Barboza <danielhb413@gmail.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
14 #include "qemu-common.h"
15 #include "hw/ppc/spapr_numa.h"
16 #include "hw/pci-host/spapr.h"
17 #include "hw/ppc/fdt.h"
19 /* Moved from hw/ppc/spapr_pci_nvlink2.c */
20 #define SPAPR_GPU_NUMA_ID (cpu_to_be32(1))
23 * Retrieves max_dist_ref_points of the current NUMA affinity.
25 static int get_max_dist_ref_points(SpaprMachineState
*spapr
)
27 if (spapr_ovec_test(spapr
->ov5_cas
, OV5_FORM2_AFFINITY
)) {
28 return FORM2_DIST_REF_POINTS
;
31 return FORM1_DIST_REF_POINTS
;
35 * Retrieves numa_assoc_size of the current NUMA affinity.
37 static int get_numa_assoc_size(SpaprMachineState
*spapr
)
39 if (spapr_ovec_test(spapr
->ov5_cas
, OV5_FORM2_AFFINITY
)) {
40 return FORM2_NUMA_ASSOC_SIZE
;
43 return FORM1_NUMA_ASSOC_SIZE
;
47 * Retrieves vcpu_assoc_size of the current NUMA affinity.
49 * vcpu_assoc_size is the size of ibm,associativity array
50 * for CPUs, which has an extra element (vcpu_id) in the end.
52 static int get_vcpu_assoc_size(SpaprMachineState
*spapr
)
54 return get_numa_assoc_size(spapr
) + 1;
58 * Retrieves the ibm,associativity array of NUMA node 'node_id'
59 * for the current NUMA affinity.
61 static const uint32_t *get_associativity(SpaprMachineState
*spapr
, int node_id
)
63 if (spapr_ovec_test(spapr
->ov5_cas
, OV5_FORM2_AFFINITY
)) {
64 return spapr
->FORM2_assoc_array
[node_id
];
66 return spapr
->FORM1_assoc_array
[node_id
];
70 * Wrapper that returns node distance from ms->numa_state->nodes
71 * after handling edge cases where the distance might be absent.
73 static int get_numa_distance(MachineState
*ms
, int src
, int dst
)
75 NodeInfo
*numa_info
= ms
->numa_state
->nodes
;
76 int ret
= numa_info
[src
].distance
[dst
];
83 * In case QEMU adds a default NUMA single node when the user
84 * did not add any, or where the user did not supply distances,
85 * the distance will be absent (zero). Return local/remote
86 * distance in this case.
89 return NUMA_DISTANCE_MIN
;
92 return NUMA_DISTANCE_DEFAULT
;
95 static bool spapr_numa_is_symmetrical(MachineState
*ms
)
97 int nb_numa_nodes
= ms
->numa_state
->num_nodes
;
100 for (src
= 0; src
< nb_numa_nodes
; src
++) {
101 for (dst
= src
; dst
< nb_numa_nodes
; dst
++) {
102 if (get_numa_distance(ms
, src
, dst
) !=
103 get_numa_distance(ms
, dst
, src
)) {
113 * NVLink2-connected GPU RAM needs to be placed on a separate NUMA node.
114 * We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is
115 * called from vPHB reset handler so we initialize the counter here.
116 * If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM
117 * must be equally distant from any other node.
118 * The final value of spapr->gpu_numa_id is going to be written to
119 * max-associativity-domains in spapr_build_fdt().
121 unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState
*machine
)
123 return MAX(1, machine
->numa_state
->num_nodes
);
127 * This function will translate the user distances into
128 * what the kernel understand as possible values: 10
129 * (local distance), 20, 40, 80 and 160, and return the equivalent
130 * NUMA level for each. Current heuristic is:
131 * - local distance (10) returns numa_level = 0x4, meaning there is
132 * no rounding for local distance
133 * - distances between 11 and 30 inclusive -> rounded to 20,
135 * - distances between 31 and 60 inclusive -> rounded to 40,
137 * - distances between 61 and 120 inclusive -> rounded to 80,
139 * - everything above 120 returns numa_level = 0 to indicate that
140 * there is no match. This will be calculated as disntace = 160
141 * by the kernel (as of v5.9)
143 static uint8_t spapr_numa_get_numa_level(uint8_t distance
)
145 if (distance
== 10) {
147 } else if (distance
> 11 && distance
<= 30) {
149 } else if (distance
> 31 && distance
<= 60) {
151 } else if (distance
> 61 && distance
<= 120) {
158 static void spapr_numa_define_FORM1_domains(SpaprMachineState
*spapr
)
160 MachineState
*ms
= MACHINE(spapr
);
161 int nb_numa_nodes
= ms
->numa_state
->num_nodes
;
165 * Fill all associativity domains of non-zero NUMA nodes with
166 * node_id. This is required because the default value (0) is
167 * considered a match with associativity domains of node 0.
169 for (i
= 1; i
< nb_numa_nodes
; i
++) {
170 for (j
= 1; j
< FORM1_DIST_REF_POINTS
; j
++) {
171 spapr
->FORM1_assoc_array
[i
][j
] = cpu_to_be32(i
);
175 for (src
= 0; src
< nb_numa_nodes
; src
++) {
176 for (dst
= src
; dst
< nb_numa_nodes
; dst
++) {
178 * This is how the associativity domain between A and B
181 * - get the distance D between them
182 * - get the correspondent NUMA level 'n_level' for D
183 * - all associativity arrays were initialized with their own
184 * numa_ids, and we're calculating the distance in node_id
185 * ascending order, starting from node id 0 (the first node
186 * retrieved by numa_state). This will have a cascade effect in
187 * the algorithm because the associativity domains that node 0
188 * defines will be carried over to other nodes, and node 1
189 * associativities will be carried over after taking node 0
190 * associativities into account, and so on. This happens because
191 * we'll assign assoc_src as the associativity domain of dst
192 * as well, for all NUMA levels beyond and including n_level.
194 * The PPC kernel expects the associativity domains of node 0 to
195 * be always 0, and this algorithm will grant that by default.
197 uint8_t distance
= get_numa_distance(ms
, src
, dst
);
198 uint8_t n_level
= spapr_numa_get_numa_level(distance
);
202 * n_level = 0 means that the distance is greater than our last
203 * rounded value (120). In this case there is no NUMA level match
204 * between src and dst and we can skip the remaining of the loop.
206 * The Linux kernel will assume that the distance between src and
207 * dst, in this case of no match, is 10 (local distance) doubled
208 * for each NUMA it didn't match. We have FORM1_DIST_REF_POINTS
209 * levels (4), so this gives us 10*2*2*2*2 = 160.
211 * This logic can be seen in the Linux kernel source code, as of
212 * v5.9, in arch/powerpc/mm/numa.c, function __node_distance().
219 * We must assign all assoc_src to dst, starting from n_level
220 * and going up to 0x1.
222 for (i
= n_level
; i
> 0; i
--) {
223 assoc_src
= spapr
->FORM1_assoc_array
[src
][i
];
224 spapr
->FORM1_assoc_array
[dst
][i
] = assoc_src
;
231 static void spapr_numa_FORM1_affinity_check(MachineState
*machine
)
236 * Check we don't have a memory-less/cpu-less NUMA node
237 * Firmware relies on the existing memory/cpu topology to provide the
238 * NUMA topology to the kernel.
239 * And the linux kernel needs to know the NUMA topology at start
240 * to be able to hotplug CPUs later.
242 if (machine
->numa_state
->num_nodes
) {
243 for (i
= 0; i
< machine
->numa_state
->num_nodes
; ++i
) {
244 /* check for memory-less node */
245 if (machine
->numa_state
->nodes
[i
].node_mem
== 0) {
248 /* check for cpu-less node */
250 PowerPCCPU
*cpu
= POWERPC_CPU(cs
);
251 if (cpu
->node_id
== i
) {
256 /* memory-less and cpu-less node */
259 "Memory-less/cpu-less nodes are not supported with FORM1 NUMA (node %d)", i
);
266 if (!spapr_numa_is_symmetrical(machine
)) {
268 "Asymmetrical NUMA topologies aren't supported in the pSeries machine using FORM1 NUMA");
274 * Set NUMA machine state data based on FORM1 affinity semantics.
276 static void spapr_numa_FORM1_affinity_init(SpaprMachineState
*spapr
,
277 MachineState
*machine
)
279 SpaprMachineClass
*smc
= SPAPR_MACHINE_GET_CLASS(spapr
);
280 int nb_numa_nodes
= machine
->numa_state
->num_nodes
;
281 int i
, j
, max_nodes_with_gpus
;
284 * For all associativity arrays: first position is the size,
285 * position FORM1_DIST_REF_POINTS is always the numa_id,
286 * represented by the index 'i'.
288 * This will break on sparse NUMA setups, when/if QEMU starts
289 * to support it, because there will be no more guarantee that
290 * 'i' will be a valid node_id set by the user.
292 for (i
= 0; i
< nb_numa_nodes
; i
++) {
293 spapr
->FORM1_assoc_array
[i
][0] = cpu_to_be32(FORM1_DIST_REF_POINTS
);
294 spapr
->FORM1_assoc_array
[i
][FORM1_DIST_REF_POINTS
] = cpu_to_be32(i
);
298 * Initialize NVLink GPU associativity arrays. We know that
299 * the first GPU will take the first available NUMA id, and
300 * we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine.
301 * At this point we're not sure if there are GPUs or not, but
302 * let's initialize the associativity arrays and allow NVLink
303 * GPUs to be handled like regular NUMA nodes later on.
305 max_nodes_with_gpus
= nb_numa_nodes
+ NVGPU_MAX_NUM
;
307 for (i
= nb_numa_nodes
; i
< max_nodes_with_gpus
; i
++) {
308 spapr
->FORM1_assoc_array
[i
][0] = cpu_to_be32(FORM1_DIST_REF_POINTS
);
310 for (j
= 1; j
< FORM1_DIST_REF_POINTS
; j
++) {
311 uint32_t gpu_assoc
= smc
->pre_5_1_assoc_refpoints
?
312 SPAPR_GPU_NUMA_ID
: cpu_to_be32(i
);
313 spapr
->FORM1_assoc_array
[i
][j
] = gpu_assoc
;
316 spapr
->FORM1_assoc_array
[i
][FORM1_DIST_REF_POINTS
] = cpu_to_be32(i
);
320 * Guests pseries-5.1 and older uses zeroed associativity domains,
321 * i.e. no domain definition based on NUMA distance input.
323 * Same thing with guests that have only one NUMA node.
325 if (smc
->pre_5_2_numa_associativity
||
326 machine
->numa_state
->num_nodes
<= 1) {
330 spapr_numa_define_FORM1_domains(spapr
);
334 * Init NUMA FORM2 machine state data
336 static void spapr_numa_FORM2_affinity_init(SpaprMachineState
*spapr
)
341 * For all resources but CPUs, FORM2 associativity arrays will
342 * be a size 2 array with the following format:
344 * ibm,associativity = {1, numa_id}
346 * CPUs will write an additional 'vcpu_id' on top of the arrays
347 * being initialized here. 'numa_id' is represented by the
348 * index 'i' of the loop.
350 * Given that this initialization is also valid for GPU associativity
351 * arrays, handle everything in one single step by populating the
352 * arrays up to NUMA_NODES_MAX_NUM.
354 for (i
= 0; i
< NUMA_NODES_MAX_NUM
; i
++) {
355 spapr
->FORM2_assoc_array
[i
][0] = cpu_to_be32(1);
356 spapr
->FORM2_assoc_array
[i
][1] = cpu_to_be32(i
);
360 void spapr_numa_associativity_init(SpaprMachineState
*spapr
,
361 MachineState
*machine
)
363 spapr_numa_FORM1_affinity_init(spapr
, machine
);
364 spapr_numa_FORM2_affinity_init(spapr
);
367 void spapr_numa_associativity_check(SpaprMachineState
*spapr
)
370 * FORM2 does not have any restrictions we need to handle
371 * at CAS time, for now.
373 if (spapr_ovec_test(spapr
->ov5_cas
, OV5_FORM2_AFFINITY
)) {
377 spapr_numa_FORM1_affinity_check(MACHINE(spapr
));
380 void spapr_numa_write_associativity_dt(SpaprMachineState
*spapr
, void *fdt
,
381 int offset
, int nodeid
)
383 const uint32_t *associativity
= get_associativity(spapr
, nodeid
);
385 _FDT((fdt_setprop(fdt
, offset
, "ibm,associativity",
387 get_numa_assoc_size(spapr
) * sizeof(uint32_t))));
390 static uint32_t *spapr_numa_get_vcpu_assoc(SpaprMachineState
*spapr
,
393 const uint32_t *associativity
= get_associativity(spapr
, cpu
->node_id
);
394 int max_distance_ref_points
= get_max_dist_ref_points(spapr
);
395 int vcpu_assoc_size
= get_vcpu_assoc_size(spapr
);
396 uint32_t *vcpu_assoc
= g_new(uint32_t, vcpu_assoc_size
);
397 int index
= spapr_get_vcpu_id(cpu
);
400 * VCPUs have an extra 'cpu_id' value in ibm,associativity
401 * compared to other resources. Increment the size at index
402 * 0, put cpu_id last, then copy the remaining associativity
405 vcpu_assoc
[0] = cpu_to_be32(max_distance_ref_points
+ 1);
406 vcpu_assoc
[vcpu_assoc_size
- 1] = cpu_to_be32(index
);
407 memcpy(vcpu_assoc
+ 1, associativity
+ 1,
408 (vcpu_assoc_size
- 2) * sizeof(uint32_t));
413 int spapr_numa_fixup_cpu_dt(SpaprMachineState
*spapr
, void *fdt
,
414 int offset
, PowerPCCPU
*cpu
)
416 g_autofree
uint32_t *vcpu_assoc
= NULL
;
417 int vcpu_assoc_size
= get_vcpu_assoc_size(spapr
);
419 vcpu_assoc
= spapr_numa_get_vcpu_assoc(spapr
, cpu
);
421 /* Advertise NUMA via ibm,associativity */
422 return fdt_setprop(fdt
, offset
, "ibm,associativity", vcpu_assoc
,
423 vcpu_assoc_size
* sizeof(uint32_t));
427 int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState
*spapr
, void *fdt
,
430 MachineState
*machine
= MACHINE(spapr
);
431 int max_distance_ref_points
= get_max_dist_ref_points(spapr
);
432 int nb_numa_nodes
= machine
->numa_state
->num_nodes
;
433 int nr_nodes
= nb_numa_nodes
? nb_numa_nodes
: 1;
434 g_autofree
uint32_t *int_buf
= NULL
;
438 /* ibm,associativity-lookup-arrays */
439 int_buf
= g_malloc0((nr_nodes
* max_distance_ref_points
+ 2) *
442 int_buf
[0] = cpu_to_be32(nr_nodes
);
443 /* Number of entries per associativity list */
444 int_buf
[1] = cpu_to_be32(max_distance_ref_points
);
446 for (i
= 0; i
< nr_nodes
; i
++) {
448 * For the lookup-array we use the ibm,associativity array of the
449 * current NUMA affinity, without the first element (size).
451 const uint32_t *associativity
= get_associativity(spapr
, i
);
452 memcpy(cur_index
, ++associativity
,
453 sizeof(uint32_t) * max_distance_ref_points
);
454 cur_index
+= max_distance_ref_points
;
457 return fdt_setprop(fdt
, offset
, "ibm,associativity-lookup-arrays",
458 int_buf
, (cur_index
- int_buf
) * sizeof(uint32_t));
461 static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState
*spapr
,
464 MachineState
*ms
= MACHINE(spapr
);
465 SpaprMachineClass
*smc
= SPAPR_MACHINE_GET_CLASS(spapr
);
466 uint32_t number_nvgpus_nodes
= spapr
->gpu_numa_id
-
467 spapr_numa_initial_nvgpu_numa_id(ms
);
468 uint32_t refpoints
[] = {
474 uint32_t nr_refpoints
= ARRAY_SIZE(refpoints
);
475 uint32_t maxdomain
= ms
->numa_state
->num_nodes
+ number_nvgpus_nodes
;
476 uint32_t maxdomains
[] = {
478 cpu_to_be32(maxdomain
),
479 cpu_to_be32(maxdomain
),
480 cpu_to_be32(maxdomain
),
481 cpu_to_be32(maxdomain
)
484 if (smc
->pre_5_2_numa_associativity
||
485 ms
->numa_state
->num_nodes
<= 1) {
486 uint32_t legacy_refpoints
[] = {
491 uint32_t legacy_maxdomain
= spapr
->gpu_numa_id
> 1 ? 1 : 0;
492 uint32_t legacy_maxdomains
[] = {
494 cpu_to_be32(legacy_maxdomain
),
495 cpu_to_be32(legacy_maxdomain
),
496 cpu_to_be32(legacy_maxdomain
),
497 cpu_to_be32(spapr
->gpu_numa_id
),
500 G_STATIC_ASSERT(sizeof(legacy_refpoints
) <= sizeof(refpoints
));
501 G_STATIC_ASSERT(sizeof(legacy_maxdomains
) <= sizeof(maxdomains
));
505 memcpy(refpoints
, legacy_refpoints
, sizeof(legacy_refpoints
));
506 memcpy(maxdomains
, legacy_maxdomains
, sizeof(legacy_maxdomains
));
508 /* pseries-5.0 and older reference-points array is {0x4, 0x4} */
509 if (smc
->pre_5_1_assoc_refpoints
) {
514 _FDT(fdt_setprop(fdt
, rtas
, "ibm,associativity-reference-points",
515 refpoints
, nr_refpoints
* sizeof(refpoints
[0])));
517 _FDT(fdt_setprop(fdt
, rtas
, "ibm,max-associativity-domains",
518 maxdomains
, sizeof(maxdomains
)));
521 static void spapr_numa_FORM2_write_rtas_tables(SpaprMachineState
*spapr
,
524 MachineState
*ms
= MACHINE(spapr
);
525 int nb_numa_nodes
= ms
->numa_state
->num_nodes
;
526 int distance_table_entries
= nb_numa_nodes
* nb_numa_nodes
;
527 g_autofree
uint32_t *lookup_index_table
= NULL
;
528 g_autofree
uint8_t *distance_table
= NULL
;
529 int src
, dst
, i
, distance_table_size
;
532 * ibm,numa-lookup-index-table: array with length and a
533 * list of NUMA ids present in the guest.
535 lookup_index_table
= g_new0(uint32_t, nb_numa_nodes
+ 1);
536 lookup_index_table
[0] = cpu_to_be32(nb_numa_nodes
);
538 for (i
= 0; i
< nb_numa_nodes
; i
++) {
539 lookup_index_table
[i
+ 1] = cpu_to_be32(i
);
542 _FDT(fdt_setprop(fdt
, rtas
, "ibm,numa-lookup-index-table",
544 (nb_numa_nodes
+ 1) * sizeof(uint32_t)));
547 * ibm,numa-distance-table: contains all node distances. First
548 * element is the size of the table as uint32, followed up
549 * by all the uint8 distances from the first NUMA node, then all
550 * distances from the second NUMA node and so on.
552 * ibm,numa-lookup-index-table is used by guest to navigate this
553 * array because NUMA ids can be sparse (node 0 is the first,
554 * node 8 is the second ...).
556 distance_table_size
= distance_table_entries
* sizeof(uint8_t) +
558 distance_table
= g_new0(uint8_t, distance_table_size
);
559 stl_be_p(distance_table
, distance_table_entries
);
561 /* Skip the uint32_t array length at the start */
562 i
= sizeof(uint32_t);
564 for (src
= 0; src
< nb_numa_nodes
; src
++) {
565 for (dst
= 0; dst
< nb_numa_nodes
; dst
++) {
566 distance_table
[i
++] = get_numa_distance(ms
, src
, dst
);
570 _FDT(fdt_setprop(fdt
, rtas
, "ibm,numa-distance-table",
571 distance_table
, distance_table_size
));
575 * This helper could be compressed in a single function with
576 * FORM1 logic since we're setting the same DT values, with the
577 * difference being a call to spapr_numa_FORM2_write_rtas_tables()
578 * in the end. The separation was made to avoid clogging FORM1 code
579 * which already has to deal with compat modes from previous
580 * QEMU machine types.
582 static void spapr_numa_FORM2_write_rtas_dt(SpaprMachineState
*spapr
,
585 MachineState
*ms
= MACHINE(spapr
);
586 uint32_t number_nvgpus_nodes
= spapr
->gpu_numa_id
-
587 spapr_numa_initial_nvgpu_numa_id(ms
);
590 * In FORM2, ibm,associativity-reference-points will point to
591 * the element in the ibm,associativity array that contains the
592 * primary domain index (for FORM2, the first element).
594 * This value (in our case, the numa-id) is then used as an index
595 * to retrieve all other attributes of the node (distance,
596 * bandwidth, latency) via ibm,numa-lookup-index-table and other
597 * ibm,numa-*-table properties.
599 uint32_t refpoints
[] = { cpu_to_be32(1) };
601 uint32_t maxdomain
= ms
->numa_state
->num_nodes
+ number_nvgpus_nodes
;
602 uint32_t maxdomains
[] = { cpu_to_be32(1), cpu_to_be32(maxdomain
) };
604 _FDT(fdt_setprop(fdt
, rtas
, "ibm,associativity-reference-points",
605 refpoints
, sizeof(refpoints
)));
607 _FDT(fdt_setprop(fdt
, rtas
, "ibm,max-associativity-domains",
608 maxdomains
, sizeof(maxdomains
)));
610 spapr_numa_FORM2_write_rtas_tables(spapr
, fdt
, rtas
);
614 * Helper that writes ibm,associativity-reference-points and
615 * max-associativity-domains in the RTAS pointed by @rtas
618 void spapr_numa_write_rtas_dt(SpaprMachineState
*spapr
, void *fdt
, int rtas
)
620 if (spapr_ovec_test(spapr
->ov5_cas
, OV5_FORM2_AFFINITY
)) {
621 spapr_numa_FORM2_write_rtas_dt(spapr
, fdt
, rtas
);
625 spapr_numa_FORM1_write_rtas_dt(spapr
, fdt
, rtas
);
628 static target_ulong
h_home_node_associativity(PowerPCCPU
*cpu
,
629 SpaprMachineState
*spapr
,
633 g_autofree
uint32_t *vcpu_assoc
= NULL
;
634 target_ulong flags
= args
[0];
635 target_ulong procno
= args
[1];
638 int vcpu_assoc_size
= get_vcpu_assoc_size(spapr
);
640 /* only support procno from H_REGISTER_VPA */
645 tcpu
= spapr_find_cpu(procno
);
651 * Given that we want to be flexible with the sizes and indexes,
652 * we must consider that there is a hard limit of how many
653 * associativities domain we can fit in R4 up to R9, which would be
654 * 12 associativity domains for vcpus. Assert and bail if that's
657 g_assert((vcpu_assoc_size
- 1) <= 12);
659 vcpu_assoc
= spapr_numa_get_vcpu_assoc(spapr
, tcpu
);
660 /* assoc_idx starts at 1 to skip associativity size */
663 #define ASSOCIATIVITY(a, b) (((uint64_t)(a) << 32) | \
664 ((uint64_t)(b) & 0xffffffff))
666 for (idx
= 0; idx
< 6; idx
++) {
670 * vcpu_assoc[] will contain the associativity domains for tcpu,
671 * including tcpu->node_id and procno, meaning that we don't
672 * need to use these variables here.
674 * We'll read 2 values at a time to fill up the ASSOCIATIVITY()
675 * macro. The ternary will fill the remaining registers with -1
676 * after we went through vcpu_assoc[].
678 a
= assoc_idx
< vcpu_assoc_size
?
679 be32_to_cpu(vcpu_assoc
[assoc_idx
++]) : -1;
680 b
= assoc_idx
< vcpu_assoc_size
?
681 be32_to_cpu(vcpu_assoc
[assoc_idx
++]) : -1;
683 args
[idx
] = ASSOCIATIVITY(a
, b
);
690 static void spapr_numa_register_types(void)
692 /* Virtual Processor Home Node */
693 spapr_register_hypercall(H_HOME_NODE_ASSOCIATIVITY
,
694 h_home_node_associativity
);
697 type_init(spapr_numa_register_types
)