4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25 #include <sys/systm.h>
26 #include <sys/types.h>
27 #include <sys/param.h>
28 #include <sys/thread.h>
29 #include <sys/cpuvar.h>
31 #include <sys/cmn_err.h>
32 #include <sys/policy.h>
33 #include <sys/group.h>
36 #include <sys/cpu_pm.h>
37 #include <sys/cap_util.h>
40 * Processor Groups: Hardware sharing relationship layer
42 * This file implements an extension to Processor Groups to capture
43 * hardware sharing relationships existing between logical CPUs. Examples of
44 * hardware sharing relationships include shared caches on some CMT
45 * procesoor architectures, or shared local memory controllers on NUMA
46 * based system architectures.
48 * The pghw_t structure represents the extended PG. The first member
49 * of the structure is the generic pg_t with the pghw specific members
50 * following. The generic pg_t *must* remain the first member of the
51 * structure as the code uses casting of structure references to access
52 * the generic pg_t structure elements.
54 * In addition to the generic CPU grouping, physical PGs have a hardware
55 * sharing relationship enumerated "type", and an instance id. The enumerated
56 * type is defined by the pghw_type_t enumeration, while the instance id
57 * uniquely identifies the sharing instance from among others of the same
58 * hardware sharing type.
60 * The physical PGs are organized into an overall hierarchy, and are tracked
61 * in a number of different per CPU, and per pghw_type_t type groups.
68 * || ============================
69 * ||\\-----------------------// \\ \\
70 * || | hwset (PGC_HW_CHIP) | ------------- -------------
71 * || | (group_t) | | pghw_t | | pghw_t |
72 * || ----------------------- | chip 0 | | chip 1 |
73 * || ------------- -------------
74 * || \\ \\ \\ \\ \\ \\ \\ \\
75 * || cpu cpu cpu cpu cpu cpu cpu cpu
77 * || ============================
78 * ||\\-----------------------// \\ \\
79 * || | hwset (PGC_HW_IPIPE)| ------------- -------------
80 * || | (group_t) | | pghw_t | | pghw_t |
81 * || ----------------------- | ipipe 0 | | ipipe 1 |
82 * || ------------- -------------
88 * The top level pg_hw is a group of "hwset" groups. Each hwset holds of group
89 * of physical PGs of the same hardware sharing type. Within each hwset, the
90 * PG's instance id uniquely identifies the grouping relationshsip among other
91 * groupings of the same sharing type. The instance id for a grouping is
92 * platform defined, and in some cases may be used by platform code as a handle
93 * to search for a particular relationship instance.
95 * Each physical PG (by virtue of the embedded pg_t) contains a group of CPUs
96 * that participate in the sharing relationship. Each CPU also has associated
97 * with it a grouping tracking the PGs in which the CPU belongs. This can be
98 * used to iterate over the various relationships in which the CPU participates
99 * (the CPU's chip, cache, lgroup, etc.).
101 * The hwsets are created dynamically as new hardware sharing relationship types
102 * are instantiated. They are never destroyed, as once a given relationship
103 * type appears in the system, it is quite likely that at least one instance of
104 * that relationship will always persist as long as the system is running.
107 static group_t
*pg_hw
; /* top level pg hw group */
114 kstat_named_t pg_class
;
115 kstat_named_t pg_ncpus
;
116 kstat_named_t pg_instance_id
;
118 kstat_named_t pg_policy
;
120 { "id", KSTAT_DATA_INT32
},
121 { "pg_class", KSTAT_DATA_STRING
},
122 { "ncpus", KSTAT_DATA_UINT32
},
123 { "instance_id", KSTAT_DATA_UINT32
},
124 { "hardware", KSTAT_DATA_STRING
},
125 { "policy", KSTAT_DATA_STRING
},
128 kmutex_t pghw_kstat_lock
;
131 * Capacity and Utilization PG kstats
133 * These kstats are updated one at a time, so we can have a single scratch space
138 * pg_id PG ID for PG described by this kstat
140 * pg_parent Parent PG ID. The value -1 means "no parent".
142 * pg_ncpus Number of CPUs within this PG
144 * pg_cpus String describing CPUs within this PG
146 * pg_relationship Name of sharing relationship for this PG
148 * pg_generation Generation value that increases whenever any CPU leaves
149 * or joins PG. Two kstat snapshots for the same
150 * CPU may only be compared if they have the same
153 * pg_hw_util Running value of PG utilization for the sharing
156 * pg_hw_util_time_running
157 * Total time spent collecting CU data. The time may be
158 * less than wall time if CU counters were stopped for
161 * pg_hw_util_time_stopped Total time the CU counters were stopped.
163 * pg_hw_util_rate Utilization rate, expressed in operations per second.
165 * pg_hw_util_rate_max Maximum observed value of utilization rate.
167 struct pghw_cu_kstat
{
169 kstat_named_t pg_parent_id
;
170 kstat_named_t pg_ncpus
;
171 kstat_named_t pg_generation
;
172 kstat_named_t pg_hw_util
;
173 kstat_named_t pg_hw_util_time_running
;
174 kstat_named_t pg_hw_util_time_stopped
;
175 kstat_named_t pg_hw_util_rate
;
176 kstat_named_t pg_hw_util_rate_max
;
177 kstat_named_t pg_cpus
;
178 kstat_named_t pg_relationship
;
180 { "pg_id", KSTAT_DATA_INT32
},
181 { "parent_pg_id", KSTAT_DATA_INT32
},
182 { "ncpus", KSTAT_DATA_UINT32
},
183 { "generation", KSTAT_DATA_UINT32
},
184 { "hw_util", KSTAT_DATA_UINT64
},
185 { "hw_util_time_running", KSTAT_DATA_UINT64
},
186 { "hw_util_time_stopped", KSTAT_DATA_UINT64
},
187 { "hw_util_rate", KSTAT_DATA_UINT64
},
188 { "hw_util_rate_max", KSTAT_DATA_UINT64
},
189 { "cpus", KSTAT_DATA_STRING
},
190 { "relationship", KSTAT_DATA_STRING
},
194 * Calculate the string size to represent NCPUS. Allow 5 digits for each CPU ID
195 * plus one space per CPU plus NUL byte in the end. This is only an estimate,
196 * since we try to compress CPU ranges as x-y. In the worst case the string
197 * representation of CPUs may be truncated.
199 #define CPUSTR_LEN(ncpus) ((ncpus) * 6)
202 * Maximum length of the string that represents list of CPUs
204 static int pg_cpulist_maxlen
= 0;
206 static void pghw_kstat_create(pghw_t
*);
207 static int pghw_kstat_update(kstat_t
*, int);
208 static int pghw_cu_kstat_update(kstat_t
*, int);
209 static int cpu2id(void *);
214 static group_t
*pghw_set_create(pghw_type_t
);
215 static void pghw_set_add(group_t
*, pghw_t
*);
216 static void pghw_set_remove(group_t
*, pghw_t
*);
218 static void pghw_cpulist_alloc(pghw_t
*);
219 static int cpu2id(void *);
220 static pgid_t
pghw_parent_id(pghw_t
*);
223 * Initialize the physical portion of a hardware PG
226 pghw_init(pghw_t
*pg
, cpu_t
*cp
, pghw_type_t hw
)
230 if ((hwset
= pghw_set_lookup(hw
)) == NULL
) {
232 * Haven't seen this hardware type yet
234 hwset
= pghw_set_create(hw
);
237 pghw_set_add(hwset
, pg
);
239 pg
->pghw_generation
= 0;
241 pg_plat_hw_instance_id(cp
, hw
);
242 pghw_kstat_create(pg
);
245 * Hardware sharing relationship specific initialization
247 switch (pg
->pghw_hw
) {
248 case PGHW_POW_ACTIVE
:
250 (pghw_handle_t
)cpupm_domain_init(cp
, CPUPM_DTYPE_ACTIVE
);
254 (pghw_handle_t
)cpupm_domain_init(cp
, CPUPM_DTYPE_IDLE
);
257 pg
->pghw_handle
= (pghw_handle_t
)NULL
;
262 * Teardown the physical portion of a physical PG
265 pghw_fini(pghw_t
*pg
)
271 hwset
= pghw_set_lookup(pg
->pghw_hw
);
272 ASSERT(hwset
!= NULL
);
274 pghw_set_remove(hwset
, pg
);
275 pg
->pghw_instance
= (id_t
)PGHW_INSTANCE_ANON
;
276 pg
->pghw_hw
= (pghw_type_t
)-1;
278 if (pg
->pghw_kstat
!= NULL
)
279 kstat_delete(pg
->pghw_kstat
);
284 * PG is removed from CMT hierarchy
287 pghw_cmt_fini(pghw_t
*pg
)
290 * Destroy string representation of CPUs
292 if (pg
->pghw_cpulist
!= NULL
) {
293 kmem_free(pg
->pghw_cpulist
,
294 pg
->pghw_cpulist_len
);
295 pg
->pghw_cpulist
= NULL
;
301 if (pg
->pghw_cu_kstat
!= NULL
) {
302 kstat_delete(pg
->pghw_cu_kstat
);
303 pg
->pghw_cu_kstat
= NULL
;
308 * Find an existing physical PG in which to place
309 * the given CPU for the specified hardware sharing
313 pghw_place_cpu(cpu_t
*cp
, pghw_type_t hw
)
317 if ((hwset
= pghw_set_lookup(hw
)) == NULL
) {
321 return ((pghw_t
*)pg_cpu_find_pg(cp
, hwset
));
325 * Find the pg representing the hw sharing relationship in which
329 pghw_find_pg(cpu_t
*cp
, pghw_type_t hw
)
335 while ((pg
= group_iterate(&cp
->cpu_pg
->pgs
, &i
)) != NULL
) {
336 if (pg
->pghw_hw
== hw
)
343 * Find the PG of the given hardware sharing relationship
344 * type with the given instance id
347 pghw_find_by_instance(id_t id
, pghw_type_t hw
)
353 set
= pghw_set_lookup(hw
);
358 while ((pg
= group_iterate(set
, &i
)) != NULL
) {
359 if (pg
->pghw_instance
== id
)
366 * CPUs physical ID cache creation / destruction
367 * The cache's elements are initialized to the CPU's id
370 pghw_physid_create(cpu_t
*cp
)
374 cp
->cpu_physid
= kmem_alloc(sizeof (cpu_physid_t
), KM_SLEEP
);
376 for (i
= 0; i
< (sizeof (cpu_physid_t
) / sizeof (id_t
)); i
++) {
377 ((id_t
*)cp
->cpu_physid
)[i
] = cp
->cpu_id
;
382 pghw_physid_destroy(cpu_t
*cp
)
384 if (cp
->cpu_physid
) {
385 kmem_free(cp
->cpu_physid
, sizeof (cpu_physid_t
));
386 cp
->cpu_physid
= NULL
;
391 * Create a new, empty hwset.
392 * This routine may block, and must not be called from any
393 * paused CPU context.
396 pghw_set_create(pghw_type_t hw
)
402 * Create the top level PG hw group if it doesn't already exist
403 * This is a "set" of hardware sets, that is ordered (and indexed)
404 * by the pghw_type_t enum.
407 pg_hw
= kmem_alloc(sizeof (group_t
), KM_SLEEP
);
409 group_expand(pg_hw
, (uint_t
)PGHW_NUM_COMPONENTS
);
413 * Create the new hwset
414 * Add it to the top level pg_hw group.
416 g
= kmem_alloc(sizeof (group_t
), KM_SLEEP
);
419 ret
= group_add_at(pg_hw
, g
, (uint_t
)hw
);
426 * Find the hwset associated with the given hardware sharing type
429 pghw_set_lookup(pghw_type_t hw
)
436 hwset
= GROUP_ACCESS(pg_hw
, (uint_t
)hw
);
441 * Add a PG to a hwset
444 pghw_set_add(group_t
*hwset
, pghw_t
*pg
)
446 (void) group_add(hwset
, pg
, GRP_RESIZE
);
450 * Remove a PG from a hwset
453 pghw_set_remove(group_t
*hwset
, pghw_t
*pg
)
457 result
= group_remove(hwset
, pg
, GRP_RESIZE
);
462 * Return a string name given a pg_hw sharing type
465 pghw_type_string(pghw_type_t hw
)
469 return ("Integer Pipeline");
473 return ("Floating Point Unit");
475 return ("Data Pipe to memory");
480 case PGHW_POW_ACTIVE
:
481 return ("CPU PM Active Power Domain");
483 return ("CPU PM Idle Power Domain");
490 * Create / Update routines for PG hw kstats
492 * It is the intention of these kstats to provide some level
493 * of informational / debugging observability into the types
494 * and nature of the system's detected hardware sharing relationships
497 pghw_kstat_create(pghw_t
*pg
)
499 char *sharing
= pghw_type_string(pg
->pghw_hw
);
500 char name
[KSTAT_STRLEN
+ 1];
503 * Canonify PG name to conform to kstat name rules
505 (void) strncpy(name
, pghw_type_string(pg
->pghw_hw
), KSTAT_STRLEN
+ 1);
506 strident_canon(name
, KSTAT_STRLEN
+ 1);
509 * Create a hardware performance kstat
511 if ((pg
->pghw_kstat
= kstat_create("pg", ((pg_t
*)pg
)->pg_id
,
514 sizeof (pghw_kstat
) / sizeof (kstat_named_t
),
515 KSTAT_FLAG_VIRTUAL
)) != NULL
) {
516 /* Class string, hw string, and policy string */
517 pg
->pghw_kstat
->ks_data_size
+= PG_CLASS_NAME_MAX
;
518 pg
->pghw_kstat
->ks_data_size
+= PGHW_KSTAT_STR_LEN_MAX
;
519 pg
->pghw_kstat
->ks_data_size
+= PGHW_KSTAT_STR_LEN_MAX
;
520 pg
->pghw_kstat
->ks_lock
= &pghw_kstat_lock
;
521 pg
->pghw_kstat
->ks_data
= &pghw_kstat
;
522 pg
->pghw_kstat
->ks_update
= pghw_kstat_update
;
523 pg
->pghw_kstat
->ks_private
= pg
;
524 kstat_install(pg
->pghw_kstat
);
527 if (pg_cpulist_maxlen
== 0)
528 pg_cpulist_maxlen
= CPUSTR_LEN(max_ncpus
);
531 * Create a physical pg kstat
533 if ((pg
->pghw_cu_kstat
= kstat_create("pg_hw_perf", ((pg_t
*)pg
)->pg_id
,
534 name
, "processor_group",
536 sizeof (pghw_cu_kstat
) / sizeof (kstat_named_t
),
537 KSTAT_FLAG_VIRTUAL
)) != NULL
) {
538 pg
->pghw_cu_kstat
->ks_lock
= &pghw_kstat_lock
;
539 pg
->pghw_cu_kstat
->ks_data
= &pghw_cu_kstat
;
540 pg
->pghw_cu_kstat
->ks_update
= pghw_cu_kstat_update
;
541 pg
->pghw_cu_kstat
->ks_private
= pg
;
542 pg
->pghw_cu_kstat
->ks_data_size
+= strlen(sharing
) + 1;
543 /* Allow space for CPU strings */
544 pg
->pghw_cu_kstat
->ks_data_size
+= PGHW_KSTAT_STR_LEN_MAX
;
545 pg
->pghw_cu_kstat
->ks_data_size
+= pg_cpulist_maxlen
;
546 kstat_install(pg
->pghw_cu_kstat
);
551 pghw_kstat_update(kstat_t
*ksp
, int rw
)
553 struct pghw_kstat
*pgsp
= &pghw_kstat
;
554 pghw_t
*pg
= ksp
->ks_private
;
556 if (rw
== KSTAT_WRITE
)
559 pgsp
->pg_id
.value
.ui32
= ((pg_t
*)pg
)->pg_id
;
560 pgsp
->pg_ncpus
.value
.ui32
= GROUP_SIZE(&((pg_t
*)pg
)->pg_cpus
);
561 pgsp
->pg_instance_id
.value
.ui32
= pg
->pghw_instance
;
562 kstat_named_setstr(&pgsp
->pg_class
, ((pg_t
*)pg
)->pg_class
->pgc_name
);
563 kstat_named_setstr(&pgsp
->pg_hw
, pghw_type_string(pg
->pghw_hw
));
564 kstat_named_setstr(&pgsp
->pg_policy
, pg_policy_name((pg_t
*)pg
));
569 pghw_cu_kstat_update(kstat_t
*ksp
, int rw
)
571 struct pghw_cu_kstat
*pgsp
= &pghw_cu_kstat
;
572 pghw_t
*pg
= ksp
->ks_private
;
573 pghw_util_t
*hw_util
= &pg
->pghw_stats
;
574 boolean_t has_cpc_privilege
;
576 if (rw
== KSTAT_WRITE
)
580 * Check whether the caller has priv_cpc_cpu privilege. If he doesn't,
581 * he will not get hardware utilization data.
584 has_cpc_privilege
= (secpolicy_cpc_cpu(crgetcred()) == 0);
586 pgsp
->pg_id
.value
.i32
= ((pg_t
*)pg
)->pg_id
;
587 pgsp
->pg_parent_id
.value
.i32
= (int)pghw_parent_id(pg
);
589 pgsp
->pg_ncpus
.value
.ui32
= GROUP_SIZE(&((pg_t
*)pg
)->pg_cpus
);
592 * Allocate memory for the string representing the list of CPUs in PG.
593 * This memory should persist past the call to pghw_cu_kstat_update()
594 * since the kstat snapshot routine will reference this memory.
596 pghw_cpulist_alloc(pg
);
598 if (pg
->pghw_kstat_gen
!= pg
->pghw_generation
) {
600 * PG kstat generation number is out of sync with PG's
601 * generation mumber. It means that some CPUs could have joined
602 * or left PG and it is not possible to compare the numbers
603 * obtained before and after the generation change.
605 * Reset the maximum utilization rate and start computing it
608 hw_util
->pghw_util
= 0;
609 hw_util
->pghw_rate_max
= 0;
610 pg
->pghw_kstat_gen
= pg
->pghw_generation
;
614 * We can't block on CPU lock because when PG is destroyed (under
615 * cpu_lock) it tries to delete this kstat and it will wait for us to
616 * complete which will never happen since we are waiting for cpu_lock to
617 * drop. Deadlocks are fun!
619 if (mutex_tryenter(&cpu_lock
)) {
620 if (pg
->pghw_cpulist
!= NULL
&&
621 *(pg
->pghw_cpulist
) == '\0') {
622 (void) group2intlist(&(((pg_t
*)pg
)->pg_cpus
),
623 pg
->pghw_cpulist
, pg
->pghw_cpulist_len
, cpu2id
);
626 if (has_cpc_privilege
)
629 mutex_exit(&cpu_lock
);
632 pgsp
->pg_generation
.value
.ui32
= pg
->pghw_kstat_gen
;
633 if (pg
->pghw_cpulist
!= NULL
)
634 kstat_named_setstr(&pgsp
->pg_cpus
, pg
->pghw_cpulist
);
636 kstat_named_setstr(&pgsp
->pg_cpus
, "");
638 kstat_named_setstr(&pgsp
->pg_relationship
,
639 pghw_type_string(pg
->pghw_hw
));
641 if (has_cpc_privilege
) {
642 pgsp
->pg_hw_util
.value
.ui64
= hw_util
->pghw_util
;
643 pgsp
->pg_hw_util_time_running
.value
.ui64
=
644 hw_util
->pghw_time_running
;
645 pgsp
->pg_hw_util_time_stopped
.value
.ui64
=
646 hw_util
->pghw_time_stopped
;
647 pgsp
->pg_hw_util_rate
.value
.ui64
= hw_util
->pghw_rate
;
648 pgsp
->pg_hw_util_rate_max
.value
.ui64
= hw_util
->pghw_rate_max
;
650 pgsp
->pg_hw_util
.value
.ui64
= 0;
651 pgsp
->pg_hw_util_time_running
.value
.ui64
= 0;
652 pgsp
->pg_hw_util_time_stopped
.value
.ui64
= 0;
653 pgsp
->pg_hw_util_rate
.value
.ui64
= 0;
654 pgsp
->pg_hw_util_rate_max
.value
.ui64
= 0;
661 * Update the string representation of CPUs in PG (pg->pghw_cpulist).
662 * The string representation is used for kstats.
664 * The string is allocated if it has not already been or if it is already
665 * allocated and PG has more CPUs now. If PG has smaller or equal number of
666 * CPUs, but the actual CPUs may have changed, the string is reset to the empty
667 * string causes the string representation to be recreated. The pghw_generation
668 * field is used to detect whether CPUs within the pg may have changed.
671 pghw_cpulist_alloc(pghw_t
*pg
)
673 uint_t ncpus
= GROUP_SIZE(&((pg_t
*)pg
)->pg_cpus
);
674 size_t len
= CPUSTR_LEN(ncpus
);
677 * If the pghw_cpulist string is already allocated we need to make sure
678 * that it has sufficient length. Also if the set of CPUs may have
679 * changed, we need to re-generate the string.
681 if (pg
->pghw_cpulist
!= NULL
&&
682 pg
->pghw_kstat_gen
!= pg
->pghw_generation
) {
683 if (len
<= pg
->pghw_cpulist_len
) {
685 * There is sufficient space in the pghw_cpulist for
686 * the new set of CPUs. Just clear the string to trigger
687 * re-generation of list of CPUs
689 *(pg
->pghw_cpulist
) = '\0';
692 * There is, potentially, insufficient space in
693 * pghw_cpulist, so reallocate the string.
695 ASSERT(strlen(pg
->pghw_cpulist
) < pg
->pghw_cpulist_len
);
696 kmem_free(pg
->pghw_cpulist
, pg
->pghw_cpulist_len
);
697 pg
->pghw_cpulist
= NULL
;
698 pg
->pghw_cpulist_len
= 0;
702 if (pg
->pghw_cpulist
== NULL
) {
704 * Allocate space to hold cpulist.
706 * Length can not be bigger that the maximum space we have
707 * allowed for the kstat buffer
709 if (len
> pg_cpulist_maxlen
)
710 len
= pg_cpulist_maxlen
;
712 pg
->pghw_cpulist
= kmem_zalloc(len
, KM_NOSLEEP
);
713 if (pg
->pghw_cpulist
!= NULL
)
714 pg
->pghw_cpulist_len
= len
;
722 cpu_t
*cp
= (cpu_t
*)v
;
730 * Return parent ID or -1 if there is no parent.
731 * All hardware PGs are currently also CMT PGs, but for safety we check the
732 * class matches cmt before we upcast the pghw pointer to pg_cmt_t.
735 pghw_parent_id(pghw_t
*pghw
)
737 pg_t
*pg
= (pg_t
*)pghw
;
738 pgid_t parent_id
= -1;
740 if (pg
!= NULL
&& strcmp(pg
->pg_class
->pgc_name
, "cmt") == 0) {
741 pg_cmt_t
*cmt
= (pg_cmt_t
*)pg
;
742 pg_t
*parent
= (pg_t
*)cmt
->cmt_parent
;
744 parent_id
= parent
->pg_id
;