4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
26 #include <sys/types.h>
27 #include <sys/systm.h>
28 #include <sys/archsystm.h>
29 #include <sys/machparam.h>
30 #include <sys/machsystm.h>
32 #include <sys/elf_SPARC.h>
33 #include <vm/hat_sfmmu.h>
35 #include <vm/vm_dep.h>
36 #include <sys/cpuvar.h>
37 #include <sys/async.h>
38 #include <sys/cmn_err.h>
39 #include <sys/debug.h>
40 #include <sys/dditypes.h>
41 #include <sys/sunddi.h>
42 #include <sys/cpu_module.h>
43 #include <sys/prom_debug.h>
44 #include <sys/vmsystm.h>
45 #include <sys/prom_plat.h>
46 #include <sys/sysmacros.h>
47 #include <sys/intreg.h>
48 #include <sys/machtrap.h>
49 #include <sys/ontrap.h>
50 #include <sys/ivintr.h>
51 #include <sys/atomic.h>
52 #include <sys/panic.h>
53 #include <sys/dtrace.h>
54 #include <sys/simulate.h>
55 #include <sys/fault.h>
56 #include <sys/niagara2regs.h>
58 #include <sys/trapstat.h>
59 #include <sys/mutex_impl.h>
61 uint_t root_phys_addr_lo_mask
= 0xffffffffU
;
62 #if defined(NIAGARA2_IMPL)
63 char cpu_module_name
[] = "SUNW,UltraSPARC-T2";
64 #elif defined(VFALLS_IMPL)
65 char cpu_module_name
[] = "SUNW,UltraSPARC-T2+";
66 #elif defined(KT_IMPL)
67 char cpu_module_name
[] = "SPARC-T3";
71 * Hypervisor services information for the NIAGARA2 and Victoria Falls
74 static boolean_t cpu_hsvc_available
= B_TRUE
;
75 static uint64_t cpu_sup_minor
; /* Supported minor number */
76 #if defined(NIAGARA2_IMPL)
77 static hsvc_info_t cpu_hsvc
= {
78 HSVC_REV_1
, NULL
, HSVC_GROUP_NIAGARA2_CPU
, NIAGARA2_HSVC_MAJOR
,
79 NIAGARA2_HSVC_MINOR
, cpu_module_name
81 #elif defined(VFALLS_IMPL)
82 static hsvc_info_t cpu_hsvc
= {
83 HSVC_REV_1
, NULL
, HSVC_GROUP_VFALLS_CPU
, VFALLS_HSVC_MAJOR
,
84 VFALLS_HSVC_MINOR
, cpu_module_name
86 #elif defined(KT_IMPL)
87 static hsvc_info_t cpu_hsvc
= {
88 HSVC_REV_1
, NULL
, HSVC_GROUP_KT_CPU
, KT_HSVC_MAJOR
,
89 KT_HSVC_MINOR
, cpu_module_name
96 extern int mmu_exported_pagesize_mask
;
97 extern int cpc_has_overflow_intr
;
98 extern size_t contig_mem_prealloc_base_size
;
102 * Negotiate the API version for Niagara2 specific hypervisor
105 status
= hsvc_register(&cpu_hsvc
, &cpu_sup_minor
);
107 cmn_err(CE_WARN
, "%s: cannot negotiate hypervisor services "
108 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d",
109 cpu_hsvc
.hsvc_modname
, cpu_hsvc
.hsvc_group
,
110 cpu_hsvc
.hsvc_major
, cpu_hsvc
.hsvc_minor
, status
);
111 cpu_hsvc_available
= B_FALSE
;
115 * The setup common to all CPU modules is done in cpu_setup_common
118 cpu_setup_common(NULL
);
121 * Initialize the cpu_hwcap_flags for N2 and VF if it is not already
122 * set in cpu_setup_common() by the hwcap MD info. Note that this MD
123 * info may not be available for N2/VF.
125 if (cpu_hwcap_flags
== 0) {
128 * This should not happen since hwcap MD info is always
129 * available for KT platforms.
131 ASSERT(cpu_hwcap_flags
!= 0); /* panic in DEBUG mode */
132 cpu_hwcap_flags
|= AV_SPARC_VIS3
| AV_SPARC_HPC
| AV_SPARC_FMAF
;
134 cpu_hwcap_flags
|= AV_SPARC_VIS
| AV_SPARC_VIS2
|
135 AV_SPARC_ASI_BLK_INIT
| AV_SPARC_POPC
;
138 cache
|= (CACHE_PTAG
| CACHE_IOCOHERENT
);
140 if ((mmu_exported_pagesize_mask
&
141 DEFAULT_SUN4V_MMU_PAGESIZE_MASK
) !=
142 DEFAULT_SUN4V_MMU_PAGESIZE_MASK
)
143 cmn_err(CE_PANIC
, "machine description"
144 " does not have required sun4v page sizes"
145 " 8K, 64K and 4M: MD mask is 0x%x",
146 mmu_exported_pagesize_mask
);
149 * Niagara2 supports a 48-bit subset of the full 64-bit virtual
150 * address space. Virtual addresses between 0x0000800000000000
151 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole"
152 * and must never be mapped. In addition, software must not use
153 * pages within 4GB of the VA hole as instruction pages to
154 * avoid problems with prefetching into the VA hole.
156 hole_start
= (caddr_t
)((1ull << (va_bits
- 1)) - (1ull << 32));
157 hole_end
= (caddr_t
)((0ull - (1ull << (va_bits
- 1))) + (1ull << 32));
160 * Niagara2 has a performance counter overflow interrupt
162 cpc_has_overflow_intr
= 1;
165 * Enable 4M pages for OOB.
167 max_uheap_lpsize
= MMU_PAGESIZE4M
;
168 max_ustack_lpsize
= MMU_PAGESIZE4M
;
169 max_privmap_lpsize
= MMU_PAGESIZE4M
;
171 #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB
173 * Use CPU Makefile specific compile time define (if exists)
174 * to add to the contig preallocation size.
176 contig_mem_prealloc_base_size
= MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB
);
181 * Set the magic constants of the implementation.
184 cpu_fiximp(struct cpu_node
*cpunode
)
187 * The Cache node is optional in MD. Therefore in case "Cache"
188 * node does not exists in MD, set the default L2 cache associativity,
191 if (cpunode
->ecache_size
== 0)
192 cpunode
->ecache_size
= L2CACHE_SIZE
;
193 if (cpunode
->ecache_linesize
== 0)
194 cpunode
->ecache_linesize
= L2CACHE_LINESIZE
;
195 if (cpunode
->ecache_associativity
== 0)
196 cpunode
->ecache_associativity
= L2CACHE_ASSOCIATIVITY
;
200 cpu_map_exec_units(struct cpu
*cp
)
202 ASSERT(MUTEX_HELD(&cpu_lock
));
205 * The cpu_ipipe and cpu_fpu fields are initialized based on
206 * the execution unit sharing information from the MD. They
207 * default to the CPU id in the absence of such information.
209 cp
->cpu_m
.cpu_ipipe
= cpunodes
[cp
->cpu_id
].exec_unit_mapping
;
210 if (cp
->cpu_m
.cpu_ipipe
== NO_EU_MAPPING_FOUND
)
211 cp
->cpu_m
.cpu_ipipe
= (id_t
)(cp
->cpu_id
);
213 cp
->cpu_m
.cpu_fpu
= cpunodes
[cp
->cpu_id
].fpu_mapping
;
214 if (cp
->cpu_m
.cpu_fpu
== NO_EU_MAPPING_FOUND
)
215 cp
->cpu_m
.cpu_fpu
= (id_t
)(cp
->cpu_id
);
218 * Niagara 2 defines the core to be at the FPU level
220 cp
->cpu_m
.cpu_core
= cp
->cpu_m
.cpu_fpu
;
223 * The cpu_chip field is initialized based on the information
224 * in the MD and assume that all cpus within a chip
225 * share the same L2 cache. If no such info is available, we
226 * set the cpu to belong to the defacto chip 0.
228 cp
->cpu_m
.cpu_mpipe
= cpunodes
[cp
->cpu_id
].l2_cache_mapping
;
229 if (cp
->cpu_m
.cpu_mpipe
== NO_L2_CACHE_MAPPING_FOUND
)
230 cp
->cpu_m
.cpu_mpipe
= CPU_L2_CACHEID_INVALID
;
232 cp
->cpu_m
.cpu_chip
= cpunodes
[cp
->cpu_id
].l2_cache_mapping
;
233 if (cp
->cpu_m
.cpu_chip
== NO_L2_CACHE_MAPPING_FOUND
)
234 cp
->cpu_m
.cpu_chip
= CPU_CHIPID_INVALID
;
240 cpu_init_private(struct cpu
*cp
)
242 extern void niagara_kstat_init(void);
244 ASSERT(MUTEX_HELD(&cpu_lock
));
246 cpu_map_exec_units(cp
);
248 if ((cpucnt
++ == 0) && (cpu_hsvc_available
== B_TRUE
))
249 (void) niagara_kstat_init();
251 mutex_delay
= rdccr_delay
;
256 cpu_uninit_private(struct cpu
*cp
)
258 extern void niagara_kstat_fini(void);
260 ASSERT(MUTEX_HELD(&cpu_lock
));
261 if ((--cpucnt
== 0) && (cpu_hsvc_available
== B_TRUE
))
262 (void) niagara_kstat_fini();
266 * On Niagara2, any flush will cause all preceding stores to be
267 * synchronized wrt the i$, regardless of address or ASI. In fact,
268 * the address is ignored, so we always flush address 0.
272 dtrace_flush_sec(uintptr_t addr
)
278 * Trapstat support for Niagara2 processor
279 * The Niagara2 provides HWTW support for TSB lookup and with HWTW
280 * enabled no TSB hit information will be available. Therefore setting
281 * the time spent in TLB miss handler for TSB hits to 0.
284 cpu_trapstat_conf(int cmd
)
289 case CPU_TSTATCONF_INIT
:
290 case CPU_TSTATCONF_FINI
:
291 case CPU_TSTATCONF_ENABLE
:
292 case CPU_TSTATCONF_DISABLE
:
302 cpu_trapstat_data(void *buf
, uint_t tstat_pgszs
)
304 tstat_pgszdata_t
*tstatp
= (tstat_pgszdata_t
*)buf
;
307 for (i
= 0; i
< tstat_pgszs
; i
++, tstatp
++) {
308 tstatp
->tpgsz_kernel
.tmode_itlb
.ttlb_tlb
.tmiss_count
= 0;
309 tstatp
->tpgsz_kernel
.tmode_itlb
.ttlb_tlb
.tmiss_time
= 0;
310 tstatp
->tpgsz_user
.tmode_itlb
.ttlb_tlb
.tmiss_count
= 0;
311 tstatp
->tpgsz_user
.tmode_itlb
.ttlb_tlb
.tmiss_time
= 0;
312 tstatp
->tpgsz_kernel
.tmode_dtlb
.ttlb_tlb
.tmiss_count
= 0;
313 tstatp
->tpgsz_kernel
.tmode_dtlb
.ttlb_tlb
.tmiss_time
= 0;
314 tstatp
->tpgsz_user
.tmode_dtlb
.ttlb_tlb
.tmiss_count
= 0;
315 tstatp
->tpgsz_user
.tmode_dtlb
.ttlb_tlb
.tmiss_time
= 0;
320 * Page coloring support for hashed cache index mode
324 * Node id bits from machine description (MD). Node id distinguishes
325 * local versus remote memory. Because of MPO, page allocation does
326 * not cross node boundaries. Therefore, remove the node id bits from
327 * the color, since they are fixed. Either bit 30, or 31:30 in
328 * Victoria Falls processors.
329 * The number of node id bits is always 0 in Niagara2.
331 typedef struct n2color
{
332 uchar_t nnbits
; /* number of node id bits */
333 uchar_t nnmask
; /* mask for node id bits */
334 uchar_t lomask
; /* mask for bits below node id */
335 uchar_t lobits
; /* number of bits below node id */
338 n2color_t n2color
[MMU_PAGE_SIZES
];
339 static uchar_t nhbits
[] = {7, 7, 6, 5, 5, 5};
342 * Remove node id bits from color bits 32:28.
343 * This will reduce the number of colors.
344 * No change if number of node bits is zero.
347 n2_hash2color(uint_t color
, uchar_t szc
)
349 n2color_t m
= n2color
[szc
];
352 color
= ((color
>> m
.nnbits
) & ~m
.lomask
) | (color
& m
.lomask
);
353 ASSERT((color
& ~(hw_page_array
[szc
].hp_colors
- 1)) == 0);
360 * Restore node id bits into page color.
361 * This will increase the number of colors to match N2.
362 * No change if number of node bits is zero.
365 n2_color2hash(uint_t color
, uchar_t szc
, uint_t node
)
367 n2color_t m
= n2color
[szc
];
370 color
= ((color
& ~m
.lomask
) << m
.nnbits
) | (color
& m
.lomask
);
371 color
|= (node
& m
.nnmask
) << m
.lobits
;
377 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */
380 * iterator NULL means pfn is VA, do not adjust ra_to_pa
381 * iterator (-1) means pfn is RA, need to convert to PA
382 * iterator non-null means pfn is RA, use ra_to_pa
385 page_pfn_2_color_cpu(pfn_t pfn
, uchar_t szc
, void *cookie
)
387 mem_node_iterator_t
*it
= cookie
;
390 ASSERT(szc
<= TTE256M
);
392 if (it
== ((mem_node_iterator_t
*)(-1))) {
393 pfn
= plat_rapfn_to_papfn(pfn
);
394 } else if (it
!= NULL
) {
395 ASSERT(pfn
>= it
->mi_mblock_base
&& pfn
<= it
->mi_mblock_end
);
396 pfn
= pfn
+ it
->mi_ra_to_pa
;
398 pfn
= PFN_BASE(pfn
, szc
);
399 color
= ((pfn
>> 15) ^ pfn
) & 0x1f;
402 color
= (color
<< 2) | ((pfn
>> 5) & 0x3);
404 color
>>= 1; /* 19 */
406 return (n2_hash2color(color
, szc
));
410 page_papfn_2_color_cpu(pfn_t papfn
, uchar_t szc
)
414 ASSERT(szc
<= TTE256M
);
416 papfn
= PFN_BASE(papfn
, szc
);
417 color
= ((papfn
>> 15) ^ papfn
) & 0x1f;
420 color
= (color
<< 2) | ((papfn
>> 5) & 0x3);
422 color
>>= 1; /* 19 */
428 #error TTE256M is not 5
432 page_get_nsz_color_mask_cpu(uchar_t szc
, uint_t mask
)
434 static uint_t ni2_color_masks
[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f};
435 ASSERT(szc
< TTE256M
);
436 mask
= n2_color2hash(mask
, szc
, 0);
437 mask
&= ni2_color_masks
[szc
];
438 if (szc
== TTE64K
|| szc
== TTE512K
)
440 return (n2_hash2color(mask
, szc
+ 1));
444 page_get_nsz_color_cpu(uchar_t szc
, uint_t color
)
446 ASSERT(szc
< TTE256M
);
447 color
= n2_color2hash(color
, szc
, 0);
448 if (szc
== TTE64K
|| szc
== TTE512K
)
450 return (n2_hash2color(color
, szc
+ 1));
454 page_get_color_shift_cpu(uchar_t szc
, uchar_t nszc
)
458 ASSERT(nszc
<= TTE256M
);
460 s
= nhbits
[szc
] - n2color
[szc
].nnbits
;
461 s
-= nhbits
[nszc
] - n2color
[nszc
].nnbits
;
467 page_convert_color_cpu(uint_t ncolor
, uchar_t szc
, uchar_t nszc
)
472 ASSERT(nszc
<= TTE256M
);
473 ncolor
= n2_color2hash(ncolor
, nszc
, 0);
474 color
= ncolor
<< (nhbits
[szc
] - nhbits
[nszc
]);
475 color
= n2_hash2color(color
, szc
);
479 #define PAPFN_2_MNODE(pfn) \
480 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift)
484 page_next_pfn_for_color_cpu(pfn_t pfn
, uchar_t szc
, uint_t color
,
485 uint_t ceq_mask
, uint_t color_mask
, void *cookie
)
487 mem_node_iterator_t
*it
= cookie
;
488 pfn_t pstep
= PNUM_SIZE(szc
);
489 pfn_t npfn
, pfn_ceq_mask
, pfn_color
;
490 pfn_t tmpmask
, mask
= (pfn_t
)-1;
493 ASSERT((color
& ~ceq_mask
) == 0);
494 ASSERT(pfn
>= it
->mi_mblock_base
&& pfn
<= it
->mi_mblock_end
);
496 /* convert RA to PA for accurate color calculation */
498 /* first call after it, so cache these values */
499 it
->mi_hash_ceq_mask
=
500 n2_color2hash(ceq_mask
, szc
, it
->mi_mnode_mask
);
502 n2_color2hash(color
, szc
, it
->mi_mnode
);
505 ASSERT(it
->mi_hash_ceq_mask
==
506 n2_color2hash(ceq_mask
, szc
, it
->mi_mnode_mask
));
507 ASSERT(it
->mi_hash_color
==
508 n2_color2hash(color
, szc
, it
->mi_mnode
));
510 ceq_mask
= it
->mi_hash_ceq_mask
;
511 color
= it
->mi_hash_color
;
512 pfn
+= it
->mi_ra_to_pa
;
514 /* restart here when we switch memblocks */
516 pfnmn
= PAPFN_2_MNODE(pfn
);
517 if ((((page_papfn_2_color_cpu(pfn
, szc
) ^ color
) & ceq_mask
) == 0) &&
518 (pfnmn
== it
->mi_mnode
)) {
520 /* we start from the page with correct color and mnode */
521 if (szc
>= TTE512K
) {
523 /* page color is PA[32:28] */
524 pfn_ceq_mask
= ceq_mask
<< 15;
526 /* page color is PA[32:28].PA[19:19] */
527 pfn_ceq_mask
= ((ceq_mask
& 1) << 6) |
528 ((ceq_mask
>> 1) << 15);
531 * Preserve mnode bits in case they are not part of the
532 * color mask (eg., 8GB interleave, mnode bits 34:33).
534 pfn_ceq_mask
|= it
->mi_mnode_pfn_mask
;
535 npfn
= ADD_MASKED(pfn
, pstep
, pfn_ceq_mask
, mask
);
539 * We deal 64K or 8K page. Check if we could the
540 * satisfy the request without changing PA[32:28]
542 pfn_ceq_mask
= ((ceq_mask
& 3) << 5) | (ceq_mask
>> 2);
543 pfn_ceq_mask
|= it
->mi_mnode_pfn_mask
;
544 npfn
= ADD_MASKED(pfn
, pstep
, pfn_ceq_mask
, mask
);
546 if ((((npfn
^ pfn
) >> 15) & 0x1f) == 0)
550 * for next pfn we have to change bits PA[32:28]
551 * set PA[63:28] and PA[19:18] of the next pfn
553 npfn
= (pfn
>> 15) << 15;
554 npfn
|= (ceq_mask
& color
& 3) << 5;
555 pfn_ceq_mask
= (szc
== TTE8K
) ? 0 :
556 (ceq_mask
& 0x1c) << 13;
557 pfn_ceq_mask
|= it
->mi_mnode_pfn_mask
;
558 npfn
= ADD_MASKED(npfn
, (1 << 15), pfn_ceq_mask
, mask
);
561 * set bits PA[17:13] to match the color
563 npfn
|= ((npfn
>> 15) ^ (color
>> 2)) & (ceq_mask
>> 2);
569 * we start from the page with incorrect color - rare case
571 if (szc
>= TTE512K
) {
573 /* page color is in bits PA[32:28] */
574 npfn
= ((pfn
>> 20) << 20) | (color
<< 15);
575 pfn_ceq_mask
= (ceq_mask
<< 15) | 0x7fff;
577 /* try get the right color by changing bit PA[19:19] */
579 pfnmn
= PAPFN_2_MNODE(npfn
);
580 if ((((page_papfn_2_color_cpu(npfn
, szc
) ^ color
) &
581 ceq_mask
) == 0) && (pfnmn
== it
->mi_mnode
))
584 /* page color is PA[32:28].PA[19:19] */
585 pfn_ceq_mask
= ((ceq_mask
& 1) << 6) |
586 ((ceq_mask
>> 1) << 15) | (0xff << 7);
587 pfn_color
= ((color
& 1) << 6) | ((color
>> 1) << 15);
588 npfn
= ((pfn
>> 20) << 20) | pfn_color
;
591 /* Fix mnode if necessary */
592 if ((pfnmn
= PAPFN_2_MNODE(npfn
)) != it
->mi_mnode
)
593 npfn
+= ((it
->mi_mnode
- pfnmn
) & it
->mi_mnode_mask
) <<
594 it
->mi_mnode_pfn_shift
;
597 * Preserve mnode bits in case they are not part of the color
598 * mask eg 8GB interleave, mnode bits 34:33).
600 pfn_ceq_mask
|= it
->mi_mnode_pfn_mask
;
601 while (npfn
<= pfn
) {
602 npfn
= ADD_MASKED(npfn
, pstep
, pfn_ceq_mask
, mask
);
608 * We deal 64K or 8K page of incorrect color.
609 * Try correcting color without changing PA[32:28]
611 pfn_ceq_mask
= ((ceq_mask
& 3) << 5) | (ceq_mask
>> 2);
612 pfn_color
= ((color
& 3) << 5) | (color
>> 2);
613 if (pfnmn
== it
->mi_mnode
) {
614 npfn
= (pfn
& ~(pfn_t
)0x7f);
615 npfn
|= (((pfn
>> 15) & 0x1f) ^ pfn_color
) & pfn_ceq_mask
;
616 npfn
= (szc
== TTE64K
) ? (npfn
& ~(pfn_t
)0x7) : npfn
;
618 if (((page_papfn_2_color_cpu(npfn
, szc
) ^ color
) &
620 /* the color is fixed - find the next page */
621 pfn_ceq_mask
|= it
->mi_mnode_pfn_mask
;
622 while (npfn
<= pfn
) {
623 npfn
= ADD_MASKED(npfn
, pstep
, pfn_ceq_mask
,
626 if ((((npfn
^ pfn
) >> 15) & 0x1f) == 0)
631 /* to fix the color need to touch PA[32:28] */
632 npfn
= (szc
== TTE8K
) ? ((pfn
>> 15) << 15) :
633 (((pfn
>> 18) << 18) | ((color
& 0x1c) << 13));
635 /* fix mnode if input pfn is in the wrong mnode. */
636 if ((pfnmn
= PAPFN_2_MNODE(npfn
)) != it
->mi_mnode
) {
637 npfn
+= ((it
->mi_mnode
- pfnmn
) & it
->mi_mnode_mask
) <<
638 it
->mi_mnode_pfn_shift
;
641 tmpmask
= (szc
== TTE8K
) ? 0 : (ceq_mask
& 0x1c) << 13;
642 tmpmask
|= it
->mi_mnode_pfn_mask
;
644 while (npfn
<= pfn
) {
645 npfn
= ADD_MASKED(npfn
, (1 << 15), tmpmask
, mask
);
648 /* set bits PA[19:13] to match the color */
649 npfn
|= (((npfn
>> 15) & 0x1f) ^ pfn_color
) & pfn_ceq_mask
;
650 npfn
= (szc
== TTE64K
) ? (npfn
& ~(pfn_t
)0x7) : npfn
;
653 ASSERT(((page_papfn_2_color_cpu(npfn
, szc
) ^ color
) & ceq_mask
) == 0);
654 ASSERT(PAPFN_2_MNODE(npfn
) == it
->mi_mnode
);
657 npfn
-= it
->mi_ra_to_pa
;
659 /* check for possible memblock switch */
660 if (npfn
> it
->mi_mblock_end
) {
661 pfn
= plat_mem_node_iterator_init(npfn
, it
->mi_mnode
, szc
, it
,
663 if (pfn
== (pfn_t
)-1)
665 ASSERT(pfn
>= it
->mi_mblock_base
&& pfn
<= it
->mi_mblock_end
);
666 pfn
+= it
->mi_ra_to_pa
;
675 * VF encodes node_id for an L-group in either bit 30 or 31:30,
676 * which effectively reduces the number of colors available per mnode.
679 page_coloring_init_cpu()
686 mem_node_iterator_t it
;
687 static uchar_t idmask
[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f};
689 for (i
= 0; i
< max_mem_nodes
; i
++) {
690 memset(&it
, 0, sizeof (it
));
691 if (plat_mem_node_iterator_init(0, i
, 0, &it
, 1) != (pfn_t
)-1)
694 ASSERT(i
< max_mem_nodes
);
695 for (i
= 0; i
< mmu_page_sizes
; i
++) {
696 (void) memset(&m
, 0, sizeof (m
));
697 id
= it
.mi_mnode_pfn_mask
>> 15; /* node id mask */
702 m
.nnbits
= hi
- lo
+ 1;
703 m
.nnmask
= (1 << m
.nnbits
) - 1;
705 m
.lomask
= (1 << (lo
- 1)) - 1;
708 hw_page_array
[i
].hp_colors
= 1 << (nhbits
[i
] - m
.nnbits
);
714 * group colorequiv colors on N2 by low order bits of the color first
717 page_set_colorequiv_arr_cpu(void)
719 static uint_t nequiv_shades_log2
[MMU_PAGE_SIZES
] = {2, 5, 0, 0, 0, 0};
721 nequiv_shades_log2
[1] -= n2color
[1].nnbits
;
722 if (colorequiv
> 1) {
724 uint_t sv_a
= lowbit(colorequiv
) - 1;
729 for (i
= 0; i
< MMU_PAGE_SIZES
; i
++) {
733 if ((colors
= hw_page_array
[i
].hp_colors
) <= 1)
735 while ((colors
>> a
) == 0)
737 if (a
> (colorequivszc
[i
] & 0xf) +
738 (colorequivszc
[i
] >> 4)) {
739 if (a
<= nequiv_shades_log2
[i
]) {
740 colorequivszc
[i
] = (uchar_t
)a
;
743 ((a
- nequiv_shades_log2
[i
]) << 4) |
744 nequiv_shades_log2
[i
];