1 /* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
2 because MTRRs can span upto 40 bits (36bits on most modern x86) */
3 #include <linux/init.h>
4 #include <linux/slab.h>
6 #include <linux/module.h>
10 #include <asm/system.h>
11 #include <asm/cpufeature.h>
12 #include <asm/tlbflush.h>
16 struct mtrr_var_range
*var_ranges
;
17 mtrr_type fixed_ranges
[NUM_FIXED_RANGES
];
18 unsigned char enabled
;
19 unsigned char have_fixed
;
23 struct fixed_range_block
{
24 int base_msr
; /* start address of an MTRR block */
25 int ranges
; /* number of MTRRs in this block */
28 static struct fixed_range_block fixed_range_blocks
[] = {
29 { MTRRfix64K_00000_MSR
, 1 }, /* one 64k MTRR */
30 { MTRRfix16K_80000_MSR
, 2 }, /* two 16k MTRRs */
31 { MTRRfix4K_C0000_MSR
, 8 }, /* eight 4k MTRRs */
35 static unsigned long smp_changes_mask
;
36 static struct mtrr_state mtrr_state
= {};
38 #undef MODULE_PARAM_PREFIX
39 #define MODULE_PARAM_PREFIX "mtrr."
41 static __initdata
int mtrr_show
;
42 module_param_named(show
, mtrr_show
, bool, 0);
44 /* Get the MSR pair relating to a var range */
46 get_mtrr_var_range(unsigned int index
, struct mtrr_var_range
*vr
)
48 rdmsr(MTRRphysBase_MSR(index
), vr
->base_lo
, vr
->base_hi
);
49 rdmsr(MTRRphysMask_MSR(index
), vr
->mask_lo
, vr
->mask_hi
);
53 get_fixed_ranges(mtrr_type
* frs
)
55 unsigned int *p
= (unsigned int *) frs
;
58 rdmsr(MTRRfix64K_00000_MSR
, p
[0], p
[1]);
60 for (i
= 0; i
< 2; i
++)
61 rdmsr(MTRRfix16K_80000_MSR
+ i
, p
[2 + i
* 2], p
[3 + i
* 2]);
62 for (i
= 0; i
< 8; i
++)
63 rdmsr(MTRRfix4K_C0000_MSR
+ i
, p
[6 + i
* 2], p
[7 + i
* 2]);
66 void mtrr_save_fixed_ranges(void *info
)
68 get_fixed_ranges(mtrr_state
.fixed_ranges
);
71 static void __init
print_fixed(unsigned base
, unsigned step
, const mtrr_type
*types
)
75 for (i
= 0; i
< 8; ++i
, ++types
, base
+= step
)
76 printk(KERN_INFO
"MTRR %05X-%05X %s\n", base
, base
+ step
- 1, mtrr_attrib_to_str(*types
));
79 /* Grab all of the MTRR state for this CPU into *state */
80 void __init
get_mtrr_state(void)
83 struct mtrr_var_range
*vrs
;
86 if (!mtrr_state
.var_ranges
) {
87 mtrr_state
.var_ranges
= kmalloc(num_var_ranges
* sizeof (struct mtrr_var_range
),
89 if (!mtrr_state
.var_ranges
)
92 vrs
= mtrr_state
.var_ranges
;
94 rdmsr(MTRRcap_MSR
, lo
, dummy
);
95 mtrr_state
.have_fixed
= (lo
>> 8) & 1;
97 for (i
= 0; i
< num_var_ranges
; i
++)
98 get_mtrr_var_range(i
, &vrs
[i
]);
99 if (mtrr_state
.have_fixed
)
100 get_fixed_ranges(mtrr_state
.fixed_ranges
);
102 rdmsr(MTRRdefType_MSR
, lo
, dummy
);
103 mtrr_state
.def_type
= (lo
& 0xff);
104 mtrr_state
.enabled
= (lo
& 0xc00) >> 10;
109 printk(KERN_INFO
"MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state
.def_type
));
110 if (mtrr_state
.have_fixed
) {
111 printk(KERN_INFO
"MTRR fixed ranges %sabled:\n",
112 mtrr_state
.enabled
& 1 ? "en" : "dis");
113 print_fixed(0x00000, 0x10000, mtrr_state
.fixed_ranges
+ 0);
114 for (i
= 0; i
< 2; ++i
)
115 print_fixed(0x80000 + i
* 0x20000, 0x04000, mtrr_state
.fixed_ranges
+ (i
+ 1) * 8);
116 for (i
= 0; i
< 8; ++i
)
117 print_fixed(0xC0000 + i
* 0x08000, 0x01000, mtrr_state
.fixed_ranges
+ (i
+ 3) * 8);
119 printk(KERN_INFO
"MTRR variable ranges %sabled:\n",
120 mtrr_state
.enabled
& 2 ? "en" : "dis");
121 high_width
= ((size_or_mask
? ffs(size_or_mask
) - 1 : 32) - (32 - PAGE_SHIFT
) + 3) / 4;
122 for (i
= 0; i
< num_var_ranges
; ++i
) {
123 if (mtrr_state
.var_ranges
[i
].mask_lo
& (1 << 11))
124 printk(KERN_INFO
"MTRR %u base %0*X%05X000 mask %0*X%05X000 %s\n",
127 mtrr_state
.var_ranges
[i
].base_hi
,
128 mtrr_state
.var_ranges
[i
].base_lo
>> 12,
130 mtrr_state
.var_ranges
[i
].mask_hi
,
131 mtrr_state
.var_ranges
[i
].mask_lo
>> 12,
132 mtrr_attrib_to_str(mtrr_state
.var_ranges
[i
].base_lo
& 0xff));
134 printk(KERN_INFO
"MTRR %u disabled\n", i
);
139 /* Some BIOS's are fucked and don't set all MTRRs the same! */
140 void __init
mtrr_state_warn(void)
142 unsigned long mask
= smp_changes_mask
;
146 if (mask
& MTRR_CHANGE_MASK_FIXED
)
147 printk(KERN_WARNING
"mtrr: your CPUs had inconsistent fixed MTRR settings\n");
148 if (mask
& MTRR_CHANGE_MASK_VARIABLE
)
149 printk(KERN_WARNING
"mtrr: your CPUs had inconsistent variable MTRR settings\n");
150 if (mask
& MTRR_CHANGE_MASK_DEFTYPE
)
151 printk(KERN_WARNING
"mtrr: your CPUs had inconsistent MTRRdefType settings\n");
152 printk(KERN_INFO
"mtrr: probably your BIOS does not setup all CPUs.\n");
153 printk(KERN_INFO
"mtrr: corrected configuration.\n");
156 /* Doesn't attempt to pass an error out to MTRR users
157 because it's quite complicated in some cases and probably not
158 worth it because the best error handling is to ignore it. */
159 void mtrr_wrmsr(unsigned msr
, unsigned a
, unsigned b
)
161 if (wrmsr_safe(msr
, a
, b
) < 0)
163 "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
164 smp_processor_id(), msr
, a
, b
);
168 * Enable and allow read/write of extended fixed-range MTRR bits on K8 CPUs
169 * see AMD publication no. 24593, chapter 3.2.1 for more information
171 static inline void k8_enable_fixed_iorrs(void)
175 rdmsr(MSR_K8_SYSCFG
, lo
, hi
);
176 mtrr_wrmsr(MSR_K8_SYSCFG
, lo
177 | K8_MTRRFIXRANGE_DRAM_ENABLE
178 | K8_MTRRFIXRANGE_DRAM_MODIFY
, hi
);
182 * Checks and updates an fixed-range MTRR if it differs from the value it
183 * should have. If K8 extenstions are wanted, update the K8 SYSCFG MSR also.
184 * see AMD publication no. 24593, chapter 7.8.1, page 233 for more information
185 * \param msr MSR address of the MTTR which should be checked and updated
186 * \param changed pointer which indicates whether the MTRR needed to be changed
187 * \param msrwords pointer to the MSR values which the MSR should have
189 static void set_fixed_range(int msr
, int * changed
, unsigned int * msrwords
)
195 if (lo
!= msrwords
[0] || hi
!= msrwords
[1]) {
196 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_AMD
&&
197 boot_cpu_data
.x86
== 15 &&
198 ((msrwords
[0] | msrwords
[1]) & K8_MTRR_RDMEM_WRMEM_MASK
))
199 k8_enable_fixed_iorrs();
200 mtrr_wrmsr(msr
, msrwords
[0], msrwords
[1]);
205 int generic_get_free_region(unsigned long base
, unsigned long size
, int replace_reg
)
206 /* [SUMMARY] Get a free MTRR.
207 <base> The starting (base) address of the region.
208 <size> The size (in bytes) of the region.
209 [RETURNS] The index of the region on success, else -1 on error.
214 unsigned long lbase
, lsize
;
216 max
= num_var_ranges
;
217 if (replace_reg
>= 0 && replace_reg
< max
)
219 for (i
= 0; i
< max
; ++i
) {
220 mtrr_if
->get(i
, &lbase
, &lsize
, <ype
);
227 static void generic_get_mtrr(unsigned int reg
, unsigned long *base
,
228 unsigned long *size
, mtrr_type
*type
)
230 unsigned int mask_lo
, mask_hi
, base_lo
, base_hi
;
232 rdmsr(MTRRphysMask_MSR(reg
), mask_lo
, mask_hi
);
233 if ((mask_lo
& 0x800) == 0) {
234 /* Invalid (i.e. free) range */
241 rdmsr(MTRRphysBase_MSR(reg
), base_lo
, base_hi
);
243 /* Work out the shifted address mask. */
244 mask_lo
= size_or_mask
| mask_hi
<< (32 - PAGE_SHIFT
)
245 | mask_lo
>> PAGE_SHIFT
;
247 /* This works correctly if size is a power of two, i.e. a
250 *base
= base_hi
<< (32 - PAGE_SHIFT
) | base_lo
>> PAGE_SHIFT
;
251 *type
= base_lo
& 0xff;
255 * Checks and updates the fixed-range MTRRs if they differ from the saved set
256 * \param frs pointer to fixed-range MTRR values, saved by get_fixed_ranges()
258 static int set_fixed_ranges(mtrr_type
* frs
)
260 unsigned long long *saved
= (unsigned long long *) frs
;
264 while (fixed_range_blocks
[++block
].ranges
)
265 for (range
=0; range
< fixed_range_blocks
[block
].ranges
; range
++)
266 set_fixed_range(fixed_range_blocks
[block
].base_msr
+ range
,
267 &changed
, (unsigned int *) saved
++);
272 /* Set the MSR pair relating to a var range. Returns TRUE if
274 static int set_mtrr_var_ranges(unsigned int index
, struct mtrr_var_range
*vr
)
279 rdmsr(MTRRphysBase_MSR(index
), lo
, hi
);
280 if ((vr
->base_lo
& 0xfffff0ffUL
) != (lo
& 0xfffff0ffUL
)
281 || (vr
->base_hi
& (size_and_mask
>> (32 - PAGE_SHIFT
))) !=
282 (hi
& (size_and_mask
>> (32 - PAGE_SHIFT
)))) {
283 mtrr_wrmsr(MTRRphysBase_MSR(index
), vr
->base_lo
, vr
->base_hi
);
287 rdmsr(MTRRphysMask_MSR(index
), lo
, hi
);
289 if ((vr
->mask_lo
& 0xfffff800UL
) != (lo
& 0xfffff800UL
)
290 || (vr
->mask_hi
& (size_and_mask
>> (32 - PAGE_SHIFT
))) !=
291 (hi
& (size_and_mask
>> (32 - PAGE_SHIFT
)))) {
292 mtrr_wrmsr(MTRRphysMask_MSR(index
), vr
->mask_lo
, vr
->mask_hi
);
298 static u32 deftype_lo
, deftype_hi
;
300 static unsigned long set_mtrr_state(void)
301 /* [SUMMARY] Set the MTRR state for this CPU.
302 <state> The MTRR state information to read.
303 <ctxt> Some relevant CPU context.
304 [NOTE] The CPU must already be in a safe state for MTRR changes.
305 [RETURNS] 0 if no changes made, else a mask indication what was changed.
309 unsigned long change_mask
= 0;
311 for (i
= 0; i
< num_var_ranges
; i
++)
312 if (set_mtrr_var_ranges(i
, &mtrr_state
.var_ranges
[i
]))
313 change_mask
|= MTRR_CHANGE_MASK_VARIABLE
;
315 if (mtrr_state
.have_fixed
&& set_fixed_ranges(mtrr_state
.fixed_ranges
))
316 change_mask
|= MTRR_CHANGE_MASK_FIXED
;
318 /* Set_mtrr_restore restores the old value of MTRRdefType,
319 so to set it we fiddle with the saved value */
320 if ((deftype_lo
& 0xff) != mtrr_state
.def_type
321 || ((deftype_lo
& 0xc00) >> 10) != mtrr_state
.enabled
) {
322 deftype_lo
= (deftype_lo
& ~0xcff) | mtrr_state
.def_type
| (mtrr_state
.enabled
<< 10);
323 change_mask
|= MTRR_CHANGE_MASK_DEFTYPE
;
330 static unsigned long cr4
= 0;
331 static DEFINE_SPINLOCK(set_atomicity_lock
);
334 * Since we are disabling the cache don't allow any interrupts - they
335 * would run extremely slow and would only increase the pain. The caller must
336 * ensure that local interrupts are disabled and are reenabled after post_set()
340 static void prepare_set(void) __acquires(set_atomicity_lock
)
344 /* Note that this is not ideal, since the cache is only flushed/disabled
345 for this CPU while the MTRRs are changed, but changing this requires
346 more invasive changes to the way the kernel boots */
348 spin_lock(&set_atomicity_lock
);
350 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
351 cr0
= read_cr0() | 0x40000000; /* set CD flag */
355 /* Save value of CR4 and clear Page Global Enable (bit 7) */
358 write_cr4(cr4
& ~X86_CR4_PGE
);
361 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
364 /* Save MTRR state */
365 rdmsr(MTRRdefType_MSR
, deftype_lo
, deftype_hi
);
367 /* Disable MTRRs, and set the default type to uncached */
368 mtrr_wrmsr(MTRRdefType_MSR
, deftype_lo
& ~0xcff, deftype_hi
);
371 static void post_set(void) __releases(set_atomicity_lock
)
373 /* Flush TLBs (no need to flush caches - they are disabled) */
376 /* Intel (P6) standard MTRRs */
377 mtrr_wrmsr(MTRRdefType_MSR
, deftype_lo
, deftype_hi
);
380 write_cr0(read_cr0() & 0xbfffffff);
382 /* Restore value of CR4 */
385 spin_unlock(&set_atomicity_lock
);
388 static void generic_set_all(void)
390 unsigned long mask
, count
;
393 local_irq_save(flags
);
396 /* Actually set the state */
397 mask
= set_mtrr_state();
400 local_irq_restore(flags
);
402 /* Use the atomic bitops to update the global mask */
403 for (count
= 0; count
< sizeof mask
* 8; ++count
) {
405 set_bit(count
, &smp_changes_mask
);
411 static void generic_set_mtrr(unsigned int reg
, unsigned long base
,
412 unsigned long size
, mtrr_type type
)
413 /* [SUMMARY] Set variable MTRR register on the local CPU.
414 <reg> The register to set.
415 <base> The base address of the region.
416 <size> The size of the region. If this is 0 the region is disabled.
417 <type> The type of the region.
418 <do_safe> If TRUE, do the change safely. If FALSE, safety measures should
424 struct mtrr_var_range
*vr
;
426 vr
= &mtrr_state
.var_ranges
[reg
];
428 local_irq_save(flags
);
432 /* The invalid bit is kept in the mask, so we simply clear the
433 relevant mask register to disable a range. */
434 mtrr_wrmsr(MTRRphysMask_MSR(reg
), 0, 0);
435 memset(vr
, 0, sizeof(struct mtrr_var_range
));
437 vr
->base_lo
= base
<< PAGE_SHIFT
| type
;
438 vr
->base_hi
= (base
& size_and_mask
) >> (32 - PAGE_SHIFT
);
439 vr
->mask_lo
= -size
<< PAGE_SHIFT
| 0x800;
440 vr
->mask_hi
= (-size
& size_and_mask
) >> (32 - PAGE_SHIFT
);
442 mtrr_wrmsr(MTRRphysBase_MSR(reg
), vr
->base_lo
, vr
->base_hi
);
443 mtrr_wrmsr(MTRRphysMask_MSR(reg
), vr
->mask_lo
, vr
->mask_hi
);
447 local_irq_restore(flags
);
450 int generic_validate_add_page(unsigned long base
, unsigned long size
, unsigned int type
)
452 unsigned long lbase
, last
;
454 /* For Intel PPro stepping <= 7, must be 4 MiB aligned
455 and not touch 0x70000000->0x7003FFFF */
456 if (is_cpu(INTEL
) && boot_cpu_data
.x86
== 6 &&
457 boot_cpu_data
.x86_model
== 1 &&
458 boot_cpu_data
.x86_mask
<= 7) {
459 if (base
& ((1 << (22 - PAGE_SHIFT
)) - 1)) {
460 printk(KERN_WARNING
"mtrr: base(0x%lx000) is not 4 MiB aligned\n", base
);
463 if (!(base
+ size
< 0x70000 || base
> 0x7003F) &&
464 (type
== MTRR_TYPE_WRCOMB
465 || type
== MTRR_TYPE_WRBACK
)) {
466 printk(KERN_WARNING
"mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
472 printk(KERN_WARNING
"mtrr: cannot set region below 1 MiB (0x%lx000,0x%lx000)\n",
476 /* Check upper bits of base and last are equal and lower bits are 0
477 for base and 1 for last */
478 last
= base
+ size
- 1;
479 for (lbase
= base
; !(lbase
& 1) && (last
& 1);
480 lbase
= lbase
>> 1, last
= last
>> 1) ;
482 printk(KERN_WARNING
"mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
490 static int generic_have_wrcomb(void)
492 unsigned long config
, dummy
;
493 rdmsr(MTRRcap_MSR
, config
, dummy
);
494 return (config
& (1 << 10));
497 int positive_have_wrcomb(void)
502 /* generic structure...
504 struct mtrr_ops generic_mtrr_ops
= {
506 .set_all
= generic_set_all
,
507 .get
= generic_get_mtrr
,
508 .get_free_region
= generic_get_free_region
,
509 .set
= generic_set_mtrr
,
510 .validate_add_page
= generic_validate_add_page
,
511 .have_wrcomb
= generic_have_wrcomb
,