1 /* Generic MTRR (Memory Type Range Register) driver.
3 Copyright (C) 1997-2000 Richard Gooch
4 Copyright (c) 2002 Patrick Mochel
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Library General Public
8 License as published by the Free Software Foundation; either
9 version 2 of the License, or (at your option) any later version.
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Library General Public License for more details.
16 You should have received a copy of the GNU Library General Public
17 License along with this library; if not, write to the Free
18 Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 Richard Gooch may be reached by email at rgooch@atnf.csiro.au
21 The postal address is:
22 Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
24 Source: "Pentium Pro Family Developer's Manual, Volume 3:
25 Operating System Writer's Guide" (Intel document number 242692),
28 This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
30 Source: Intel Architecture Software Developers Manual, Volume 3:
31 System Programming Guide; Section 9.11. (1997 edition - PPro).
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/pci.h>
37 #include <linux/smp.h>
38 #include <linux/cpu.h>
42 #include <asm/uaccess.h>
43 #include <asm/processor.h>
47 #define MTRR_VERSION "2.0 (20020519)"
49 u32 num_var_ranges
= 0;
51 unsigned int *usage_table
;
52 static DECLARE_MUTEX(main_lock
);
54 u32 size_or_mask
, size_and_mask
;
56 static struct mtrr_ops
* mtrr_ops
[X86_VENDOR_NUM
] = {};
58 struct mtrr_ops
* mtrr_if
= NULL
;
60 static void set_mtrr(unsigned int reg
, unsigned long base
,
61 unsigned long size
, mtrr_type type
);
63 extern int arr3_protected
;
65 void set_mtrr_ops(struct mtrr_ops
* ops
)
67 if (ops
->vendor
&& ops
->vendor
< X86_VENDOR_NUM
)
68 mtrr_ops
[ops
->vendor
] = ops
;
71 /* Returns non-zero if we have the write-combining memory type */
72 static int have_wrcomb(void)
77 if ((dev
= pci_get_class(PCI_CLASS_BRIDGE_HOST
<< 8, NULL
)) != NULL
) {
78 /* ServerWorks LE chipsets < rev 6 have problems with write-combining
79 Don't allow it and leave room for other chipsets to be tagged */
80 if (dev
->vendor
== PCI_VENDOR_ID_SERVERWORKS
&&
81 dev
->device
== PCI_DEVICE_ID_SERVERWORKS_LE
) {
82 pci_read_config_byte(dev
, PCI_CLASS_REVISION
, &rev
);
84 printk(KERN_INFO
"mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
89 /* Intel 450NX errata # 23. Non ascending cacheline evictions to
90 write combining memory may resulting in data corruption */
91 if (dev
->vendor
== PCI_VENDOR_ID_INTEL
&&
92 dev
->device
== PCI_DEVICE_ID_INTEL_82451NX
) {
93 printk(KERN_INFO
"mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
99 return (mtrr_if
->have_wrcomb
? mtrr_if
->have_wrcomb() : 0);
102 /* This function returns the number of variable MTRRs */
103 static void __init
set_num_var_ranges(void)
105 unsigned long config
= 0, dummy
;
108 rdmsr(MTRRcap_MSR
, config
, dummy
);
109 } else if (is_cpu(AMD
))
111 else if (is_cpu(CYRIX
) || is_cpu(CENTAUR
))
113 num_var_ranges
= config
& 0xff;
116 static void __init
init_table(void)
120 max
= num_var_ranges
;
121 if ((usage_table
= kmalloc(max
* sizeof *usage_table
, GFP_KERNEL
))
123 printk(KERN_ERR
"mtrr: could not allocate\n");
126 for (i
= 0; i
< max
; i
++)
130 struct set_mtrr_data
{
133 unsigned long smp_base
;
134 unsigned long smp_size
;
135 unsigned int smp_reg
;
141 static void ipi_handler(void *info
)
142 /* [SUMMARY] Synchronisation handler. Executed by "other" CPUs.
146 struct set_mtrr_data
*data
= info
;
149 local_irq_save(flags
);
151 atomic_dec(&data
->count
);
152 while(!atomic_read(&data
->gate
))
155 /* The master has cleared me to execute */
156 if (data
->smp_reg
!= ~0U)
157 mtrr_if
->set(data
->smp_reg
, data
->smp_base
,
158 data
->smp_size
, data
->smp_type
);
162 atomic_dec(&data
->count
);
163 while(atomic_read(&data
->gate
))
166 atomic_dec(&data
->count
);
167 local_irq_restore(flags
);
173 * set_mtrr - update mtrrs on all processors
174 * @reg: mtrr in question
179 * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
181 * 1. Send IPI to do the following:
182 * 2. Disable Interrupts
183 * 3. Wait for all procs to do so
184 * 4. Enter no-fill cache mode
188 * 8. Disable all range registers
189 * 9. Update the MTRRs
190 * 10. Enable all range registers
191 * 11. Flush all TLBs and caches again
192 * 12. Enter normal cache mode and reenable caching
194 * 14. Wait for buddies to catch up
195 * 15. Enable interrupts.
197 * What does that mean for us? Well, first we set data.count to the number
198 * of CPUs. As each CPU disables interrupts, it'll decrement it once. We wait
199 * until it hits 0 and proceed. We set the data.gate flag and reset data.count.
200 * Meanwhile, they are waiting for that flag to be set. Once it's set, each
201 * CPU goes through the transition of updating MTRRs. The CPU vendors may each do it
202 * differently, so we call mtrr_if->set() callback and let them take care of it.
203 * When they're done, they again decrement data->count and wait for data.gate to
205 * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag.
206 * Everyone then enables interrupts and we all continue on.
208 * Note that the mechanism is the same for UP systems, too; all the SMP stuff
211 static void set_mtrr(unsigned int reg
, unsigned long base
,
212 unsigned long size
, mtrr_type type
)
214 struct set_mtrr_data data
;
218 data
.smp_base
= base
;
219 data
.smp_size
= size
;
220 data
.smp_type
= type
;
221 atomic_set(&data
.count
, num_booting_cpus() - 1);
222 atomic_set(&data
.gate
,0);
224 /* Start the ball rolling on other CPUs */
225 if (smp_call_function(ipi_handler
, &data
, 1, 0) != 0)
226 panic("mtrr: timed out waiting for other CPUs\n");
228 local_irq_save(flags
);
230 while(atomic_read(&data
.count
))
233 /* ok, reset count and toggle gate */
234 atomic_set(&data
.count
, num_booting_cpus() - 1);
235 atomic_set(&data
.gate
,1);
237 /* do our MTRR business */
240 * We use this same function to initialize the mtrrs on boot.
241 * The state of the boot cpu's mtrrs has been saved, and we want
242 * to replicate across all the APs.
243 * If we're doing that @reg is set to something special...
246 mtrr_if
->set(reg
,base
,size
,type
);
248 /* wait for the others */
249 while(atomic_read(&data
.count
))
252 atomic_set(&data
.count
, num_booting_cpus() - 1);
253 atomic_set(&data
.gate
,0);
256 * Wait here for everyone to have seen the gate change
257 * So we're the last ones to touch 'data'
259 while(atomic_read(&data
.count
))
262 local_irq_restore(flags
);
266 * mtrr_add_page - Add a memory type region
267 * @base: Physical base address of region in pages (4 KB)
268 * @size: Physical size of region in pages (4 KB)
269 * @type: Type of MTRR desired
270 * @increment: If this is true do usage counting on the region
272 * Memory type region registers control the caching on newer Intel and
273 * non Intel processors. This function allows drivers to request an
274 * MTRR is added. The details and hardware specifics of each processor's
275 * implementation are hidden from the caller, but nevertheless the
276 * caller should expect to need to provide a power of two size on an
277 * equivalent power of two boundary.
279 * If the region cannot be added either because all regions are in use
280 * or the CPU cannot support it a negative value is returned. On success
281 * the register number for this entry is returned, but should be treated
284 * On a multiprocessor machine the changes are made to all processors.
285 * This is required on x86 by the Intel processors.
287 * The available types are
289 * %MTRR_TYPE_UNCACHABLE - No caching
291 * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
293 * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
295 * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
297 * BUGS: Needs a quiet flag for the cases where drivers do not mind
298 * failures and do not wish system log messages to be sent.
301 int mtrr_add_page(unsigned long base
, unsigned long size
,
302 unsigned int type
, char increment
)
313 if ((error
= mtrr_if
->validate_add_page(base
,size
,type
)))
316 if (type
>= MTRR_NUM_TYPES
) {
317 printk(KERN_WARNING
"mtrr: type: %u invalid\n", type
);
321 /* If the type is WC, check that this processor supports it */
322 if ((type
== MTRR_TYPE_WRCOMB
) && !have_wrcomb()) {
324 "mtrr: your processor doesn't support write-combining\n");
328 if (base
& size_or_mask
|| size
& size_or_mask
) {
329 printk(KERN_WARNING
"mtrr: base or size exceeds the MTRR width\n");
335 /* No CPU hotplug when we change MTRR entries */
337 /* Search for existing MTRR */
339 for (i
= 0; i
< num_var_ranges
; ++i
) {
340 mtrr_if
->get(i
, &lbase
, &lsize
, <ype
);
341 if (base
>= lbase
+ lsize
)
343 if ((base
< lbase
) && (base
+ size
<= lbase
))
345 /* At this point we know there is some kind of overlap/enclosure */
346 if ((base
< lbase
) || (base
+ size
> lbase
+ lsize
)) {
348 "mtrr: 0x%lx000,0x%lx000 overlaps existing"
349 " 0x%lx000,0x%x000\n", base
, size
, lbase
,
353 /* New region is enclosed by an existing region */
355 if (type
== MTRR_TYPE_UNCACHABLE
)
357 printk (KERN_WARNING
"mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
358 base
, size
, mtrr_attrib_to_str(ltype
),
359 mtrr_attrib_to_str(type
));
367 /* Search for an empty MTRR */
368 i
= mtrr_if
->get_free_region(base
, size
);
370 set_mtrr(i
, base
, size
, type
);
373 printk(KERN_INFO
"mtrr: no more MTRRs available\n");
377 unlock_cpu_hotplug();
381 static int mtrr_check(unsigned long base
, unsigned long size
)
383 if ((base
& (PAGE_SIZE
- 1)) || (size
& (PAGE_SIZE
- 1))) {
385 "mtrr: size and base must be multiples of 4 kiB\n");
387 "mtrr: size: 0x%lx base: 0x%lx\n", size
, base
);
395 * mtrr_add - Add a memory type region
396 * @base: Physical base address of region
397 * @size: Physical size of region
398 * @type: Type of MTRR desired
399 * @increment: If this is true do usage counting on the region
401 * Memory type region registers control the caching on newer Intel and
402 * non Intel processors. This function allows drivers to request an
403 * MTRR is added. The details and hardware specifics of each processor's
404 * implementation are hidden from the caller, but nevertheless the
405 * caller should expect to need to provide a power of two size on an
406 * equivalent power of two boundary.
408 * If the region cannot be added either because all regions are in use
409 * or the CPU cannot support it a negative value is returned. On success
410 * the register number for this entry is returned, but should be treated
413 * On a multiprocessor machine the changes are made to all processors.
414 * This is required on x86 by the Intel processors.
416 * The available types are
418 * %MTRR_TYPE_UNCACHABLE - No caching
420 * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
422 * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
424 * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
426 * BUGS: Needs a quiet flag for the cases where drivers do not mind
427 * failures and do not wish system log messages to be sent.
431 mtrr_add(unsigned long base
, unsigned long size
, unsigned int type
,
434 if (mtrr_check(base
, size
))
436 return mtrr_add_page(base
>> PAGE_SHIFT
, size
>> PAGE_SHIFT
, type
,
441 * mtrr_del_page - delete a memory type region
442 * @reg: Register returned by mtrr_add
443 * @base: Physical base address
444 * @size: Size of region
446 * If register is supplied then base and size are ignored. This is
447 * how drivers should call it.
449 * Releases an MTRR region. If the usage count drops to zero the
450 * register is freed and the region returns to default state.
451 * On success the register is returned, on failure a negative error
455 int mtrr_del_page(int reg
, unsigned long base
, unsigned long size
)
466 max
= num_var_ranges
;
467 /* No CPU hotplug when we change MTRR entries */
471 /* Search for existing MTRR */
472 for (i
= 0; i
< max
; ++i
) {
473 mtrr_if
->get(i
, &lbase
, &lsize
, <ype
);
474 if (lbase
== base
&& lsize
== size
) {
480 printk(KERN_DEBUG
"mtrr: no MTRR for %lx000,%lx000 found\n", base
,
486 printk(KERN_WARNING
"mtrr: register: %d too big\n", reg
);
489 if (is_cpu(CYRIX
) && !use_intel()) {
490 if ((reg
== 3) && arr3_protected
) {
491 printk(KERN_WARNING
"mtrr: ARR3 cannot be changed\n");
495 mtrr_if
->get(reg
, &lbase
, &lsize
, <ype
);
497 printk(KERN_WARNING
"mtrr: MTRR %d not used\n", reg
);
500 if (usage_table
[reg
] < 1) {
501 printk(KERN_WARNING
"mtrr: reg: %d has count=0\n", reg
);
504 if (--usage_table
[reg
] < 1)
505 set_mtrr(reg
, 0, 0, 0);
509 unlock_cpu_hotplug();
513 * mtrr_del - delete a memory type region
514 * @reg: Register returned by mtrr_add
515 * @base: Physical base address
516 * @size: Size of region
518 * If register is supplied then base and size are ignored. This is
519 * how drivers should call it.
521 * Releases an MTRR region. If the usage count drops to zero the
522 * register is freed and the region returns to default state.
523 * On success the register is returned, on failure a negative error
528 mtrr_del(int reg
, unsigned long base
, unsigned long size
)
530 if (mtrr_check(base
, size
))
532 return mtrr_del_page(reg
, base
>> PAGE_SHIFT
, size
>> PAGE_SHIFT
);
535 EXPORT_SYMBOL(mtrr_add
);
536 EXPORT_SYMBOL(mtrr_del
);
539 * These should be called implicitly, but we can't yet until all the initcall
542 extern void amd_init_mtrr(void);
543 extern void cyrix_init_mtrr(void);
544 extern void centaur_init_mtrr(void);
546 static void __init
init_ifs(void)
553 /* The suspend/resume methods are only for CPU without MTRR. CPU using generic
554 * MTRR driver doesn't require this
562 static struct mtrr_value
* mtrr_state
;
564 static int mtrr_save(struct sys_device
* sysdev
, u32 state
)
567 int size
= num_var_ranges
* sizeof(struct mtrr_value
);
569 mtrr_state
= kmalloc(size
,GFP_ATOMIC
);
571 memset(mtrr_state
,0,size
);
575 for (i
= 0; i
< num_var_ranges
; i
++) {
577 &mtrr_state
[i
].lbase
,
578 &mtrr_state
[i
].lsize
,
579 &mtrr_state
[i
].ltype
);
584 static int mtrr_restore(struct sys_device
* sysdev
)
588 for (i
= 0; i
< num_var_ranges
; i
++) {
589 if (mtrr_state
[i
].lsize
)
593 mtrr_state
[i
].ltype
);
601 static struct sysdev_driver mtrr_sysdev_driver
= {
602 .suspend
= mtrr_save
,
603 .resume
= mtrr_restore
,
608 * mtrr_bp_init - initialize mtrrs on the boot CPU
610 * This needs to be called early; before any of the other CPUs are
611 * initialized (i.e. before smp_init()).
614 void __init
mtrr_bp_init(void)
619 mtrr_if
= &generic_mtrr_ops
;
620 size_or_mask
= 0xff000000; /* 36 bits */
621 size_and_mask
= 0x00f00000;
623 /* This is an AMD specific MSR, but we assume(hope?) that
624 Intel will implement it to when they extend the address
626 if (cpuid_eax(0x80000000) >= 0x80000008) {
628 phys_addr
= cpuid_eax(0x80000008) & 0xff;
629 size_or_mask
= ~((1 << (phys_addr
- PAGE_SHIFT
)) - 1);
630 size_and_mask
= ~size_or_mask
& 0xfff00000;
631 } else if (boot_cpu_data
.x86_vendor
== X86_VENDOR_CENTAUR
&&
632 boot_cpu_data
.x86
== 6) {
633 /* VIA C* family have Intel style MTRRs, but
635 size_or_mask
= 0xfff00000; /* 32 bits */
639 switch (boot_cpu_data
.x86_vendor
) {
641 if (cpu_has_k6_mtrr
) {
642 /* Pre-Athlon (K6) AMD CPU MTRRs */
643 mtrr_if
= mtrr_ops
[X86_VENDOR_AMD
];
644 size_or_mask
= 0xfff00000; /* 32 bits */
648 case X86_VENDOR_CENTAUR
:
649 if (cpu_has_centaur_mcr
) {
650 mtrr_if
= mtrr_ops
[X86_VENDOR_CENTAUR
];
651 size_or_mask
= 0xfff00000; /* 32 bits */
655 case X86_VENDOR_CYRIX
:
656 if (cpu_has_cyrix_arr
) {
657 mtrr_if
= mtrr_ops
[X86_VENDOR_CYRIX
];
658 size_or_mask
= 0xfff00000; /* 32 bits */
666 printk(KERN_INFO
"mtrr: v%s\n",MTRR_VERSION
);
669 set_num_var_ranges();
676 void mtrr_ap_init(void)
680 if (!mtrr_if
|| !use_intel())
683 * Ideally we should hold main_lock here to avoid mtrr entries changed,
684 * but this routine will be called in cpu boot time, holding the lock
685 * breaks it. This routine is called in two cases: 1.very earily time
686 * of software resume, when there absolutely isn't mtrr entry changes;
687 * 2.cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug lock to
688 * prevent mtrr entry changes
690 local_irq_save(flags
);
694 local_irq_restore(flags
);
697 static int __init
mtrr_init_finialize(void)
704 /* The CPUs haven't MTRR and seemes not support SMP. They have
705 * specific drivers, we use a tricky method to support
706 * suspend/resume for them.
707 * TBD: is there any system with such CPU which supports
708 * suspend/resume? if no, we should remove the code.
710 sysdev_driver_register(&cpu_sysdev_class
,
711 &mtrr_sysdev_driver
);
715 subsys_initcall(mtrr_init_finialize
);