2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
6 * (C) Copyright 1995 1996 Linus Torvalds
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
16 #include <asm/cacheflush.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
30 unsigned long __phys_addr(unsigned long x
)
32 if (x
>= __START_KERNEL_map
)
33 return x
- __START_KERNEL_map
+ phys_base
;
34 return x
- PAGE_OFFSET
;
36 EXPORT_SYMBOL(__phys_addr
);
38 static inline int phys_addr_valid(unsigned long addr
)
40 return addr
< (1UL << boot_cpu_data
.x86_phys_bits
);
45 static inline int phys_addr_valid(unsigned long addr
)
52 int page_is_ram(unsigned long pagenr
)
54 unsigned long addr
, end
;
58 * A special case is the first 4Kb of memory;
59 * This is a BIOS owned area, not kernel ram, but generally
60 * not listed as such in the E820 table.
66 * Second special case: Some BIOSen report the PC BIOS
67 * area (640->1Mb) as ram even though it is not.
69 if (pagenr
>= (BIOS_BEGIN
>> PAGE_SHIFT
) &&
70 pagenr
< (BIOS_END
>> PAGE_SHIFT
))
73 for (i
= 0; i
< e820
.nr_map
; i
++) {
77 if (e820
.map
[i
].type
!= E820_RAM
)
79 addr
= (e820
.map
[i
].addr
+ PAGE_SIZE
-1) >> PAGE_SHIFT
;
80 end
= (e820
.map
[i
].addr
+ e820
.map
[i
].size
) >> PAGE_SHIFT
;
83 if ((pagenr
>= addr
) && (pagenr
< end
))
90 * Fix up the linear direct mapping of the kernel to avoid cache attribute
93 static int ioremap_change_attr(unsigned long vaddr
, unsigned long size
,
94 enum ioremap_mode mode
)
96 unsigned long nrpages
= size
>> PAGE_SHIFT
;
100 case IOR_MODE_UNCACHED
:
102 err
= set_memory_uc(vaddr
, nrpages
);
104 case IOR_MODE_CACHED
:
105 err
= set_memory_wb(vaddr
, nrpages
);
113 * Remap an arbitrary physical address space into the kernel virtual
114 * address space. Needed when the kernel wants to access high addresses
117 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
118 * have to convert them into an offset in a page-aligned mapping, but the
119 * caller shouldn't need to know that small detail.
121 static void __iomem
*__ioremap(resource_size_t phys_addr
, unsigned long size
,
122 enum ioremap_mode mode
)
124 unsigned long pfn
, offset
, last_addr
, vaddr
;
125 struct vm_struct
*area
;
128 /* Don't allow wraparound or zero size */
129 last_addr
= phys_addr
+ size
- 1;
130 if (!size
|| last_addr
< phys_addr
)
133 if (!phys_addr_valid(phys_addr
)) {
134 printk(KERN_WARNING
"ioremap: invalid physical address %lx\n",
141 * Don't remap the low PCI/ISA area, it's always mapped..
143 if (phys_addr
>= ISA_START_ADDRESS
&& last_addr
< ISA_END_ADDRESS
)
144 return (__force
void __iomem
*)phys_to_virt(phys_addr
);
147 * Don't allow anybody to remap normal RAM that we're using..
149 for (pfn
= phys_addr
>> PAGE_SHIFT
;
150 (pfn
<< PAGE_SHIFT
) < last_addr
; pfn
++) {
152 int is_ram
= page_is_ram(pfn
);
154 if (is_ram
&& pfn_valid(pfn
) && !PageReserved(pfn_to_page(pfn
)))
156 WARN_ON_ONCE(is_ram
);
160 case IOR_MODE_UNCACHED
:
162 prot
= PAGE_KERNEL_NOCACHE
;
164 case IOR_MODE_CACHED
:
170 * Mappings have to be page-aligned
172 offset
= phys_addr
& ~PAGE_MASK
;
173 phys_addr
&= PAGE_MASK
;
174 size
= PAGE_ALIGN(last_addr
+1) - phys_addr
;
179 area
= get_vm_area(size
, VM_IOREMAP
);
182 area
->phys_addr
= phys_addr
;
183 vaddr
= (unsigned long) area
->addr
;
184 if (ioremap_page_range(vaddr
, vaddr
+ size
, phys_addr
, prot
)) {
189 if (ioremap_change_attr(vaddr
, size
, mode
) < 0) {
194 return (void __iomem
*) (vaddr
+ offset
);
198 * ioremap_nocache - map bus memory into CPU space
199 * @offset: bus address of the memory
200 * @size: size of the resource to map
202 * ioremap_nocache performs a platform specific sequence of operations to
203 * make bus memory CPU accessible via the readb/readw/readl/writeb/
204 * writew/writel functions and the other mmio helpers. The returned
205 * address is not guaranteed to be usable directly as a virtual
208 * This version of ioremap ensures that the memory is marked uncachable
209 * on the CPU as well as honouring existing caching rules from things like
210 * the PCI bus. Note that there are other caches and buffers on many
211 * busses. In particular driver authors should read up on PCI writes
213 * It's useful if some control registers are in such an area and
214 * write combining or read caching is not desirable:
216 * Must be freed with iounmap.
218 void __iomem
*ioremap_nocache(resource_size_t phys_addr
, unsigned long size
)
220 return __ioremap(phys_addr
, size
, IOR_MODE_UNCACHED
);
222 EXPORT_SYMBOL(ioremap_nocache
);
224 void __iomem
*ioremap_cache(resource_size_t phys_addr
, unsigned long size
)
226 return __ioremap(phys_addr
, size
, IOR_MODE_CACHED
);
228 EXPORT_SYMBOL(ioremap_cache
);
231 * iounmap - Free a IO remapping
232 * @addr: virtual address from ioremap_*
234 * Caller must ensure there is only one unmapping for the same pointer.
236 void iounmap(volatile void __iomem
*addr
)
238 struct vm_struct
*p
, *o
;
240 if ((void __force
*)addr
<= high_memory
)
244 * __ioremap special-cases the PCI/ISA range by not instantiating a
245 * vm_area and by simply returning an address into the kernel mapping
246 * of ISA space. So handle that here.
248 if (addr
>= phys_to_virt(ISA_START_ADDRESS
) &&
249 addr
< phys_to_virt(ISA_END_ADDRESS
))
252 addr
= (volatile void __iomem
*)
253 (PAGE_MASK
& (unsigned long __force
)addr
);
255 /* Use the vm area unlocked, assuming the caller
256 ensures there isn't another iounmap for the same address
257 in parallel. Reuse of the virtual address is prevented by
258 leaving it in the global lists until we're done with it.
259 cpa takes care of the direct mappings. */
260 read_lock(&vmlist_lock
);
261 for (p
= vmlist
; p
; p
= p
->next
) {
265 read_unlock(&vmlist_lock
);
268 printk(KERN_ERR
"iounmap: bad address %p\n", addr
);
273 /* Finally remove it */
274 o
= remove_vm_area((void *)addr
);
275 BUG_ON(p
!= o
|| o
== NULL
);
278 EXPORT_SYMBOL(iounmap
);
282 int __initdata early_ioremap_debug
;
284 static int __init
early_ioremap_debug_setup(char *str
)
286 early_ioremap_debug
= 1;
290 early_param("early_ioremap_debug", early_ioremap_debug_setup
);
292 static __initdata
int after_paging_init
;
293 static pte_t bm_pte
[PAGE_SIZE
/sizeof(pte_t
)]
294 __section(.bss
.page_aligned
);
296 static inline pmd_t
* __init
early_ioremap_pmd(unsigned long addr
)
298 /* Don't assume we're using swapper_pg_dir at this point */
299 pgd_t
*base
= __va(read_cr3());
300 pgd_t
*pgd
= &base
[pgd_index(addr
)];
301 pud_t
*pud
= pud_offset(pgd
, addr
);
302 pmd_t
*pmd
= pmd_offset(pud
, addr
);
307 static inline pte_t
* __init
early_ioremap_pte(unsigned long addr
)
309 return &bm_pte
[pte_index(addr
)];
312 void __init
early_ioremap_init(void)
316 if (early_ioremap_debug
)
317 printk(KERN_INFO
"early_ioremap_init()\n");
319 pmd
= early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
));
320 memset(bm_pte
, 0, sizeof(bm_pte
));
321 pmd_populate_kernel(&init_mm
, pmd
, bm_pte
);
324 * The boot-ioremap range spans multiple pmds, for which
325 * we are not prepared:
327 if (pmd
!= early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END
))) {
329 printk(KERN_WARNING
"pmd %p != %p\n",
330 pmd
, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END
)));
331 printk(KERN_WARNING
"fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
332 fix_to_virt(FIX_BTMAP_BEGIN
));
333 printk(KERN_WARNING
"fix_to_virt(FIX_BTMAP_END): %08lx\n",
334 fix_to_virt(FIX_BTMAP_END
));
336 printk(KERN_WARNING
"FIX_BTMAP_END: %d\n", FIX_BTMAP_END
);
337 printk(KERN_WARNING
"FIX_BTMAP_BEGIN: %d\n",
342 void __init
early_ioremap_clear(void)
346 if (early_ioremap_debug
)
347 printk(KERN_INFO
"early_ioremap_clear()\n");
349 pmd
= early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
));
351 paravirt_release_pt(__pa(bm_pte
) >> PAGE_SHIFT
);
355 void __init
early_ioremap_reset(void)
357 enum fixed_addresses idx
;
358 unsigned long addr
, phys
;
361 after_paging_init
= 1;
362 for (idx
= FIX_BTMAP_BEGIN
; idx
>= FIX_BTMAP_END
; idx
--) {
363 addr
= fix_to_virt(idx
);
364 pte
= early_ioremap_pte(addr
);
365 if (pte_present(*pte
)) {
366 phys
= pte_val(*pte
) & PAGE_MASK
;
367 set_fixmap(idx
, phys
);
372 static void __init
__early_set_fixmap(enum fixed_addresses idx
,
373 unsigned long phys
, pgprot_t flags
)
375 unsigned long addr
= __fix_to_virt(idx
);
378 if (idx
>= __end_of_fixed_addresses
) {
382 pte
= early_ioremap_pte(addr
);
383 if (pgprot_val(flags
))
384 set_pte(pte
, pfn_pte(phys
>> PAGE_SHIFT
, flags
));
386 pte_clear(NULL
, addr
, pte
);
387 __flush_tlb_one(addr
);
390 static inline void __init
early_set_fixmap(enum fixed_addresses idx
,
393 if (after_paging_init
)
394 set_fixmap(idx
, phys
);
396 __early_set_fixmap(idx
, phys
, PAGE_KERNEL
);
399 static inline void __init
early_clear_fixmap(enum fixed_addresses idx
)
401 if (after_paging_init
)
404 __early_set_fixmap(idx
, 0, __pgprot(0));
408 int __initdata early_ioremap_nested
;
410 static int __init
check_early_ioremap_leak(void)
412 if (!early_ioremap_nested
)
416 "Debug warning: early ioremap leak of %d areas detected.\n",
417 early_ioremap_nested
);
419 "please boot with early_ioremap_debug and report the dmesg.\n");
424 late_initcall(check_early_ioremap_leak
);
426 void __init
*early_ioremap(unsigned long phys_addr
, unsigned long size
)
428 unsigned long offset
, last_addr
;
429 unsigned int nrpages
, nesting
;
430 enum fixed_addresses idx0
, idx
;
432 WARN_ON(system_state
!= SYSTEM_BOOTING
);
434 nesting
= early_ioremap_nested
;
435 if (early_ioremap_debug
) {
436 printk(KERN_INFO
"early_ioremap(%08lx, %08lx) [%d] => ",
437 phys_addr
, size
, nesting
);
441 /* Don't allow wraparound or zero size */
442 last_addr
= phys_addr
+ size
- 1;
443 if (!size
|| last_addr
< phys_addr
) {
448 if (nesting
>= FIX_BTMAPS_NESTING
) {
452 early_ioremap_nested
++;
454 * Mappings have to be page-aligned
456 offset
= phys_addr
& ~PAGE_MASK
;
457 phys_addr
&= PAGE_MASK
;
458 size
= PAGE_ALIGN(last_addr
) - phys_addr
;
461 * Mappings have to fit in the FIX_BTMAP area.
463 nrpages
= size
>> PAGE_SHIFT
;
464 if (nrpages
> NR_FIX_BTMAPS
) {
472 idx0
= FIX_BTMAP_BEGIN
- NR_FIX_BTMAPS
*nesting
;
474 while (nrpages
> 0) {
475 early_set_fixmap(idx
, phys_addr
);
476 phys_addr
+= PAGE_SIZE
;
480 if (early_ioremap_debug
)
481 printk(KERN_CONT
"%08lx + %08lx\n", offset
, fix_to_virt(idx0
));
483 return (void *) (offset
+ fix_to_virt(idx0
));
486 void __init
early_iounmap(void *addr
, unsigned long size
)
488 unsigned long virt_addr
;
489 unsigned long offset
;
490 unsigned int nrpages
;
491 enum fixed_addresses idx
;
492 unsigned int nesting
;
494 nesting
= --early_ioremap_nested
;
495 WARN_ON(nesting
< 0);
497 if (early_ioremap_debug
) {
498 printk(KERN_INFO
"early_iounmap(%p, %08lx) [%d]\n", addr
,
503 virt_addr
= (unsigned long)addr
;
504 if (virt_addr
< fix_to_virt(FIX_BTMAP_BEGIN
)) {
508 offset
= virt_addr
& ~PAGE_MASK
;
509 nrpages
= PAGE_ALIGN(offset
+ size
- 1) >> PAGE_SHIFT
;
511 idx
= FIX_BTMAP_BEGIN
- NR_FIX_BTMAPS
*nesting
;
512 while (nrpages
> 0) {
513 early_clear_fixmap(idx
);
519 void __this_fixmap_does_not_exist(void)
524 #endif /* CONFIG_X86_32 */