2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
6 * (C) Copyright 1995 1996 Linus Torvalds
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
16 #include <asm/cacheflush.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
30 unsigned long __phys_addr(unsigned long x
)
32 if (x
>= __START_KERNEL_map
)
33 return x
- __START_KERNEL_map
+ phys_base
;
34 return x
- PAGE_OFFSET
;
36 EXPORT_SYMBOL(__phys_addr
);
40 int page_is_ram(unsigned long pagenr
)
42 unsigned long addr
, end
;
45 for (i
= 0; i
< e820
.nr_map
; i
++) {
49 if (e820
.map
[i
].type
!= E820_RAM
)
51 addr
= (e820
.map
[i
].addr
+ PAGE_SIZE
-1) >> PAGE_SHIFT
;
52 end
= (e820
.map
[i
].addr
+ e820
.map
[i
].size
) >> PAGE_SHIFT
;
55 * Sanity check: Some BIOSen report areas as RAM that
56 * are not. Notably the 640->1Mb area, which is the
59 if (addr
>= (BIOS_BEGIN
>> PAGE_SHIFT
) &&
60 end
< (BIOS_END
>> PAGE_SHIFT
))
63 if ((pagenr
>= addr
) && (pagenr
< end
))
70 * Fix up the linear direct mapping of the kernel to avoid cache attribute
73 static int ioremap_change_attr(unsigned long paddr
, unsigned long size
,
74 enum ioremap_mode mode
)
76 unsigned long vaddr
= (unsigned long)__va(paddr
);
77 unsigned long nrpages
= size
>> PAGE_SHIFT
;
81 /* No change for pages after the last mapping */
82 if ((paddr
+ size
- 1) >= (max_pfn_mapped
<< PAGE_SHIFT
))
86 * If there is no identity map for this address,
87 * change_page_attr_addr is unnecessary
89 if (!lookup_address(vaddr
, &level
))
93 case IOR_MODE_UNCACHED
:
95 err
= set_memory_uc(vaddr
, nrpages
);
98 err
= set_memory_wb(vaddr
, nrpages
);
106 * Remap an arbitrary physical address space into the kernel virtual
107 * address space. Needed when the kernel wants to access high addresses
110 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
111 * have to convert them into an offset in a page-aligned mapping, but the
112 * caller shouldn't need to know that small detail.
114 static void __iomem
*__ioremap(unsigned long phys_addr
, unsigned long size
,
115 enum ioremap_mode mode
)
118 struct vm_struct
*area
;
119 unsigned long pfn
, offset
, last_addr
;
122 /* Don't allow wraparound or zero size */
123 last_addr
= phys_addr
+ size
- 1;
124 if (!size
|| last_addr
< phys_addr
)
128 * Don't remap the low PCI/ISA area, it's always mapped..
130 if (phys_addr
>= ISA_START_ADDRESS
&& last_addr
< ISA_END_ADDRESS
)
131 return (__force
void __iomem
*)phys_to_virt(phys_addr
);
134 * Don't allow anybody to remap normal RAM that we're using..
136 for (pfn
= phys_addr
>> PAGE_SHIFT
; pfn
< max_pfn_mapped
&&
137 (pfn
<< PAGE_SHIFT
) < last_addr
; pfn
++) {
138 if (page_is_ram(pfn
) && pfn_valid(pfn
) &&
139 !PageReserved(pfn_to_page(pfn
)))
144 case IOR_MODE_UNCACHED
:
146 prot
= PAGE_KERNEL_NOCACHE
;
148 case IOR_MODE_CACHED
:
154 * Mappings have to be page-aligned
156 offset
= phys_addr
& ~PAGE_MASK
;
157 phys_addr
&= PAGE_MASK
;
158 size
= PAGE_ALIGN(last_addr
+1) - phys_addr
;
163 area
= get_vm_area(size
, VM_IOREMAP
);
166 area
->phys_addr
= phys_addr
;
167 addr
= (void __iomem
*) area
->addr
;
168 if (ioremap_page_range((unsigned long)addr
, (unsigned long)addr
+ size
,
170 remove_vm_area((void *)(PAGE_MASK
& (unsigned long) addr
));
174 if (ioremap_change_attr(phys_addr
, size
, mode
) < 0) {
179 return (void __iomem
*) (offset
+ (char __iomem
*)addr
);
183 * ioremap_nocache - map bus memory into CPU space
184 * @offset: bus address of the memory
185 * @size: size of the resource to map
187 * ioremap_nocache performs a platform specific sequence of operations to
188 * make bus memory CPU accessible via the readb/readw/readl/writeb/
189 * writew/writel functions and the other mmio helpers. The returned
190 * address is not guaranteed to be usable directly as a virtual
193 * This version of ioremap ensures that the memory is marked uncachable
194 * on the CPU as well as honouring existing caching rules from things like
195 * the PCI bus. Note that there are other caches and buffers on many
196 * busses. In particular driver authors should read up on PCI writes
198 * It's useful if some control registers are in such an area and
199 * write combining or read caching is not desirable:
201 * Must be freed with iounmap.
203 void __iomem
*ioremap_nocache(unsigned long phys_addr
, unsigned long size
)
205 return __ioremap(phys_addr
, size
, IOR_MODE_UNCACHED
);
207 EXPORT_SYMBOL(ioremap_nocache
);
209 void __iomem
*ioremap_cache(unsigned long phys_addr
, unsigned long size
)
211 return __ioremap(phys_addr
, size
, IOR_MODE_CACHED
);
213 EXPORT_SYMBOL(ioremap_cache
);
216 * iounmap - Free a IO remapping
217 * @addr: virtual address from ioremap_*
219 * Caller must ensure there is only one unmapping for the same pointer.
221 void iounmap(volatile void __iomem
*addr
)
223 struct vm_struct
*p
, *o
;
225 if ((void __force
*)addr
<= high_memory
)
229 * __ioremap special-cases the PCI/ISA range by not instantiating a
230 * vm_area and by simply returning an address into the kernel mapping
231 * of ISA space. So handle that here.
233 if (addr
>= phys_to_virt(ISA_START_ADDRESS
) &&
234 addr
< phys_to_virt(ISA_END_ADDRESS
))
237 addr
= (volatile void __iomem
*)
238 (PAGE_MASK
& (unsigned long __force
)addr
);
240 /* Use the vm area unlocked, assuming the caller
241 ensures there isn't another iounmap for the same address
242 in parallel. Reuse of the virtual address is prevented by
243 leaving it in the global lists until we're done with it.
244 cpa takes care of the direct mappings. */
245 read_lock(&vmlist_lock
);
246 for (p
= vmlist
; p
; p
= p
->next
) {
250 read_unlock(&vmlist_lock
);
253 printk(KERN_ERR
"iounmap: bad address %p\n", addr
);
258 /* Reset the direct mapping. Can block */
259 ioremap_change_attr(p
->phys_addr
, p
->size
, IOR_MODE_CACHED
);
261 /* Finally remove it */
262 o
= remove_vm_area((void *)addr
);
263 BUG_ON(p
!= o
|| o
== NULL
);
266 EXPORT_SYMBOL(iounmap
);
270 int __initdata early_ioremap_debug
;
272 static int __init
early_ioremap_debug_setup(char *str
)
274 early_ioremap_debug
= 1;
278 early_param("early_ioremap_debug", early_ioremap_debug_setup
);
280 static __initdata
int after_paging_init
;
281 static __initdata
unsigned long bm_pte
[1024]
282 __attribute__((aligned(PAGE_SIZE
)));
284 static inline unsigned long * __init
early_ioremap_pgd(unsigned long addr
)
286 return (unsigned long *)swapper_pg_dir
+ ((addr
>> 22) & 1023);
289 static inline unsigned long * __init
early_ioremap_pte(unsigned long addr
)
291 return bm_pte
+ ((addr
>> PAGE_SHIFT
) & 1023);
294 void __init
early_ioremap_init(void)
298 if (early_ioremap_debug
)
299 printk(KERN_INFO
"early_ioremap_init()\n");
301 pgd
= early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN
));
302 *pgd
= __pa(bm_pte
) | _PAGE_TABLE
;
303 memset(bm_pte
, 0, sizeof(bm_pte
));
305 * The boot-ioremap range spans multiple pgds, for which
306 * we are not prepared:
308 if (pgd
!= early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END
))) {
310 printk(KERN_WARNING
"pgd %p != %p\n",
311 pgd
, early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END
)));
312 printk(KERN_WARNING
"fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
313 fix_to_virt(FIX_BTMAP_BEGIN
));
314 printk(KERN_WARNING
"fix_to_virt(FIX_BTMAP_END): %08lx\n",
315 fix_to_virt(FIX_BTMAP_END
));
317 printk(KERN_WARNING
"FIX_BTMAP_END: %d\n", FIX_BTMAP_END
);
318 printk(KERN_WARNING
"FIX_BTMAP_BEGIN: %d\n",
323 void __init
early_ioremap_clear(void)
327 if (early_ioremap_debug
)
328 printk(KERN_INFO
"early_ioremap_clear()\n");
330 pgd
= early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN
));
332 paravirt_release_pt(__pa(pgd
) >> PAGE_SHIFT
);
336 void __init
early_ioremap_reset(void)
338 enum fixed_addresses idx
;
339 unsigned long *pte
, phys
, addr
;
341 after_paging_init
= 1;
342 for (idx
= FIX_BTMAP_BEGIN
; idx
>= FIX_BTMAP_END
; idx
--) {
343 addr
= fix_to_virt(idx
);
344 pte
= early_ioremap_pte(addr
);
345 if (*pte
& _PAGE_PRESENT
) {
346 phys
= *pte
& PAGE_MASK
;
347 set_fixmap(idx
, phys
);
352 static void __init
__early_set_fixmap(enum fixed_addresses idx
,
353 unsigned long phys
, pgprot_t flags
)
355 unsigned long *pte
, addr
= __fix_to_virt(idx
);
357 if (idx
>= __end_of_fixed_addresses
) {
361 pte
= early_ioremap_pte(addr
);
362 if (pgprot_val(flags
))
363 *pte
= (phys
& PAGE_MASK
) | pgprot_val(flags
);
366 __flush_tlb_one(addr
);
369 static inline void __init
early_set_fixmap(enum fixed_addresses idx
,
372 if (after_paging_init
)
373 set_fixmap(idx
, phys
);
375 __early_set_fixmap(idx
, phys
, PAGE_KERNEL
);
378 static inline void __init
early_clear_fixmap(enum fixed_addresses idx
)
380 if (after_paging_init
)
383 __early_set_fixmap(idx
, 0, __pgprot(0));
387 int __initdata early_ioremap_nested
;
389 static int __init
check_early_ioremap_leak(void)
391 if (!early_ioremap_nested
)
395 "Debug warning: early ioremap leak of %d areas detected.\n",
396 early_ioremap_nested
);
398 "please boot with early_ioremap_debug and report the dmesg.\n");
403 late_initcall(check_early_ioremap_leak
);
405 void __init
*early_ioremap(unsigned long phys_addr
, unsigned long size
)
407 unsigned long offset
, last_addr
;
408 unsigned int nrpages
, nesting
;
409 enum fixed_addresses idx0
, idx
;
411 WARN_ON(system_state
!= SYSTEM_BOOTING
);
413 nesting
= early_ioremap_nested
;
414 if (early_ioremap_debug
) {
415 printk(KERN_INFO
"early_ioremap(%08lx, %08lx) [%d] => ",
416 phys_addr
, size
, nesting
);
420 /* Don't allow wraparound or zero size */
421 last_addr
= phys_addr
+ size
- 1;
422 if (!size
|| last_addr
< phys_addr
) {
427 if (nesting
>= FIX_BTMAPS_NESTING
) {
431 early_ioremap_nested
++;
433 * Mappings have to be page-aligned
435 offset
= phys_addr
& ~PAGE_MASK
;
436 phys_addr
&= PAGE_MASK
;
437 size
= PAGE_ALIGN(last_addr
) - phys_addr
;
440 * Mappings have to fit in the FIX_BTMAP area.
442 nrpages
= size
>> PAGE_SHIFT
;
443 if (nrpages
> NR_FIX_BTMAPS
) {
451 idx0
= FIX_BTMAP_BEGIN
- NR_FIX_BTMAPS
*nesting
;
453 while (nrpages
> 0) {
454 early_set_fixmap(idx
, phys_addr
);
455 phys_addr
+= PAGE_SIZE
;
459 if (early_ioremap_debug
)
460 printk(KERN_CONT
"%08lx + %08lx\n", offset
, fix_to_virt(idx0
));
462 return (void *) (offset
+ fix_to_virt(idx0
));
465 void __init
early_iounmap(void *addr
, unsigned long size
)
467 unsigned long virt_addr
;
468 unsigned long offset
;
469 unsigned int nrpages
;
470 enum fixed_addresses idx
;
471 unsigned int nesting
;
473 nesting
= --early_ioremap_nested
;
474 WARN_ON(nesting
< 0);
476 if (early_ioremap_debug
) {
477 printk(KERN_INFO
"early_iounmap(%p, %08lx) [%d]\n", addr
,
482 virt_addr
= (unsigned long)addr
;
483 if (virt_addr
< fix_to_virt(FIX_BTMAP_BEGIN
)) {
487 offset
= virt_addr
& ~PAGE_MASK
;
488 nrpages
= PAGE_ALIGN(offset
+ size
- 1) >> PAGE_SHIFT
;
490 idx
= FIX_BTMAP_BEGIN
- NR_FIX_BTMAPS
*nesting
;
491 while (nrpages
> 0) {
492 early_clear_fixmap(idx
);
498 void __this_fixmap_does_not_exist(void)
503 #endif /* CONFIG_X86_32 */