arch/x86/mm/ioremap.c

   1 /*
   2  * Re-map IO memory to kernel address space so that we can access it.
   3  * This is needed for high PCI addresses that aren't mapped in the
   4  * 640k-1MB IO memory area on PC's
   5  *
   6  * (C) Copyright 1995 1996 Linus Torvalds
   7  */
   8
   9 #include <linux/bootmem.h>
  10 #include <linux/init.h>
  11 #include <linux/io.h>
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include <linux/vmalloc.h>
  15
  16 #include <asm/cacheflush.h>
  17 #include <asm/e820.h>
  18 #include <asm/fixmap.h>
  19 #include <asm/pgtable.h>
  20 #include <asm/tlbflush.h>
  21 #include <asm/pgalloc.h>
  22
  23 enum ioremap_mode {
  24         IOR_MODE_UNCACHED,
  25         IOR_MODE_CACHED,
  26 };
  27
  28 #ifdef CONFIG_X86_64
  29
  30 unsigned long __phys_addr(unsigned long x)
  31 {
  32         if (x >= __START_KERNEL_map)
  33                 return x - __START_KERNEL_map + phys_base;
  34         return x - PAGE_OFFSET;
  35 }
  36 EXPORT_SYMBOL(__phys_addr);
  37
  38 #endif
  39
  40 int page_is_ram(unsigned long pagenr)
  41 {
  42         unsigned long addr, end;
  43         int i;
  44
  45         for (i = 0; i < e820.nr_map; i++) {
  46                 /*
  47                  * Not usable memory:
  48                  */
  49                 if (e820.map[i].type != E820_RAM)
  50                         continue;
  51                 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
  52                 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
  53
  54                 /*
  55                  * Sanity check: Some BIOSen report areas as RAM that
  56                  * are not. Notably the 640->1Mb area, which is the
  57                  * PCI BIOS area.
  58                  */
  59                 if (addr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
  60                     end < (BIOS_END >> PAGE_SHIFT))
  61                         continue;
  62
  63                 if ((pagenr >= addr) && (pagenr < end))
  64                         return 1;
  65         }
  66         return 0;
  67 }
  68
  69 /*
  70  * Fix up the linear direct mapping of the kernel to avoid cache attribute
  71  * conflicts.
  72  */
  73 static int ioremap_change_attr(unsigned long paddr, unsigned long size,
  74                                enum ioremap_mode mode)
  75 {
  76         unsigned long vaddr = (unsigned long)__va(paddr);
  77         unsigned long nrpages = size >> PAGE_SHIFT;
  78         unsigned int level;
  79         int err;
  80
  81         /* No change for pages after the last mapping */
  82         if ((paddr + size - 1) >= (max_pfn_mapped << PAGE_SHIFT))
  83                 return 0;
  84
  85         /*
  86          * If there is no identity map for this address,
  87          * change_page_attr_addr is unnecessary
  88          */
  89         if (!lookup_address(vaddr, &level))
  90                 return 0;
  91
  92         switch (mode) {
  93         case IOR_MODE_UNCACHED:
  94         default:
  95                 err = set_memory_uc(vaddr, nrpages);
  96                 break;
  97         case IOR_MODE_CACHED:
  98                 err = set_memory_wb(vaddr, nrpages);
  99                 break;
 100         }
 101
 102         return err;
 103 }
 104
 105 /*
 106  * Remap an arbitrary physical address space into the kernel virtual
 107  * address space. Needed when the kernel wants to access high addresses
 108  * directly.
 109  *
 110  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 111  * have to convert them into an offset in a page-aligned mapping, but the
 112  * caller shouldn't need to know that small detail.
 113  */
 114 static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
 115                                enum ioremap_mode mode)
 116 {
 117         void __iomem *addr;
 118         struct vm_struct *area;
 119         unsigned long offset, last_addr;
 120         pgprot_t prot;
 121
 122         /* Don't allow wraparound or zero size */
 123         last_addr = phys_addr + size - 1;
 124         if (!size || last_addr < phys_addr)
 125                 return NULL;
 126
 127         /*
 128          * Don't remap the low PCI/ISA area, it's always mapped..
 129          */
 130         if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
 131                 return (__force void __iomem *)phys_to_virt(phys_addr);
 132
 133         /*
 134          * Don't allow anybody to remap normal RAM that we're using..
 135          */
 136         for (offset = phys_addr >> PAGE_SHIFT; offset < max_pfn_mapped &&
 137              (offset << PAGE_SHIFT) < last_addr; offset++) {
 138                 if (page_is_ram(offset))
 139                         return NULL;
 140         }
 141
 142         switch (mode) {
 143         case IOR_MODE_UNCACHED:
 144         default:
 145                 prot = PAGE_KERNEL_NOCACHE;
 146                 break;
 147         case IOR_MODE_CACHED:
 148                 prot = PAGE_KERNEL;
 149                 break;
 150         }
 151
 152         /*
 153          * Mappings have to be page-aligned
 154          */
 155         offset = phys_addr & ~PAGE_MASK;
 156         phys_addr &= PAGE_MASK;
 157         size = PAGE_ALIGN(last_addr+1) - phys_addr;
 158
 159         /*
 160          * Ok, go for it..
 161          */
 162         area = get_vm_area(size, VM_IOREMAP);
 163         if (!area)
 164                 return NULL;
 165         area->phys_addr = phys_addr;
 166         addr = (void __iomem *) area->addr;
 167         if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
 168                                phys_addr, prot)) {
 169                 remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr));
 170                 return NULL;
 171         }
 172
 173         if (ioremap_change_attr(phys_addr, size, mode) < 0) {
 174                 vunmap(addr);
 175                 return NULL;
 176         }
 177
 178         return (void __iomem *) (offset + (char __iomem *)addr);
 179 }
 180
 181 /**
 182  * ioremap_nocache     -   map bus memory into CPU space
 183  * @offset:    bus address of the memory
 184  * @size:      size of the resource to map
 185  *
 186  * ioremap_nocache performs a platform specific sequence of operations to
 187  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 188  * writew/writel functions and the other mmio helpers. The returned
 189  * address is not guaranteed to be usable directly as a virtual
 190  * address.
 191  *
 192  * This version of ioremap ensures that the memory is marked uncachable
 193  * on the CPU as well as honouring existing caching rules from things like
 194  * the PCI bus. Note that there are other caches and buffers on many
 195  * busses. In particular driver authors should read up on PCI writes
 196  *
 197  * It's useful if some control registers are in such an area and
 198  * write combining or read caching is not desirable:
 199  *
 200  * Must be freed with iounmap.
 201  */
 202 void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
 203 {
 204         return __ioremap(phys_addr, size, IOR_MODE_UNCACHED);
 205 }
 206 EXPORT_SYMBOL(ioremap_nocache);
 207
 208 void __iomem *ioremap_cache(unsigned long phys_addr, unsigned long size)
 209 {
 210         return __ioremap(phys_addr, size, IOR_MODE_CACHED);
 211 }
 212 EXPORT_SYMBOL(ioremap_cache);
 213
 214 /**
 215  * iounmap - Free a IO remapping
 216  * @addr: virtual address from ioremap_*
 217  *
 218  * Caller must ensure there is only one unmapping for the same pointer.
 219  */
 220 void iounmap(volatile void __iomem *addr)
 221 {
 222         struct vm_struct *p, *o;
 223
 224         if ((void __force *)addr <= high_memory)
 225                 return;
 226
 227         /*
 228          * __ioremap special-cases the PCI/ISA range by not instantiating a
 229          * vm_area and by simply returning an address into the kernel mapping
 230          * of ISA space.   So handle that here.
 231          */
 232         if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
 233             addr < phys_to_virt(ISA_END_ADDRESS))
 234                 return;
 235
 236         addr = (volatile void __iomem *)
 237                 (PAGE_MASK & (unsigned long __force)addr);
 238
 239         /* Use the vm area unlocked, assuming the caller
 240            ensures there isn't another iounmap for the same address
 241            in parallel. Reuse of the virtual address is prevented by
 242            leaving it in the global lists until we're done with it.
 243            cpa takes care of the direct mappings. */
 244         read_lock(&vmlist_lock);
 245         for (p = vmlist; p; p = p->next) {
 246                 if (p->addr == addr)
 247                         break;
 248         }
 249         read_unlock(&vmlist_lock);
 250
 251         if (!p) {
 252                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
 253                 dump_stack();
 254                 return;
 255         }
 256
 257         /* Reset the direct mapping. Can block */
 258         ioremap_change_attr(p->phys_addr, p->size, IOR_MODE_CACHED);
 259
 260         /* Finally remove it */
 261         o = remove_vm_area((void *)addr);
 262         BUG_ON(p != o || o == NULL);
 263         kfree(p);
 264 }
 265 EXPORT_SYMBOL(iounmap);
 266
 267 #ifdef CONFIG_X86_32
 268
 269 int __initdata early_ioremap_debug;
 270
 271 static int __init early_ioremap_debug_setup(char *str)
 272 {
 273         early_ioremap_debug = 1;
 274
 275         return 0;
 276 }
 277 early_param("early_ioremap_debug", early_ioremap_debug_setup);
 278
 279 static __initdata int after_paging_init;
 280 static __initdata unsigned long bm_pte[1024]
 281                                 __attribute__((aligned(PAGE_SIZE)));
 282
 283 static inline unsigned long * __init early_ioremap_pgd(unsigned long addr)
 284 {
 285         return (unsigned long *)swapper_pg_dir + ((addr >> 22) & 1023);
 286 }
 287
 288 static inline unsigned long * __init early_ioremap_pte(unsigned long addr)
 289 {
 290         return bm_pte + ((addr >> PAGE_SHIFT) & 1023);
 291 }
 292
 293 void __init early_ioremap_init(void)
 294 {
 295         unsigned long *pgd;
 296
 297         if (early_ioremap_debug)
 298                 printk(KERN_INFO "early_ioremap_init()\n");
 299
 300         pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
 301         *pgd = __pa(bm_pte) | _PAGE_TABLE;
 302         memset(bm_pte, 0, sizeof(bm_pte));
 303         /*
 304          * The boot-ioremap range spans multiple pgds, for which
 305          * we are not prepared:
 306          */
 307         if (pgd != early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END))) {
 308                 WARN_ON(1);
 309                 printk(KERN_WARNING "pgd %p != %p\n",
 310                        pgd, early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END)));
 311                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 312                        fix_to_virt(FIX_BTMAP_BEGIN));
 313                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
 314                        fix_to_virt(FIX_BTMAP_END));
 315
 316                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
 317                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
 318                        FIX_BTMAP_BEGIN);
 319         }
 320 }
 321
 322 void __init early_ioremap_clear(void)
 323 {
 324         unsigned long *pgd;
 325
 326         if (early_ioremap_debug)
 327                 printk(KERN_INFO "early_ioremap_clear()\n");
 328
 329         pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
 330         *pgd = 0;
 331         paravirt_release_pt(__pa(pgd) >> PAGE_SHIFT);
 332         __flush_tlb_all();
 333 }
 334
 335 void __init early_ioremap_reset(void)
 336 {
 337         enum fixed_addresses idx;
 338         unsigned long *pte, phys, addr;
 339
 340         after_paging_init = 1;
 341         for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
 342                 addr = fix_to_virt(idx);
 343                 pte = early_ioremap_pte(addr);
 344                 if (*pte & _PAGE_PRESENT) {
 345                         phys = *pte & PAGE_MASK;
 346                         set_fixmap(idx, phys);
 347                 }
 348         }
 349 }
 350
 351 static void __init __early_set_fixmap(enum fixed_addresses idx,
 352                                    unsigned long phys, pgprot_t flags)
 353 {
 354         unsigned long *pte, addr = __fix_to_virt(idx);
 355
 356         if (idx >= __end_of_fixed_addresses) {
 357                 BUG();
 358                 return;
 359         }
 360         pte = early_ioremap_pte(addr);
 361         if (pgprot_val(flags))
 362                 *pte = (phys & PAGE_MASK) | pgprot_val(flags);
 363         else
 364                 *pte = 0;
 365         __flush_tlb_one(addr);
 366 }
 367
 368 static inline void __init early_set_fixmap(enum fixed_addresses idx,
 369                                         unsigned long phys)
 370 {
 371         if (after_paging_init)
 372                 set_fixmap(idx, phys);
 373         else
 374                 __early_set_fixmap(idx, phys, PAGE_KERNEL);
 375 }
 376
 377 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
 378 {
 379         if (after_paging_init)
 380                 clear_fixmap(idx);
 381         else
 382                 __early_set_fixmap(idx, 0, __pgprot(0));
 383 }
 384
 385
 386 int __initdata early_ioremap_nested;
 387
 388 static int __init check_early_ioremap_leak(void)
 389 {
 390         if (!early_ioremap_nested)
 391                 return 0;
 392
 393         printk(KERN_WARNING
 394                "Debug warning: early ioremap leak of %d areas detected.\n",
 395                early_ioremap_nested);
 396         printk(KERN_WARNING
 397                "please boot with early_ioremap_debug and report the dmesg.\n");
 398         WARN_ON(1);
 399
 400         return 1;
 401 }
 402 late_initcall(check_early_ioremap_leak);
 403
 404 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
 405 {
 406         unsigned long offset, last_addr;
 407         unsigned int nrpages, nesting;
 408         enum fixed_addresses idx0, idx;
 409
 410         WARN_ON(system_state != SYSTEM_BOOTING);
 411
 412         nesting = early_ioremap_nested;
 413         if (early_ioremap_debug) {
 414                 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
 415                        phys_addr, size, nesting);
 416                 dump_stack();
 417         }
 418
 419         /* Don't allow wraparound or zero size */
 420         last_addr = phys_addr + size - 1;
 421         if (!size || last_addr < phys_addr) {
 422                 WARN_ON(1);
 423                 return NULL;
 424         }
 425
 426         if (nesting >= FIX_BTMAPS_NESTING) {
 427                 WARN_ON(1);
 428                 return NULL;
 429         }
 430         early_ioremap_nested++;
 431         /*
 432          * Mappings have to be page-aligned
 433          */
 434         offset = phys_addr & ~PAGE_MASK;
 435         phys_addr &= PAGE_MASK;
 436         size = PAGE_ALIGN(last_addr) - phys_addr;
 437
 438         /*
 439          * Mappings have to fit in the FIX_BTMAP area.
 440          */
 441         nrpages = size >> PAGE_SHIFT;
 442         if (nrpages > NR_FIX_BTMAPS) {
 443                 WARN_ON(1);
 444                 return NULL;
 445         }
 446
 447         /*
 448          * Ok, go for it..
 449          */
 450         idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 451         idx = idx0;
 452         while (nrpages > 0) {
 453                 early_set_fixmap(idx, phys_addr);
 454                 phys_addr += PAGE_SIZE;
 455                 --idx;
 456                 --nrpages;
 457         }
 458         if (early_ioremap_debug)
 459                 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
 460
 461         return (void *) (offset + fix_to_virt(idx0));
 462 }
 463
 464 void __init early_iounmap(void *addr, unsigned long size)
 465 {
 466         unsigned long virt_addr;
 467         unsigned long offset;
 468         unsigned int nrpages;
 469         enum fixed_addresses idx;
 470         unsigned int nesting;
 471
 472         nesting = --early_ioremap_nested;
 473         WARN_ON(nesting < 0);
 474
 475         if (early_ioremap_debug) {
 476                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
 477                        size, nesting);
 478                 dump_stack();
 479         }
 480
 481         virt_addr = (unsigned long)addr;
 482         if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
 483                 WARN_ON(1);
 484                 return;
 485         }
 486         offset = virt_addr & ~PAGE_MASK;
 487         nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
 488
 489         idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 490         while (nrpages > 0) {
 491                 early_clear_fixmap(idx);
 492                 --idx;
 493                 --nrpages;
 494         }
 495 }
 496
 497 void __this_fixmap_does_not_exist(void)
 498 {
 499         WARN_ON(1);
 500 }
 501
 502 #endif /* CONFIG_X86_32 */