release/src-rt-6.x.4708/linux/linux-2.6.36/arch/arm/include/asm/cacheflush.h

   1 /* Modified by Broadcom Corp. Portions Copyright (c) Broadcom Corp, 2012. */
   2 /*
   3  *  arch/arm/include/asm/cacheflush.h
   4  *
   5  *  Copyright (C) 1999-2002 Russell King
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #ifndef _ASMARM_CACHEFLUSH_H
  12 #define _ASMARM_CACHEFLUSH_H
  13
  14 #include <linux/mm.h>
  15
  16 #include <asm/glue.h>
  17 #include <asm/shmparam.h>
  18 #include <asm/cachetype.h>
  19 #include <asm/outercache.h>
  20
  21 #define CACHE_COLOUR(vaddr)     ((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT)
  22
  23 /*
  24  *      Cache Model
  25  *      ===========
  26  */
  27 #undef _CACHE
  28 #undef MULTI_CACHE
  29
  30 #if defined(CONFIG_CPU_CACHE_V3)
  31 # ifdef _CACHE
  32 #  define MULTI_CACHE 1
  33 # else
  34 #  define _CACHE v3
  35 # endif
  36 #endif
  37
  38 #if defined(CONFIG_CPU_CACHE_V4)
  39 # ifdef _CACHE
  40 #  define MULTI_CACHE 1
  41 # else
  42 #  define _CACHE v4
  43 # endif
  44 #endif
  45
  46 #if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
  47         defined(CONFIG_CPU_ARM925T) || defined(CONFIG_CPU_ARM1020) || \
  48         defined(CONFIG_CPU_ARM1026)
  49 # define MULTI_CACHE 1
  50 #endif
  51
  52 #if defined(CONFIG_CPU_FA526)
  53 # ifdef _CACHE
  54 #  define MULTI_CACHE 1
  55 # else
  56 #  define _CACHE fa
  57 # endif
  58 #endif
  59
  60 #if defined(CONFIG_CPU_ARM926T)
  61 # ifdef _CACHE
  62 #  define MULTI_CACHE 1
  63 # else
  64 #  define _CACHE arm926
  65 # endif
  66 #endif
  67
  68 #if defined(CONFIG_CPU_ARM940T)
  69 # ifdef _CACHE
  70 #  define MULTI_CACHE 1
  71 # else
  72 #  define _CACHE arm940
  73 # endif
  74 #endif
  75
  76 #if defined(CONFIG_CPU_ARM946E)
  77 # ifdef _CACHE
  78 #  define MULTI_CACHE 1
  79 # else
  80 #  define _CACHE arm946
  81 # endif
  82 #endif
  83
  84 #if defined(CONFIG_CPU_CACHE_V4WB)
  85 # ifdef _CACHE
  86 #  define MULTI_CACHE 1
  87 # else
  88 #  define _CACHE v4wb
  89 # endif
  90 #endif
  91
  92 #if defined(CONFIG_CPU_XSCALE)
  93 # ifdef _CACHE
  94 #  define MULTI_CACHE 1
  95 # else
  96 #  define _CACHE xscale
  97 # endif
  98 #endif
  99
 100 #if defined(CONFIG_CPU_XSC3)
 101 # ifdef _CACHE
 102 #  define MULTI_CACHE 1
 103 # else
 104 #  define _CACHE xsc3
 105 # endif
 106 #endif
 107
 108 #if defined(CONFIG_CPU_MOHAWK)
 109 # ifdef _CACHE
 110 #  define MULTI_CACHE 1
 111 # else
 112 #  define _CACHE mohawk
 113 # endif
 114 #endif
 115
 116 #if defined(CONFIG_CPU_FEROCEON)
 117 # define MULTI_CACHE 1
 118 #endif
 119
 120 #if defined(CONFIG_CPU_V6)
 121 //# ifdef _CACHE
 122 #  define MULTI_CACHE 1
 123 //# else
 124 //#  define _CACHE v6
 125 //# endif
 126 #endif
 127
 128 #if defined(CONFIG_CPU_V7)
 129 //# ifdef _CACHE
 130 #  define MULTI_CACHE 1
 131 //# else
 132 //#  define _CACHE v7
 133 //# endif
 134 #endif
 135
 136 #if !defined(_CACHE) && !defined(MULTI_CACHE)
 137 #error Unknown cache maintainence model
 138 #endif
 139
 140 /*
 141  * This flag is used to indicate that the page pointed to by a pte
 142  * is dirty and requires cleaning before returning it to the user.
 143  */
 144 #define PG_dcache_dirty PG_arch_1
 145
 146 /*
 147  *      MM Cache Management
 148  *      ===================
 149  *
 150  *      The arch/arm/mm/cache-*.S and arch/arm/mm/proc-*.S files
 151  *      implement these methods.
 152  *
 153  *      Start addresses are inclusive and end addresses are exclusive;
 154  *      start addresses should be rounded down, end addresses up.
 155  *
 156  *      See Documentation/cachetlb.txt for more information.
 157  *      Please note that the implementation of these, and the required
 158  *      effects are cache-type (VIVT/VIPT/PIPT) specific.
 159  *
 160  *      flush_kern_all()
 161  *
 162  *              Unconditionally clean and invalidate the entire cache.
 163  *
 164  *      flush_user_all()
 165  *
 166  *              Clean and invalidate all user space cache entries
 167  *              before a change of page tables.
 168  *
 169  *      flush_user_range(start, end, flags)
 170  *
 171  *              Clean and invalidate a range of cache entries in the
 172  *              specified address space before a change of page tables.
 173  *              - start - user start address (inclusive, page aligned)
 174  *              - end   - user end address   (exclusive, page aligned)
 175  *              - flags - vma->vm_flags field
 176  *
 177  *      coherent_kern_range(start, end)
 178  *
 179  *              Ensure coherency between the Icache and the Dcache in the
 180  *              region described by start, end.  If you have non-snooping
 181  *              Harvard caches, you need to implement this function.
 182  *              - start  - virtual start address
 183  *              - end    - virtual end address
 184  *
 185  *      coherent_user_range(start, end)
 186  *
 187  *              Ensure coherency between the Icache and the Dcache in the
 188  *              region described by start, end.  If you have non-snooping
 189  *              Harvard caches, you need to implement this function.
 190  *              - start  - virtual start address
 191  *              - end    - virtual end address
 192  *
 193  *      flush_kern_dcache_area(kaddr, size)
 194  *
 195  *              Ensure that the data held in page is written back.
 196  *              - kaddr  - page address
 197  *              - size   - region size
 198  *
 199  *      DMA Cache Coherency
 200  *      ===================
 201  *
 202  *      dma_flush_range(start, end)
 203  *
 204  *              Clean and invalidate the specified virtual address range.
 205  *              - start  - virtual start address
 206  *              - end    - virtual end address
 207  */
 208
 209 struct cpu_cache_fns {
 210         void (*flush_kern_all)(void);
 211         void (*flush_user_all)(void);
 212         void (*flush_user_range)(unsigned long, unsigned long, unsigned int);
 213
 214         void (*coherent_kern_range)(unsigned long, unsigned long);
 215         void (*coherent_user_range)(unsigned long, unsigned long);
 216         void (*flush_kern_dcache_area)(void *, size_t);
 217
 218         void (*dma_map_area)(const void *, size_t, int);
 219         void (*dma_unmap_area)(const void *, size_t, int);
 220
 221         void (*dma_flush_range)(const void *, const void *);
 222 };
 223
 224 /*
 225  * Select the calling method
 226  */
 227 #ifdef MULTI_CACHE
 228
 229 extern struct cpu_cache_fns cpu_cache;
 230
 231 #define __cpuc_flush_kern_all           cpu_cache.flush_kern_all
 232 #define __cpuc_flush_user_all           cpu_cache.flush_user_all
 233 #define __cpuc_flush_user_range         cpu_cache.flush_user_range
 234 #define __cpuc_coherent_kern_range      cpu_cache.coherent_kern_range
 235 #define __cpuc_coherent_user_range      cpu_cache.coherent_user_range
 236 #define __cpuc_flush_dcache_area        cpu_cache.flush_kern_dcache_area
 237
 238 /*
 239  * These are private to the dma-mapping API.  Do not use directly.
 240  * Their sole purpose is to ensure that data held in the cache
 241  * is visible to DMA, or data written by DMA to system memory is
 242  * visible to the CPU.
 243  */
 244 #define dmac_map_area                   cpu_cache.dma_map_area
 245 #define dmac_unmap_area         cpu_cache.dma_unmap_area
 246 #define dmac_flush_range                cpu_cache.dma_flush_range
 247
 248 #else
 249
 250 #define __cpuc_flush_kern_all           __glue(_CACHE,_flush_kern_cache_all)
 251 #define __cpuc_flush_user_all           __glue(_CACHE,_flush_user_cache_all)
 252 #define __cpuc_flush_user_range         __glue(_CACHE,_flush_user_cache_range)
 253 #define __cpuc_coherent_kern_range      __glue(_CACHE,_coherent_kern_range)
 254 #define __cpuc_coherent_user_range      __glue(_CACHE,_coherent_user_range)
 255 #define __cpuc_flush_dcache_area        __glue(_CACHE,_flush_kern_dcache_area)
 256
 257 extern void __cpuc_flush_kern_all(void);
 258 extern void __cpuc_flush_user_all(void);
 259 extern void __cpuc_flush_user_range(unsigned long, unsigned long, unsigned int);
 260 extern void __cpuc_coherent_kern_range(unsigned long, unsigned long);
 261 extern void __cpuc_coherent_user_range(unsigned long, unsigned long);
 262 extern void __cpuc_flush_dcache_area(void *, size_t);
 263
 264 /*
 265  * These are private to the dma-mapping API.  Do not use directly.
 266  * Their sole purpose is to ensure that data held in the cache
 267  * is visible to DMA, or data written by DMA to system memory is
 268  * visible to the CPU.
 269  */
 270 #define dmac_map_area                   __glue(_CACHE,_dma_map_area)
 271 #define dmac_unmap_area         __glue(_CACHE,_dma_unmap_area)
 272 #define dmac_flush_range                __glue(_CACHE,_dma_flush_range)
 273
 274 extern void dmac_map_area(const void *, size_t, int);
 275 extern void dmac_unmap_area(const void *, size_t, int);
 276 extern void dmac_flush_range(const void *, const void *);
 277
 278 #endif
 279
 280 /*
 281  * Copy user data from/to a page which is mapped into a different
 282  * processes address space.  Really, we want to allow our "user
 283  * space" model to handle this.
 284  */
 285 extern void copy_to_user_page(struct vm_area_struct *, struct page *,
 286         unsigned long, void *, const void *, unsigned long);
 287 #define copy_from_user_page(vma, page, vaddr, dst, src, len) \
 288         do {                                                    \
 289                 memcpy(dst, src, len);                          \
 290         } while (0)
 291
 292 /*
 293  * Convert calls to our calling convention.
 294  */
 295 #define flush_cache_all()               __cpuc_flush_kern_all()
 296
 297 static inline void vivt_flush_cache_mm(struct mm_struct *mm)
 298 {
 299         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)))
 300                 __cpuc_flush_user_all();
 301 }
 302
 303 static inline void
 304 vivt_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 305 {
 306         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
 307                 __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
 308                                         vma->vm_flags);
 309 }
 310
 311 static inline void
 312 vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
 313 {
 314         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
 315                 unsigned long addr = user_addr & PAGE_MASK;
 316                 __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
 317         }
 318 }
 319
 320 #ifndef CONFIG_CPU_CACHE_VIPT
 321 #define flush_cache_mm(mm) \
 322                 vivt_flush_cache_mm(mm)
 323 #define flush_cache_range(vma,start,end) \
 324                 vivt_flush_cache_range(vma,start,end)
 325 #define flush_cache_page(vma,addr,pfn) \
 326                 vivt_flush_cache_page(vma,addr,pfn)
 327 #else
 328 extern void flush_cache_mm(struct mm_struct *mm);
 329 extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 330 extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn);
 331 #endif
 332
 333 #define flush_cache_dup_mm(mm) flush_cache_mm(mm)
 334
 335 /*
 336  * flush_cache_user_range is used when we want to ensure that the
 337  * Harvard caches are synchronised for the user space address range.
 338  * This is used for the ARM private sys_cacheflush system call.
 339  */
 340 #define flush_cache_user_range(vma,start,end) \
 341         __cpuc_coherent_user_range((start) & PAGE_MASK, PAGE_ALIGN(end))
 342
 343 /*
 344  * Perform necessary cache operations to ensure that data previously
 345  * stored within this range of addresses can be executed by the CPU.
 346  */
 347 #define flush_icache_range(s,e)         __cpuc_coherent_kern_range(s,e)
 348
 349 /*
 350  * Perform necessary cache operations to ensure that the TLB will
 351  * see data written in the specified area.
 352  */
 353 #define clean_dcache_area(start,size)   cpu_dcache_clean_area(start, size)
 354
 355 /*
 356  * flush_dcache_page is used when the kernel has written to the page
 357  * cache page at virtual address page->virtual.
 358  *
 359  * If this page isn't mapped (ie, page_mapping == NULL), or it might
 360  * have userspace mappings, then we _must_ always clean + invalidate
 361  * the dcache entries associated with the kernel mapping.
 362  *
 363  * Otherwise we can defer the operation, and clean the cache when we are
 364  * about to change to user space.  This is the same method as used on SPARC64.
 365  * See update_mmu_cache for the user space part.
 366  */
 367 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
 368 extern void flush_dcache_page(struct page *);
 369
 370 static inline void __flush_icache_all(void)
 371 {
 372 #ifdef CONFIG_ARM_ERRATA_411920
 373         extern void v6_icache_inval_all(void);
 374         v6_icache_inval_all();
 375 #elif defined(CONFIG_SMP) && __LINUX_ARM_ARCH__ >= 7
 376         asm("mcr        p15, 0, %0, c7, c1, 0   @ invalidate I-cache inner shareable\n"
 377             :
 378             : "r" (0));
 379 #else
 380         asm("mcr        p15, 0, %0, c7, c5, 0   @ invalidate I-cache\n"
 381             :
 382             : "r" (0));
 383 #endif
 384 }
 385 static inline void flush_kernel_vmap_range(void *addr, int size)
 386 {
 387         if ((cache_is_vivt() || cache_is_vipt_aliasing()))
 388           __cpuc_flush_dcache_area(addr, (size_t)size);
 389 }
 390 static inline void invalidate_kernel_vmap_range(void *addr, int size)
 391 {
 392         if ((cache_is_vivt() || cache_is_vipt_aliasing()))
 393           __cpuc_flush_dcache_area(addr, (size_t)size);
 394 }
 395
 396 #define ARCH_HAS_FLUSH_ANON_PAGE
 397 static inline void flush_anon_page(struct vm_area_struct *vma,
 398                          struct page *page, unsigned long vmaddr)
 399 {
 400         extern void __flush_anon_page(struct vm_area_struct *vma,
 401                                 struct page *, unsigned long);
 402         if (PageAnon(page))
 403                 __flush_anon_page(vma, page, vmaddr);
 404 }
 405
 406 #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
 407 static inline void flush_kernel_dcache_page(struct page *page)
 408 {
 409         /* highmem pages are always flushed upon kunmap already */
 410         if ((cache_is_vivt() || cache_is_vipt_aliasing()) && !PageHighMem(page))
 411                 __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 412 }
 413
 414 #define flush_dcache_mmap_lock(mapping) \
 415         spin_lock_irq(&(mapping)->tree_lock)
 416 #define flush_dcache_mmap_unlock(mapping) \
 417         spin_unlock_irq(&(mapping)->tree_lock)
 418
 419 #define flush_icache_user_range(vma,page,addr,len) \
 420         flush_dcache_page(page)
 421
 422 /*
 423  * We don't appear to need to do anything here.  In fact, if we did, we'd
 424  * duplicate cache flushing elsewhere performed by flush_dcache_page().
 425  */
 426 #define flush_icache_page(vma,page)     do { } while (0)
 427
 428 /*
 429  * flush_cache_vmap() is used when creating mappings (eg, via vmap,
 430  * vmalloc, ioremap etc) in kernel space for pages.  On non-VIPT
 431  * caches, since the direct-mappings of these pages may contain cached
 432  * data, we need to do a full cache flush to ensure that writebacks
 433  * don't corrupt data placed into these pages via the new mappings.
 434  */
 435 static inline void flush_cache_vmap(unsigned long start, unsigned long end)
 436 {
 437         if (!cache_is_vipt_nonaliasing())
 438                 flush_cache_all();
 439         else
 440                 /*
 441                  * set_pte_at() called from vmap_pte_range() does not
 442                  * have a DSB after cleaning the cache line.
 443                  */
 444                 dsb();
 445 }
 446
 447 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 448 {
 449         if (!cache_is_vipt_nonaliasing())
 450                 flush_cache_all();
 451 }
 452
 453 #endif