arch/arm/mm/flush.c

   1 /*
   2  *  linux/arch/arm/mm/flush.c
   3  *
   4  *  Copyright (C) 1995-2002 Russell King
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/module.h>
  11 #include <linux/mm.h>
  12 #include <linux/pagemap.h>
  13
  14 #include <asm/cacheflush.h>
  15 #include <asm/system.h>
  16
  17 #ifdef CONFIG_MMU
  18
  19 #include <asm/tlbflush.h>
  20
  21 #include "mm.h"
  22
  23 #ifdef CONFIG_CPU_CACHE_VIPT
  24
  25 #define ALIAS_FLUSH_START       0xffff4000
  26
  27 static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
  28 {
  29         unsigned long to = ALIAS_FLUSH_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  30         const int zero = 0;
  31
  32         set_pte(TOP_PTE(to), pfn_pte(pfn, PAGE_KERNEL));
  33         flush_tlb_kernel_page(to);
  34
  35         asm(    "mcrr   p15, 0, %1, %0, c14\n"
  36         "       mcr     p15, 0, %2, c7, c10, 4\n"
  37         "       mcr     p15, 0, %2, c7, c5, 0\n"
  38             :
  39             : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES), "r" (zero)
  40             : "cc");
  41 }
  42
  43 void flush_cache_mm(struct mm_struct *mm)
  44 {
  45         if (cache_is_vivt()) {
  46                 if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
  47                         __cpuc_flush_user_all();
  48                 return;
  49         }
  50
  51         if (cache_is_vipt_aliasing()) {
  52                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  53                 "       mcr     p15, 0, %0, c7, c5, 0\n"
  54                 "       mcr     p15, 0, %0, c7, c10, 4"
  55                     :
  56                     : "r" (0)
  57                     : "cc");
  58         }
  59 }
  60
  61 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  62 {
  63         if (cache_is_vivt()) {
  64                 if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
  65                         __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
  66                                                 vma->vm_flags);
  67                 return;
  68         }
  69
  70         if (cache_is_vipt_aliasing()) {
  71                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  72                 "       mcr     p15, 0, %0, c7, c5, 0\n"
  73                 "       mcr     p15, 0, %0, c7, c10, 4"
  74                     :
  75                     : "r" (0)
  76                     : "cc");
  77         }
  78 }
  79
  80 void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  81 {
  82         if (cache_is_vivt()) {
  83                 if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
  84                         unsigned long addr = user_addr & PAGE_MASK;
  85                         __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
  86                 }
  87                 return;
  88         }
  89
  90         if (cache_is_vipt_aliasing())
  91                 flush_pfn_alias(pfn, user_addr);
  92 }
  93
  94 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
  95                          unsigned long uaddr, void *kaddr,
  96                          unsigned long len, int write)
  97 {
  98         if (cache_is_vivt()) {
  99                 if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
 100                         unsigned long addr = (unsigned long)kaddr;
 101                         __cpuc_coherent_kern_range(addr, addr + len);
 102                 }
 103                 return;
 104         }
 105
 106         if (cache_is_vipt_aliasing()) {
 107                 flush_pfn_alias(page_to_pfn(page), uaddr);
 108                 return;
 109         }
 110
 111         /* VIPT non-aliasing cache */
 112         if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask) &&
 113             vma->vm_flags & VM_EXEC) {
 114                 unsigned long addr = (unsigned long)kaddr;
 115                 /* only flushing the kernel mapping on non-aliasing VIPT */
 116                 __cpuc_coherent_kern_range(addr, addr + len);
 117         }
 118 }
 119 #else
 120 #define flush_pfn_alias(pfn,vaddr)      do { } while (0)
 121 #endif
 122 #else
 123 #define flush_pfn_alias(pfn,vaddr)      do { } while (0)
 124 #endif /* CONFIG_MMU */
 125
 126 void __flush_dcache_page(struct address_space *mapping, struct page *page)
 127 {
 128         /*
 129          * Writeback any data associated with the kernel mapping of this
 130          * page.  This ensures that data in the physical page is mutually
 131          * coherent with the kernels mapping.
 132          */
 133         __cpuc_flush_dcache_page(page_address(page));
 134
 135         /*
 136          * If this is a page cache page, and we have an aliasing VIPT cache,
 137          * we only need to do one flush - which would be at the relevant
 138          * userspace colour, which is congruent with page->index.
 139          */
 140         if (mapping && cache_is_vipt_aliasing())
 141                 flush_pfn_alias(page_to_pfn(page),
 142                                 page->index << PAGE_CACHE_SHIFT);
 143 }
 144
 145 static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
 146 {
 147         struct mm_struct *mm = current->active_mm;
 148         struct vm_area_struct *mpnt;
 149         struct prio_tree_iter iter;
 150         pgoff_t pgoff;
 151
 152         /*
 153          * There are possible user space mappings of this page:
 154          * - VIVT cache: we need to also write back and invalidate all user
 155          *   data in the current VM view associated with this page.
 156          * - aliasing VIPT: we only need to find one mapping of this page.
 157          */
 158         pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
 159
 160         flush_dcache_mmap_lock(mapping);
 161         vma_prio_tree_foreach(mpnt, &iter, &mapping->i_mmap, pgoff, pgoff) {
 162                 unsigned long offset;
 163
 164                 /*
 165                  * If this VMA is not in our MM, we can ignore it.
 166                  */
 167                 if (mpnt->vm_mm != mm)
 168                         continue;
 169                 if (!(mpnt->vm_flags & VM_MAYSHARE))
 170                         continue;
 171                 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
 172                 flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
 173         }
 174         flush_dcache_mmap_unlock(mapping);
 175 }
 176
 177 /*
 178  * Ensure cache coherency between kernel mapping and userspace mapping
 179  * of this page.
 180  *
 181  * We have three cases to consider:
 182  *  - VIPT non-aliasing cache: fully coherent so nothing required.
 183  *  - VIVT: fully aliasing, so we need to handle every alias in our
 184  *          current VM view.
 185  *  - VIPT aliasing: need to handle one alias in our current VM view.
 186  *
 187  * If we need to handle aliasing:
 188  *  If the page only exists in the page cache and there are no user
 189  *  space mappings, we can be lazy and remember that we may have dirty
 190  *  kernel cache lines for later.  Otherwise, we assume we have
 191  *  aliasing mappings.
 192  *
 193  * Note that we disable the lazy flush for SMP.
 194  */
 195 void flush_dcache_page(struct page *page)
 196 {
 197         struct address_space *mapping = page_mapping(page);
 198
 199 #ifndef CONFIG_SMP
 200         if (mapping && !mapping_mapped(mapping))
 201                 set_bit(PG_dcache_dirty, &page->flags);
 202         else
 203 #endif
 204         {
 205                 __flush_dcache_page(mapping, page);
 206                 if (mapping && cache_is_vivt())
 207                         __flush_dcache_aliases(mapping, page);
 208         }
 209 }
 210 EXPORT_SYMBOL(flush_dcache_page);