arch/s390/mm/gup.c

   1 /*
   2  *  Lockless get_user_pages_fast for s390
   3  *
   4  *  Copyright IBM Corp. 2010
   5  *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
   6  */
   7 #include <linux/sched.h>
   8 #include <linux/mm.h>
   9 #include <linux/hugetlb.h>
  10 #include <linux/vmstat.h>
  11 #include <linux/pagemap.h>
  12 #include <linux/rwsem.h>
  13 #include <asm/pgtable.h>
  14
  15 /*
  16  * The performance critical leaf functions are made noinline otherwise gcc
  17  * inlines everything into a single function which results in too much
  18  * register pressure.
  19  */
  20 static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
  21                 unsigned long end, int write, struct page **pages, int *nr)
  22 {
  23         unsigned long mask;
  24         pte_t *ptep, pte;
  25         struct page *page;
  26
  27         mask = (write ? _PAGE_RO : 0) | _PAGE_INVALID | _PAGE_SPECIAL;
  28
  29         ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr);
  30         do {
  31                 pte = *ptep;
  32                 barrier();
  33                 if ((pte_val(pte) & mask) != 0)
  34                         return 0;
  35                 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
  36                 page = pte_page(pte);
  37                 if (!page_cache_get_speculative(page))
  38                         return 0;
  39                 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
  40                         put_page(page);
  41                         return 0;
  42                 }
  43                 pages[*nr] = page;
  44                 (*nr)++;
  45
  46         } while (ptep++, addr += PAGE_SIZE, addr != end);
  47
  48         return 1;
  49 }
  50
  51 static inline void get_huge_page_tail(struct page *page)
  52 {
  53         /*
  54          * __split_huge_page_refcount() cannot run
  55          * from under us.
  56          */
  57         VM_BUG_ON(page_mapcount(page) < 0);
  58         VM_BUG_ON(atomic_read(&page->_count) != 0);
  59         atomic_inc(&page->_mapcount);
  60 }
  61
  62 static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
  63                 unsigned long end, int write, struct page **pages, int *nr)
  64 {
  65         unsigned long mask, result;
  66         struct page *head, *page, *tail;
  67         int refs;
  68
  69         result = write ? 0 : _SEGMENT_ENTRY_RO;
  70         mask = result | _SEGMENT_ENTRY_INV;
  71         if ((pmd_val(pmd) & mask) != result)
  72                 return 0;
  73         VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
  74
  75         refs = 0;
  76         head = pmd_page(pmd);
  77         page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
  78         tail = page;
  79         do {
  80                 VM_BUG_ON(compound_head(page) != head);
  81                 pages[*nr] = page;
  82                 (*nr)++;
  83                 page++;
  84                 refs++;
  85         } while (addr += PAGE_SIZE, addr != end);
  86
  87         if (!page_cache_add_speculative(head, refs)) {
  88                 *nr -= refs;
  89                 return 0;
  90         }
  91
  92         if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
  93                 *nr -= refs;
  94                 while (refs--)
  95                         put_page(head);
  96         } else {
  97                 /*
  98                  * Any tail page need their mapcount reference taken
  99                  * before we return.
 100                  */
 101                 while (refs--) {
 102                         if (PageTail(tail))
 103                                 get_huge_page_tail(tail);
 104                         tail++;
 105                 }
 106         }
 107
 108         return 1;
 109 }
 110
 111
 112 static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
 113                 unsigned long end, int write, struct page **pages, int *nr)
 114 {
 115         unsigned long next;
 116         pmd_t *pmdp, pmd;
 117
 118         pmdp = (pmd_t *) pudp;
 119 #ifdef CONFIG_64BIT
 120         if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 121                 pmdp = (pmd_t *) pud_deref(pud);
 122         pmdp += pmd_index(addr);
 123 #endif
 124         do {
 125                 pmd = *pmdp;
 126                 barrier();
 127                 next = pmd_addr_end(addr, end);
 128                 if (pmd_none(pmd))
 129                         return 0;
 130                 if (unlikely(pmd_huge(pmd))) {
 131                         if (!gup_huge_pmd(pmdp, pmd, addr, next,
 132                                           write, pages, nr))
 133                                 return 0;
 134                 } else if (!gup_pte_range(pmdp, pmd, addr, next,
 135                                           write, pages, nr))
 136                         return 0;
 137         } while (pmdp++, addr = next, addr != end);
 138
 139         return 1;
 140 }
 141
 142 static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
 143                 unsigned long end, int write, struct page **pages, int *nr)
 144 {
 145         unsigned long next;
 146         pud_t *pudp, pud;
 147
 148         pudp = (pud_t *) pgdp;
 149 #ifdef CONFIG_64BIT
 150         if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
 151                 pudp = (pud_t *) pgd_deref(pgd);
 152         pudp += pud_index(addr);
 153 #endif
 154         do {
 155                 pud = *pudp;
 156                 barrier();
 157                 next = pud_addr_end(addr, end);
 158                 if (pud_none(pud))
 159                         return 0;
 160                 if (!gup_pmd_range(pudp, pud, addr, next, write, pages, nr))
 161                         return 0;
 162         } while (pudp++, addr = next, addr != end);
 163
 164         return 1;
 165 }
 166
 167 /**
 168  * get_user_pages_fast() - pin user pages in memory
 169  * @start:      starting user address
 170  * @nr_pages:   number of pages from start to pin
 171  * @write:      whether pages will be written to
 172  * @pages:      array that receives pointers to the pages pinned.
 173  *              Should be at least nr_pages long.
 174  *
 175  * Attempt to pin user pages in memory without taking mm->mmap_sem.
 176  * If not successful, it will fall back to taking the lock and
 177  * calling get_user_pages().
 178  *
 179  * Returns number of pages pinned. This may be fewer than the number
 180  * requested. If nr_pages is 0 or negative, returns 0. If no pages
 181  * were pinned, returns -errno.
 182  */
 183 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
 184                         struct page **pages)
 185 {
 186         struct mm_struct *mm = current->mm;
 187         unsigned long addr, len, end;
 188         unsigned long next;
 189         pgd_t *pgdp, pgd;
 190         int nr = 0;
 191
 192         start &= PAGE_MASK;
 193         addr = start;
 194         len = (unsigned long) nr_pages << PAGE_SHIFT;
 195         end = start + len;
 196         if (end < start)
 197                 goto slow_irqon;
 198
 199         /*
 200          * local_irq_disable() doesn't prevent pagetable teardown, but does
 201          * prevent the pagetables from being freed on s390.
 202          *
 203          * So long as we atomically load page table pointers versus teardown,
 204          * we can follow the address down to the the page and take a ref on it.
 205          */
 206         local_irq_disable();
 207         pgdp = pgd_offset(mm, addr);
 208         do {
 209                 pgd = *pgdp;
 210                 barrier();
 211                 next = pgd_addr_end(addr, end);
 212                 if (pgd_none(pgd))
 213                         goto slow;
 214                 if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
 215                         goto slow;
 216         } while (pgdp++, addr = next, addr != end);
 217         local_irq_enable();
 218
 219         VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
 220         return nr;
 221
 222         {
 223                 int ret;
 224 slow:
 225                 local_irq_enable();
 226 slow_irqon:
 227                 /* Try to get the remaining pages with get_user_pages */
 228                 start += nr << PAGE_SHIFT;
 229                 pages += nr;
 230
 231                 down_read(&mm->mmap_sem);
 232                 ret = get_user_pages(current, mm, start,
 233                         (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
 234                 up_read(&mm->mmap_sem);
 235
 236                 /* Have to be a bit careful with return values */
 237                 if (nr > 0) {
 238                         if (ret < 0)
 239                                 ret = nr;
 240                         else
 241                                 ret += nr;
 242                 }
 243
 244                 return ret;
 245         }
 246 }