2 * SPARC64 Huge TLB page support.
4 * Copyright (C) 2002, 2003 David S. Miller (davem@redhat.com)
7 #include <linux/config.h>
8 #include <linux/init.h>
9 #include <linux/module.h>
12 #include <linux/hugetlb.h>
13 #include <linux/pagemap.h>
14 #include <linux/smp_lock.h>
15 #include <linux/slab.h>
16 #include <linux/sysctl.h>
19 #include <asm/pgalloc.h>
21 #include <asm/tlbflush.h>
22 #include <asm/cacheflush.h>
24 static pte_t
*huge_pte_alloc(struct mm_struct
*mm
, unsigned long addr
)
30 pgd
= pgd_offset(mm
, addr
);
32 pmd
= pmd_alloc(mm
, pgd
, addr
);
34 pte
= pte_alloc_map(mm
, pmd
, addr
);
39 static pte_t
*huge_pte_offset(struct mm_struct
*mm
, unsigned long addr
)
45 pgd
= pgd_offset(mm
, addr
);
47 pmd
= pmd_offset(pgd
, addr
);
49 pte
= pte_offset_map(pmd
, addr
);
54 #define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0)
56 static void set_huge_pte(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
57 struct page
*page
, pte_t
* page_table
, int write_access
)
62 mm
->rss
+= (HPAGE_SIZE
/ PAGE_SIZE
);
65 entry
= pte_mkwrite(pte_mkdirty(mk_pte(page
,
68 entry
= pte_wrprotect(mk_pte(page
, vma
->vm_page_prot
));
69 entry
= pte_mkyoung(entry
);
72 for (i
= 0; i
< (1 << HUGETLB_PAGE_ORDER
); i
++) {
73 set_pte(page_table
, entry
);
76 pte_val(entry
) += PAGE_SIZE
;
81 * This function checks for proper alignment of input addr and len parameters.
83 int is_aligned_hugepage_range(unsigned long addr
, unsigned long len
)
85 if (len
& ~HPAGE_MASK
)
87 if (addr
& ~HPAGE_MASK
)
92 int copy_hugetlb_page_range(struct mm_struct
*dst
, struct mm_struct
*src
,
93 struct vm_area_struct
*vma
)
95 pte_t
*src_pte
, *dst_pte
, entry
;
97 unsigned long addr
= vma
->vm_start
;
98 unsigned long end
= vma
->vm_end
;
102 dst_pte
= huge_pte_alloc(dst
, addr
);
105 src_pte
= huge_pte_offset(src
, addr
);
106 BUG_ON(!src_pte
|| pte_none(*src_pte
));
108 ptepage
= pte_page(entry
);
110 for (i
= 0; i
< (1 << HUGETLB_PAGE_ORDER
); i
++) {
111 set_pte(dst_pte
, entry
);
112 pte_val(entry
) += PAGE_SIZE
;
115 dst
->rss
+= (HPAGE_SIZE
/ PAGE_SIZE
);
124 int follow_hugetlb_page(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
125 struct page
**pages
, struct vm_area_struct
**vmas
,
126 unsigned long *position
, int *length
, int i
)
128 unsigned long vaddr
= *position
;
129 int remainder
= *length
;
131 WARN_ON(!is_vm_hugetlb_page(vma
));
133 while (vaddr
< vma
->vm_end
&& remainder
) {
138 pte
= huge_pte_offset(mm
, vaddr
);
140 /* hugetlb should be locked, and hence, prefaulted */
141 BUG_ON(!pte
|| pte_none(*pte
));
143 page
= pte_page(*pte
);
145 WARN_ON(!PageCompound(page
));
165 struct page
*follow_huge_addr(struct mm_struct
*mm
,
166 unsigned long address
, int write
)
168 return ERR_PTR(-EINVAL
);
171 int pmd_huge(pmd_t pmd
)
176 struct page
*follow_huge_pmd(struct mm_struct
*mm
, unsigned long address
,
177 pmd_t
*pmd
, int write
)
182 void unmap_hugepage_range(struct vm_area_struct
*vma
,
183 unsigned long start
, unsigned long end
)
185 struct mm_struct
*mm
= vma
->vm_mm
;
186 unsigned long address
;
191 BUG_ON(start
& (HPAGE_SIZE
- 1));
192 BUG_ON(end
& (HPAGE_SIZE
- 1));
194 for (address
= start
; address
< end
; address
+= HPAGE_SIZE
) {
195 pte
= huge_pte_offset(mm
, address
);
199 page
= pte_page(*pte
);
201 for (i
= 0; i
< (1 << HUGETLB_PAGE_ORDER
); i
++) {
206 mm
->rss
-= (end
- start
) >> PAGE_SHIFT
;
207 flush_tlb_range(vma
, start
, end
);
210 int hugetlb_prefault(struct address_space
*mapping
, struct vm_area_struct
*vma
)
212 struct mm_struct
*mm
= current
->mm
;
216 BUG_ON(vma
->vm_start
& ~HPAGE_MASK
);
217 BUG_ON(vma
->vm_end
& ~HPAGE_MASK
);
219 spin_lock(&mm
->page_table_lock
);
220 for (addr
= vma
->vm_start
; addr
< vma
->vm_end
; addr
+= HPAGE_SIZE
) {
222 pte_t
*pte
= huge_pte_alloc(mm
, addr
);
232 idx
= ((addr
- vma
->vm_start
) >> HPAGE_SHIFT
)
233 + (vma
->vm_pgoff
>> (HPAGE_SHIFT
- PAGE_SHIFT
));
234 page
= find_get_page(mapping
, idx
);
236 /* charge the fs quota first */
237 if (hugetlb_get_quota(mapping
)) {
241 page
= alloc_huge_page();
243 hugetlb_put_quota(mapping
);
247 ret
= add_to_page_cache(page
, mapping
, idx
, GFP_ATOMIC
);
251 hugetlb_put_quota(mapping
);
252 free_huge_page(page
);
256 set_huge_pte(mm
, vma
, page
, pte
, vma
->vm_flags
& VM_WRITE
);
259 spin_unlock(&mm
->page_table_lock
);