6 #include <linux/swap.h>
7 #include <asm/processor.h>
8 #include <asm/pgalloc.h>
10 #include <asm/tlbflush.h>
15 #define TLB_NR_PTRS 508
23 void *array
[TLB_NR_PTRS
];
26 DECLARE_PER_CPU(struct mmu_gather
, mmu_gathers
);
28 static inline struct mmu_gather
*tlb_gather_mmu(struct mm_struct
*mm
,
29 unsigned int full_mm_flush
)
31 struct mmu_gather
*tlb
= &get_cpu_var(mmu_gathers
);
34 tlb
->fullmm
= full_mm_flush
;
36 tlb
->nr_pxds
= TLB_NR_PTRS
;
42 static inline void tlb_flush_mmu(struct mmu_gather
*tlb
,
43 unsigned long start
, unsigned long end
)
45 if (!tlb
->fullmm
&& (tlb
->nr_ptes
> 0 || tlb
->nr_pxds
< TLB_NR_PTRS
))
46 __tlb_flush_mm(tlb
->mm
);
47 while (tlb
->nr_ptes
> 0)
48 pte_free(tlb
->mm
, tlb
->array
[--tlb
->nr_ptes
]);
49 while (tlb
->nr_pxds
< TLB_NR_PTRS
)
50 /* pgd_free frees the pointer as region or segment table */
51 pgd_free(tlb
->mm
, tlb
->array
[tlb
->nr_pxds
++]);
54 static inline void tlb_finish_mmu(struct mmu_gather
*tlb
,
55 unsigned long start
, unsigned long end
)
57 tlb_flush_mmu(tlb
, start
, end
);
59 /* keep the page table cache within bounds */
62 put_cpu_var(mmu_gathers
);
66 * Release the page cache reference for a pte removed by
67 * tlb_ptep_clear_flush. In both flush modes the tlb fo a page cache page
68 * has already been freed, so just do free_page_and_swap_cache.
70 static inline void tlb_remove_page(struct mmu_gather
*tlb
, struct page
*page
)
72 free_page_and_swap_cache(page
);
76 * pte_free_tlb frees a pte table and clears the CRSTE for the
77 * page table from the tlb.
79 static inline void pte_free_tlb(struct mmu_gather
*tlb
, pgtable_t pte
,
80 unsigned long address
)
83 tlb
->array
[tlb
->nr_ptes
++] = pte
;
84 if (tlb
->nr_ptes
>= tlb
->nr_pxds
)
85 tlb_flush_mmu(tlb
, 0, 0);
87 pte_free(tlb
->mm
, pte
);
91 * pmd_free_tlb frees a pmd table and clears the CRSTE for the
92 * segment table entry from the tlb.
93 * If the mm uses a two level page table the single pmd is freed
94 * as the pgd. pmd_free_tlb checks the asce_limit against 2GB
95 * to avoid the double free of the pmd in this case.
97 static inline void pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
,
98 unsigned long address
)
101 if (tlb
->mm
->context
.asce_limit
<= (1UL << 31))
104 tlb
->array
[--tlb
->nr_pxds
] = pmd
;
105 if (tlb
->nr_ptes
>= tlb
->nr_pxds
)
106 tlb_flush_mmu(tlb
, 0, 0);
108 pmd_free(tlb
->mm
, pmd
);
113 * pud_free_tlb frees a pud table and clears the CRSTE for the
114 * region third table entry from the tlb.
115 * If the mm uses a three level page table the single pud is freed
116 * as the pgd. pud_free_tlb checks the asce_limit against 4TB
117 * to avoid the double free of the pud in this case.
119 static inline void pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
,
120 unsigned long address
)
123 if (tlb
->mm
->context
.asce_limit
<= (1UL << 42))
126 tlb
->array
[--tlb
->nr_pxds
] = pud
;
127 if (tlb
->nr_ptes
>= tlb
->nr_pxds
)
128 tlb_flush_mmu(tlb
, 0, 0);
130 pud_free(tlb
->mm
, pud
);
134 #define tlb_start_vma(tlb, vma) do { } while (0)
135 #define tlb_end_vma(tlb, vma) do { } while (0)
136 #define tlb_remove_tlb_entry(tlb, ptep, addr) do { } while (0)
137 #define tlb_migrate_finish(mm) do { } while (0)
139 #endif /* _S390_TLB_H */