mm/page_table_check.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright (c) 2021, Google LLC.
   5  * Pasha Tatashin <pasha.tatashin@soleen.com>
   6  */
   7 #include <linux/kstrtox.h>
   8 #include <linux/mm.h>
   9 #include <linux/page_table_check.h>
  10 #include <linux/swap.h>
  11 #include <linux/swapops.h>
  12
  13 #undef pr_fmt
  14 #define pr_fmt(fmt)     "page_table_check: " fmt
  15
  16 struct page_table_check {
  17         atomic_t anon_map_count;
  18         atomic_t file_map_count;
  19 };
  20
  21 static bool __page_table_check_enabled __initdata =
  22                                 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
  23
  24 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
  25 EXPORT_SYMBOL(page_table_check_disabled);
  26
  27 static int __init early_page_table_check_param(char *buf)
  28 {
  29         return kstrtobool(buf, &__page_table_check_enabled);
  30 }
  31
  32 early_param("page_table_check", early_page_table_check_param);
  33
  34 static bool __init need_page_table_check(void)
  35 {
  36         return __page_table_check_enabled;
  37 }
  38
  39 static void __init init_page_table_check(void)
  40 {
  41         if (!__page_table_check_enabled)
  42                 return;
  43         static_branch_disable(&page_table_check_disabled);
  44 }
  45
  46 struct page_ext_operations page_table_check_ops = {
  47         .size = sizeof(struct page_table_check),
  48         .need = need_page_table_check,
  49         .init = init_page_table_check,
  50         .need_shared_flags = false,
  51 };
  52
  53 static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
  54 {
  55         BUG_ON(!page_ext);
  56         return page_ext_data(page_ext, &page_table_check_ops);
  57 }
  58
  59 /*
  60  * An entry is removed from the page table, decrement the counters for that page
  61  * verify that it is of correct type and counters do not become negative.
  62  */
  63 static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
  64 {
  65         struct page_ext *page_ext;
  66         struct page *page;
  67         unsigned long i;
  68         bool anon;
  69
  70         if (!pfn_valid(pfn))
  71                 return;
  72
  73         page = pfn_to_page(pfn);
  74         page_ext = page_ext_get(page);
  75
  76         if (!page_ext)
  77                 return;
  78
  79         BUG_ON(PageSlab(page));
  80         anon = PageAnon(page);
  81
  82         for (i = 0; i < pgcnt; i++) {
  83                 struct page_table_check *ptc = get_page_table_check(page_ext);
  84
  85                 if (anon) {
  86                         BUG_ON(atomic_read(&ptc->file_map_count));
  87                         BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
  88                 } else {
  89                         BUG_ON(atomic_read(&ptc->anon_map_count));
  90                         BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
  91                 }
  92                 page_ext = page_ext_next(page_ext);
  93         }
  94         page_ext_put(page_ext);
  95 }
  96
  97 /*
  98  * A new entry is added to the page table, increment the counters for that page
  99  * verify that it is of correct type and is not being mapped with a different
 100  * type to a different process.
 101  */
 102 static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
 103                                  bool rw)
 104 {
 105         struct page_ext *page_ext;
 106         struct page *page;
 107         unsigned long i;
 108         bool anon;
 109
 110         if (!pfn_valid(pfn))
 111                 return;
 112
 113         page = pfn_to_page(pfn);
 114         page_ext = page_ext_get(page);
 115
 116         if (!page_ext)
 117                 return;
 118
 119         BUG_ON(PageSlab(page));
 120         anon = PageAnon(page);
 121
 122         for (i = 0; i < pgcnt; i++) {
 123                 struct page_table_check *ptc = get_page_table_check(page_ext);
 124
 125                 if (anon) {
 126                         BUG_ON(atomic_read(&ptc->file_map_count));
 127                         BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
 128                 } else {
 129                         BUG_ON(atomic_read(&ptc->anon_map_count));
 130                         BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
 131                 }
 132                 page_ext = page_ext_next(page_ext);
 133         }
 134         page_ext_put(page_ext);
 135 }
 136
 137 /*
 138  * page is on free list, or is being allocated, verify that counters are zeroes
 139  * crash if they are not.
 140  */
 141 void __page_table_check_zero(struct page *page, unsigned int order)
 142 {
 143         struct page_ext *page_ext;
 144         unsigned long i;
 145
 146         BUG_ON(PageSlab(page));
 147
 148         page_ext = page_ext_get(page);
 149
 150         if (!page_ext)
 151                 return;
 152
 153         for (i = 0; i < (1ul << order); i++) {
 154                 struct page_table_check *ptc = get_page_table_check(page_ext);
 155
 156                 BUG_ON(atomic_read(&ptc->anon_map_count));
 157                 BUG_ON(atomic_read(&ptc->file_map_count));
 158                 page_ext = page_ext_next(page_ext);
 159         }
 160         page_ext_put(page_ext);
 161 }
 162
 163 void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
 164 {
 165         if (&init_mm == mm)
 166                 return;
 167
 168         if (pte_user_accessible_page(pte)) {
 169                 page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
 170         }
 171 }
 172 EXPORT_SYMBOL(__page_table_check_pte_clear);
 173
 174 void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
 175 {
 176         if (&init_mm == mm)
 177                 return;
 178
 179         if (pmd_user_accessible_page(pmd)) {
 180                 page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
 181         }
 182 }
 183 EXPORT_SYMBOL(__page_table_check_pmd_clear);
 184
 185 void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
 186 {
 187         if (&init_mm == mm)
 188                 return;
 189
 190         if (pud_user_accessible_page(pud)) {
 191                 page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
 192         }
 193 }
 194 EXPORT_SYMBOL(__page_table_check_pud_clear);
 195
 196 /* Whether the swap entry cached writable information */
 197 static inline bool swap_cached_writable(swp_entry_t entry)
 198 {
 199         return is_writable_device_exclusive_entry(entry) ||
 200             is_writable_device_private_entry(entry) ||
 201             is_writable_migration_entry(entry);
 202 }
 203
 204 static inline void page_table_check_pte_flags(pte_t pte)
 205 {
 206         if (pte_present(pte) && pte_uffd_wp(pte))
 207                 WARN_ON_ONCE(pte_write(pte));
 208         else if (is_swap_pte(pte) && pte_swp_uffd_wp(pte))
 209                 WARN_ON_ONCE(swap_cached_writable(pte_to_swp_entry(pte)));
 210 }
 211
 212 void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
 213                 unsigned int nr)
 214 {
 215         unsigned int i;
 216
 217         if (&init_mm == mm)
 218                 return;
 219
 220         page_table_check_pte_flags(pte);
 221
 222         for (i = 0; i < nr; i++)
 223                 __page_table_check_pte_clear(mm, ptep_get(ptep + i));
 224         if (pte_user_accessible_page(pte))
 225                 page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
 226 }
 227 EXPORT_SYMBOL(__page_table_check_ptes_set);
 228
 229 static inline void page_table_check_pmd_flags(pmd_t pmd)
 230 {
 231         if (pmd_present(pmd) && pmd_uffd_wp(pmd))
 232                 WARN_ON_ONCE(pmd_write(pmd));
 233         else if (is_swap_pmd(pmd) && pmd_swp_uffd_wp(pmd))
 234                 WARN_ON_ONCE(swap_cached_writable(pmd_to_swp_entry(pmd)));
 235 }
 236
 237 void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd)
 238 {
 239         if (&init_mm == mm)
 240                 return;
 241
 242         page_table_check_pmd_flags(pmd);
 243
 244         __page_table_check_pmd_clear(mm, *pmdp);
 245         if (pmd_user_accessible_page(pmd)) {
 246                 page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT,
 247                                      pmd_write(pmd));
 248         }
 249 }
 250 EXPORT_SYMBOL(__page_table_check_pmd_set);
 251
 252 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud)
 253 {
 254         if (&init_mm == mm)
 255                 return;
 256
 257         __page_table_check_pud_clear(mm, *pudp);
 258         if (pud_user_accessible_page(pud)) {
 259                 page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT,
 260                                      pud_write(pud));
 261         }
 262 }
 263 EXPORT_SYMBOL(__page_table_check_pud_set);
 264
 265 void __page_table_check_pte_clear_range(struct mm_struct *mm,
 266                                         unsigned long addr,
 267                                         pmd_t pmd)
 268 {
 269         if (&init_mm == mm)
 270                 return;
 271
 272         if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
 273                 pte_t *ptep = pte_offset_map(&pmd, addr);
 274                 unsigned long i;
 275
 276                 if (WARN_ON(!ptep))
 277                         return;
 278                 for (i = 0; i < PTRS_PER_PTE; i++) {
 279                         __page_table_check_pte_clear(mm, ptep_get(ptep));
 280                         addr += PAGE_SIZE;
 281                         ptep++;
 282                 }
 283                 pte_unmap(ptep - PTRS_PER_PTE);
 284         }
 285 }