arch/s390/mm/vmem.c

   1 /*
   2  *  arch/s390/mm/vmem.c
   3  *
   4  *    Copyright IBM Corp. 2006
   5  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   6  */
   7
   8 #include <linux/bootmem.h>
   9 #include <linux/pfn.h>
  10 #include <linux/mm.h>
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <asm/pgalloc.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/setup.h>
  16 #include <asm/tlbflush.h>
  17
  18 static DEFINE_MUTEX(vmem_mutex);
  19
  20 struct memory_segment {
  21         struct list_head list;
  22         unsigned long start;
  23         unsigned long size;
  24 };
  25
  26 static LIST_HEAD(mem_segs);
  27
  28 void __meminit memmap_init(unsigned long size, int nid, unsigned long zone,
  29                            unsigned long start_pfn)
  30 {
  31         struct page *start, *end;
  32         struct page *map_start, *map_end;
  33         int i;
  34
  35         start = pfn_to_page(start_pfn);
  36         end = start + size;
  37
  38         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
  39                 unsigned long cstart, cend;
  40
  41                 cstart = PFN_DOWN(memory_chunk[i].addr);
  42                 cend = cstart + PFN_DOWN(memory_chunk[i].size);
  43
  44                 map_start = mem_map + cstart;
  45                 map_end = mem_map + cend;
  46
  47                 if (map_start < start)
  48                         map_start = start;
  49                 if (map_end > end)
  50                         map_end = end;
  51
  52                 map_start -= ((unsigned long) map_start & (PAGE_SIZE - 1))
  53                         / sizeof(struct page);
  54                 map_end += ((PFN_ALIGN((unsigned long) map_end)
  55                              - (unsigned long) map_end)
  56                             / sizeof(struct page));
  57
  58                 if (map_start < map_end)
  59                         memmap_init_zone((unsigned long)(map_end - map_start),
  60                                          nid, zone, page_to_pfn(map_start),
  61                                          MEMMAP_EARLY);
  62         }
  63 }
  64
  65 static void __ref *vmem_alloc_pages(unsigned int order)
  66 {
  67         if (slab_is_available())
  68                 return (void *)__get_free_pages(GFP_KERNEL, order);
  69         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  70 }
  71
  72 #define vmem_pud_alloc()        ({ BUG(); ((pud_t *) NULL); })
  73
  74 static inline pmd_t *vmem_pmd_alloc(void)
  75 {
  76         pmd_t *pmd = NULL;
  77
  78 #ifdef CONFIG_64BIT
  79         pmd = vmem_alloc_pages(2);
  80         if (!pmd)
  81                 return NULL;
  82         clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE*4);
  83 #endif
  84         return pmd;
  85 }
  86
  87 static pte_t __init_refok *vmem_pte_alloc(void)
  88 {
  89         pte_t *pte;
  90
  91         if (slab_is_available())
  92                 pte = (pte_t *) page_table_alloc(&init_mm);
  93         else
  94                 pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t));
  95         if (!pte)
  96                 return NULL;
  97         clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY,
  98                     PTRS_PER_PTE * sizeof(pte_t));
  99         return pte;
 100 }
 101
 102 /*
 103  * Add a physical memory range to the 1:1 mapping.
 104  */
 105 static int vmem_add_range(unsigned long start, unsigned long size)
 106 {
 107         unsigned long address;
 108         pgd_t *pg_dir;
 109         pud_t *pu_dir;
 110         pmd_t *pm_dir;
 111         pte_t *pt_dir;
 112         pte_t  pte;
 113         int ret = -ENOMEM;
 114
 115         for (address = start; address < start + size; address += PAGE_SIZE) {
 116                 pg_dir = pgd_offset_k(address);
 117                 if (pgd_none(*pg_dir)) {
 118                         pu_dir = vmem_pud_alloc();
 119                         if (!pu_dir)
 120                                 goto out;
 121                         pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
 122                 }
 123
 124                 pu_dir = pud_offset(pg_dir, address);
 125                 if (pud_none(*pu_dir)) {
 126                         pm_dir = vmem_pmd_alloc();
 127                         if (!pm_dir)
 128                                 goto out;
 129                         pud_populate_kernel(&init_mm, pu_dir, pm_dir);
 130                 }
 131
 132                 pm_dir = pmd_offset(pu_dir, address);
 133                 if (pmd_none(*pm_dir)) {
 134                         pt_dir = vmem_pte_alloc();
 135                         if (!pt_dir)
 136                                 goto out;
 137                         pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
 138                 }
 139
 140                 pt_dir = pte_offset_kernel(pm_dir, address);
 141                 pte = pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL);
 142                 *pt_dir = pte;
 143         }
 144         ret = 0;
 145 out:
 146         flush_tlb_kernel_range(start, start + size);
 147         return ret;
 148 }
 149
 150 /*
 151  * Remove a physical memory range from the 1:1 mapping.
 152  * Currently only invalidates page table entries.
 153  */
 154 static void vmem_remove_range(unsigned long start, unsigned long size)
 155 {
 156         unsigned long address;
 157         pgd_t *pg_dir;
 158         pud_t *pu_dir;
 159         pmd_t *pm_dir;
 160         pte_t *pt_dir;
 161         pte_t  pte;
 162
 163         pte_val(pte) = _PAGE_TYPE_EMPTY;
 164         for (address = start; address < start + size; address += PAGE_SIZE) {
 165                 pg_dir = pgd_offset_k(address);
 166                 pu_dir = pud_offset(pg_dir, address);
 167                 if (pud_none(*pu_dir))
 168                         continue;
 169                 pm_dir = pmd_offset(pu_dir, address);
 170                 if (pmd_none(*pm_dir))
 171                         continue;
 172                 pt_dir = pte_offset_kernel(pm_dir, address);
 173                 *pt_dir = pte;
 174         }
 175         flush_tlb_kernel_range(start, start + size);
 176 }
 177
 178 /*
 179  * Add a backed mem_map array to the virtual mem_map array.
 180  */
 181 static int vmem_add_mem_map(unsigned long start, unsigned long size)
 182 {
 183         unsigned long address, start_addr, end_addr;
 184         struct page *map_start, *map_end;
 185         pgd_t *pg_dir;
 186         pud_t *pu_dir;
 187         pmd_t *pm_dir;
 188         pte_t *pt_dir;
 189         pte_t  pte;
 190         int ret = -ENOMEM;
 191
 192         map_start = VMEM_MAP + PFN_DOWN(start);
 193         map_end = VMEM_MAP + PFN_DOWN(start + size);
 194
 195         start_addr = (unsigned long) map_start & PAGE_MASK;
 196         end_addr = PFN_ALIGN((unsigned long) map_end);
 197
 198         for (address = start_addr; address < end_addr; address += PAGE_SIZE) {
 199                 pg_dir = pgd_offset_k(address);
 200                 if (pgd_none(*pg_dir)) {
 201                         pu_dir = vmem_pud_alloc();
 202                         if (!pu_dir)
 203                                 goto out;
 204                         pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
 205                 }
 206
 207                 pu_dir = pud_offset(pg_dir, address);
 208                 if (pud_none(*pu_dir)) {
 209                         pm_dir = vmem_pmd_alloc();
 210                         if (!pm_dir)
 211                                 goto out;
 212                         pud_populate_kernel(&init_mm, pu_dir, pm_dir);
 213                 }
 214
 215                 pm_dir = pmd_offset(pu_dir, address);
 216                 if (pmd_none(*pm_dir)) {
 217                         pt_dir = vmem_pte_alloc();
 218                         if (!pt_dir)
 219                                 goto out;
 220                         pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
 221                 }
 222
 223                 pt_dir = pte_offset_kernel(pm_dir, address);
 224                 if (pte_none(*pt_dir)) {
 225                         unsigned long new_page;
 226
 227                         new_page =__pa(vmem_alloc_pages(0));
 228                         if (!new_page)
 229                                 goto out;
 230                         pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL);
 231                         *pt_dir = pte;
 232                 }
 233         }
 234         ret = 0;
 235 out:
 236         flush_tlb_kernel_range(start_addr, end_addr);
 237         return ret;
 238 }
 239
 240 static int vmem_add_mem(unsigned long start, unsigned long size)
 241 {
 242         int ret;
 243
 244         ret = vmem_add_mem_map(start, size);
 245         if (ret)
 246                 return ret;
 247         return vmem_add_range(start, size);
 248 }
 249
 250 /*
 251  * Add memory segment to the segment list if it doesn't overlap with
 252  * an already present segment.
 253  */
 254 static int insert_memory_segment(struct memory_segment *seg)
 255 {
 256         struct memory_segment *tmp;
 257
 258         if (seg->start + seg->size >= VMEM_MAX_PHYS ||
 259             seg->start + seg->size < seg->start)
 260                 return -ERANGE;
 261
 262         list_for_each_entry(tmp, &mem_segs, list) {
 263                 if (seg->start >= tmp->start + tmp->size)
 264                         continue;
 265                 if (seg->start + seg->size <= tmp->start)
 266                         continue;
 267                 return -ENOSPC;
 268         }
 269         list_add(&seg->list, &mem_segs);
 270         return 0;
 271 }
 272
 273 /*
 274  * Remove memory segment from the segment list.
 275  */
 276 static void remove_memory_segment(struct memory_segment *seg)
 277 {
 278         list_del(&seg->list);
 279 }
 280
 281 static void __remove_shared_memory(struct memory_segment *seg)
 282 {
 283         remove_memory_segment(seg);
 284         vmem_remove_range(seg->start, seg->size);
 285 }
 286
 287 int remove_shared_memory(unsigned long start, unsigned long size)
 288 {
 289         struct memory_segment *seg;
 290         int ret;
 291
 292         mutex_lock(&vmem_mutex);
 293
 294         ret = -ENOENT;
 295         list_for_each_entry(seg, &mem_segs, list) {
 296                 if (seg->start == start && seg->size == size)
 297                         break;
 298         }
 299
 300         if (seg->start != start || seg->size != size)
 301                 goto out;
 302
 303         ret = 0;
 304         __remove_shared_memory(seg);
 305         kfree(seg);
 306 out:
 307         mutex_unlock(&vmem_mutex);
 308         return ret;
 309 }
 310
 311 int add_shared_memory(unsigned long start, unsigned long size)
 312 {
 313         struct memory_segment *seg;
 314         struct page *page;
 315         unsigned long pfn, num_pfn, end_pfn;
 316         int ret;
 317
 318         mutex_lock(&vmem_mutex);
 319         ret = -ENOMEM;
 320         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 321         if (!seg)
 322                 goto out;
 323         seg->start = start;
 324         seg->size = size;
 325
 326         ret = insert_memory_segment(seg);
 327         if (ret)
 328                 goto out_free;
 329
 330         ret = vmem_add_mem(start, size);
 331         if (ret)
 332                 goto out_remove;
 333
 334         pfn = PFN_DOWN(start);
 335         num_pfn = PFN_DOWN(size);
 336         end_pfn = pfn + num_pfn;
 337
 338         page = pfn_to_page(pfn);
 339         memset(page, 0, num_pfn * sizeof(struct page));
 340
 341         for (; pfn < end_pfn; pfn++) {
 342                 page = pfn_to_page(pfn);
 343                 init_page_count(page);
 344                 reset_page_mapcount(page);
 345                 SetPageReserved(page);
 346                 INIT_LIST_HEAD(&page->lru);
 347         }
 348         goto out;
 349
 350 out_remove:
 351         __remove_shared_memory(seg);
 352 out_free:
 353         kfree(seg);
 354 out:
 355         mutex_unlock(&vmem_mutex);
 356         return ret;
 357 }
 358
 359 /*
 360  * map whole physical memory to virtual memory (identity mapping)
 361  * we reserve enough space in the vmalloc area for vmemmap to hotplug
 362  * additional memory segments.
 363  */
 364 void __init vmem_map_init(void)
 365 {
 366         int i;
 367
 368         INIT_LIST_HEAD(&init_mm.context.crst_list);
 369         INIT_LIST_HEAD(&init_mm.context.pgtable_list);
 370         init_mm.context.noexec = 0;
 371         NODE_DATA(0)->node_mem_map = VMEM_MAP;
 372         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++)
 373                 vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size);
 374 }
 375
 376 /*
 377  * Convert memory chunk array to a memory segment list so there is a single
 378  * list that contains both r/w memory and shared memory segments.
 379  */
 380 static int __init vmem_convert_memory_chunk(void)
 381 {
 382         struct memory_segment *seg;
 383         int i;
 384
 385         mutex_lock(&vmem_mutex);
 386         for (i = 0; i < MEMORY_CHUNKS; i++) {
 387                 if (!memory_chunk[i].size)
 388                         continue;
 389                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 390                 if (!seg)
 391                         panic("Out of memory...\n");
 392                 seg->start = memory_chunk[i].addr;
 393                 seg->size = memory_chunk[i].size;
 394                 insert_memory_segment(seg);
 395         }
 396         mutex_unlock(&vmem_mutex);
 397         return 0;
 398 }
 399
 400 core_initcall(vmem_convert_memory_chunk);