arch/s390/mm/vmem.c

   1 /*
   2  *  arch/s390/mm/vmem.c
   3  *
   4  *    Copyright IBM Corp. 2006
   5  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   6  */
   7
   8 #include <linux/bootmem.h>
   9 #include <linux/pfn.h>
  10 #include <linux/mm.h>
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <asm/pgalloc.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/setup.h>
  16 #include <asm/tlbflush.h>
  17
  18 unsigned long vmalloc_end;
  19 EXPORT_SYMBOL(vmalloc_end);
  20
  21 static struct page *vmem_map;
  22 static DEFINE_MUTEX(vmem_mutex);
  23
  24 struct memory_segment {
  25         struct list_head list;
  26         unsigned long start;
  27         unsigned long size;
  28 };
  29
  30 static LIST_HEAD(mem_segs);
  31
  32 void __meminit memmap_init(unsigned long size, int nid, unsigned long zone,
  33                            unsigned long start_pfn)
  34 {
  35         struct page *start, *end;
  36         struct page *map_start, *map_end;
  37         int i;
  38
  39         start = pfn_to_page(start_pfn);
  40         end = start + size;
  41
  42         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
  43                 unsigned long cstart, cend;
  44
  45                 cstart = PFN_DOWN(memory_chunk[i].addr);
  46                 cend = cstart + PFN_DOWN(memory_chunk[i].size);
  47
  48                 map_start = mem_map + cstart;
  49                 map_end = mem_map + cend;
  50
  51                 if (map_start < start)
  52                         map_start = start;
  53                 if (map_end > end)
  54                         map_end = end;
  55
  56                 map_start -= ((unsigned long) map_start & (PAGE_SIZE - 1))
  57                         / sizeof(struct page);
  58                 map_end += ((PFN_ALIGN((unsigned long) map_end)
  59                              - (unsigned long) map_end)
  60                             / sizeof(struct page));
  61
  62                 if (map_start < map_end)
  63                         memmap_init_zone((unsigned long)(map_end - map_start),
  64                                          nid, zone, page_to_pfn(map_start),
  65                                          MEMMAP_EARLY);
  66         }
  67 }
  68
  69 static void __init_refok *vmem_alloc_pages(unsigned int order)
  70 {
  71         if (slab_is_available())
  72                 return (void *)__get_free_pages(GFP_KERNEL, order);
  73         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  74 }
  75
  76 #define vmem_pud_alloc()        ({ BUG(); ((pud_t *) NULL); })
  77
  78 static inline pmd_t *vmem_pmd_alloc(void)
  79 {
  80         pmd_t *pmd = NULL;
  81
  82 #ifdef CONFIG_64BIT
  83         pmd = vmem_alloc_pages(2);
  84         if (!pmd)
  85                 return NULL;
  86         clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE*4);
  87 #endif
  88         return pmd;
  89 }
  90
  91 static inline pte_t *vmem_pte_alloc(void)
  92 {
  93         pte_t *pte = vmem_alloc_pages(0);
  94
  95         if (!pte)
  96                 return NULL;
  97         clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, PAGE_SIZE);
  98         return pte;
  99 }
 100
 101 /*
 102  * Add a physical memory range to the 1:1 mapping.
 103  */
 104 static int vmem_add_range(unsigned long start, unsigned long size)
 105 {
 106         unsigned long address;
 107         pgd_t *pg_dir;
 108         pud_t *pu_dir;
 109         pmd_t *pm_dir;
 110         pte_t *pt_dir;
 111         pte_t  pte;
 112         int ret = -ENOMEM;
 113
 114         for (address = start; address < start + size; address += PAGE_SIZE) {
 115                 pg_dir = pgd_offset_k(address);
 116                 if (pgd_none(*pg_dir)) {
 117                         pu_dir = vmem_pud_alloc();
 118                         if (!pu_dir)
 119                                 goto out;
 120                         pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
 121                 }
 122
 123                 pu_dir = pud_offset(pg_dir, address);
 124                 if (pud_none(*pu_dir)) {
 125                         pm_dir = vmem_pmd_alloc();
 126                         if (!pm_dir)
 127                                 goto out;
 128                         pud_populate_kernel(&init_mm, pu_dir, pm_dir);
 129                 }
 130
 131                 pm_dir = pmd_offset(pu_dir, address);
 132                 if (pmd_none(*pm_dir)) {
 133                         pt_dir = vmem_pte_alloc();
 134                         if (!pt_dir)
 135                                 goto out;
 136                         pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
 137                 }
 138
 139                 pt_dir = pte_offset_kernel(pm_dir, address);
 140                 pte = pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL);
 141                 *pt_dir = pte;
 142         }
 143         ret = 0;
 144 out:
 145         flush_tlb_kernel_range(start, start + size);
 146         return ret;
 147 }
 148
 149 /*
 150  * Remove a physical memory range from the 1:1 mapping.
 151  * Currently only invalidates page table entries.
 152  */
 153 static void vmem_remove_range(unsigned long start, unsigned long size)
 154 {
 155         unsigned long address;
 156         pgd_t *pg_dir;
 157         pud_t *pu_dir;
 158         pmd_t *pm_dir;
 159         pte_t *pt_dir;
 160         pte_t  pte;
 161
 162         pte_val(pte) = _PAGE_TYPE_EMPTY;
 163         for (address = start; address < start + size; address += PAGE_SIZE) {
 164                 pg_dir = pgd_offset_k(address);
 165                 pu_dir = pud_offset(pg_dir, address);
 166                 if (pud_none(*pu_dir))
 167                         continue;
 168                 pm_dir = pmd_offset(pu_dir, address);
 169                 if (pmd_none(*pm_dir))
 170                         continue;
 171                 pt_dir = pte_offset_kernel(pm_dir, address);
 172                 *pt_dir = pte;
 173         }
 174         flush_tlb_kernel_range(start, start + size);
 175 }
 176
 177 /*
 178  * Add a backed mem_map array to the virtual mem_map array.
 179  */
 180 static int vmem_add_mem_map(unsigned long start, unsigned long size)
 181 {
 182         unsigned long address, start_addr, end_addr;
 183         struct page *map_start, *map_end;
 184         pgd_t *pg_dir;
 185         pud_t *pu_dir;
 186         pmd_t *pm_dir;
 187         pte_t *pt_dir;
 188         pte_t  pte;
 189         int ret = -ENOMEM;
 190
 191         map_start = vmem_map + PFN_DOWN(start);
 192         map_end = vmem_map + PFN_DOWN(start + size);
 193
 194         start_addr = (unsigned long) map_start & PAGE_MASK;
 195         end_addr = PFN_ALIGN((unsigned long) map_end);
 196
 197         for (address = start_addr; address < end_addr; address += PAGE_SIZE) {
 198                 pg_dir = pgd_offset_k(address);
 199                 if (pgd_none(*pg_dir)) {
 200                         pu_dir = vmem_pud_alloc();
 201                         if (!pu_dir)
 202                                 goto out;
 203                         pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
 204                 }
 205
 206                 pu_dir = pud_offset(pg_dir, address);
 207                 if (pud_none(*pu_dir)) {
 208                         pm_dir = vmem_pmd_alloc();
 209                         if (!pm_dir)
 210                                 goto out;
 211                         pud_populate_kernel(&init_mm, pu_dir, pm_dir);
 212                 }
 213
 214                 pm_dir = pmd_offset(pu_dir, address);
 215                 if (pmd_none(*pm_dir)) {
 216                         pt_dir = vmem_pte_alloc();
 217                         if (!pt_dir)
 218                                 goto out;
 219                         pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
 220                 }
 221
 222                 pt_dir = pte_offset_kernel(pm_dir, address);
 223                 if (pte_none(*pt_dir)) {
 224                         unsigned long new_page;
 225
 226                         new_page =__pa(vmem_alloc_pages(0));
 227                         if (!new_page)
 228                                 goto out;
 229                         pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL);
 230                         *pt_dir = pte;
 231                 }
 232         }
 233         ret = 0;
 234 out:
 235         flush_tlb_kernel_range(start_addr, end_addr);
 236         return ret;
 237 }
 238
 239 static int vmem_add_mem(unsigned long start, unsigned long size)
 240 {
 241         int ret;
 242
 243         ret = vmem_add_range(start, size);
 244         if (ret)
 245                 return ret;
 246         return vmem_add_mem_map(start, size);
 247 }
 248
 249 /*
 250  * Add memory segment to the segment list if it doesn't overlap with
 251  * an already present segment.
 252  */
 253 static int insert_memory_segment(struct memory_segment *seg)
 254 {
 255         struct memory_segment *tmp;
 256
 257         if (PFN_DOWN(seg->start + seg->size) > max_pfn ||
 258             seg->start + seg->size < seg->start)
 259                 return -ERANGE;
 260
 261         list_for_each_entry(tmp, &mem_segs, list) {
 262                 if (seg->start >= tmp->start + tmp->size)
 263                         continue;
 264                 if (seg->start + seg->size <= tmp->start)
 265                         continue;
 266                 return -ENOSPC;
 267         }
 268         list_add(&seg->list, &mem_segs);
 269         return 0;
 270 }
 271
 272 /*
 273  * Remove memory segment from the segment list.
 274  */
 275 static void remove_memory_segment(struct memory_segment *seg)
 276 {
 277         list_del(&seg->list);
 278 }
 279
 280 static void __remove_shared_memory(struct memory_segment *seg)
 281 {
 282         remove_memory_segment(seg);
 283         vmem_remove_range(seg->start, seg->size);
 284 }
 285
 286 int remove_shared_memory(unsigned long start, unsigned long size)
 287 {
 288         struct memory_segment *seg;
 289         int ret;
 290
 291         mutex_lock(&vmem_mutex);
 292
 293         ret = -ENOENT;
 294         list_for_each_entry(seg, &mem_segs, list) {
 295                 if (seg->start == start && seg->size == size)
 296                         break;
 297         }
 298
 299         if (seg->start != start || seg->size != size)
 300                 goto out;
 301
 302         ret = 0;
 303         __remove_shared_memory(seg);
 304         kfree(seg);
 305 out:
 306         mutex_unlock(&vmem_mutex);
 307         return ret;
 308 }
 309
 310 int add_shared_memory(unsigned long start, unsigned long size)
 311 {
 312         struct memory_segment *seg;
 313         struct page *page;
 314         unsigned long pfn, num_pfn, end_pfn;
 315         int ret;
 316
 317         mutex_lock(&vmem_mutex);
 318         ret = -ENOMEM;
 319         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 320         if (!seg)
 321                 goto out;
 322         seg->start = start;
 323         seg->size = size;
 324
 325         ret = insert_memory_segment(seg);
 326         if (ret)
 327                 goto out_free;
 328
 329         ret = vmem_add_mem(start, size);
 330         if (ret)
 331                 goto out_remove;
 332
 333         pfn = PFN_DOWN(start);
 334         num_pfn = PFN_DOWN(size);
 335         end_pfn = pfn + num_pfn;
 336
 337         page = pfn_to_page(pfn);
 338         memset(page, 0, num_pfn * sizeof(struct page));
 339
 340         for (; pfn < end_pfn; pfn++) {
 341                 page = pfn_to_page(pfn);
 342                 init_page_count(page);
 343                 reset_page_mapcount(page);
 344                 SetPageReserved(page);
 345                 INIT_LIST_HEAD(&page->lru);
 346         }
 347         goto out;
 348
 349 out_remove:
 350         __remove_shared_memory(seg);
 351 out_free:
 352         kfree(seg);
 353 out:
 354         mutex_unlock(&vmem_mutex);
 355         return ret;
 356 }
 357
 358 /*
 359  * map whole physical memory to virtual memory (identity mapping)
 360  */
 361 void __init vmem_map_init(void)
 362 {
 363         unsigned long map_size;
 364         int i;
 365
 366         map_size = ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page);
 367         vmalloc_end = PFN_ALIGN(VMALLOC_END_INIT) - PFN_ALIGN(map_size);
 368         vmem_map = (struct page *) vmalloc_end;
 369         NODE_DATA(0)->node_mem_map = vmem_map;
 370
 371         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++)
 372                 vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size);
 373 }
 374
 375 /*
 376  * Convert memory chunk array to a memory segment list so there is a single
 377  * list that contains both r/w memory and shared memory segments.
 378  */
 379 static int __init vmem_convert_memory_chunk(void)
 380 {
 381         struct memory_segment *seg;
 382         int i;
 383
 384         mutex_lock(&vmem_mutex);
 385         for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
 386                 if (!memory_chunk[i].size)
 387                         continue;
 388                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 389                 if (!seg)
 390                         panic("Out of memory...\n");
 391                 seg->start = memory_chunk[i].addr;
 392                 seg->size = memory_chunk[i].size;
 393                 insert_memory_segment(seg);
 394         }
 395         mutex_unlock(&vmem_mutex);
 396         return 0;
 397 }
 398
 399 core_initcall(vmem_convert_memory_chunk);