?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
some vm to accomodate needing to have a region search spot be
[newos.git] / kernel / vm / vm.c
blobcfcc71993ff55f76dbb7c110e1232dee04f3dd9f
1 /*
2 ** Copyright 2001-2008, Travis Geiselbrecht. All rights reserved.
3 ** Distributed under the terms of the NewOS License.
4 */
5 #include <kernel/kernel.h>
6 #include <kernel/vm.h>
7 #include <kernel/vm_priv.h>
8 #include <kernel/vm_page.h>
9 #include <kernel/vm_cache.h>
10 #include <kernel/vm_store_anonymous_noswap.h>
11 #include <kernel/vm_store_device.h>
12 #include <kernel/vm_store_null.h>
13 #include <kernel/vm_store_vnode.h>
14 #include <kernel/heap.h>
15 #include <kernel/debug.h>
16 #include <kernel/console.h>
17 #include <kernel/int.h>
18 #include <kernel/smp.h>
19 #include <kernel/sem.h>
20 #include <kernel/lock.h>
21 #include <kernel/khash.h>
22 #include <kernel/time.h>
23 #include <newos/errors.h>
24
25 #include <boot/stage2.h>
26
27 #include <kernel/arch/cpu.h>
28 #include <kernel/arch/vm.h>
29
30 #include <string.h>
31 #include <ctype.h>
32 #include <stdlib.h>
33 #include <stdio.h>
34
35
36 /* global data about the vm */
37 vm_info_t vm_info;
38
39 static vm_address_space *kernel_aspace;
40
41 #define REGION_HASH_TABLE_SIZE 1024
42 static region_id next_region_id;
43 static void *region_table;
44 static sem_id region_hash_sem;
45
46 #define ASPACE_HASH_TABLE_SIZE 1024
47 static aspace_id next_aspace_id;
48 static void *aspace_table;
49 static sem_id aspace_hash_sem;
50
51 static spinlock_t max_commit_lock;
52
53 // function declarations
54 static vm_region *_vm_create_region_struct(vm_address_space *aspace, const char *name, int wiring, int lock);
55 static int map_backing_store(vm_address_space *aspace, vm_store *store, void **vaddr,
56 off_t offset, addr_t size, int addr_type, int wiring, int lock, int mapping, vm_region **_region, const char *region_name);
57 static int vm_soft_fault(addr_t address, bool is_write, bool is_user);
58 static vm_region *vm_virtual_map_lookup(vm_virtual_map *map, addr_t address);
59
60 static int region_compare(void *_r, const void *key)
61 {
62 vm_region *r = _r;
63 const region_id *id = key;
64
65 #if DEBUG > 1
66 VERIFY_VM_REGION(r);
67 #endif
68
69 if(r->id == *id)
70 return 0;
71 else
72 return -1;
73 }
74
75 static unsigned int region_hash(void *_r, const void *key, unsigned int range)
76 {
77 vm_region *r = _r;
78 const region_id *id = key;
79
80 #if DEBUG > 1
81 if(r != NULL)
82 VERIFY_VM_REGION(r);
83 #endif
84
85 if(r != NULL)
86 return (r->id % range);
87 else
88 return (*id % range);
89 }
90
91 static int aspace_compare(void *_a, const void *key)
92 {
93 vm_address_space *aspace = _a;
94 const aspace_id *id = key;
95
96 #if DEBUG > 1
97 VERIFY_VM_ASPACE(aspace);
98 #endif
99
100 if(aspace->id == *id)
101 return 0;
102 else
103 return -1;
104 }
105
106 static unsigned int aspace_hash(void *_a, const void *key, unsigned int range)
107 {
108 vm_address_space *aspace = _a;
109 const aspace_id *id = key;
110
111 #if DEBUG > 1
112 if(aspace != NULL)
113 VERIFY_VM_ASPACE(aspace);
114 #endif
115
116 if(aspace != NULL)
117 return (aspace->id % range);
118 else
119 return (*id % range);
120 }
121
122 vm_address_space *vm_get_aspace_by_id(aspace_id aid)
123 {
124 vm_address_space *aspace;
125
126 sem_acquire(aspace_hash_sem, READ_COUNT);
127 aspace = hash_lookup(aspace_table, &aid);
128 if(aspace) {
129 VERIFY_VM_ASPACE(aspace);
130 atomic_add(&aspace->ref_count, 1);
131 }
132 sem_release(aspace_hash_sem, READ_COUNT);
133
134 return aspace;
135 }
136
137 vm_region *vm_get_region_by_id(region_id rid)
138 {
139 vm_region *region;
140
141 sem_acquire(region_hash_sem, READ_COUNT);
142 region = hash_lookup(region_table, &rid);
143 if(region) {
144 VERIFY_VM_REGION(region);
145 atomic_add(&region->ref_count, 1);
146 }
147 sem_release(region_hash_sem, READ_COUNT);
148
149 return region;
150 }
151
152 region_id vm_find_region_by_name(aspace_id aid, const char *name)
153 {
154 vm_region *region = NULL;
155 vm_address_space *aspace;
156 region_id id = ERR_NOT_FOUND;
157
158 ASSERT(name != NULL);
159
160 aspace = vm_get_aspace_by_id(aid);
161 if(aspace == NULL)
162 return ERR_VM_INVALID_ASPACE;
163
164 sem_acquire(aspace->virtual_map.sem, READ_COUNT);
165
166 region = aspace->virtual_map.region_list;
167 while(region != NULL) {
168 VERIFY_VM_REGION(region);
169 ASSERT(region->name != NULL);
170 if(strcmp(region->name, name) == 0) {
171 id = region->id;
172 break;
173 }
174 region = region->aspace_next;
175 }
176
177 sem_release(aspace->virtual_map.sem, READ_COUNT);
178 vm_put_aspace(aspace);
179 return id;
180 }
181
182 static vm_region *_vm_create_region_struct(vm_address_space *aspace, const char *name, int wiring, int lock)
183 {
184 vm_region *region = NULL;
185
186 VERIFY_VM_ASPACE(aspace);
187 ASSERT(name != NULL);
188
189 region = (vm_region *)kmalloc(sizeof(vm_region));
190 if(region == NULL)
191 return NULL;
192 region->name = (char *)kmalloc(strlen(name) + 1);
193 if(region->name == NULL) {
194 kfree(region);
195 return NULL;
196 }
197 strcpy(region->name, name);
198 region->magic = VM_REGION_MAGIC;
199 region->id = atomic_add(&next_region_id, 1);
200 region->base = 0;
201 region->size = 0;
202 region->lock = lock;
203 region->wiring = wiring;
204 region->ref_count = 1;
205
206 region->cache_ref = NULL;
207 region->cache_offset = 0;
208
209 region->aspace = aspace;
210 region->aspace_next = NULL;
211 region->map = &aspace->virtual_map;
212 list_clear_node(&region->cache_node);
213 region->hash_next = NULL;
214
215 return region;
216 }
217
218 // must be called with this address space's virtual_map.sem held
219 static int find_and_insert_region_slot(vm_virtual_map *map, addr_t start, addr_t size, addr_t end, int addr_type, vm_region *region)
220 {
221 vm_region *last_r = NULL;
222 vm_region *next_r;
223 bool foundspot = false;
224
225 // dprintf("find_and_insert_region_slot: map %p, start 0x%lx, size %ld, end 0x%lx, addr_type %d, region %p\n",
226 // map, start, size, end, addr_type, region);
227 // dprintf("map->base 0x%lx, map->alloc_base 0x%lx, map->size 0x%lx\n", map->base, map->alloc_base, map->size);
228
229 // do some sanity checking
230 if(start < map->base || size == 0 || (end - 1) > (map->base + (map->size - 1)) || start + size > end)
231 return ERR_VM_BAD_ADDRESS;
232
233 // walk up to the spot where we should start searching
234 next_r = map->region_list;
235 while(next_r) {
236 if(next_r->base >= start + size) {
237 // we have a winner
238 break;
239 }
240 last_r = next_r;
241 next_r = next_r->aspace_next;
242 }
243
244 #if 0
245 dprintf("last_r %p, next_r %p\n", last_r, next_r);
246 if(last_r) dprintf("last_r->base 0x%lx, last_r->size 0x%lx\n", last_r->base, last_r->size);
247 if(next_r) dprintf("next_r->base 0x%lx, next_r->size 0x%lx\n", next_r->base, next_r->size);
248 #endif
249
250 switch(addr_type) {
251 case REGION_ADDR_ANY_ADDRESS:
252 // find a hole big enough for a new region
253 if(!last_r) {
254 // see if we can build it at the beginning of the virtual map
255 if(!next_r || (next_r->base >= map->base + size)) {
256 foundspot = true;
257 region->base = map->base;
258 break;
259 }
260 last_r = next_r;
261 next_r = next_r->aspace_next;
262 }
263 // keep walking
264 while(next_r) {
265 if(next_r->base >= last_r->base + last_r->size + size) {
266 // we found a spot
267 foundspot = true;
268 region->base = last_r->base + last_r->size;
269 break;
270 }
271 last_r = next_r;
272 next_r = next_r->aspace_next;
273 }
274 if((map->base + (map->size - 1)) >= (last_r->base + last_r->size + (size - 1))) {
275 // found a spot
276 foundspot = true;
277 region->base = last_r->base + last_r->size;
278 break;
279 }
280 break;
281 case REGION_ADDR_EXACT_ADDRESS:
282 // see if we can create it exactly here
283 if(!last_r) {
284 if(!next_r || (next_r->base >= start + size)) {
285 foundspot = true;
286 region->base = start;
287 break;
288 }
289 } else {
290 if(next_r) {
291 if(last_r->base + last_r->size <= start && next_r->base >= start + size) {
292 foundspot = true;
293 region->base = start;
294 break;
295 }
296 } else {
297 if((last_r->base + (last_r->size - 1)) <= start - 1) {
298 foundspot = true;
299 region->base = start;
300 }
301 }
302 }
303 break;
304 default:
305 return ERR_INVALID_ARGS;
306 }
307
308 if(foundspot) {
309 region->size = size;
310 // dprintf("found spot: base 0x%lx, size 0x%lx\n", region->base, region->size);
311 if(last_r) {
312 region->aspace_next = last_r->aspace_next;
313 last_r->aspace_next = region;
314 } else {
315 region->aspace_next = map->region_list;
316 map->region_list = region;
317 }
318 map->change_count++;
319 return NO_ERROR;
320 } else {
321 return ERR_VM_NO_REGION_SLOT;
322 }
323 }
324
325 // a ref to the cache holding this store must be held before entering here
326 static int map_backing_store(vm_address_space *aspace, vm_store *store, void **vaddr,
327 off_t offset, addr_t size, int addr_type, int wiring, int lock, int mapping, vm_region **_region, const char *region_name)
328 {
329 vm_cache *cache;
330 vm_cache_ref *cache_ref;
331 vm_region *region;
332 vm_cache *nu_cache;
333 vm_cache_ref *nu_cache_ref = NULL;
334 vm_store *nu_store;
335
336 int err;
337
338 VERIFY_VM_ASPACE(aspace);
339 VERIFY_VM_STORE(store);
340
341 // dprintf("map_backing_store: aspace %p, store %p, *vaddr %p, offset 0x%Lx, size 0x%lx, addr_type %d, wiring %d, lock %d, _region %p, region_name '%s'\n",
342 // aspace, store, *vaddr, offset, size, addr_type, wiring, lock, _region, region_name);
343
344 region = _vm_create_region_struct(aspace, region_name, wiring, lock);
345 if(!region)
346 return ERR_NO_MEMORY;
347
348 cache = store->cache;
349 VERIFY_VM_CACHE(cache);
350 cache_ref = cache->ref;
351 VERIFY_VM_CACHE_REF(cache_ref);
352
353 // if this is a private map, we need to create a new cache & store object
354 // pair to handle the private copies of pages as they are written to
355 if(mapping == REGION_PRIVATE_MAP) {
356 // create an anonymous store object
357 nu_store = vm_store_create_anonymous_noswap();
358 if(nu_store == NULL)
359 panic("map_backing_store: vm_create_store_anonymous_noswap returned NULL");
360 nu_cache = vm_cache_create(nu_store);
361 if(nu_cache == NULL)
362 panic("map_backing_store: vm_cache_create returned NULL");
363 nu_cache_ref = vm_cache_ref_create(nu_cache);
364 if(nu_cache_ref == NULL)
365 panic("map_backing_store: vm_cache_ref_create returned NULL");
366 nu_cache->temporary = 1;
367 nu_cache->scan_skip = cache->scan_skip;
368
369 nu_cache->source = cache;
370
371 // grab a ref to the cache object we're now linked to as a source
372 vm_cache_acquire_ref(cache_ref, true);
373
374 cache = nu_cache;
375 cache_ref = cache->ref;
376 store = nu_store;
377 }
378
379 mutex_lock(&cache_ref->lock);
380 if(store->committed_size < offset + size) {
381 // try to commit more memory
382 off_t old_store_commitment = store->committed_size;
383 off_t commitment = (store->ops->commit)(store, offset + size);
384 if(commitment < offset + size) {
385 if(cache->temporary) {
386 int_disable_interrupts();
387 acquire_spinlock(&max_commit_lock);
388
389 if(vm_info.max_commit - old_store_commitment + commitment < offset + size) {
390 release_spinlock(&max_commit_lock);
391 int_restore_interrupts();
392 mutex_unlock(&cache_ref->lock);
393 err = ERR_VM_WOULD_OVERCOMMIT;
394 goto err1a;
395 }
396
397 // dprintf("map_backing_store: adding %d to max_commit\n",
398 // (commitment - old_store_commitment) - (offset + size - cache->committed_size));
399
400 vm_info.max_commit += (commitment - old_store_commitment) - (offset + size - cache->virtual_size);
401 cache->virtual_size = offset + size;
402 release_spinlock(&max_commit_lock);
403 int_restore_interrupts();
404 } else {
405 mutex_unlock(&cache_ref->lock);
406 err = ERR_NO_MEMORY;
407 goto err1a;
408 }
409 }
410 }
411
412 mutex_unlock(&cache_ref->lock);
413
414 vm_cache_acquire_ref(cache_ref, true);
415
416 sem_acquire(aspace->virtual_map.sem, WRITE_COUNT);
417
418 // check to see if this aspace has entered DELETE state
419 if(aspace->state == VM_ASPACE_STATE_DELETION) {
420 // okay, someone is trying to delete this aspace now, so we can't
421 // insert the region, so back out
422 err = ERR_VM_INVALID_ASPACE;
423 goto err1b;
424 }
425
426 {
427 addr_t search_addr, search_end;
428
429 if(addr_type == REGION_ADDR_EXACT_ADDRESS) {
430 search_addr = (addr_t)*vaddr;
431 search_end = (addr_t)*vaddr + size;
432 } else if(addr_type == REGION_ADDR_ANY_ADDRESS) {
433 search_addr = aspace->virtual_map.alloc_base;
434 search_end = aspace->virtual_map.alloc_base +
435 ((aspace->virtual_map.size - 1) - (aspace->virtual_map.alloc_base - aspace->virtual_map.base));
436 } else {
437 err = ERR_INVALID_ARGS;
438 goto err1b;
439 }
440
441 err = find_and_insert_region_slot(&aspace->virtual_map, search_addr, size, search_end, addr_type, region);
442 if(err < 0)
443 goto err1b;
444 *vaddr = (addr_t *)region->base;
445 }
446
447 // attach the cache to the region
448 region->cache_ref = cache_ref;
449 region->cache_offset = offset;
450 // point the cache back to the region
451 vm_cache_insert_region(cache_ref, region);
452
453 // insert the region in the global region hash table
454 sem_acquire(region_hash_sem, WRITE_COUNT);
455 hash_insert(region_table, region);
456 sem_release(region_hash_sem, WRITE_COUNT);
457
458 // grab a ref to the aspace (the region holds this)
459 atomic_add(&aspace->ref_count, 1);
460
461 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
462
463 *_region = region;
464
465 return NO_ERROR;
466
467 err1b:
468 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
469 vm_cache_release_ref(cache_ref);
470 goto err;
471 err1a:
472 if(nu_cache_ref) {
473 // had never acquired it's initial ref, so acquire and then release it
474 // this should clean up all the objects it references
475 vm_cache_acquire_ref(cache_ref, true);
476 vm_cache_release_ref(cache_ref);
477 }
478 err:
479 kfree(region->name);
480 kfree(region);
481 return err;
482 }
483
484 region_id user_vm_create_anonymous_region(char *uname, void **uaddress, int addr_type,
485 addr_t size, int wiring, int lock)
486 {
487 char name[SYS_MAX_OS_NAME_LEN];
488 void *address;
489 int rc, rc2;
490
491 if(is_kernel_address(uname))
492 return ERR_VM_BAD_USER_MEMORY;
493
494 rc = user_strncpy(name, uname, SYS_MAX_OS_NAME_LEN-1);
495 if(rc < 0)
496 return rc;
497 name[SYS_MAX_OS_NAME_LEN-1] = 0;
498
499 rc = user_memcpy(&address, uaddress, sizeof(address));
500 if(rc < 0)
501 return rc;
502
503 rc = vm_create_anonymous_region(vm_get_current_user_aspace_id(), name, &address, addr_type, size, wiring, lock);
504 if(rc < 0)
505 return rc;
506
507 rc2 = user_memcpy(uaddress, &address, sizeof(address));
508 if(rc2 < 0)
509 return rc2;
510
511 return rc;
512 }
513
514 region_id vm_create_anonymous_region(aspace_id aid, char *name, void **address, int addr_type,
515 addr_t size, int wiring, int lock)
516 {
517 int err;
518 vm_region *region;
519 vm_cache *cache;
520 vm_store *store;
521 vm_address_space *aspace;
522 vm_cache_ref *cache_ref;
523
524 // dprintf("create_anonymous_region: name '%s', type %d, size 0x%lx, wiring %d, lock %d\n",
525 // name, addr_type, size, wiring, lock);
526
527 if(addr_type != REGION_ADDR_ANY_ADDRESS && addr_type != REGION_ADDR_EXACT_ADDRESS)
528 return ERR_INVALID_ARGS;
529 switch(wiring) {
530 case REGION_WIRING_WIRED:
531 case REGION_WIRING_WIRED_ALREADY:
532 case REGION_WIRING_WIRED_CONTIG:
533 case REGION_WIRING_LAZY:
534 break;
535 default:
536 return ERR_INVALID_ARGS;
537 }
538 aspace = vm_get_aspace_by_id(aid);
539 if(aspace == NULL)
540 return ERR_VM_INVALID_ASPACE;
541
542 size = PAGE_ALIGN(size);
543
544 // create an anonymous store object
545 store = vm_store_create_anonymous_noswap();
546 if(store == NULL)
547 panic("vm_create_anonymous_region: vm_create_store_anonymous_noswap returned NULL");
548 cache = vm_cache_create(store);
549 if(cache == NULL)
550 panic("vm_create_anonymous_region: vm_cache_create returned NULL");
551 cache_ref = vm_cache_ref_create(cache);
552 if(cache_ref == NULL)
553 panic("vm_create_anonymous_region: vm_cache_ref_create returned NULL");
554 cache->temporary = 1;
555
556 switch(wiring) {
557 case REGION_WIRING_WIRED:
558 case REGION_WIRING_WIRED_ALREADY:
559 case REGION_WIRING_WIRED_CONTIG:
560 cache->scan_skip = 1;
561 break;
562 case REGION_WIRING_LAZY:
563 cache->scan_skip = 0;
564 break;
565 }
566
567 // dprintf("create_anonymous_region: calling map_backing store\n");
568
569 vm_cache_acquire_ref(cache_ref, true);
570 err = map_backing_store(aspace, store, address, 0, size, addr_type, wiring, lock, REGION_NO_PRIVATE_MAP, &region, name);
571 vm_cache_release_ref(cache_ref);
572 if(err < 0) {
573 vm_put_aspace(aspace);
574 return err;
575 }
576
577 // dprintf("create_anonymous_region: done calling map_backing store\n");
578
579 cache_ref = store->cache->ref;
580 switch(wiring) {
581 case REGION_WIRING_LAZY:
582 break; // do nothing
583 case REGION_WIRING_WIRED: {
584 // pages aren't mapped at this point, but we just simulate a fault on
585 // every page, which should allocate them
586 addr_t va;
587 // XXX remove
588 for(va = region->base; va < region->base + region->size; va += PAGE_SIZE) {
589 // dprintf("mapping wired pages: region %p, cache_ref %p %p, address 0x%lx\n", region, cache_ref, region->cache_ref, va);
590 vm_soft_fault(va, false, false);
591 }
592 break;
593 }
594 case REGION_WIRING_WIRED_ALREADY: {
595 // the pages should already be mapped. This is only really useful during
596 // boot time. Find the appropriate vm_page objects and stick them in
597 // the cache object.
598 addr_t va;
599 addr_t pa;
600 unsigned int flags;
601 int err;
602 vm_page *page;
603 off_t offset = 0;
604
605 mutex_lock(&cache_ref->lock);
606 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
607 for(va = region->base; va < region->base + region->size; va += PAGE_SIZE, offset += PAGE_SIZE) {
608 err = (*aspace->translation_map.ops->query)(&aspace->translation_map,
609 va, &pa, &flags);
610 if(err < 0) {
611 // dprintf("vm_create_anonymous_region: error looking up mapping for va 0x%x\n", va);
612 continue;
613 }
614 // dprintf("vm_create_anonymous_region: looked up page at va 0x%lx. pa 0x%lx\n", va, pa);
615 page = vm_lookup_page(pa / PAGE_SIZE);
616 if(page == NULL) {
617 // dprintf("vm_create_anonymous_region: error looking up vm_page structure for pa 0x%x\n", pa);
618 continue;
619 }
620 atomic_add(&page->ref_count, 1);
621 vm_page_set_state(page, PAGE_STATE_WIRED);
622 vm_cache_insert_page(cache_ref, page, offset);
623 }
624 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
625 mutex_unlock(&cache_ref->lock);
626 break;
627 }
628 case REGION_WIRING_WIRED_CONTIG: {
629 addr_t va;
630 addr_t phys_addr;
631 int err;
632 vm_page *page;
633 off_t offset = 0;
634
635 page = vm_page_allocate_page_run(PAGE_STATE_CLEAR, ROUNDUP(region->size, PAGE_SIZE) / PAGE_SIZE);
636 if(page == NULL) {
637 // XXX back out of this
638 panic("couldn't allocate page run of size %ld\n", region->size);
639 }
640 phys_addr = page->ppn * PAGE_SIZE;
641
642 mutex_lock(&cache_ref->lock);
643 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
644 for(va = region->base; va < region->base + region->size; va += PAGE_SIZE, offset += PAGE_SIZE, phys_addr += PAGE_SIZE) {
645 page = vm_lookup_page(phys_addr / PAGE_SIZE);
646 if(page == NULL) {
647 panic("couldn't lookup physical page just allocated\n");
648 }
649 atomic_add(&page->ref_count, 1);
650 err = (*aspace->translation_map.ops->map)(&aspace->translation_map, va, phys_addr, lock);
651 if(err < 0) {
652 panic("couldn't map physical page in page run\n");
653 }
654 vm_page_set_state(page, PAGE_STATE_WIRED);
655 vm_cache_insert_page(cache_ref, page, offset);
656 }
657 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
658 mutex_unlock(&cache_ref->lock);
659
660 break;
661 }
662 default:
663 ;
664 }
665 vm_put_aspace(aspace);
666 // dprintf("create_anonymous_region: done\n");
667 if(region)
668 return region->id;
669 else
670 return ERR_NO_MEMORY;
671 }
672
673 region_id vm_map_physical_memory(aspace_id aid, char *name, void **address, int addr_type,
674 addr_t size, int lock, addr_t phys_addr)
675 {
676 vm_region *region;
677 vm_cache *cache;
678 vm_cache_ref *cache_ref;
679 vm_store *store;
680 addr_t map_offset;
681 int err;
682
683 vm_address_space *aspace = vm_get_aspace_by_id(aid);
684 if(aspace == NULL)
685 return ERR_VM_INVALID_ASPACE;
686
687 // if the physical address is somewhat inside a page,
688 // move the actual region down to align on a page boundary
689 map_offset = phys_addr % PAGE_SIZE;
690 size += map_offset;
691 phys_addr -= map_offset;
692
693 size = PAGE_ALIGN(size);
694
695 // create an device store object
696 store = vm_store_create_device(phys_addr);
697 if(store == NULL)
698 panic("vm_map_physical_memory: vm_store_create_device returned NULL");
699 cache = vm_cache_create(store);
700 if(cache == NULL)
701 panic("vm_map_physical_memory: vm_cache_create returned NULL");
702 cache_ref = vm_cache_ref_create(cache);
703 if(cache_ref == NULL)
704 panic("vm_map_physical_memory: vm_cache_ref_create returned NULL");
705
706 // tell the page scanner to skip over this region, it's pages are special
707 cache->scan_skip = 1;
708
709 vm_cache_acquire_ref(cache_ref, true);
710 err = map_backing_store(aspace, store, address, 0, size, addr_type, 0, lock, REGION_NO_PRIVATE_MAP, &region, name);
711 vm_cache_release_ref(cache_ref);
712 vm_put_aspace(aspace);
713
714 if(err < 0) {
715 return err;
716 }
717
718 // modify the pointer returned to be offset back into the new region
719 // the same way the physical address in was offset
720 (*address) = (void *)(((addr_t)*address) + map_offset);
721 return region->id;
722 }
723
724 region_id vm_create_null_region(aspace_id aid, char *name, void **address, int addr_type, addr_t size)
725 {
726 vm_region *region;
727 vm_cache *cache;
728 vm_cache_ref *cache_ref;
729 vm_store *store;
730 int err;
731
732 vm_address_space *aspace = vm_get_aspace_by_id(aid);
733 if(aspace == NULL)
734 return ERR_VM_INVALID_ASPACE;
735
736 size = PAGE_ALIGN(size);
737
738 // create an null store object
739 store = vm_store_create_null();
740 if(store == NULL)
741 panic("vm_create_null_region: vm_store_create_null returned NULL");
742 cache = vm_cache_create(store);
743 if(cache == NULL)
744 panic("vm_create_null_region: vm_cache_create returned NULL");
745 cache_ref = vm_cache_ref_create(cache);
746 if(cache_ref == NULL)
747 panic("vm_create_null_region: vm_cache_ref_create returned NULL");
748 // tell the page scanner to skip over this region, no pages will be mapped here
749 cache->scan_skip = 1;
750
751 vm_cache_acquire_ref(cache_ref, true);
752 err = map_backing_store(aspace, store, address, 0, size, addr_type, 0, LOCK_RO, REGION_NO_PRIVATE_MAP, &region, name);
753 vm_cache_release_ref(cache_ref);
754 vm_put_aspace(aspace);
755 if(err < 0)
756 return err;
757
758 return region->id;
759 }
760
761 static region_id _vm_map_file(aspace_id aid, char *name, void **address, int addr_type,
762 addr_t size, int lock, int mapping, const char *path, off_t offset, bool kernel)
763 {
764 vm_region *region;
765 vm_cache *cache;
766 vm_cache_ref *cache_ref;
767 vm_store *store;
768 void *v;
769 int err;
770
771 vm_address_space *aspace = vm_get_aspace_by_id(aid);
772 if(aspace == NULL)
773 return ERR_VM_INVALID_ASPACE;
774
775 offset = ROUNDOWN(offset, PAGE_SIZE);
776 size = PAGE_ALIGN(size);
777
778 restart:
779 // get the vnode for the object, this also grabs a ref to it
780 err = vfs_get_vnode_from_path(path, kernel, &v);
781 if(err < 0) {
782 vm_put_aspace(aspace);
783 return err;
784 }
785
786 cache_ref = vfs_get_cache_ptr(v);
787 if(!cache_ref) {
788 // create a vnode store object
789 store = vm_store_create_vnode(v);
790 if(store == NULL)
791 panic("vm_map_file: couldn't create vnode store");
792 cache = vm_cache_create(store);
793 if(cache == NULL)
794 panic("vm_map_physical_memory: vm_cache_create returned NULL");
795 cache_ref = vm_cache_ref_create(cache);
796 if(cache_ref == NULL)
797 panic("vm_map_physical_memory: vm_cache_ref_create returned NULL");
798
799 // acquire the cache ref once to represent the ref that the vnode will have
800 // this is one of the only places where we dont want to ref to ripple down to the store
801 vm_cache_acquire_ref(cache_ref, false);
802
803 // try to set the cache ptr in the vnode
804 if(vfs_set_cache_ptr(v, cache_ref) < 0) {
805 // the cache pointer was set between here and then
806 // this can only happen if someone else tries to map it
807 // at the same time. Rare enough to not worry about the
808 // performance impact of undoing what we just did and retrying
809
810 // this will delete the cache object and release the ref to the vnode we have
811 vm_cache_release_ref(cache_ref);
812 goto restart;
813 }
814 } else {
815 VERIFY_VM_CACHE_REF(cache_ref);
816 cache = cache_ref->cache;
817 VERIFY_VM_CACHE(cache);
818 store = cache->store;
819 VERIFY_VM_STORE(store);
820 }
821
822 // acquire a ref to the cache before we do work on it. Dont ripple the ref acquision to the vnode
823 // below because we'll have to release it later anyway, since we grabbed a ref to the vnode at
824 // vfs_get_vnode_from_path(). This puts the ref counts in sync.
825 vm_cache_acquire_ref(cache_ref, false);
826 err = map_backing_store(aspace, store, address, offset, size, addr_type, 0, lock, mapping, &region, name);
827 vm_cache_release_ref(cache_ref);
828 vm_put_aspace(aspace);
829 if(err < 0)
830 return err;
831
832 // modify the pointer returned to be offset back into the new region
833 // the same way the physical address in was offset
834 return region->id;
835 }
836
837 region_id vm_map_file(aspace_id aid, char *name, void **address, int addr_type,
838 addr_t size, int lock, int mapping, const char *path, off_t offset)
839 {
840 return _vm_map_file(aid, name, address, addr_type, size, lock, mapping, path, offset, true);
841 }
842
843 region_id user_vm_map_file(char *uname, void **uaddress, int addr_type,
844 addr_t size, int lock, int mapping, const char *upath, off_t offset)
845 {
846 char name[SYS_MAX_OS_NAME_LEN];
847 void *address;
848 char path[SYS_MAX_PATH_LEN];
849 int rc, rc2;
850
851 if(is_kernel_address(uname))
852 return ERR_VM_BAD_USER_MEMORY;
853
854 if(is_kernel_address(uaddress))
855 return ERR_VM_BAD_USER_MEMORY;
856
857 if(is_kernel_address(upath))
858 return ERR_VM_BAD_USER_MEMORY;
859
860 rc = user_strncpy(name, uname, SYS_MAX_OS_NAME_LEN-1);
861 if(rc < 0)
862 return rc;
863 name[SYS_MAX_OS_NAME_LEN-1] = 0;
864
865 rc = user_strncpy(path, upath, SYS_MAX_PATH_LEN-1);
866 if(rc < 0)
867 return rc;
868 path[SYS_MAX_PATH_LEN-1] = 0;
869
870 rc = user_memcpy(&address, uaddress, sizeof(address));
871 if(rc < 0)
872 return rc;
873
874 rc = _vm_map_file(vm_get_current_user_aspace_id(), name, &address, addr_type, size, lock, mapping, path, offset, false);
875 if(rc < 0)
876 return rc;
877
878 rc2 = user_memcpy(uaddress, &address, sizeof(address));
879 if(rc2 < 0)
880 return rc2;
881
882 return rc;
883 }
884
885 region_id user_vm_clone_region(char *uname, void **uaddress, int addr_type,
886 region_id source_region, int mapping, int lock)
887 {
888 char name[SYS_MAX_OS_NAME_LEN];
889 void *address;
890 int rc, rc2;
891
892 if(is_kernel_address(uname))
893 return ERR_VM_BAD_USER_MEMORY;
894
895 if(is_kernel_address(uaddress))
896 return ERR_VM_BAD_USER_MEMORY;
897
898 rc = user_strncpy(name, uname, SYS_MAX_OS_NAME_LEN-1);
899 if(rc < 0)
900 return rc;
901 name[SYS_MAX_OS_NAME_LEN-1] = 0;
902
903 rc = user_memcpy(&address, uaddress, sizeof(address));
904 if(rc < 0)
905 return rc;
906
907 rc = vm_clone_region(vm_get_current_user_aspace_id(), name, &address, addr_type, source_region, mapping, lock);
908 if(rc < 0)
909 return rc;
910
911 rc2 = user_memcpy(uaddress, &address, sizeof(address));
912 if(rc2 < 0)
913 return rc2;
914
915 return rc;
916 }
917
918 region_id vm_clone_region(aspace_id aid, char *name, void **address, int addr_type,
919 region_id source_region, int mapping, int lock)
920 {
921 vm_region *new_region;
922 vm_region *src_region;
923 int err;
924
925 vm_address_space *aspace = vm_get_aspace_by_id(aid);
926 if(aspace == NULL)
927 return ERR_VM_INVALID_ASPACE;
928
929 src_region = vm_get_region_by_id(source_region);
930 if(src_region == NULL) {
931 vm_put_aspace(aspace);
932 return ERR_VM_INVALID_REGION;
933 }
934
935 vm_cache_acquire_ref(src_region->cache_ref, true);
936 err = map_backing_store(aspace, src_region->cache_ref->cache->store, address, src_region->cache_offset, src_region->size,
937 addr_type, src_region->wiring, lock, mapping, &new_region, name);
938 vm_cache_release_ref(src_region->cache_ref);
939
940 // release the ref on the old region
941 vm_put_region(src_region);
942
943 vm_put_aspace(aspace);
944
945 if(err < 0)
946 return err;
947 else
948 return new_region->id;
949 }
950
951 static int __vm_delete_region(vm_address_space *aspace, vm_region *region)
952 {
953 VERIFY_VM_ASPACE(aspace);
954 VERIFY_VM_REGION(region);
955
956 if(region->aspace == aspace)
957 vm_put_region(region);
958 return NO_ERROR;
959 }
960
961 static int _vm_delete_region(vm_address_space *aspace, region_id rid)
962 {
963 vm_region *region;
964
965 // dprintf("vm_delete_region: aspace id 0x%x, region id 0x%x\n", aspace->id, rid);
966
967 VERIFY_VM_ASPACE(aspace);
968
969 region = vm_get_region_by_id(rid);
970 if(region == NULL)
971 return ERR_VM_INVALID_REGION;
972
973 __vm_delete_region(aspace, region);
974 vm_put_region(region);
975
976 return 0;
977 }
978
979 int user_vm_delete_region(region_id rid)
980 {
981 return vm_delete_region(vm_get_current_user_aspace_id(), rid);
982 }
983
984 int vm_delete_region(aspace_id aid, region_id rid)
985 {
986 vm_address_space *aspace;
987 int err;
988
989 aspace = vm_get_aspace_by_id(aid);
990 if(aspace == NULL)
991 return ERR_VM_INVALID_ASPACE;
992
993 err = _vm_delete_region(aspace, rid);
994 vm_put_aspace(aspace);
995 return err;
996 }
997
998 static void _vm_put_region(vm_region *region, bool aspace_locked)
999 {
1000 vm_region *temp, *last = NULL;
1001 vm_address_space *aspace;
1002 bool removeit = false;
1003
1004 VERIFY_VM_REGION(region);
1005
1006 sem_acquire(region_hash_sem, WRITE_COUNT);
1007 if(atomic_add(&region->ref_count, -1) == 1) {
1008 hash_remove(region_table, region);
1009 removeit = true;
1010 }
1011 sem_release(region_hash_sem, WRITE_COUNT);
1012
1013 if(!removeit)
1014 return;
1015
1016 aspace = region->aspace;
1017 VERIFY_VM_ASPACE(aspace);
1018
1019 // remove the region from the aspace's virtual map
1020 if(!aspace_locked)
1021 sem_acquire(aspace->virtual_map.sem, WRITE_COUNT);
1022 temp = aspace->virtual_map.region_list;
1023 while(temp != NULL) {
1024 if(region == temp) {
1025 if(last != NULL) {
1026 last->aspace_next = temp->aspace_next;
1027 } else {
1028 aspace->virtual_map.region_list = temp->aspace_next;
1029 }
1030 aspace->virtual_map.change_count++;
1031 break;
1032 }
1033 last = temp;
1034 temp = temp->aspace_next;
1035 }
1036 if(region == aspace->virtual_map.region_hint)
1037 aspace->virtual_map.region_hint = NULL;
1038 if(!aspace_locked)
1039 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
1040
1041 if(temp == NULL)
1042 panic("vm_region_release_ref: region not found in aspace's region_list\n");
1043
1044 vm_cache_remove_region(region->cache_ref, region);
1045 vm_cache_release_ref(region->cache_ref);
1046
1047 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
1048 (*aspace->translation_map.ops->unmap)(&aspace->translation_map, region->base,
1049 region->base + (region->size - 1));
1050 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
1051
1052 // now we can give up the last ref to the aspace
1053 vm_put_aspace(aspace);
1054
1055 if(region->name)
1056 kfree(region->name);
1057 kfree(region);
1058
1059 return;
1060 }
1061
1062 void vm_put_region(vm_region *region)
1063 {
1064 return _vm_put_region(region, false);
1065 }
1066
1067 int user_vm_get_region_info(region_id id, vm_region_info *uinfo)
1068 {
1069 vm_region_info info;
1070 int rc, rc2;
1071
1072 if(is_kernel_address(uinfo))
1073 return ERR_VM_BAD_USER_MEMORY;
1074
1075 rc = vm_get_region_info(id, &info);
1076 if(rc < 0)
1077 return rc;
1078
1079 rc2 = user_memcpy(uinfo, &info, sizeof(info));
1080 if(rc2 < 0)
1081 return rc2;
1082
1083 return rc;
1084 }
1085
1086 int vm_get_region_info(region_id id, vm_region_info *info)
1087 {
1088 vm_region *region;
1089
1090 if(info == NULL)
1091 return ERR_INVALID_ARGS;
1092
1093 region = vm_get_region_by_id(id);
1094 if(region == NULL)
1095 return ERR_VM_INVALID_REGION;
1096
1097 info->id = region->id;
1098 info->base = region->base;
1099 info->size = region->size;
1100 info->lock = region->lock;
1101 info->wiring = region->wiring;
1102 strncpy(info->name, region->name, SYS_MAX_OS_NAME_LEN-1);
1103 info->name[SYS_MAX_OS_NAME_LEN-1] = 0;
1104
1105 vm_put_region(region);
1106
1107 return 0;
1108 }
1109
1110 int vm_get_page_mapping(aspace_id aid, addr_t vaddr, addr_t *paddr)
1111 {
1112 vm_address_space *aspace;
1113 unsigned int null_flags;
1114 int err;
1115
1116 aspace = vm_get_aspace_by_id(aid);
1117 if(aspace == NULL)
1118 return ERR_VM_INVALID_ASPACE;
1119
1120 err = aspace->translation_map.ops->query(&aspace->translation_map,
1121 vaddr, paddr, &null_flags);
1122 vm_put_aspace(aspace);
1123 return err;
1124 }
1125
1126 static void display_mem(int argc, char **argv)
1127 {
1128 int item_size;
1129 int display_width;
1130 int num = 1;
1131 addr_t address;
1132 int i;
1133 int j;
1134
1135 if(argc < 2) {
1136 dprintf("not enough arguments\n");
1137 return;
1138 }
1139
1140 address = atoul(argv[1]);
1141
1142 if(argc >= 3) {
1143 num = -1;
1144 num = atoi(argv[2]);
1145 }
1146
1147 // build the format string
1148 if(strcmp(argv[0], "db") == 0) {
1149 item_size = 1;
1150 display_width = 16;
1151 } else if(strcmp(argv[0], "ds") == 0) {
1152 item_size = 2;
1153 display_width = 8;
1154 } else if(strcmp(argv[0], "dw") == 0) {
1155 item_size = 4;
1156 display_width = 4;
1157 } else {
1158 dprintf("display_mem called in an invalid way!\n");
1159 return;
1160 }
1161
1162 dprintf("[0x%lx] '", address);
1163 for(j=0; j<min(display_width, num) * item_size; j++) {
1164 char c = *((char *)address + j);
1165 if(!isalnum(c)) {
1166 c = '.';
1167 }
1168 dprintf("%c", c);
1169 }
1170 dprintf("'");
1171 for(i=0; i<num; i++) {
1172 if((i % display_width) == 0 && i != 0) {
1173 dprintf("\n[0x%lx] '", address + i * item_size);
1174 for(j=0; j<min(display_width, (num-i)) * item_size; j++) {
1175 char c = *((char *)address + i * item_size + j);
1176 if(!isalnum(c)) {
1177 c = '.';
1178 }
1179 dprintf("%c", c);
1180 }
1181 dprintf("'");
1182 }
1183
1184 switch(item_size) {
1185 case 1:
1186 dprintf(" 0x%02x", *((uint8 *)address + i));
1187 break;
1188 case 2:
1189 dprintf(" 0x%04x", *((uint16 *)address + i));
1190 break;
1191 case 4:
1192 dprintf(" 0x%08x", *((uint32 *)address + i));
1193 break;
1194 default:
1195 dprintf("huh?\n");
1196 }
1197 }
1198 dprintf("\n");
1199 }
1200
1201 static void dump_cache_ref(int argc, char **argv)
1202 {
1203 addr_t address;
1204 vm_region *region;
1205 vm_cache_ref *cache_ref;
1206
1207 if(argc < 2) {
1208 dprintf("cache_ref: not enough arguments\n");
1209 return;
1210 }
1211 if(strlen(argv[1]) < 2 || argv[1][0] != '0' || argv[1][1] != 'x') {
1212 dprintf("cache_ref: invalid argument, pass address\n");
1213 return;
1214 }
1215
1216 address = atoul(argv[1]);
1217 cache_ref = (vm_cache_ref *)address;
1218
1219 dprintf("cache_ref at %p:\n", cache_ref);
1220 dprintf("magic: 0x%x ", cache_ref->magic);
1221 if(cache_ref->magic == VM_CACHE_REF_MAGIC)
1222 dprintf("(GOOD)\n");
1223 else
1224 dprintf("(BAD!)\n");
1225 dprintf("cache: %p\n", cache_ref->cache);
1226 dprintf("lock.holder: %d\n", cache_ref->lock.holder);
1227 dprintf("lock.sem: 0x%x\n", cache_ref->lock.sem);
1228 dprintf("region_list:\n");
1229 list_for_every_entry(&cache_ref->region_list_head, region, vm_region, cache_node) {
1230 dprintf(" region 0x%x: ", region->id);
1231 dprintf("base_addr = 0x%lx ", region->base);
1232 dprintf("size = 0x%lx ", region->size);
1233 dprintf("name = '%s' ", region->name);
1234 dprintf("lock = 0x%x\n", region->lock);
1235 }
1236 dprintf("ref_count: %d\n", cache_ref->ref_count);
1237 }
1238
1239 static const char *page_state_to_text(int state)
1240 {
1241 switch(state) {
1242 case PAGE_STATE_ACTIVE:
1243 return "active";
1244 case PAGE_STATE_INACTIVE:
1245 return "inactive";
1246 case PAGE_STATE_BUSY:
1247 return "busy";
1248 case PAGE_STATE_MODIFIED:
1249 return "modified";
1250 case PAGE_STATE_MODIFIED_TEMPORARY:
1251 return "modified_temporary";
1252 case PAGE_STATE_FREE:
1253 return "free";
1254 case PAGE_STATE_CLEAR:
1255 return "clear";
1256 case PAGE_STATE_WIRED:
1257 return "wired";
1258 case PAGE_STATE_UNUSED:
1259 return "unused";
1260 default:
1261 return "unknown";
1262 }
1263 }
1264
1265 static void dump_cache(int argc, char **argv)
1266 {
1267 addr_t address;
1268 vm_cache *cache;
1269 vm_page *page;
1270
1271 if(argc < 2) {
1272 dprintf("cache: not enough arguments\n");
1273 return;
1274 }
1275 if(strlen(argv[1]) < 2 || argv[1][0] != '0' || argv[1][1] != 'x') {
1276 dprintf("cache: invalid argument, pass address\n");
1277 return;
1278 }
1279
1280 address = atoul(argv[1]);
1281 cache = (vm_cache *)address;
1282
1283 dprintf("cache at %p:\n", cache);
1284 dprintf("magic: 0x%x ", cache->magic);
1285 if(cache->magic == VM_CACHE_MAGIC)
1286 dprintf("(GOOD)\n");
1287 else
1288 dprintf("(BAD!)\n");
1289 dprintf("cache_ref: %p\n", cache->ref);
1290 dprintf("source: %p\n", cache->source);
1291 dprintf("store: %p\n", cache->store);
1292 dprintf("temporary: %d\n", cache->temporary);
1293 dprintf("scan_skip: %d\n", cache->scan_skip);
1294 dprintf("virtual_size: 0x%Lx\n", cache->virtual_size);
1295 dprintf("page_list:\n");
1296 list_for_every_entry(&cache->page_list_head, page, vm_page, cache_node) {
1297 if(page->type == PAGE_TYPE_PHYSICAL)
1298 dprintf(" %p ppn 0x%lx offset 0x%Lx type %d state %d (%s) ref_count %d\n",
1299 page, page->ppn, page->offset, page->type, page->state, page_state_to_text(page->state), page->ref_count);
1300 else if(page->type == PAGE_TYPE_DUMMY)
1301 dprintf(" %p DUMMY PAGE state %d (%s)\n", page, page->state, page_state_to_text(page->state));
1302 else
1303 dprintf(" %p UNKNOWN PAGE type %d\n", page, page->type);
1304 }
1305 }
1306
1307 static void _dump_region(vm_region *region)
1308 {
1309 dprintf("dump of region at %p:\n", region);
1310 dprintf("magic: 0x%x ", region->magic);
1311 if(region->magic == VM_REGION_MAGIC)
1312 dprintf("(GOOD)\n");
1313 else
1314 dprintf("(BAD!)\n");
1315 dprintf("name: '%s'\n", region->name);
1316 dprintf("id: 0x%x\n", region->id);
1317 dprintf("base: 0x%lx\n", region->base);
1318 dprintf("size: 0x%lx\n", region->size);
1319 dprintf("lock: 0x%x\n", region->lock);
1320 dprintf("wiring: 0x%x\n", region->wiring);
1321 dprintf("ref_count: %d\n", region->ref_count);
1322 dprintf("cache_offset: 0x%Lx\n", region->cache_offset);
1323 dprintf("cache_ref: %p\n", region->cache_ref);
1324 dprintf("aspace: %p\n", region->aspace);
1325 dprintf("aspace_next: %p\n", region->aspace_next);
1326 dprintf("cache_node.prev: %p\n", region->cache_node.prev);
1327 dprintf("cache_node.next: %p\n", region->cache_node.next);
1328 dprintf("hash_next: %p\n", region->hash_next);
1329 }
1330
1331 static void dump_region(int argc, char **argv)
1332 {
1333 vm_region *region;
1334
1335 if(argc < 2) {
1336 dprintf("region: not enough arguments\n");
1337 return;
1338 }
1339
1340 // if the argument looks like a hex number, treat it as such
1341 if(strlen(argv[1]) > 2 && argv[1][0] == '0' && argv[1][1] == 'x') {
1342 unsigned long num = atoul(argv[1]);
1343 region_id id = num;
1344
1345 region = hash_lookup(region_table, &id);
1346 if(region == NULL) {
1347 dprintf("invalid region id\n");
1348 } else {
1349 _dump_region(region);
1350 }
1351 return;
1352 } else {
1353 // walk through the region list, looking for the arguments as a name
1354 struct hash_iterator iter;
1355
1356 hash_open(region_table, &iter);
1357 while((region = hash_next(region_table, &iter)) != NULL) {
1358 if(region->name != NULL && strcmp(argv[1], region->name) == 0) {
1359 _dump_region(region);
1360 }
1361 }
1362 }
1363 }
1364
1365 static void dump_region_list(int argc, char **argv)
1366 {
1367 vm_region *region;
1368 struct hash_iterator iter;
1369
1370 dprintf("addr\tid\t%32s\tbase\t\tsize\tlock\twiring\n", "name");
1371
1372 hash_open(region_table, &iter);
1373 while((region = hash_next(region_table, &iter)) != NULL) {
1374 dprintf("%p\t0x%x\t%32s\t0x%lx\t\t0x%lx\t%d\t%d\n",
1375 region, region->id, region->name, region->base, region->size, region->lock, region->wiring);
1376 }
1377 hash_close(region_table, &iter, false);
1378 }
1379
1380 static void _dump_aspace(vm_address_space *aspace)
1381 {
1382 vm_region *region;
1383
1384 dprintf("dump of address space at %p:\n", aspace);
1385 dprintf("magic: 0x%x ", aspace->magic);
1386 if(aspace->magic == VM_ASPACE_MAGIC)
1387 dprintf("(GOOD)\n");
1388 else
1389 dprintf("(BAD!)\n");
1390 dprintf("name: '%s'\n", aspace->name);
1391 dprintf("id: 0x%x\n", aspace->id);
1392 dprintf("ref_count: %d\n", aspace->ref_count);
1393 dprintf("fault_count: %d\n", aspace->fault_count);
1394 dprintf("state: %d\n", aspace->state);
1395 dprintf("scan_va: 0x%lx\n", aspace->scan_va);
1396 dprintf("working_set_size: 0x%lx\n", aspace->working_set_size);
1397 dprintf("max_working_set: 0x%lx\n", aspace->max_working_set);
1398 dprintf("min_working_set: 0x%lx\n", aspace->min_working_set);
1399 dprintf("last_working_set_adjust: %Ld\n", aspace->last_working_set_adjust);
1400 dprintf("hash_next: %p\n", aspace->hash_next);
1401 dprintf("translation_map: %p\n", &aspace->translation_map);
1402 dprintf("virtual_map.base: 0x%lx\n", aspace->virtual_map.base);
1403 dprintf("virtual_map.alloc_base: 0x%lx\n", aspace->virtual_map.alloc_base);
1404 dprintf("virtual_map.size: 0x%lx\n", aspace->virtual_map.size);
1405 dprintf("virtual_map.change_count: 0x%x\n", aspace->virtual_map.change_count);
1406 dprintf("virtual_map.sem: 0x%x\n", aspace->virtual_map.sem);
1407 dprintf("virtual_map.region_hint: %p\n", aspace->virtual_map.region_hint);
1408 dprintf("virtual_map.region_list:\n");
1409 for(region = aspace->virtual_map.region_list; region != NULL; region = region->aspace_next) {
1410 dprintf(" region 0x%x: ", region->id);
1411 dprintf("base_addr = 0x%lx ", region->base);
1412 dprintf("size = 0x%lx ", region->size);
1413 dprintf("name = '%s' ", region->name);
1414 dprintf("lock = 0x%x\n", region->lock);
1415 }
1416 }
1417
1418 static void dump_aspace(int argc, char **argv)
1419 {
1420 vm_address_space *aspace;
1421
1422 if(argc < 2) {
1423 dprintf("aspace: not enough arguments\n");
1424 return;
1425 }
1426
1427 // if the argument looks like a hex number, treat it as such
1428 if(strlen(argv[1]) > 2 && argv[1][0] == '0' && argv[1][1] == 'x') {
1429 unsigned long num = atoul(argv[1]);
1430 aspace_id id = num;
1431
1432 aspace = hash_lookup(aspace_table, &id);
1433 if(aspace == NULL) {
1434 dprintf("invalid aspace id\n");
1435 } else {
1436 _dump_aspace(aspace);
1437 }
1438 return;
1439 } else {
1440 // walk through the aspace list, looking for the arguments as a name
1441 struct hash_iterator iter;
1442
1443 hash_open(aspace_table, &iter);
1444 while((aspace = hash_next(aspace_table, &iter)) != NULL) {
1445 if(aspace->name != NULL && strcmp(argv[1], aspace->name) == 0) {
1446 _dump_aspace(aspace);
1447 }
1448 }
1449 }
1450 }
1451
1452 static void dump_aspace_list(int argc, char **argv)
1453 {
1454 vm_address_space *as;
1455 struct hash_iterator iter;
1456
1457 dprintf("addr\tid\t%32s\tbase\t\tsize\n", "name");
1458
1459 hash_open(aspace_table, &iter);
1460 while((as = hash_next(aspace_table, &iter)) != NULL) {
1461 dprintf("%p\t0x%x\t%32s\t0x%lx\t\t0x%lx\n",
1462 as, as->id, as->name, as->virtual_map.base, as->virtual_map.size);
1463 }
1464 hash_close(aspace_table, &iter, false);
1465 }
1466
1467 vm_address_space *vm_get_kernel_aspace(void)
1468 {
1469 VERIFY_VM_ASPACE(kernel_aspace);
1470
1471 /* we can treat this one a little differently since it can't be deleted */
1472 sem_acquire(aspace_hash_sem, READ_COUNT);
1473 atomic_add(&kernel_aspace->ref_count, 1);
1474 sem_release(aspace_hash_sem, READ_COUNT);
1475 return kernel_aspace;
1476 }
1477
1478 aspace_id vm_get_kernel_aspace_id(void)
1479 {
1480 VERIFY_VM_ASPACE(kernel_aspace);
1481 return kernel_aspace->id;
1482 }
1483
1484 vm_address_space *vm_get_current_user_aspace(void)
1485 {
1486 return vm_get_aspace_by_id(vm_get_current_user_aspace_id());
1487 }
1488
1489 aspace_id vm_get_current_user_aspace_id(void)
1490 {
1491 struct thread *t = thread_get_current_thread();
1492
1493 if(t)
1494 return t->proc->aspace_id;
1495 else
1496 return -1;
1497 }
1498
1499 void vm_put_aspace(vm_address_space *aspace)
1500 {
1501 bool removeit = false;
1502
1503 VERIFY_VM_ASPACE(aspace);
1504
1505 sem_acquire(aspace_hash_sem, WRITE_COUNT);
1506 if(atomic_add(&aspace->ref_count, -1) == 1) {
1507 hash_remove(aspace_table, aspace);
1508 removeit = true;
1509 }
1510 sem_release(aspace_hash_sem, WRITE_COUNT);
1511
1512 if(!removeit)
1513 return;
1514
1515 // dprintf("vm_put_aspace: reached zero ref, deleting aspace\n");
1516
1517 if(aspace == kernel_aspace)
1518 panic("vm_put_aspace: tried to delete the kernel aspace!\n");
1519
1520 if(aspace->state != VM_ASPACE_STATE_DELETION)
1521 panic("vm_put_apsace: removed the last ref to aspace %p that's not in the DELETION state\n", aspace);
1522
1523 if(aspace->virtual_map.region_list)
1524 panic("vm_put_aspace: aspace at %p has zero ref count, but region list isn't empty!\n", aspace);
1525
1526 (*aspace->translation_map.ops->destroy)(&aspace->translation_map);
1527
1528 kfree(aspace->name);
1529 sem_delete(aspace->virtual_map.sem);
1530 kfree(aspace);
1531
1532 return;
1533 }
1534
1535 aspace_id vm_create_aspace(const char *name, addr_t base, addr_t alloc_base, addr_t size, bool kernel)
1536 {
1537 vm_address_space *aspace;
1538 int err;
1539
1540 aspace = (vm_address_space *)kmalloc(sizeof(vm_address_space));
1541 if(aspace == NULL) {
1542 err = ERR_NO_MEMORY;
1543 goto err;
1544 }
1545
1546 aspace->name = (char *)kmalloc(strlen(name) + 1);
1547 if(aspace->name == NULL ) {
1548 err = ERR_NO_MEMORY;
1549 goto err1;
1550 }
1551 strcpy(aspace->name, name);
1552
1553 aspace->magic = VM_ASPACE_MAGIC;
1554 aspace->id = next_aspace_id++;
1555 aspace->ref_count = 1;
1556 aspace->state = VM_ASPACE_STATE_NORMAL;
1557 aspace->fault_count = 0;
1558 aspace->scan_va = base;
1559 aspace->working_set_size = kernel ? DEFAULT_KERNEL_WORKING_SET : DEFAULT_WORKING_SET;
1560 aspace->max_working_set = DEFAULT_MAX_WORKING_SET;
1561 aspace->min_working_set = DEFAULT_MIN_WORKING_SET;
1562 aspace->last_working_set_adjust = system_time();
1563
1564 // initialize the corresponding translation map
1565 err = vm_translation_map_create(&aspace->translation_map, kernel);
1566 if(err < 0)
1567 goto err2;
1568
1569 // initialize the virtual map
1570 aspace->virtual_map.base = base;
1571 aspace->virtual_map.alloc_base = alloc_base;
1572 aspace->virtual_map.size = size;
1573 aspace->virtual_map.region_list = NULL;
1574 aspace->virtual_map.region_hint = NULL;
1575 aspace->virtual_map.change_count = 0;
1576 aspace->virtual_map.sem = sem_create(WRITE_COUNT, "aspacelock");
1577 aspace->virtual_map.aspace = aspace;
1578
1579 // add the aspace to the global hash table
1580 sem_acquire(aspace_hash_sem, WRITE_COUNT);
1581 hash_insert(aspace_table, aspace);
1582 sem_release(aspace_hash_sem, WRITE_COUNT);
1583
1584 return aspace->id;
1585
1586 err2:
1587 kfree(aspace->name);
1588 err1:
1589 kfree(aspace);
1590 err:
1591 return err;
1592 }
1593
1594 int vm_delete_aspace(aspace_id aid)
1595 {
1596 vm_region *region;
1597 vm_region *next;
1598 vm_address_space *aspace;
1599
1600 aspace = vm_get_aspace_by_id(aid);
1601 if(aspace == NULL)
1602 return ERR_VM_INVALID_ASPACE;
1603
1604 // dprintf("vm_delete_aspace: called on aspace 0x%x\n", aid);
1605
1606 // put this aspace in the deletion state
1607 // this guarantees that no one else will add regions to the list
1608 sem_acquire(aspace->virtual_map.sem, WRITE_COUNT);
1609 if(aspace->state == VM_ASPACE_STATE_DELETION) {
1610 // abort, someone else is already deleting this aspace
1611 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
1612 vm_put_aspace(aspace);
1613 return NO_ERROR;
1614 }
1615 aspace->state = VM_ASPACE_STATE_DELETION;
1616
1617 // delete all the regions in this aspace
1618 region = aspace->virtual_map.region_list;
1619 while(region) {
1620 VERIFY_VM_REGION(region);
1621 next = region->aspace_next;
1622 // decrement the ref on this region, may actually push the ref < 0, but that's okay
1623 _vm_put_region(region, true);
1624 region = next;
1625 }
1626
1627 // unlock
1628 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
1629
1630 // release two refs on the address space
1631 vm_put_aspace(aspace);
1632 vm_put_aspace(aspace);
1633
1634 return NO_ERROR;
1635 }
1636
1637 int vm_aspace_walk_start(struct hash_iterator *i)
1638 {
1639 hash_open(aspace_table, i);
1640 return 0;
1641 }
1642
1643 vm_address_space *vm_aspace_walk_next(struct hash_iterator *i)
1644 {
1645 vm_address_space *aspace;
1646
1647 sem_acquire(aspace_hash_sem, READ_COUNT);
1648 aspace = hash_next(aspace_table, i);
1649 if(aspace) {
1650 VERIFY_VM_ASPACE(aspace);
1651 atomic_add(&aspace->ref_count, 1);
1652 }
1653 sem_release(aspace_hash_sem, READ_COUNT);
1654 return aspace;
1655 }
1656
1657 int vm_init(kernel_args *ka)
1658 {
1659 int err = 0;
1660 unsigned int i;
1661 addr_t heap_base;
1662 addr_t heap_size;
1663 void *null_addr;
1664
1665 dprintf("vm_init: entry\n");
1666 kprintf("initializing vm system...\n");
1667
1668 err = vm_translation_map_module_init(ka);
1669 err = arch_vm_init(ka);
1670
1671 // initialize some globals
1672 kernel_aspace = NULL;
1673 next_region_id = 0;
1674 region_hash_sem = -1;
1675 next_aspace_id = 0;
1676 aspace_hash_sem = -1;
1677 vm_info.max_commit = 0; // will be increased in vm_page_init
1678 max_commit_lock = 0;
1679 memset(&vm_info, 0, sizeof(vm_info));
1680
1681 // figure the size of memory
1682 vm_page_init(ka);
1683
1684 // map in the new heap and initialize it
1685 heap_size = ROUNDUP(vm_get_mem_size() / 32, 1*1024*1024);
1686 if(heap_size > 16*1024*1024)
1687 heap_size = 16*1024*1024;
1688 heap_base = vm_alloc_from_ka_struct(ka, heap_size, LOCK_KERNEL|LOCK_RW);
1689 dprintf("heap at 0x%lx, size 0x%lx\n", heap_base, heap_size);
1690 kprintf("creating kernel heap at 0x%lx, size 0x%lx\n", heap_base, heap_size);
1691 heap_init(heap_base, heap_size);
1692
1693 // initialize the free page list and page allocator
1694 vm_page_init_postheap(ka);
1695
1696 // initialize the hash table that stores the pages mapped to caches
1697 vm_cache_init(ka);
1698
1699 // create the region and address space hash tables
1700 aspace_table = hash_init(ASPACE_HASH_TABLE_SIZE, offsetof(vm_address_space, hash_next),
1701 &aspace_compare, &aspace_hash);
1702 if(aspace_table == NULL)
1703 panic("vm_init: error creating aspace hash table\n");
1704
1705 region_table = hash_init(REGION_HASH_TABLE_SIZE, offsetof(vm_region, hash_next),
1706 &region_compare, &region_hash);
1707 if(region_table == NULL)
1708 panic("vm_init: error creating aspace hash table\n");
1709
1710 // create the initial kernel address space
1711 {
1712 aspace_id aid;
1713 aid = vm_create_aspace("kernel_land", KERNEL_BASE, KERNEL_ALLOC_BASE, KERNEL_SIZE, true);
1714 if(aid < 0)
1715 panic("vm_init: error creating kernel address space!\n");
1716 kernel_aspace = vm_get_aspace_by_id(aid);
1717 vm_put_aspace(kernel_aspace);
1718 }
1719
1720 // do any further initialization that the architecture dependant layers may need now
1721 vm_translation_map_module_init2(ka);
1722 arch_vm_init2(ka);
1723 vm_page_init2(ka);
1724
1725 // allocate regions to represent stuff that already exists
1726 null_addr = (void *)ROUNDOWN(heap_base, PAGE_SIZE);
1727 vm_create_anonymous_region(vm_get_kernel_aspace_id(), "kernel_heap", &null_addr, REGION_ADDR_EXACT_ADDRESS,
1728 heap_size, REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1729
1730 null_addr = (void *)ROUNDOWN(ka->kernel_seg0_addr.start, PAGE_SIZE);
1731 vm_create_anonymous_region(vm_get_kernel_aspace_id(), "kernel_seg0", &null_addr, REGION_ADDR_EXACT_ADDRESS,
1732 PAGE_ALIGN(ka->kernel_seg0_addr.size), REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1733
1734 if(ka->kernel_seg1_addr.size > 0) {
1735 null_addr = (void *)ROUNDOWN(ka->kernel_seg1_addr.start, PAGE_SIZE);
1736 vm_create_anonymous_region(vm_get_kernel_aspace_id(), "kernel_seg1", &null_addr, REGION_ADDR_EXACT_ADDRESS,
1737 PAGE_ALIGN(ka->kernel_seg1_addr.size), REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1738 }
1739 for(i=0; i < ka->num_cpus; i++) {
1740 char temp[64];
1741
1742 sprintf(temp, "idle_thread%d_kstack", i);
1743 null_addr = (void *)ka->cpu_kstack[i].start;
1744 vm_create_anonymous_region(vm_get_kernel_aspace_id(), temp, &null_addr, REGION_ADDR_EXACT_ADDRESS,
1745 ka->cpu_kstack[i].size, REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1746 }
1747
1748 arch_vm_init_existing_maps(ka);
1749
1750 // map in the boot dir
1751 {
1752 void *null;
1753 vm_map_physical_memory(vm_get_kernel_aspace_id(), "bootdir", &null, REGION_ADDR_ANY_ADDRESS,
1754 ka->bootdir_addr.size, LOCK_RO|LOCK_KERNEL, ka->bootdir_addr.start);
1755 }
1756
1757 arch_vm_init_endvm(ka);
1758
1759 // add some debugger commands
1760 dbg_add_command(&dump_region_list, "regions", "Dump a list of all regions");
1761 dbg_add_command(&dump_region, "region", "Dump info about a particular region");
1762 dbg_add_command(&dump_aspace_list, "aspaces", "Dump a list of all address spaces");
1763 dbg_add_command(&dump_aspace, "aspace", "Dump info about a particular address space");
1764 dbg_add_command(&dump_cache_ref, "cache_ref", "Dump cache_ref data structure");
1765 dbg_add_command(&dump_cache, "cache", "Dump cache_ref data structure");
1766 // dbg_add_command(&display_mem, "dl", "dump memory long words (64-bit)");
1767 dbg_add_command(&display_mem, "dw", "dump memory words (32-bit)");
1768 dbg_add_command(&display_mem, "ds", "dump memory shorts (16-bit)");
1769 dbg_add_command(&display_mem, "db", "dump memory bytes (8-bit)");
1770
1771 dprintf("vm_init: exit\n");
1772
1773 return err;
1774 }
1775
1776 int vm_init_postsem(kernel_args *ka)
1777 {
1778 vm_region *region;
1779
1780 // have the heap finish it's initialization
1781 heap_init_postsem(ka);
1782
1783 // fill in all of the semaphores that were not allocated before
1784 // since we're still single threaded and only the kernel address space exists,
1785 // it isn't that hard to find all of the ones we need to create
1786 vm_translation_map_module_init_post_sem(ka);
1787 kernel_aspace->virtual_map.sem = sem_create(WRITE_COUNT, "kernel_aspacelock");
1788 recursive_lock_create(&kernel_aspace->translation_map.lock);
1789
1790 for(region = kernel_aspace->virtual_map.region_list; region; region = region->aspace_next) {
1791 if(region->cache_ref->lock.sem < 0) {
1792 mutex_init(&region->cache_ref->lock, "cache_ref_mutex");
1793 }
1794 }
1795
1796 region_hash_sem = sem_create(WRITE_COUNT, "region_hash_sem");
1797 aspace_hash_sem = sem_create(WRITE_COUNT, "aspace_hash_sem");
1798
1799 return 0;
1800 }
1801
1802 int vm_init_postthread(kernel_args *ka)
1803 {
1804 vm_page_init_postthread(ka);
1805
1806 vm_daemon_init();
1807
1808 return 0;
1809 }
1810
1811 int vm_page_fault(addr_t address, addr_t fault_address, bool is_write, bool is_user, addr_t *newip)
1812 {
1813 int err;
1814
1815 // dprintf("vm_page_fault: page fault at 0x%lx, ip 0x%lx\n", address, fault_address);
1816
1817 *newip = 0;
1818
1819 err = vm_soft_fault(address, is_write, is_user);
1820 if(err < 0) {
1821 dprintf("vm_page_fault: vm_soft_fault returned error %d on fault at 0x%lx, ip 0x%lx, write %d, user %d, thread 0x%x\n",
1822 err, address, fault_address, is_write, is_user, thread_get_current_thread_id());
1823 if(!is_user) {
1824 struct thread *t = thread_get_current_thread();
1825 if(t && t->fault_handler != 0) {
1826 // this will cause the arch dependant page fault handler to
1827 // modify the IP on the interrupt frame or whatever to return
1828 // to this address
1829 *newip = t->fault_handler;
1830 } else {
1831 // unhandled page fault in the kernel
1832 panic("vm_page_fault: unhandled page fault in kernel space at 0x%lx, ip 0x%lx\n",
1833 address, fault_address);
1834 }
1835 } else {
1836 dprintf("vm_page_fault: killing process 0x%x\n", thread_get_current_thread()->proc->id);
1837 proc_kill_proc(thread_get_current_thread()->proc->id);
1838 }
1839 }
1840
1841 return INT_NO_RESCHEDULE;
1842 }
1843
1844 #define TRACE_PFAULT 0
1845
1846 #if TRACE_PFAULT
1847 #define TRACE dprintf("in pfault at line %d\n", __LINE__)
1848 #else
1849 #define TRACE
1850 #endif
1851
1852 static int vm_soft_fault(addr_t address, bool is_write, bool is_user)
1853 {
1854 vm_address_space *aspace;
1855 vm_virtual_map *map;
1856 vm_region *region;
1857 vm_cache_ref *cache_ref;
1858 vm_cache_ref *last_cache_ref;
1859 vm_cache_ref *top_cache_ref;
1860 off_t cache_offset;
1861 vm_page dummy_page;
1862 vm_page *page = NULL;
1863 int change_count;
1864 int err;
1865
1866 // dprintf("vm_soft_fault: thid 0x%x address 0x%x, is_write %d, is_user %d\n",
1867 // thread_get_current_thread_id(), address, is_write, is_user);
1868
1869 atomic_add(&vm_info.page_faults, 1);
1870
1871 address = ROUNDOWN(address, PAGE_SIZE);
1872
1873 if(is_kernel_address(address)) {
1874 aspace = vm_get_kernel_aspace();
1875 } else if(is_user_address(address)) {
1876 aspace = vm_get_current_user_aspace();
1877 if(aspace == NULL) {
1878 if(is_user == false) {
1879 dprintf("vm_soft_fault: kernel thread accessing invalid user memory!\n");
1880 return ERR_VM_PF_FATAL;
1881 } else {
1882 // XXX weird state.
1883 panic("vm_soft_fault: non kernel thread accessing user memory that doesn't exist!\n");
1884 }
1885 }
1886 } else {
1887 // the hit was probably in the 64k DMZ between kernel and user space
1888 // this keeps a user space thread from passing a buffer that crosses into kernel space
1889 return ERR_VM_PF_FATAL;
1890 }
1891 map = &aspace->virtual_map;
1892 atomic_add(&aspace->fault_count, 1);
1893
1894 sem_acquire(map->sem, READ_COUNT);
1895 region = vm_virtual_map_lookup(map, address);
1896 if(region == NULL) {
1897 sem_release(map->sem, READ_COUNT);
1898 vm_put_aspace(aspace);
1899 dprintf("vm_soft_fault: va 0x%lx not covered by region in address space\n", address);
1900 return ERR_VM_PF_BAD_ADDRESS; // BAD_ADDRESS
1901 }
1902
1903 // check permissions
1904 if(is_user && (region->lock & LOCK_KERNEL) == LOCK_KERNEL) {
1905 sem_release(map->sem, READ_COUNT);
1906 vm_put_aspace(aspace);
1907 dprintf("user access on kernel region\n");
1908 return ERR_VM_PF_BAD_PERM; // BAD_PERMISSION
1909 }
1910 if(is_write && (region->lock & LOCK_RW) == 0) {
1911 sem_release(map->sem, READ_COUNT);
1912 vm_put_aspace(aspace);
1913 dprintf("write access attempted on read-only region\n");
1914 return ERR_VM_PF_BAD_PERM; // BAD_PERMISSION
1915 }
1916
1917 TRACE;
1918
1919 top_cache_ref = region->cache_ref;
1920 VERIFY_VM_CACHE_REF(top_cache_ref);
1921 cache_offset = address - region->base + region->cache_offset;
1922 vm_cache_acquire_ref(top_cache_ref, true);
1923 change_count = map->change_count;
1924 sem_release(map->sem, READ_COUNT);
1925
1926 VERIFY_VM_CACHE(top_cache_ref->cache);
1927 VERIFY_VM_STORE(top_cache_ref->cache->store);
1928
1929 // see if this cache has a fault handler
1930 if(top_cache_ref->cache->store->ops->fault) {
1931 int err = (*top_cache_ref->cache->store->ops->fault)(top_cache_ref->cache->store, aspace, cache_offset);
1932 vm_cache_release_ref(top_cache_ref);
1933 vm_put_aspace(aspace);
1934 return err;
1935 }
1936
1937 TRACE;
1938
1939 dummy_page.magic = VM_PAGE_MAGIC;
1940 dummy_page.state = PAGE_STATE_INACTIVE;
1941 dummy_page.type = PAGE_TYPE_DUMMY;
1942
1943 last_cache_ref = top_cache_ref;
1944 for(cache_ref = top_cache_ref; cache_ref; cache_ref = (cache_ref->cache->source) ? cache_ref->cache->source->ref : NULL) {
1945 VERIFY_VM_CACHE_REF(cache_ref);
1946 mutex_lock(&cache_ref->lock);
1947
1948 TRACE;
1949
1950 for(;;) {
1951 page = vm_cache_lookup_page(cache_ref, cache_offset);
1952 if(page != NULL && page->state != PAGE_STATE_BUSY) {
1953 vm_page_set_state(page, PAGE_STATE_BUSY);
1954 mutex_unlock(&cache_ref->lock);
1955 break;
1956 }
1957
1958 if(page == NULL)
1959 break;
1960
1961 TRACE;
1962
1963 // page must be busy
1964 mutex_unlock(&cache_ref->lock);
1965 thread_snooze(20000);
1966 mutex_lock(&cache_ref->lock);
1967 }
1968
1969 TRACE;
1970
1971 if(page != NULL)
1972 break;
1973
1974 TRACE;
1975
1976 // insert this dummy page here to keep other threads from faulting on the
1977 // same address and chasing us up the cache chain
1978 if(cache_ref == top_cache_ref) {
1979 dummy_page.state = PAGE_STATE_BUSY;
1980 vm_cache_insert_page(cache_ref, &dummy_page, cache_offset);
1981 }
1982
1983 VERIFY_VM_CACHE(cache_ref->cache);
1984 VERIFY_VM_STORE(cache_ref->cache->store);
1985
1986 // see if the vm_store has it
1987 if(cache_ref->cache->store->ops->has_page) {
1988 if(cache_ref->cache->store->ops->has_page(cache_ref->cache->store, cache_offset)) {
1989 IOVECS(vecs, 1);
1990
1991 TRACE;
1992
1993 mutex_unlock(&cache_ref->lock);
1994
1995 vecs->num = 1;
1996 vecs->total_len = PAGE_SIZE;
1997 vecs->vec[0].len = PAGE_SIZE;
1998
1999 page = vm_page_allocate_page(PAGE_STATE_FREE);
2000 (*aspace->translation_map.ops->get_physical_page)(page->ppn * PAGE_SIZE, (addr_t *)&vecs->vec[0].start, PHYSICAL_PAGE_CAN_WAIT);
2001 // handle errors here
2002 err = cache_ref->cache->store->ops->read(cache_ref->cache->store, cache_offset, vecs);
2003 (*aspace->translation_map.ops->put_physical_page)((addr_t)vecs->vec[0].start);
2004
2005 mutex_lock(&cache_ref->lock);
2006
2007 if(cache_ref == top_cache_ref) {
2008 vm_cache_remove_page(cache_ref, &dummy_page);
2009 dummy_page.state = PAGE_STATE_INACTIVE;
2010 }
2011 vm_cache_insert_page(cache_ref, page, cache_offset);
2012 mutex_unlock(&cache_ref->lock);
2013 break;
2014 }
2015 }
2016 mutex_unlock(&cache_ref->lock);
2017 last_cache_ref = cache_ref;
2018 TRACE;
2019 }
2020
2021 TRACE;
2022
2023 // we rolled off the end of the cache chain, so we need to decide which
2024 // cache will get the new page we're about to create
2025 if(!cache_ref) {
2026 if(!is_write)
2027 cache_ref = last_cache_ref; // put it in the deepest cache
2028 else
2029 cache_ref = top_cache_ref; // put it in the topmost cache
2030 }
2031
2032 TRACE;
2033
2034 if(page == NULL) {
2035 // still haven't found a page, so zero out a new one
2036 page = vm_page_allocate_page(PAGE_STATE_CLEAR);
2037 // dprintf("vm_soft_fault: just allocated page 0x%x\n", page->ppn);
2038 mutex_lock(&cache_ref->lock);
2039 if(dummy_page.state == PAGE_STATE_BUSY && dummy_page.cache_ref == cache_ref) {
2040 vm_cache_remove_page(cache_ref, &dummy_page);
2041 dummy_page.state = PAGE_STATE_INACTIVE;
2042 }
2043 vm_cache_insert_page(cache_ref, page, cache_offset);
2044 mutex_unlock(&cache_ref->lock);
2045 if(dummy_page.state == PAGE_STATE_BUSY) {
2046 vm_cache_ref *temp_cache = dummy_page.cache_ref;
2047 mutex_lock(&temp_cache->lock);
2048 vm_cache_remove_page(temp_cache, &dummy_page);
2049 mutex_unlock(&temp_cache->lock);
2050 dummy_page.state = PAGE_STATE_INACTIVE;
2051 }
2052 }
2053
2054 TRACE;
2055
2056 if(page->cache_ref != top_cache_ref && is_write) {
2057 // now we have a page that has the data we want, but in the wrong cache object
2058 // so we need to copy it and stick it into the top cache
2059 vm_page *src_page = page;
2060 void *src, *dest;
2061
2062 page = vm_page_allocate_page(PAGE_STATE_FREE);
2063
2064 // try to get a mapping for the src and dest page so we can copy it
2065 for(;;) {
2066 (*aspace->translation_map.ops->get_physical_page)(src_page->ppn * PAGE_SIZE, (addr_t *)&src, PHYSICAL_PAGE_CAN_WAIT);
2067 err = (*aspace->translation_map.ops->get_physical_page)(page->ppn * PAGE_SIZE, (addr_t *)&dest, PHYSICAL_PAGE_NO_WAIT);
2068 if(err == NO_ERROR)
2069 break;
2070
2071 // it couldn't map the second one, so sleep and retry
2072 // keeps an extremely rare deadlock from occuring
2073 (*aspace->translation_map.ops->put_physical_page)((addr_t)src);
2074 thread_snooze(5000);
2075 }
2076
2077 memcpy(dest, src, PAGE_SIZE);
2078 (*aspace->translation_map.ops->put_physical_page)((addr_t)src);
2079 (*aspace->translation_map.ops->put_physical_page)((addr_t)dest);
2080
2081 vm_page_set_state(src_page, PAGE_STATE_ACTIVE);
2082
2083 mutex_lock(&top_cache_ref->lock);
2084 if(dummy_page.state == PAGE_STATE_BUSY && dummy_page.cache_ref == top_cache_ref) {
2085 vm_cache_remove_page(top_cache_ref, &dummy_page);
2086 dummy_page.state = PAGE_STATE_INACTIVE;
2087 }
2088 vm_cache_insert_page(top_cache_ref, page, cache_offset);
2089 mutex_unlock(&top_cache_ref->lock);
2090
2091 if(dummy_page.state == PAGE_STATE_BUSY) {
2092 vm_cache_ref *temp_cache = dummy_page.cache_ref;
2093 mutex_lock(&temp_cache->lock);
2094 vm_cache_remove_page(temp_cache, &dummy_page);
2095 mutex_unlock(&temp_cache->lock);
2096 dummy_page.state = PAGE_STATE_INACTIVE;
2097 }
2098 }
2099
2100 TRACE;
2101
2102 err = 0;
2103 sem_acquire(map->sem, READ_COUNT);
2104 if(change_count != map->change_count) {
2105 // something may have changed, see if the address is still valid
2106 region = vm_virtual_map_lookup(map, address);
2107 if(region == NULL
2108 || region->cache_ref != top_cache_ref
2109 || (address - region->base + region->cache_offset) != cache_offset) {
2110 dprintf("vm_soft_fault: address space layout changed effecting ongoing soft fault\n");
2111 err = ERR_VM_PF_BAD_ADDRESS; // BAD_ADDRESS
2112 }
2113 }
2114
2115 TRACE;
2116
2117 if(err == 0) {
2118 int new_lock = region->lock;
2119 if(page->cache_ref != top_cache_ref && !is_write)
2120 new_lock &= ~LOCK_RW;
2121
2122 atomic_add(&page->ref_count, 1);
2123 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
2124 (*aspace->translation_map.ops->map)(&aspace->translation_map, address,
2125 page->ppn * PAGE_SIZE, new_lock);
2126 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
2127 }
2128
2129 TRACE;
2130
2131 sem_release(map->sem, READ_COUNT);
2132
2133 TRACE;
2134
2135 if(dummy_page.state == PAGE_STATE_BUSY) {
2136 vm_cache_ref *temp_cache = dummy_page.cache_ref;
2137 mutex_lock(&temp_cache->lock);
2138 vm_cache_remove_page(temp_cache, &dummy_page);
2139 mutex_unlock(&temp_cache->lock);
2140 dummy_page.state = PAGE_STATE_INACTIVE;
2141 }
2142
2143 TRACE;
2144
2145 vm_page_set_state(page, PAGE_STATE_ACTIVE);
2146
2147 vm_cache_release_ref(top_cache_ref);
2148 vm_put_aspace(aspace);
2149
2150 TRACE;
2151
2152 return err;
2153 }
2154
2155 static vm_region *vm_virtual_map_lookup(vm_virtual_map *map, addr_t address)
2156 {
2157 vm_region *region;
2158
2159 // check the region_hint region first
2160 region = map->region_hint;
2161 if(region)
2162 VERIFY_VM_REGION(region);
2163 if(region && region->base <= address && (region->base + region->size) > address)
2164 return region;
2165
2166 for(region = map->region_list; region != NULL; region = region->aspace_next) {
2167 VERIFY_VM_REGION(region);
2168 if(region->base <= address && (region->base + region->size) > address)
2169 break;
2170 }
2171
2172 if(region) {
2173 map->region_hint = region;
2174 VERIFY_VM_REGION(region);
2175 }
2176 return region;
2177 }
2178
2179 int vm_get_physical_page(addr_t paddr, addr_t *vaddr, int flags)
2180 {
2181 #if DEBUG > 1
2182 VERIFY_VM_ASPACE(kernel_aspace);
2183 #endif
2184 return (*kernel_aspace->translation_map.ops->get_physical_page)(paddr, vaddr, flags);
2185 }
2186
2187 int vm_put_physical_page(addr_t vaddr)
2188 {
2189 #if DEBUG > 1
2190 VERIFY_VM_ASPACE(kernel_aspace);
2191 #endif
2192 return (*kernel_aspace->translation_map.ops->put_physical_page)(vaddr);
2193 }
2194
2195 void vm_increase_max_commit(addr_t delta)
2196 {
2197 // dprintf("vm_increase_max_commit: delta 0x%x\n", delta);
2198
2199 int_disable_interrupts();
2200 acquire_spinlock(&max_commit_lock);
2201 vm_info.max_commit += delta;
2202 release_spinlock(&max_commit_lock);
2203 int_restore_interrupts();
2204 }
2205
2206 int user_memcpy(void *to, const void *from, size_t size)
2207 {
2208 return arch_cpu_user_memcpy(to, from, size, &thread_get_current_thread()->fault_handler);
2209 }
2210
2211 int user_strcpy(char *to, const char *from)
2212 {
2213 return arch_cpu_user_strcpy(to, from, &thread_get_current_thread()->fault_handler);
2214 }
2215
2216 int user_strncpy(char *to, const char *from, size_t size)
2217 {
2218 return arch_cpu_user_strncpy(to, from, size, &thread_get_current_thread()->fault_handler);
2219 }
2220
2221 int user_memset(void *s, char c, size_t count)
2222 {
2223 return arch_cpu_user_memset(s, c, count, &thread_get_current_thread()->fault_handler);
2224 }
2225
2226 addr_t vm_get_mem_size(void)
2227 {
2228 return vm_info.physical_page_size * vm_info.physical_pages;
2229 }
2230
2231 int user_vm_get_vm_info(vm_info_t *uinfo)
2232 {
2233 return user_memcpy(uinfo, &vm_info, sizeof(vm_info));
2234 }
2235
A new operating system