1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <linux/sched.h>
32 #include <linux/highmem.h>
33 #include <linux/pagemap.h>
34 #include <linux/file.h>
35 #include <linux/swap.h>
36 #include <linux/slab.h>
37 #include "drm_cache.h"
38 #include "drm_mem_util.h"
39 #include "ttm/ttm_module.h"
40 #include "ttm/ttm_bo_driver.h"
41 #include "ttm/ttm_placement.h"
42 #include "ttm/ttm_page_alloc.h"
44 static int ttm_tt_swapin(struct ttm_tt
*ttm
);
47 * Allocates storage for pointers to the pages that back the ttm.
49 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
51 ttm
->pages
= drm_calloc_large(ttm
->num_pages
, sizeof(*ttm
->pages
));
52 ttm
->dma_address
= drm_calloc_large(ttm
->num_pages
,
53 sizeof(*ttm
->dma_address
));
56 static void ttm_tt_free_page_directory(struct ttm_tt
*ttm
)
58 drm_free_large(ttm
->pages
);
60 drm_free_large(ttm
->dma_address
);
61 ttm
->dma_address
= NULL
;
64 static void ttm_tt_free_user_pages(struct ttm_tt
*ttm
)
70 struct ttm_backend
*be
= ttm
->be
;
72 BUG_ON(!(ttm
->page_flags
& TTM_PAGE_FLAG_USER
));
73 write
= ((ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0);
74 dirty
= ((ttm
->page_flags
& TTM_PAGE_FLAG_USER_DIRTY
) != 0);
79 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
84 if (page
== ttm
->dummy_read_page
) {
89 if (write
&& dirty
&& !PageReserved(page
))
90 set_page_dirty_lock(page
);
93 ttm_mem_global_free(ttm
->glob
->mem_glob
, PAGE_SIZE
);
96 ttm
->state
= tt_unpopulated
;
97 ttm
->first_himem_page
= ttm
->num_pages
;
98 ttm
->last_lomem_page
= -1;
101 static struct page
*__ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
105 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
108 while (NULL
== (p
= ttm
->pages
[index
])) {
112 ret
= ttm_get_pages(&h
, ttm
->page_flags
, ttm
->caching_state
, 1,
113 &ttm
->dma_address
[index
]);
118 p
= list_first_entry(&h
, struct page
, lru
);
120 ret
= ttm_mem_global_alloc_page(mem_glob
, p
, false, false);
121 if (unlikely(ret
!= 0))
125 ttm
->pages
[--ttm
->first_himem_page
] = p
;
127 ttm
->pages
[++ttm
->last_lomem_page
] = p
;
135 struct page
*ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
139 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
140 ret
= ttm_tt_swapin(ttm
);
141 if (unlikely(ret
!= 0))
144 return __ttm_tt_get_page(ttm
, index
);
147 int ttm_tt_populate(struct ttm_tt
*ttm
)
151 struct ttm_backend
*be
;
154 if (ttm
->state
!= tt_unpopulated
)
157 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
158 ret
= ttm_tt_swapin(ttm
);
159 if (unlikely(ret
!= 0))
165 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
166 page
= __ttm_tt_get_page(ttm
, i
);
171 be
->func
->populate(be
, ttm
->num_pages
, ttm
->pages
,
172 ttm
->dummy_read_page
, ttm
->dma_address
);
173 ttm
->state
= tt_unbound
;
176 EXPORT_SYMBOL(ttm_tt_populate
);
179 static inline int ttm_tt_set_page_caching(struct page
*p
,
180 enum ttm_caching_state c_old
,
181 enum ttm_caching_state c_new
)
188 if (c_old
!= tt_cached
) {
189 /* p isn't in the default caching state, set it to
190 * writeback first to free its current memtype. */
192 ret
= set_pages_wb(p
, 1);
198 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
199 else if (c_new
== tt_uncached
)
200 ret
= set_pages_uc(p
, 1);
204 #else /* CONFIG_X86 */
205 static inline int ttm_tt_set_page_caching(struct page
*p
,
206 enum ttm_caching_state c_old
,
207 enum ttm_caching_state c_new
)
211 #endif /* CONFIG_X86 */
214 * Change caching policy for the linear kernel map
215 * for range of pages in a ttm.
218 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
219 enum ttm_caching_state c_state
)
222 struct page
*cur_page
;
225 if (ttm
->caching_state
== c_state
)
228 if (ttm
->state
== tt_unpopulated
) {
229 /* Change caching but don't populate */
230 ttm
->caching_state
= c_state
;
234 if (ttm
->caching_state
== tt_cached
)
235 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
237 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
238 cur_page
= ttm
->pages
[i
];
239 if (likely(cur_page
!= NULL
)) {
240 ret
= ttm_tt_set_page_caching(cur_page
,
243 if (unlikely(ret
!= 0))
248 ttm
->caching_state
= c_state
;
253 for (j
= 0; j
< i
; ++j
) {
254 cur_page
= ttm
->pages
[j
];
255 if (likely(cur_page
!= NULL
)) {
256 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
264 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
266 enum ttm_caching_state state
;
268 if (placement
& TTM_PL_FLAG_WC
)
270 else if (placement
& TTM_PL_FLAG_UNCACHED
)
275 return ttm_tt_set_caching(ttm
, state
);
277 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
279 static void ttm_tt_free_alloced_pages(struct ttm_tt
*ttm
)
284 struct page
*cur_page
;
285 struct ttm_backend
*be
= ttm
->be
;
291 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
293 cur_page
= ttm
->pages
[i
];
294 ttm
->pages
[i
] = NULL
;
296 if (page_count(cur_page
) != 1)
297 printk(KERN_ERR TTM_PFX
298 "Erroneous page count. "
300 ttm_mem_global_free_page(ttm
->glob
->mem_glob
,
302 list_add(&cur_page
->lru
, &h
);
306 ttm_put_pages(&h
, count
, ttm
->page_flags
, ttm
->caching_state
,
308 ttm
->state
= tt_unpopulated
;
309 ttm
->first_himem_page
= ttm
->num_pages
;
310 ttm
->last_lomem_page
= -1;
313 void ttm_tt_destroy(struct ttm_tt
*ttm
)
315 struct ttm_backend
*be
;
317 if (unlikely(ttm
== NULL
))
321 if (likely(be
!= NULL
)) {
322 be
->func
->destroy(be
);
326 if (likely(ttm
->pages
!= NULL
)) {
327 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
328 ttm_tt_free_user_pages(ttm
);
330 ttm_tt_free_alloced_pages(ttm
);
332 ttm_tt_free_page_directory(ttm
);
335 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
) &&
337 fput(ttm
->swap_storage
);
342 int ttm_tt_set_user(struct ttm_tt
*ttm
,
343 struct task_struct
*tsk
,
344 unsigned long start
, unsigned long num_pages
)
346 struct mm_struct
*mm
= tsk
->mm
;
348 int write
= (ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0;
349 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
351 BUG_ON(num_pages
!= ttm
->num_pages
);
352 BUG_ON((ttm
->page_flags
& TTM_PAGE_FLAG_USER
) == 0);
355 * Account user pages as lowmem pages for now.
358 ret
= ttm_mem_global_alloc(mem_glob
, num_pages
* PAGE_SIZE
,
360 if (unlikely(ret
!= 0))
363 down_read(&mm
->mmap_sem
);
364 ret
= get_user_pages(tsk
, mm
, start
, num_pages
,
365 write
, 0, ttm
->pages
, NULL
);
366 up_read(&mm
->mmap_sem
);
368 if (ret
!= num_pages
&& write
) {
369 ttm_tt_free_user_pages(ttm
);
370 ttm_mem_global_free(mem_glob
, num_pages
* PAGE_SIZE
);
376 ttm
->state
= tt_unbound
;
381 struct ttm_tt
*ttm_tt_create(struct ttm_bo_device
*bdev
, unsigned long size
,
382 uint32_t page_flags
, struct page
*dummy_read_page
)
384 struct ttm_bo_driver
*bo_driver
= bdev
->driver
;
390 ttm
= kzalloc(sizeof(*ttm
), GFP_KERNEL
);
394 ttm
->glob
= bdev
->glob
;
395 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
396 ttm
->first_himem_page
= ttm
->num_pages
;
397 ttm
->last_lomem_page
= -1;
398 ttm
->caching_state
= tt_cached
;
399 ttm
->page_flags
= page_flags
;
401 ttm
->dummy_read_page
= dummy_read_page
;
403 ttm_tt_alloc_page_directory(ttm
);
406 printk(KERN_ERR TTM_PFX
"Failed allocating page table\n");
409 ttm
->be
= bo_driver
->create_ttm_backend_entry(bdev
);
412 printk(KERN_ERR TTM_PFX
"Failed creating ttm backend entry\n");
415 ttm
->state
= tt_unpopulated
;
419 void ttm_tt_unbind(struct ttm_tt
*ttm
)
422 struct ttm_backend
*be
= ttm
->be
;
424 if (ttm
->state
== tt_bound
) {
425 ret
= be
->func
->unbind(be
);
427 ttm
->state
= tt_unbound
;
431 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
434 struct ttm_backend
*be
;
439 if (ttm
->state
== tt_bound
)
444 ret
= ttm_tt_populate(ttm
);
448 ret
= be
->func
->bind(be
, bo_mem
);
449 if (unlikely(ret
!= 0))
452 ttm
->state
= tt_bound
;
454 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
455 ttm
->page_flags
|= TTM_PAGE_FLAG_USER_DIRTY
;
458 EXPORT_SYMBOL(ttm_tt_bind
);
460 static int ttm_tt_swapin(struct ttm_tt
*ttm
)
462 struct address_space
*swap_space
;
463 struct file
*swap_storage
;
464 struct page
*from_page
;
465 struct page
*to_page
;
471 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
472 ret
= ttm_tt_set_user(ttm
, ttm
->tsk
, ttm
->start
,
474 if (unlikely(ret
!= 0))
477 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
481 swap_storage
= ttm
->swap_storage
;
482 BUG_ON(swap_storage
== NULL
);
484 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
486 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
487 from_page
= read_mapping_page(swap_space
, i
, NULL
);
488 if (IS_ERR(from_page
)) {
489 ret
= PTR_ERR(from_page
);
492 to_page
= __ttm_tt_get_page(ttm
, i
);
493 if (unlikely(to_page
== NULL
))
497 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
498 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
499 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
500 kunmap_atomic(to_virtual
, KM_USER1
);
501 kunmap_atomic(from_virtual
, KM_USER0
);
503 page_cache_release(from_page
);
506 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
))
508 ttm
->swap_storage
= NULL
;
509 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
513 ttm_tt_free_alloced_pages(ttm
);
517 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistant_swap_storage
)
519 struct address_space
*swap_space
;
520 struct file
*swap_storage
;
521 struct page
*from_page
;
522 struct page
*to_page
;
528 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
529 BUG_ON(ttm
->caching_state
!= tt_cached
);
532 * For user buffers, just unpin the pages, as there should be
536 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
537 ttm_tt_free_user_pages(ttm
);
538 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
539 ttm
->swap_storage
= NULL
;
543 if (!persistant_swap_storage
) {
544 swap_storage
= shmem_file_setup("ttm swap",
545 ttm
->num_pages
<< PAGE_SHIFT
,
547 if (unlikely(IS_ERR(swap_storage
))) {
548 printk(KERN_ERR
"Failed allocating swap storage.\n");
549 return PTR_ERR(swap_storage
);
552 swap_storage
= persistant_swap_storage
;
554 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
556 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
557 from_page
= ttm
->pages
[i
];
558 if (unlikely(from_page
== NULL
))
560 to_page
= read_mapping_page(swap_space
, i
, NULL
);
561 if (unlikely(IS_ERR(to_page
))) {
562 ret
= PTR_ERR(to_page
);
566 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
567 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
568 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
569 kunmap_atomic(to_virtual
, KM_USER1
);
570 kunmap_atomic(from_virtual
, KM_USER0
);
572 set_page_dirty(to_page
);
573 mark_page_accessed(to_page
);
574 page_cache_release(to_page
);
577 ttm_tt_free_alloced_pages(ttm
);
578 ttm
->swap_storage
= swap_storage
;
579 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
580 if (persistant_swap_storage
)
581 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTANT_SWAP
;
585 if (!persistant_swap_storage
)