1 /**************************************************************************
3 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include "ttm/ttm_bo_driver.h"
32 #include "ttm/ttm_placement.h"
34 #include <linux/highmem.h>
35 #include <linux/wait.h>
36 #include <linux/slab.h>
37 #include <linux/vmalloc.h>
38 #include <linux/module.h>
40 void ttm_bo_free_old_node(struct ttm_buffer_object
*bo
)
42 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
44 if (old_mem
->mm_node
) {
45 spin_lock(&bo
->glob
->lru_lock
);
46 drm_mm_put_block(old_mem
->mm_node
);
47 spin_unlock(&bo
->glob
->lru_lock
);
49 old_mem
->mm_node
= NULL
;
52 int ttm_bo_move_ttm(struct ttm_buffer_object
*bo
,
53 bool evict
, bool no_wait_reserve
,
54 bool no_wait_gpu
, struct ttm_mem_reg
*new_mem
)
56 struct ttm_tt
*ttm
= bo
->ttm
;
57 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
60 if (old_mem
->mem_type
!= TTM_PL_SYSTEM
) {
62 ttm_bo_free_old_node(bo
);
63 ttm_flag_masked(&old_mem
->placement
, TTM_PL_FLAG_SYSTEM
,
65 old_mem
->mem_type
= TTM_PL_SYSTEM
;
68 ret
= ttm_tt_set_placement_caching(ttm
, new_mem
->placement
);
69 if (unlikely(ret
!= 0))
72 if (new_mem
->mem_type
!= TTM_PL_SYSTEM
) {
73 ret
= ttm_tt_bind(ttm
, new_mem
);
74 if (unlikely(ret
!= 0))
79 new_mem
->mm_node
= NULL
;
83 EXPORT_SYMBOL(ttm_bo_move_ttm
);
85 int ttm_mem_io_reserve(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
89 if (!mem
->bus
.io_reserved
) {
90 mem
->bus
.io_reserved
= true;
91 ret
= bdev
->driver
->io_mem_reserve(bdev
, mem
);
92 if (unlikely(ret
!= 0))
98 void ttm_mem_io_free(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
100 if (bdev
->driver
->io_mem_reserve
) {
101 if (mem
->bus
.io_reserved
) {
102 mem
->bus
.io_reserved
= false;
103 bdev
->driver
->io_mem_free(bdev
, mem
);
108 int ttm_mem_reg_ioremap(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
,
115 ret
= ttm_mem_io_reserve(bdev
, mem
);
116 if (ret
|| !mem
->bus
.is_iomem
)
120 addr
= mem
->bus
.addr
;
122 if (mem
->placement
& TTM_PL_FLAG_WC
)
123 addr
= ioremap_wc(mem
->bus
.base
+ mem
->bus
.offset
, mem
->bus
.size
);
125 addr
= ioremap_nocache(mem
->bus
.base
+ mem
->bus
.offset
, mem
->bus
.size
);
127 ttm_mem_io_free(bdev
, mem
);
135 void ttm_mem_reg_iounmap(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
,
138 struct ttm_mem_type_manager
*man
;
140 man
= &bdev
->man
[mem
->mem_type
];
142 if (virtual && mem
->bus
.addr
== NULL
)
144 ttm_mem_io_free(bdev
, mem
);
147 static int ttm_copy_io_page(void *dst
, void *src
, unsigned long page
)
150 (uint32_t *) ((unsigned long)dst
+ (page
<< PAGE_SHIFT
));
152 (uint32_t *) ((unsigned long)src
+ (page
<< PAGE_SHIFT
));
155 for (i
= 0; i
< PAGE_SIZE
/ sizeof(uint32_t); ++i
)
156 iowrite32(ioread32(srcP
++), dstP
++);
160 static int ttm_copy_io_ttm_page(struct ttm_tt
*ttm
, void *src
,
164 struct page
*d
= ttm_tt_get_page(ttm
, page
);
170 src
= (void *)((unsigned long)src
+ (page
<< PAGE_SHIFT
));
173 dst
= kmap_atomic_prot(d
, KM_USER0
, prot
);
175 if (pgprot_val(prot
) != pgprot_val(PAGE_KERNEL
))
176 dst
= vmap(&d
, 1, 0, prot
);
183 memcpy_fromio(dst
, src
, PAGE_SIZE
);
186 kunmap_atomic(dst
, KM_USER0
);
188 if (pgprot_val(prot
) != pgprot_val(PAGE_KERNEL
))
197 static int ttm_copy_ttm_io_page(struct ttm_tt
*ttm
, void *dst
,
201 struct page
*s
= ttm_tt_get_page(ttm
, page
);
207 dst
= (void *)((unsigned long)dst
+ (page
<< PAGE_SHIFT
));
209 src
= kmap_atomic_prot(s
, KM_USER0
, prot
);
211 if (pgprot_val(prot
) != pgprot_val(PAGE_KERNEL
))
212 src
= vmap(&s
, 1, 0, prot
);
219 memcpy_toio(dst
, src
, PAGE_SIZE
);
222 kunmap_atomic(src
, KM_USER0
);
224 if (pgprot_val(prot
) != pgprot_val(PAGE_KERNEL
))
233 int ttm_bo_move_memcpy(struct ttm_buffer_object
*bo
,
234 bool evict
, bool no_wait_reserve
, bool no_wait_gpu
,
235 struct ttm_mem_reg
*new_mem
)
237 struct ttm_bo_device
*bdev
= bo
->bdev
;
238 struct ttm_mem_type_manager
*man
= &bdev
->man
[new_mem
->mem_type
];
239 struct ttm_tt
*ttm
= bo
->ttm
;
240 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
241 struct ttm_mem_reg old_copy
= *old_mem
;
247 unsigned long add
= 0;
250 ret
= ttm_mem_reg_ioremap(bdev
, old_mem
, &old_iomap
);
253 ret
= ttm_mem_reg_ioremap(bdev
, new_mem
, &new_iomap
);
257 if (old_iomap
== NULL
&& new_iomap
== NULL
)
259 if (old_iomap
== NULL
&& ttm
== NULL
)
265 if ((old_mem
->mem_type
== new_mem
->mem_type
) &&
266 (new_mem
->mm_node
->start
<
267 old_mem
->mm_node
->start
+ old_mem
->mm_node
->size
)) {
269 add
= new_mem
->num_pages
- 1;
272 for (i
= 0; i
< new_mem
->num_pages
; ++i
) {
273 page
= i
* dir
+ add
;
274 if (old_iomap
== NULL
) {
275 pgprot_t prot
= ttm_io_prot(old_mem
->placement
,
277 ret
= ttm_copy_ttm_io_page(ttm
, new_iomap
, page
,
279 } else if (new_iomap
== NULL
) {
280 pgprot_t prot
= ttm_io_prot(new_mem
->placement
,
282 ret
= ttm_copy_io_ttm_page(ttm
, old_iomap
, page
,
285 ret
= ttm_copy_io_page(new_iomap
, old_iomap
, page
);
291 ttm_bo_free_old_node(bo
);
294 new_mem
->mm_node
= NULL
;
296 if ((man
->flags
& TTM_MEMTYPE_FLAG_FIXED
) && (ttm
!= NULL
)) {
303 ttm_mem_reg_iounmap(bdev
, new_mem
, new_iomap
);
305 ttm_mem_reg_iounmap(bdev
, &old_copy
, old_iomap
);
308 EXPORT_SYMBOL(ttm_bo_move_memcpy
);
310 static void ttm_transfered_destroy(struct ttm_buffer_object
*bo
)
316 * ttm_buffer_object_transfer
318 * @bo: A pointer to a struct ttm_buffer_object.
319 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
320 * holding the data of @bo with the old placement.
322 * This is a utility function that may be called after an accelerated move
323 * has been scheduled. A new buffer object is created as a placeholder for
324 * the old data while it's being copied. When that buffer object is idle,
325 * it can be destroyed, releasing the space of the old placement.
330 static int ttm_buffer_object_transfer(struct ttm_buffer_object
*bo
,
331 struct ttm_buffer_object
**new_obj
)
333 struct ttm_buffer_object
*fbo
;
334 struct ttm_bo_device
*bdev
= bo
->bdev
;
335 struct ttm_bo_driver
*driver
= bdev
->driver
;
337 fbo
= kzalloc(sizeof(*fbo
), GFP_KERNEL
);
344 * Fix up members that we shouldn't copy directly:
345 * TODO: Explicit member copy would probably be better here.
348 spin_lock_init(&fbo
->lock
);
349 init_waitqueue_head(&fbo
->event_queue
);
350 INIT_LIST_HEAD(&fbo
->ddestroy
);
351 INIT_LIST_HEAD(&fbo
->lru
);
352 INIT_LIST_HEAD(&fbo
->swap
);
354 atomic_set(&fbo
->cpu_writers
, 0);
356 fbo
->sync_obj
= driver
->sync_obj_ref(bo
->sync_obj
);
357 kref_init(&fbo
->list_kref
);
358 kref_init(&fbo
->kref
);
359 fbo
->destroy
= &ttm_transfered_destroy
;
365 pgprot_t
ttm_io_prot(uint32_t caching_flags
, pgprot_t tmp
)
367 #if defined(__i386__) || defined(__x86_64__)
368 if (caching_flags
& TTM_PL_FLAG_WC
)
369 tmp
= pgprot_writecombine(tmp
);
370 else if (boot_cpu_data
.x86
> 3)
371 tmp
= pgprot_noncached(tmp
);
373 #elif defined(__powerpc__)
374 if (!(caching_flags
& TTM_PL_FLAG_CACHED
)) {
375 pgprot_val(tmp
) |= _PAGE_NO_CACHE
;
376 if (caching_flags
& TTM_PL_FLAG_UNCACHED
)
377 pgprot_val(tmp
) |= _PAGE_GUARDED
;
380 #if defined(__ia64__)
381 if (caching_flags
& TTM_PL_FLAG_WC
)
382 tmp
= pgprot_writecombine(tmp
);
384 tmp
= pgprot_noncached(tmp
);
386 #if defined(__sparc__)
387 if (!(caching_flags
& TTM_PL_FLAG_CACHED
))
388 tmp
= pgprot_noncached(tmp
);
392 EXPORT_SYMBOL(ttm_io_prot
);
394 static int ttm_bo_ioremap(struct ttm_buffer_object
*bo
,
395 unsigned long offset
,
397 struct ttm_bo_kmap_obj
*map
)
399 struct ttm_mem_reg
*mem
= &bo
->mem
;
401 if (bo
->mem
.bus
.addr
) {
402 map
->bo_kmap_type
= ttm_bo_map_premapped
;
403 map
->virtual = (void *)(((u8
*)bo
->mem
.bus
.addr
) + offset
);
405 map
->bo_kmap_type
= ttm_bo_map_iomap
;
406 if (mem
->placement
& TTM_PL_FLAG_WC
)
407 map
->virtual = ioremap_wc(bo
->mem
.bus
.base
+ bo
->mem
.bus
.offset
+ offset
,
410 map
->virtual = ioremap_nocache(bo
->mem
.bus
.base
+ bo
->mem
.bus
.offset
+ offset
,
413 return (!map
->virtual) ? -ENOMEM
: 0;
416 static int ttm_bo_kmap_ttm(struct ttm_buffer_object
*bo
,
417 unsigned long start_page
,
418 unsigned long num_pages
,
419 struct ttm_bo_kmap_obj
*map
)
421 struct ttm_mem_reg
*mem
= &bo
->mem
; pgprot_t prot
;
422 struct ttm_tt
*ttm
= bo
->ttm
;
427 if (num_pages
== 1 && (mem
->placement
& TTM_PL_FLAG_CACHED
)) {
429 * We're mapping a single page, and the desired
430 * page protection is consistent with the bo.
433 map
->bo_kmap_type
= ttm_bo_map_kmap
;
434 map
->page
= ttm_tt_get_page(ttm
, start_page
);
435 map
->virtual = kmap(map
->page
);
438 * Populate the part we're mapping;
440 for (i
= start_page
; i
< start_page
+ num_pages
; ++i
) {
441 d
= ttm_tt_get_page(ttm
, i
);
447 * We need to use vmap to get the desired page protection
448 * or to make the buffer object look contiguous.
450 prot
= (mem
->placement
& TTM_PL_FLAG_CACHED
) ?
452 ttm_io_prot(mem
->placement
, PAGE_KERNEL
);
453 map
->bo_kmap_type
= ttm_bo_map_vmap
;
454 map
->virtual = vmap(ttm
->pages
+ start_page
, num_pages
,
457 return (!map
->virtual) ? -ENOMEM
: 0;
460 int ttm_bo_kmap(struct ttm_buffer_object
*bo
,
461 unsigned long start_page
, unsigned long num_pages
,
462 struct ttm_bo_kmap_obj
*map
)
464 unsigned long offset
, size
;
467 BUG_ON(!list_empty(&bo
->swap
));
470 if (num_pages
> bo
->num_pages
)
472 if (start_page
> bo
->num_pages
)
474 ret
= ttm_mem_io_reserve(bo
->bdev
, &bo
->mem
);
477 if (!bo
->mem
.bus
.is_iomem
) {
478 return ttm_bo_kmap_ttm(bo
, start_page
, num_pages
, map
);
480 offset
= start_page
<< PAGE_SHIFT
;
481 size
= num_pages
<< PAGE_SHIFT
;
482 return ttm_bo_ioremap(bo
, offset
, size
, map
);
485 EXPORT_SYMBOL(ttm_bo_kmap
);
487 void ttm_bo_kunmap(struct ttm_bo_kmap_obj
*map
)
491 switch (map
->bo_kmap_type
) {
492 case ttm_bo_map_iomap
:
493 iounmap(map
->virtual);
494 ttm_mem_io_free(map
->bo
->bdev
, &map
->bo
->mem
);
496 case ttm_bo_map_vmap
:
497 vunmap(map
->virtual);
499 case ttm_bo_map_kmap
:
502 case ttm_bo_map_premapped
:
510 EXPORT_SYMBOL(ttm_bo_kunmap
);
512 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object
*bo
,
515 bool evict
, bool no_wait_reserve
,
517 struct ttm_mem_reg
*new_mem
)
519 struct ttm_bo_device
*bdev
= bo
->bdev
;
520 struct ttm_bo_driver
*driver
= bdev
->driver
;
521 struct ttm_mem_type_manager
*man
= &bdev
->man
[new_mem
->mem_type
];
522 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
524 struct ttm_buffer_object
*ghost_obj
;
525 void *tmp_obj
= NULL
;
527 spin_lock(&bo
->lock
);
529 tmp_obj
= bo
->sync_obj
;
532 bo
->sync_obj
= driver
->sync_obj_ref(sync_obj
);
533 bo
->sync_obj_arg
= sync_obj_arg
;
535 ret
= ttm_bo_wait(bo
, false, false, false);
536 spin_unlock(&bo
->lock
);
538 driver
->sync_obj_unref(&tmp_obj
);
542 ttm_bo_free_old_node(bo
);
543 if ((man
->flags
& TTM_MEMTYPE_FLAG_FIXED
) &&
545 ttm_tt_unbind(bo
->ttm
);
546 ttm_tt_destroy(bo
->ttm
);
551 * This should help pipeline ordinary buffer moves.
553 * Hang old buffer memory on a new buffer object,
554 * and leave it to be released when the GPU
555 * operation has completed.
558 set_bit(TTM_BO_PRIV_FLAG_MOVING
, &bo
->priv_flags
);
559 spin_unlock(&bo
->lock
);
561 driver
->sync_obj_unref(&tmp_obj
);
563 ret
= ttm_buffer_object_transfer(bo
, &ghost_obj
);
568 * If we're not moving to fixed memory, the TTM object
569 * needs to stay alive. Otherwhise hang it on the ghost
570 * bo to be unbound and destroyed.
573 if (!(man
->flags
& TTM_MEMTYPE_FLAG_FIXED
))
574 ghost_obj
->ttm
= NULL
;
578 ttm_bo_unreserve(ghost_obj
);
579 ttm_bo_unref(&ghost_obj
);
583 new_mem
->mm_node
= NULL
;
587 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup
);