1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <ttm/ttm_module.h>
32 #include <ttm/ttm_bo_driver.h>
33 #include <ttm/ttm_placement.h>
35 #include <linux/rbtree.h>
36 #include <linux/module.h>
37 #include <linux/uaccess.h>
39 #define TTM_BO_VM_NUM_PREFAULT 16
41 static struct ttm_buffer_object
*ttm_bo_vm_lookup_rb(struct ttm_bo_device
*bdev
,
42 unsigned long page_start
,
43 unsigned long num_pages
)
45 struct rb_node
*cur
= bdev
->addr_space_rb
.rb_node
;
46 unsigned long cur_offset
;
47 struct ttm_buffer_object
*bo
;
48 struct ttm_buffer_object
*best_bo
= NULL
;
50 while (likely(cur
!= NULL
)) {
51 bo
= rb_entry(cur
, struct ttm_buffer_object
, vm_rb
);
52 cur_offset
= bo
->vm_node
->start
;
53 if (page_start
>= cur_offset
) {
56 if (page_start
== cur_offset
)
62 if (unlikely(best_bo
== NULL
))
65 if (unlikely((best_bo
->vm_node
->start
+ best_bo
->num_pages
) <
66 (page_start
+ num_pages
)))
72 static int ttm_bo_vm_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
74 struct ttm_buffer_object
*bo
= (struct ttm_buffer_object
*)
76 struct ttm_bo_device
*bdev
= bo
->bdev
;
77 unsigned long page_offset
;
78 unsigned long page_last
;
80 struct ttm_tt
*ttm
= NULL
;
84 unsigned long address
= (unsigned long)vmf
->virtual_address
;
85 int retval
= VM_FAULT_NOPAGE
;
88 ret
= ttm_bo_reserve(bo
, true, true, false, 0);
89 if (unlikely(ret
!= 0)) {
92 return VM_FAULT_NOPAGE
;
95 if (bdev
->driver
->fault_reserve_notify
) {
96 ret
= bdev
->driver
->fault_reserve_notify(bo
);
103 retval
= VM_FAULT_NOPAGE
;
106 retval
= VM_FAULT_SIGBUS
;
112 * Wait for buffer data in transit, due to a pipelined
116 spin_lock(&bo
->lock
);
117 if (test_bit(TTM_BO_PRIV_FLAG_MOVING
, &bo
->priv_flags
)) {
118 ret
= ttm_bo_wait(bo
, false, true, false);
119 spin_unlock(&bo
->lock
);
120 if (unlikely(ret
!= 0)) {
121 retval
= (ret
!= -ERESTARTSYS
) ?
122 VM_FAULT_SIGBUS
: VM_FAULT_NOPAGE
;
126 spin_unlock(&bo
->lock
);
129 ret
= ttm_mem_io_reserve(bdev
, &bo
->mem
);
131 retval
= VM_FAULT_SIGBUS
;
135 page_offset
= ((address
- vma
->vm_start
) >> PAGE_SHIFT
) +
136 bo
->vm_node
->start
- vma
->vm_pgoff
;
137 page_last
= ((vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
) +
138 bo
->vm_node
->start
- vma
->vm_pgoff
;
140 if (unlikely(page_offset
>= bo
->num_pages
)) {
141 retval
= VM_FAULT_SIGBUS
;
146 * Strictly, we're not allowed to modify vma->vm_page_prot here,
147 * since the mmap_sem is only held in read mode. However, we
148 * modify only the caching bits of vma->vm_page_prot and
149 * consider those bits protected by
150 * the bo->mutex, as we should be the only writers.
151 * There shouldn't really be any readers of these bits except
152 * within vm_insert_mixed()? fork?
154 * TODO: Add a list of vmas to the bo, and change the
155 * vma->vm_page_prot when the object changes caching policy, with
156 * the correct locks held.
158 if (bo
->mem
.bus
.is_iomem
) {
159 vma
->vm_page_prot
= ttm_io_prot(bo
->mem
.placement
,
163 vma
->vm_page_prot
= (bo
->mem
.placement
& TTM_PL_FLAG_CACHED
) ?
164 vm_get_page_prot(vma
->vm_flags
) :
165 ttm_io_prot(bo
->mem
.placement
, vma
->vm_page_prot
);
169 * Speculatively prefault a number of pages. Only error on
173 for (i
= 0; i
< TTM_BO_VM_NUM_PREFAULT
; ++i
) {
174 if (bo
->mem
.bus
.is_iomem
)
175 pfn
= ((bo
->mem
.bus
.base
+ bo
->mem
.bus
.offset
) >> PAGE_SHIFT
) + page_offset
;
177 page
= ttm_tt_get_page(ttm
, page_offset
);
178 if (unlikely(!page
&& i
== 0)) {
179 retval
= VM_FAULT_OOM
;
181 } else if (unlikely(!page
)) {
184 pfn
= page_to_pfn(page
);
187 ret
= vm_insert_mixed(vma
, address
, pfn
);
189 * Somebody beat us to this PTE or prefaulting to
190 * an already populated PTE, or prefaulting error.
193 if (unlikely((ret
== -EBUSY
) || (ret
!= 0 && i
> 0)))
195 else if (unlikely(ret
!= 0)) {
197 (ret
== -ENOMEM
) ? VM_FAULT_OOM
: VM_FAULT_SIGBUS
;
201 address
+= PAGE_SIZE
;
202 if (unlikely(++page_offset
>= page_last
))
207 ttm_bo_unreserve(bo
);
211 static void ttm_bo_vm_open(struct vm_area_struct
*vma
)
213 struct ttm_buffer_object
*bo
=
214 (struct ttm_buffer_object
*)vma
->vm_private_data
;
216 (void)ttm_bo_reference(bo
);
219 static void ttm_bo_vm_close(struct vm_area_struct
*vma
)
221 struct ttm_buffer_object
*bo
= (struct ttm_buffer_object
*)vma
->vm_private_data
;
224 vma
->vm_private_data
= NULL
;
227 static const struct vm_operations_struct ttm_bo_vm_ops
= {
228 .fault
= ttm_bo_vm_fault
,
229 .open
= ttm_bo_vm_open
,
230 .close
= ttm_bo_vm_close
233 int ttm_bo_mmap(struct file
*filp
, struct vm_area_struct
*vma
,
234 struct ttm_bo_device
*bdev
)
236 struct ttm_bo_driver
*driver
;
237 struct ttm_buffer_object
*bo
;
240 read_lock(&bdev
->vm_lock
);
241 bo
= ttm_bo_vm_lookup_rb(bdev
, vma
->vm_pgoff
,
242 (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
);
243 if (likely(bo
!= NULL
))
244 ttm_bo_reference(bo
);
245 read_unlock(&bdev
->vm_lock
);
247 if (unlikely(bo
== NULL
)) {
248 printk(KERN_ERR TTM_PFX
249 "Could not find buffer object to map.\n");
253 driver
= bo
->bdev
->driver
;
254 if (unlikely(!driver
->verify_access
)) {
258 ret
= driver
->verify_access(bo
, filp
);
259 if (unlikely(ret
!= 0))
262 vma
->vm_ops
= &ttm_bo_vm_ops
;
265 * Note: We're transferring the bo reference to
266 * vma->vm_private_data here.
269 vma
->vm_private_data
= bo
;
270 vma
->vm_flags
|= VM_RESERVED
| VM_IO
| VM_MIXEDMAP
| VM_DONTEXPAND
;
276 EXPORT_SYMBOL(ttm_bo_mmap
);
278 int ttm_fbdev_mmap(struct vm_area_struct
*vma
, struct ttm_buffer_object
*bo
)
280 if (vma
->vm_pgoff
!= 0)
283 vma
->vm_ops
= &ttm_bo_vm_ops
;
284 vma
->vm_private_data
= ttm_bo_reference(bo
);
285 vma
->vm_flags
|= VM_RESERVED
| VM_IO
| VM_MIXEDMAP
| VM_DONTEXPAND
;
288 EXPORT_SYMBOL(ttm_fbdev_mmap
);
291 ssize_t
ttm_bo_io(struct ttm_bo_device
*bdev
, struct file
*filp
,
292 const char __user
*wbuf
, char __user
*rbuf
, size_t count
,
293 loff_t
*f_pos
, bool write
)
295 struct ttm_buffer_object
*bo
;
296 struct ttm_bo_driver
*driver
;
297 struct ttm_bo_kmap_obj map
;
298 unsigned long dev_offset
= (*f_pos
>> PAGE_SHIFT
);
299 unsigned long kmap_offset
;
300 unsigned long kmap_end
;
301 unsigned long kmap_num
;
303 unsigned int page_offset
;
306 bool no_wait
= false;
309 read_lock(&bdev
->vm_lock
);
310 bo
= ttm_bo_vm_lookup_rb(bdev
, dev_offset
, 1);
311 if (likely(bo
!= NULL
))
312 ttm_bo_reference(bo
);
313 read_unlock(&bdev
->vm_lock
);
315 if (unlikely(bo
== NULL
))
318 driver
= bo
->bdev
->driver
;
319 if (unlikely(!driver
->verify_access
)) {
324 ret
= driver
->verify_access(bo
, filp
);
325 if (unlikely(ret
!= 0))
328 kmap_offset
= dev_offset
- bo
->vm_node
->start
;
329 if (unlikely(kmap_offset
>= bo
->num_pages
)) {
334 page_offset
= *f_pos
& ~PAGE_MASK
;
335 io_size
= bo
->num_pages
- kmap_offset
;
336 io_size
= (io_size
<< PAGE_SHIFT
) - page_offset
;
340 kmap_end
= (*f_pos
+ count
- 1) >> PAGE_SHIFT
;
341 kmap_num
= kmap_end
- kmap_offset
+ 1;
343 ret
= ttm_bo_reserve(bo
, true, no_wait
, false, 0);
355 ret
= ttm_bo_kmap(bo
, kmap_offset
, kmap_num
, &map
);
356 if (unlikely(ret
!= 0)) {
357 ttm_bo_unreserve(bo
);
361 virtual = ttm_kmap_obj_virtual(&map
, &dummy
);
362 virtual += page_offset
;
365 ret
= copy_from_user(virtual, wbuf
, io_size
);
367 ret
= copy_to_user(rbuf
, virtual, io_size
);
370 ttm_bo_unreserve(bo
);
373 if (unlikely(ret
!= 0))
384 ssize_t
ttm_bo_fbdev_io(struct ttm_buffer_object
*bo
, const char __user
*wbuf
,
385 char __user
*rbuf
, size_t count
, loff_t
*f_pos
,
388 struct ttm_bo_kmap_obj map
;
389 unsigned long kmap_offset
;
390 unsigned long kmap_end
;
391 unsigned long kmap_num
;
393 unsigned int page_offset
;
396 bool no_wait
= false;
399 kmap_offset
= (*f_pos
>> PAGE_SHIFT
);
400 if (unlikely(kmap_offset
>= bo
->num_pages
))
403 page_offset
= *f_pos
& ~PAGE_MASK
;
404 io_size
= bo
->num_pages
- kmap_offset
;
405 io_size
= (io_size
<< PAGE_SHIFT
) - page_offset
;
409 kmap_end
= (*f_pos
+ count
- 1) >> PAGE_SHIFT
;
410 kmap_num
= kmap_end
- kmap_offset
+ 1;
412 ret
= ttm_bo_reserve(bo
, true, no_wait
, false, 0);
423 ret
= ttm_bo_kmap(bo
, kmap_offset
, kmap_num
, &map
);
424 if (unlikely(ret
!= 0)) {
425 ttm_bo_unreserve(bo
);
429 virtual = ttm_kmap_obj_virtual(&map
, &dummy
);
430 virtual += page_offset
;
433 ret
= copy_from_user(virtual, wbuf
, io_size
);
435 ret
= copy_to_user(rbuf
, virtual, io_size
);
438 ttm_bo_unreserve(bo
);
441 if (unlikely(ret
!= 0))