2 * Copyright © 2008 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
26 * Copyright (c) 2011 The FreeBSD Foundation
27 * All rights reserved.
29 * This software was developed by Konstantin Belousov under sponsorship from
30 * the FreeBSD Foundation.
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
56 #include <sys/resourcevar.h>
57 #include <sys/sfbuf.h>
60 #include <drm/i915_drm.h>
62 #include "intel_drv.h"
63 #include "intel_ringbuffer.h"
64 #include <linux/completion.h>
65 #include <linux/jiffies.h>
66 #include <linux/time.h>
68 static __must_check
int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object
*obj
);
69 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object
*obj
);
70 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object
*obj
);
71 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object
*obj
,
72 unsigned alignment
, bool map_and_fenceable
);
73 static int i915_gem_phys_pwrite(struct drm_device
*dev
,
74 struct drm_i915_gem_object
*obj
, uint64_t data_ptr
, uint64_t offset
,
75 uint64_t size
, struct drm_file
*file_priv
);
77 static void i915_gem_write_fence(struct drm_device
*dev
, int reg
,
78 struct drm_i915_gem_object
*obj
);
79 static void i915_gem_object_update_fence(struct drm_i915_gem_object
*obj
,
80 struct drm_i915_fence_reg
*fence
,
83 static uint32_t i915_gem_get_gtt_size(struct drm_device
*dev
, uint32_t size
,
85 static uint32_t i915_gem_get_gtt_alignment(struct drm_device
*dev
,
86 uint32_t size
, int tiling_mode
);
87 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object
*obj
,
89 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object
*obj
);
90 static void i915_gem_object_truncate(struct drm_i915_gem_object
*obj
);
92 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object
*obj
)
95 i915_gem_release_mmap(obj
);
97 /* As we do not have an associated fence register, we will force
98 * a tiling change if we ever need to acquire one.
100 obj
->fence_dirty
= false;
101 obj
->fence_reg
= I915_FENCE_REG_NONE
;
104 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object
*obj
);
105 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object
*obj
);
106 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object
*obj
);
107 static vm_page_t
i915_gem_wire_page(vm_object_t object
, vm_pindex_t pindex
);
108 static void i915_gem_process_flushing_list(struct intel_ring_buffer
*ring
,
109 uint32_t flush_domains
);
110 static void i915_gem_reset_fences(struct drm_device
*dev
);
111 static void i915_gem_lowmem(void *arg
);
113 static int i915_gem_obj_io(struct drm_device
*dev
, uint32_t handle
, uint64_t data_ptr
,
114 uint64_t size
, uint64_t offset
, enum uio_rw rw
, struct drm_file
*file
);
116 MALLOC_DEFINE(DRM_I915_GEM
, "i915gem", "Allocations from i915 gem");
117 long i915_gem_wired_pages_cnt
;
119 /* some bookkeeping */
120 static void i915_gem_info_add_obj(struct drm_i915_private
*dev_priv
,
124 dev_priv
->mm
.object_count
++;
125 dev_priv
->mm
.object_memory
+= size
;
128 static void i915_gem_info_remove_obj(struct drm_i915_private
*dev_priv
,
132 dev_priv
->mm
.object_count
--;
133 dev_priv
->mm
.object_memory
-= size
;
137 i915_gem_wait_for_error(struct drm_device
*dev
)
139 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
140 struct completion
*x
= &dev_priv
->error_completion
;
143 if (!atomic_read(&dev_priv
->mm
.wedged
))
147 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
148 * userspace. If it takes that long something really bad is going on and
149 * we should simply try to bail out and fail as gracefully as possible.
151 ret
= wait_for_completion_interruptible_timeout(x
, 10*hz
);
153 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
155 } else if (ret
< 0) {
159 if (atomic_read(&dev_priv
->mm
.wedged
)) {
160 /* GPU is hung, bump the completion count to account for
161 * the token we just consumed so that we never hit zero and
162 * end up waiting upon a subsequent completion event that
165 spin_lock(&x
->wait
.lock
);
167 spin_unlock(&x
->wait
.lock
);
172 int i915_mutex_lock_interruptible(struct drm_device
*dev
)
176 ret
= i915_gem_wait_for_error(dev
);
180 ret
= lockmgr(&dev
->dev_struct_lock
, LK_EXCLUSIVE
|LK_SLEEPFAIL
);
184 WARN_ON(i915_verify_lists(dev
));
189 i915_gem_object_is_inactive(struct drm_i915_gem_object
*obj
)
195 i915_gem_init_ioctl(struct drm_device
*dev
, void *data
,
196 struct drm_file
*file
)
198 struct drm_i915_gem_init
*args
= data
;
200 if (drm_core_check_feature(dev
, DRIVER_MODESET
))
203 if (args
->gtt_start
>= args
->gtt_end
||
204 (args
->gtt_end
| args
->gtt_start
) & (PAGE_SIZE
- 1))
207 /* GEM with user mode setting was never supported on ilk and later. */
208 if (INTEL_INFO(dev
)->gen
>= 5)
212 * XXXKIB. The second-time initialization should be guarded
215 lockmgr(&dev
->dev_lock
, LK_EXCLUSIVE
|LK_RETRY
|LK_CANRECURSE
);
216 i915_gem_do_init(dev
, args
->gtt_start
, args
->gtt_end
, args
->gtt_end
);
217 lockmgr(&dev
->dev_lock
, LK_RELEASE
);
223 i915_gem_get_aperture_ioctl(struct drm_device
*dev
, void *data
,
224 struct drm_file
*file
)
226 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
227 struct drm_i915_gem_get_aperture
*args
= data
;
228 struct drm_i915_gem_object
*obj
;
233 list_for_each_entry(obj
, &dev_priv
->mm
.gtt_list
, gtt_list
)
235 pinned
+= obj
->gtt_space
->size
;
238 args
->aper_size
= dev_priv
->mm
.gtt_total
;
239 args
->aper_available_size
= args
->aper_size
- pinned
;
245 i915_gem_create(struct drm_file
*file
, struct drm_device
*dev
, uint64_t size
,
248 struct drm_i915_gem_object
*obj
;
252 size
= roundup(size
, PAGE_SIZE
);
256 obj
= i915_gem_alloc_object(dev
, size
);
261 ret
= drm_gem_handle_create(file
, &obj
->base
, &handle
);
263 drm_gem_object_release(&obj
->base
);
264 i915_gem_info_remove_obj(dev
->dev_private
, obj
->base
.size
);
265 drm_free(obj
, DRM_I915_GEM
);
269 /* drop reference from allocate - handle holds it now */
270 drm_gem_object_unreference(&obj
->base
);
276 i915_gem_dumb_create(struct drm_file
*file
,
277 struct drm_device
*dev
,
278 struct drm_mode_create_dumb
*args
)
281 /* have to work out size/pitch and return them */
282 args
->pitch
= roundup2(args
->width
* ((args
->bpp
+ 7) / 8), 64);
283 args
->size
= args
->pitch
* args
->height
;
284 return (i915_gem_create(file
, dev
, args
->size
, &args
->handle
));
287 int i915_gem_dumb_destroy(struct drm_file
*file
,
288 struct drm_device
*dev
,
292 return (drm_gem_handle_delete(file
, handle
));
296 * Creates a new mm object and returns a handle to it.
299 i915_gem_create_ioctl(struct drm_device
*dev
, void *data
,
300 struct drm_file
*file
)
302 struct drm_i915_gem_create
*args
= data
;
304 return (i915_gem_create(file
, dev
, args
->size
, &args
->handle
));
307 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object
*obj
)
309 drm_i915_private_t
*dev_priv
;
311 dev_priv
= obj
->base
.dev
->dev_private
;
312 return (dev_priv
->mm
.bit_6_swizzle_x
== I915_BIT_6_SWIZZLE_9_10_17
&&
313 obj
->tiling_mode
!= I915_TILING_NONE
);
317 * Reads data from the object referenced by handle.
319 * On error, the contents of *data are undefined.
322 i915_gem_pread_ioctl(struct drm_device
*dev
, void *data
,
323 struct drm_file
*file
)
325 struct drm_i915_gem_pread
*args
;
328 return (i915_gem_obj_io(dev
, args
->handle
, args
->data_ptr
, args
->size
,
329 args
->offset
, UIO_READ
, file
));
333 * Writes data to the object referenced by handle.
335 * On error, the contents of the buffer that were to be modified are undefined.
338 i915_gem_pwrite_ioctl(struct drm_device
*dev
, void *data
,
339 struct drm_file
*file
)
341 struct drm_i915_gem_pwrite
*args
;
344 return (i915_gem_obj_io(dev
, args
->handle
, args
->data_ptr
, args
->size
,
345 args
->offset
, UIO_WRITE
, file
));
349 i915_gem_check_wedge(struct drm_i915_private
*dev_priv
,
352 if (atomic_read(&dev_priv
->mm
.wedged
)) {
353 struct completion
*x
= &dev_priv
->error_completion
;
354 bool recovery_complete
;
356 /* Give the error handler a chance to run. */
357 spin_lock(&x
->wait
.lock
);
358 recovery_complete
= x
->done
> 0;
359 spin_unlock(&x
->wait
.lock
);
361 /* Non-interruptible callers can't handle -EAGAIN, hence return
362 * -EIO unconditionally for these. */
366 /* Recovery complete, but still wedged means reset failure. */
367 if (recovery_complete
)
377 * Compare seqno against outstanding lazy request. Emit a request if they are
381 i915_gem_check_olr(struct intel_ring_buffer
*ring
, u32 seqno
)
385 DRM_LOCK_ASSERT(ring
->dev
);
388 if (seqno
== ring
->outstanding_lazy_request
)
389 ret
= i915_add_request(ring
, NULL
, NULL
);
395 * __wait_seqno - wait until execution of seqno has finished
396 * @ring: the ring expected to report seqno
398 * @interruptible: do an interruptible wait (normally yes)
399 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
401 * Returns 0 if the seqno was found within the alloted time. Else returns the
402 * errno with remaining time filled in timeout argument.
404 static int __wait_seqno(struct intel_ring_buffer
*ring
, u32 seqno
,
405 bool interruptible
, struct timespec
*timeout
)
407 drm_i915_private_t
*dev_priv
= ring
->dev
->dev_private
;
408 struct timespec before
, now
, wait_time
={1,0};
409 unsigned long timeout_jiffies
;
411 bool wait_forever
= true;
414 if (i915_seqno_passed(ring
->get_seqno(ring
, true), seqno
))
417 if (timeout
!= NULL
) {
418 wait_time
= *timeout
;
419 wait_forever
= false;
422 timeout_jiffies
= timespec_to_jiffies(&wait_time
);
424 if (WARN_ON(!ring
->irq_get(ring
)))
427 /* Record current time in case interrupted by signal, or wedged * */
428 getrawmonotonic(&before
);
431 (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
432 atomic_read(&dev_priv->mm.wedged))
435 end
= wait_event_interruptible_timeout(ring
->irq_queue
,
439 end
= wait_event_timeout(ring
->irq_queue
, EXIT_COND
,
442 ret
= i915_gem_check_wedge(dev_priv
, interruptible
);
445 } while (end
== 0 && wait_forever
);
447 getrawmonotonic(&now
);
453 struct timespec sleep_time
= timespec_sub(now
, before
);
454 *timeout
= timespec_sub(*timeout
, sleep_time
);
459 case -EAGAIN
: /* Wedged */
460 case -ERESTARTSYS
: /* Signal */
462 case 0: /* Timeout */
464 set_normalized_timespec(timeout
, 0, 0);
465 return -ETIMEDOUT
; /* -ETIME on Linux */
466 default: /* Completed */
467 WARN_ON(end
< 0); /* We're not aware of other errors */
473 * Waits for a sequence number to be signaled, and cleans up the
474 * request and object lists appropriately for that event.
477 i915_wait_seqno(struct intel_ring_buffer
*ring
, uint32_t seqno
)
479 drm_i915_private_t
*dev_priv
= ring
->dev
->dev_private
;
484 ret
= i915_gem_check_wedge(dev_priv
, dev_priv
->mm
.interruptible
);
488 ret
= i915_gem_check_olr(ring
, seqno
);
492 ret
= __wait_seqno(ring
, seqno
, dev_priv
->mm
.interruptible
, NULL
);
498 * Ensures that all rendering to the object has completed and the object is
499 * safe to unbind from the GTT or access from the CPU.
501 static __must_check
int
502 i915_gem_object_wait_rendering(struct drm_i915_gem_object
*obj
,
509 seqno
= obj
->last_write_seqno
;
511 seqno
= obj
->last_read_seqno
;
515 ret
= i915_wait_seqno(obj
->ring
, seqno
);
519 /* Manually manage the write flush as we may have not yet retired
522 if (obj
->last_write_seqno
&&
523 i915_seqno_passed(seqno
, obj
->last_write_seqno
)) {
524 obj
->last_write_seqno
= 0;
525 obj
->base
.write_domain
&= ~I915_GEM_GPU_DOMAINS
;
528 i915_gem_retire_requests_ring(obj
->ring
);
533 * Ensures that an object will eventually get non-busy by flushing any required
534 * write domains, emitting any outstanding lazy request and retiring and
535 * completed requests.
538 i915_gem_object_flush_active(struct drm_i915_gem_object
*obj
)
543 ret
= i915_gem_object_flush_gpu_write_domain(obj
);
547 ret
= i915_gem_check_olr(obj
->ring
, obj
->last_read_seqno
);
551 i915_gem_retire_requests_ring(obj
->ring
);
558 * Called when user space prepares to use an object with the CPU, either
559 * through the mmap ioctl's mapping or a GTT mapping.
562 i915_gem_set_domain_ioctl(struct drm_device
*dev
, void *data
,
563 struct drm_file
*file
)
565 struct drm_i915_gem_set_domain
*args
= data
;
566 struct drm_i915_gem_object
*obj
;
567 uint32_t read_domains
= args
->read_domains
;
568 uint32_t write_domain
= args
->write_domain
;
571 /* Only handle setting domains to types used by the CPU. */
572 if (write_domain
& I915_GEM_GPU_DOMAINS
)
575 if (read_domains
& I915_GEM_GPU_DOMAINS
)
578 /* Having something in the write domain implies it's in the read
579 * domain, and only that read domain. Enforce that in the request.
581 if (write_domain
!= 0 && read_domains
!= write_domain
)
584 ret
= i915_mutex_lock_interruptible(dev
);
588 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file
, args
->handle
));
589 if (&obj
->base
== NULL
) {
594 if (read_domains
& I915_GEM_DOMAIN_GTT
) {
595 ret
= i915_gem_object_set_to_gtt_domain(obj
, write_domain
!= 0);
597 /* Silently promote "you're not bound, there was nothing to do"
598 * to success, since the client was just asking us to
599 * make sure everything was done.
604 ret
= i915_gem_object_set_to_cpu_domain(obj
, write_domain
!= 0);
607 drm_gem_object_unreference(&obj
->base
);
614 * Called when user space has done writes to this buffer
617 i915_gem_sw_finish_ioctl(struct drm_device
*dev
, void *data
,
618 struct drm_file
*file
)
620 struct drm_i915_gem_sw_finish
*args
= data
;
621 struct drm_i915_gem_object
*obj
;
624 ret
= i915_mutex_lock_interruptible(dev
);
627 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file
, args
->handle
));
628 if (&obj
->base
== NULL
) {
633 /* Pinned buffers may be scanout, so flush the cache */
634 if (obj
->pin_count
!= 0)
635 i915_gem_object_flush_cpu_write_domain(obj
);
637 drm_gem_object_unreference(&obj
->base
);
644 * Maps the contents of an object, returning the address it is mapped
647 * While the mapping holds a reference on the contents of the object, it doesn't
648 * imply a ref on the object itself.
651 i915_gem_mmap_ioctl(struct drm_device
*dev
, void *data
,
652 struct drm_file
*file
)
654 struct drm_i915_gem_mmap
*args
;
655 struct drm_gem_object
*obj
;
664 obj
= drm_gem_object_lookup(dev
, file
, args
->handle
);
671 map
= &p
->p_vmspace
->vm_map
;
672 size
= round_page(args
->size
);
674 if (map
->size
+ size
> p
->p_rlimit
[RLIMIT_VMEM
].rlim_cur
) {
682 vm_object_hold(obj
->vm_obj
);
683 vm_object_reference_locked(obj
->vm_obj
);
684 vm_object_drop(obj
->vm_obj
);
686 rv
= vm_map_find(map
, obj
->vm_obj
, args
->offset
, &addr
, args
->size
,
687 PAGE_SIZE
, /* align */
689 VM_MAPTYPE_NORMAL
, /* maptype */
690 VM_PROT_READ
| VM_PROT_WRITE
, /* prot */
691 VM_PROT_READ
| VM_PROT_WRITE
, /* max */
692 MAP_SHARED
/* cow */);
693 if (rv
!= KERN_SUCCESS
) {
694 vm_object_deallocate(obj
->vm_obj
);
695 error
= -vm_mmap_to_errno(rv
);
697 args
->addr_ptr
= (uint64_t)addr
;
701 drm_gem_object_unreference(obj
);
706 * i915_gem_release_mmap - remove physical page mappings
707 * @obj: obj in question
709 * Preserve the reservation of the mmapping with the DRM core code, but
710 * relinquish ownership of the pages back to the system.
712 * It is vital that we remove the page mapping if we have mapped a tiled
713 * object through the GTT and then lose the fence register due to
714 * resource pressure. Similarly if the object has been moved out of the
715 * aperture, than pages mapped into userspace must be revoked. Removing the
716 * mapping will then trigger a page fault on the next user access, allowing
717 * fixup by i915_gem_fault().
720 i915_gem_release_mmap(struct drm_i915_gem_object
*obj
)
726 if (!obj
->fault_mappable
)
729 devobj
= cdev_pager_lookup(obj
);
730 if (devobj
!= NULL
) {
731 page_count
= OFF_TO_IDX(obj
->base
.size
);
733 VM_OBJECT_LOCK(devobj
);
734 for (i
= 0; i
< page_count
; i
++) {
735 m
= vm_page_lookup_busy_wait(devobj
, i
, TRUE
, "915unm");
738 cdev_pager_free_page(devobj
, m
);
740 VM_OBJECT_UNLOCK(devobj
);
741 vm_object_deallocate(devobj
);
744 obj
->fault_mappable
= false;
748 i915_gem_get_gtt_size(struct drm_device
*dev
, uint32_t size
, int tiling_mode
)
752 if (INTEL_INFO(dev
)->gen
>= 4 ||
753 tiling_mode
== I915_TILING_NONE
)
756 /* Previous chips need a power-of-two fence region when tiling */
757 if (INTEL_INFO(dev
)->gen
== 3)
758 gtt_size
= 1024*1024;
762 while (gtt_size
< size
)
769 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
770 * @obj: object to check
772 * Return the required GTT alignment for an object, taking into account
773 * potential fence register mapping.
776 i915_gem_get_gtt_alignment(struct drm_device
*dev
,
782 * Minimum alignment is 4k (GTT page size), but might be greater
783 * if a fence register is needed for the object.
785 if (INTEL_INFO(dev
)->gen
>= 4 ||
786 tiling_mode
== I915_TILING_NONE
)
790 * Previous chips need to be aligned to the size of the smallest
791 * fence register that can contain the object.
793 return (i915_gem_get_gtt_size(dev
, size
, tiling_mode
));
797 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
800 * @size: size of the object
801 * @tiling_mode: tiling mode of the object
803 * Return the required GTT alignment for an object, only taking into account
804 * unfenced tiled surface requirements.
807 i915_gem_get_unfenced_gtt_alignment(struct drm_device
*dev
,
812 if (tiling_mode
== I915_TILING_NONE
)
816 * Minimum alignment is 4k (GTT page size) for sane hw.
818 if (INTEL_INFO(dev
)->gen
>= 4 || IS_G33(dev
))
822 * Previous hardware however needs to be aligned to a power-of-two
823 * tile height. The simplest method for determining this is to reuse
824 * the power-of-tile object size.
826 return (i915_gem_get_gtt_size(dev
, size
, tiling_mode
));
830 i915_gem_mmap_gtt(struct drm_file
*file
,
831 struct drm_device
*dev
,
835 struct drm_i915_private
*dev_priv
;
836 struct drm_i915_gem_object
*obj
;
839 dev_priv
= dev
->dev_private
;
841 ret
= i915_mutex_lock_interruptible(dev
);
845 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file
, handle
));
846 if (&obj
->base
== NULL
) {
851 if (obj
->base
.size
> dev_priv
->mm
.gtt_mappable_end
) {
856 if (obj
->madv
!= I915_MADV_WILLNEED
) {
857 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
862 ret
= drm_gem_create_mmap_offset(&obj
->base
);
866 *offset
= DRM_GEM_MAPPING_OFF(obj
->base
.map_list
.key
) |
869 drm_gem_object_unreference(&obj
->base
);
876 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
878 * @data: GTT mapping ioctl data
879 * @file: GEM object info
881 * Simply returns the fake offset to userspace so it can mmap it.
882 * The mmap call will end up in drm_gem_mmap(), which will set things
883 * up so we can get faults in the handler above.
885 * The fault handler will take care of binding the object into the GTT
886 * (since it may have been evicted to make room for something), allocating
887 * a fence register, and mapping the appropriate aperture address into
891 i915_gem_mmap_gtt_ioctl(struct drm_device
*dev
, void *data
,
892 struct drm_file
*file
)
894 struct drm_i915_private
*dev_priv
;
895 struct drm_i915_gem_mmap_gtt
*args
= data
;
897 dev_priv
= dev
->dev_private
;
899 return (i915_gem_mmap_gtt(file
, dev
, args
->handle
, &args
->offset
));
902 /* Immediately discard the backing storage */
904 i915_gem_object_truncate(struct drm_i915_gem_object
*obj
)
908 vm_obj
= obj
->base
.vm_obj
;
909 VM_OBJECT_LOCK(vm_obj
);
910 vm_object_page_remove(vm_obj
, 0, 0, false);
911 VM_OBJECT_UNLOCK(vm_obj
);
912 obj
->madv
= __I915_MADV_PURGED
;
916 i915_gem_object_is_purgeable(struct drm_i915_gem_object
*obj
)
918 return obj
->madv
== I915_MADV_DONTNEED
;
921 static inline void vm_page_reference(vm_page_t m
)
923 vm_page_flag_set(m
, PG_REFERENCED
);
927 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object
*obj
)
932 BUG_ON(obj
->madv
== __I915_MADV_PURGED
);
934 if (obj
->tiling_mode
!= I915_TILING_NONE
)
935 i915_gem_object_save_bit_17_swizzle(obj
);
936 if (obj
->madv
== I915_MADV_DONTNEED
)
938 page_count
= obj
->base
.size
/ PAGE_SIZE
;
939 VM_OBJECT_LOCK(obj
->base
.vm_obj
);
940 #if GEM_PARANOID_CHECK_GTT
941 i915_gem_assert_pages_not_mapped(obj
->base
.dev
, obj
->pages
, page_count
);
943 for (i
= 0; i
< page_count
; i
++) {
947 if (obj
->madv
== I915_MADV_WILLNEED
)
948 vm_page_reference(m
);
949 vm_page_busy_wait(obj
->pages
[i
], FALSE
, "i915gem");
950 vm_page_unwire(obj
->pages
[i
], 1);
951 vm_page_wakeup(obj
->pages
[i
]);
952 atomic_add_long(&i915_gem_wired_pages_cnt
, -1);
954 VM_OBJECT_UNLOCK(obj
->base
.vm_obj
);
956 drm_free(obj
->pages
, DRM_I915_GEM
);
961 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object
*obj
,
964 struct drm_device
*dev
;
967 int page_count
, i
, j
;
970 KASSERT(obj
->pages
== NULL
, ("Obj already has pages"));
971 page_count
= obj
->base
.size
/ PAGE_SIZE
;
972 obj
->pages
= kmalloc(page_count
* sizeof(vm_page_t
), DRM_I915_GEM
,
974 vm_obj
= obj
->base
.vm_obj
;
975 VM_OBJECT_LOCK(vm_obj
);
976 for (i
= 0; i
< page_count
; i
++) {
977 if ((obj
->pages
[i
] = i915_gem_wire_page(vm_obj
, i
)) == NULL
)
980 VM_OBJECT_UNLOCK(vm_obj
);
981 if (i915_gem_object_needs_bit17_swizzle(obj
))
982 i915_gem_object_do_bit_17_swizzle(obj
);
986 for (j
= 0; j
< i
; j
++) {
988 vm_page_busy_wait(m
, FALSE
, "i915gem");
989 vm_page_unwire(m
, 0);
991 atomic_add_long(&i915_gem_wired_pages_cnt
, -1);
993 VM_OBJECT_UNLOCK(vm_obj
);
994 drm_free(obj
->pages
, DRM_I915_GEM
);
1000 i915_gem_object_move_to_active(struct drm_i915_gem_object
*obj
,
1001 struct intel_ring_buffer
*ring
)
1003 struct drm_device
*dev
= obj
->base
.dev
;
1004 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1005 u32 seqno
= intel_ring_get_seqno(ring
);
1007 BUG_ON(ring
== NULL
);
1010 /* Add a reference if we're newly entering the active list. */
1012 drm_gem_object_reference(&obj
->base
);
1016 /* Move from whatever list we were on to the tail of execution. */
1017 list_move_tail(&obj
->mm_list
, &dev_priv
->mm
.active_list
);
1018 list_move_tail(&obj
->ring_list
, &ring
->active_list
);
1020 obj
->last_read_seqno
= seqno
;
1022 if (obj
->fenced_gpu_access
) {
1023 obj
->last_fenced_seqno
= seqno
;
1025 /* Bump MRU to take account of the delayed flush */
1026 if (obj
->fence_reg
!= I915_FENCE_REG_NONE
) {
1027 struct drm_i915_fence_reg
*reg
;
1029 reg
= &dev_priv
->fence_regs
[obj
->fence_reg
];
1030 list_move_tail(®
->lru_list
,
1031 &dev_priv
->mm
.fence_list
);
1037 i915_gem_object_move_to_inactive(struct drm_i915_gem_object
*obj
)
1039 struct drm_device
*dev
= obj
->base
.dev
;
1040 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1042 BUG_ON(obj
->base
.write_domain
& ~I915_GEM_GPU_DOMAINS
);
1043 BUG_ON(!obj
->active
);
1045 list_move_tail(&obj
->mm_list
, &dev_priv
->mm
.inactive_list
);
1047 list_del_init(&obj
->ring_list
);
1050 obj
->last_read_seqno
= 0;
1051 obj
->last_write_seqno
= 0;
1052 obj
->base
.write_domain
= 0;
1054 obj
->last_fenced_seqno
= 0;
1055 obj
->fenced_gpu_access
= false;
1058 drm_gem_object_unreference(&obj
->base
);
1060 WARN_ON(i915_verify_lists(dev
));
1064 i915_gem_handle_seqno_wrap(struct drm_device
*dev
)
1066 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1067 struct intel_ring_buffer
*ring
;
1070 /* The hardware uses various monotonic 32-bit counters, if we
1071 * detect that they will wraparound we need to idle the GPU
1072 * and reset those counters.
1075 for_each_ring(ring
, dev_priv
, i
) {
1076 for (j
= 0; j
< ARRAY_SIZE(ring
->sync_seqno
); j
++)
1077 ret
|= ring
->sync_seqno
[j
] != 0;
1082 ret
= i915_gpu_idle(dev
);
1086 i915_gem_retire_requests(dev
);
1087 for_each_ring(ring
, dev_priv
, i
) {
1088 for (j
= 0; j
< ARRAY_SIZE(ring
->sync_seqno
); j
++)
1089 ring
->sync_seqno
[j
] = 0;
1096 i915_gem_get_seqno(struct drm_device
*dev
, u32
*seqno
)
1098 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1100 /* reserve 0 for non-seqno */
1101 if (dev_priv
->next_seqno
== 0) {
1102 int ret
= i915_gem_handle_seqno_wrap(dev
);
1106 dev_priv
->next_seqno
= 1;
1109 *seqno
= dev_priv
->next_seqno
++;
1114 i915_add_request(struct intel_ring_buffer
*ring
,
1115 struct drm_file
*file
,
1118 drm_i915_private_t
*dev_priv
= ring
->dev
->dev_private
;
1119 struct drm_i915_gem_request
*request
;
1120 u32 request_ring_position
;
1125 * Emit any outstanding flushes - execbuf can fail to emit the flush
1126 * after having emitted the batchbuffer command. Hence we need to fix
1127 * things up similar to emitting the lazy request. The difference here
1128 * is that the flush _must_ happen before the next request, no matter
1131 if (ring
->gpu_caches_dirty
) {
1132 ret
= i915_gem_flush_ring(ring
, 0, I915_GEM_GPU_DOMAINS
);
1136 ring
->gpu_caches_dirty
= false;
1139 request
= kmalloc(sizeof(*request
), DRM_I915_GEM
, M_WAITOK
| M_ZERO
);
1140 if (request
== NULL
)
1143 /* Record the position of the start of the request so that
1144 * should we detect the updated seqno part-way through the
1145 * GPU processing the request, we never over-estimate the
1146 * position of the head.
1148 request_ring_position
= intel_ring_get_tail(ring
);
1150 ret
= ring
->add_request(ring
);
1152 kfree(request
, DRM_I915_GEM
);
1156 request
->seqno
= intel_ring_get_seqno(ring
);
1157 request
->ring
= ring
;
1158 request
->tail
= request_ring_position
;
1159 request
->emitted_jiffies
= jiffies
;
1160 was_empty
= list_empty(&ring
->request_list
);
1161 list_add_tail(&request
->list
, &ring
->request_list
);
1162 request
->file_priv
= NULL
;
1165 struct drm_i915_file_private
*file_priv
= file
->driver_priv
;
1167 spin_lock(&file_priv
->mm
.lock
);
1168 request
->file_priv
= file_priv
;
1169 list_add_tail(&request
->client_list
,
1170 &file_priv
->mm
.request_list
);
1171 spin_unlock(&file_priv
->mm
.lock
);
1174 ring
->outstanding_lazy_request
= 0;
1176 if (!dev_priv
->mm
.suspended
) {
1177 if (i915_enable_hangcheck
) {
1178 mod_timer(&dev_priv
->hangcheck_timer
,
1179 round_jiffies_up(jiffies
+ DRM_I915_HANGCHECK_JIFFIES
));
1182 queue_delayed_work(dev_priv
->wq
,
1183 &dev_priv
->mm
.retire_work
,
1184 round_jiffies_up_relative(hz
));
1185 intel_mark_busy(dev_priv
->dev
);
1190 *out_seqno
= request
->seqno
;
1195 i915_gem_request_remove_from_client(struct drm_i915_gem_request
*request
)
1197 struct drm_i915_file_private
*file_priv
= request
->file_priv
;
1202 DRM_LOCK_ASSERT(request
->ring
->dev
);
1204 spin_lock(&file_priv
->mm
.lock
);
1205 if (request
->file_priv
!= NULL
) {
1206 list_del(&request
->client_list
);
1207 request
->file_priv
= NULL
;
1209 spin_unlock(&file_priv
->mm
.lock
);
1213 i915_gem_reset_ring_lists(struct drm_i915_private
*dev_priv
,
1214 struct intel_ring_buffer
*ring
)
1217 if (ring
->dev
!= NULL
)
1218 DRM_LOCK_ASSERT(ring
->dev
);
1220 while (!list_empty(&ring
->request_list
)) {
1221 struct drm_i915_gem_request
*request
;
1223 request
= list_first_entry(&ring
->request_list
,
1224 struct drm_i915_gem_request
, list
);
1226 list_del(&request
->list
);
1227 i915_gem_request_remove_from_client(request
);
1228 drm_free(request
, DRM_I915_GEM
);
1231 while (!list_empty(&ring
->active_list
)) {
1232 struct drm_i915_gem_object
*obj
;
1234 obj
= list_first_entry(&ring
->active_list
,
1235 struct drm_i915_gem_object
, ring_list
);
1237 list_del_init(&obj
->gpu_write_list
);
1238 i915_gem_object_move_to_inactive(obj
);
1242 static void i915_gem_reset_fences(struct drm_device
*dev
)
1244 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1247 for (i
= 0; i
< dev_priv
->num_fence_regs
; i
++) {
1248 struct drm_i915_fence_reg
*reg
= &dev_priv
->fence_regs
[i
];
1250 i915_gem_write_fence(dev
, i
, NULL
);
1253 i915_gem_object_fence_lost(reg
->obj
);
1257 INIT_LIST_HEAD(®
->lru_list
);
1260 INIT_LIST_HEAD(&dev_priv
->mm
.fence_list
);
1263 void i915_gem_reset(struct drm_device
*dev
)
1265 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1266 struct drm_i915_gem_object
*obj
;
1267 struct intel_ring_buffer
*ring
;
1270 for_each_ring(ring
, dev_priv
, i
)
1271 i915_gem_reset_ring_lists(dev_priv
, ring
);
1273 /* Move everything out of the GPU domains to ensure we do any
1274 * necessary invalidation upon reuse.
1276 list_for_each_entry(obj
,
1277 &dev_priv
->mm
.inactive_list
,
1280 obj
->base
.read_domains
&= ~I915_GEM_GPU_DOMAINS
;
1283 /* The fence registers are invalidated so clear them out */
1284 i915_gem_reset_fences(dev
);
1288 * This function clears the request list as sequence numbers are passed.
1291 i915_gem_retire_requests_ring(struct intel_ring_buffer
*ring
)
1295 if (list_empty(&ring
->request_list
))
1298 WARN_ON(i915_verify_lists(ring
->dev
));
1300 seqno
= ring
->get_seqno(ring
, true);
1302 while (!list_empty(&ring
->request_list
)) {
1303 struct drm_i915_gem_request
*request
;
1305 request
= list_first_entry(&ring
->request_list
,
1306 struct drm_i915_gem_request
,
1309 if (!i915_seqno_passed(seqno
, request
->seqno
))
1312 /* We know the GPU must have read the request to have
1313 * sent us the seqno + interrupt, so use the position
1314 * of tail of the request to update the last known position
1317 ring
->last_retired_head
= request
->tail
;
1319 list_del(&request
->list
);
1320 i915_gem_request_remove_from_client(request
);
1321 kfree(request
, DRM_I915_GEM
);
1324 /* Move any buffers on the active list that are no longer referenced
1325 * by the ringbuffer to the flushing/inactive lists as appropriate.
1327 while (!list_empty(&ring
->active_list
)) {
1328 struct drm_i915_gem_object
*obj
;
1330 obj
= list_first_entry(&ring
->active_list
,
1331 struct drm_i915_gem_object
,
1334 if (!i915_seqno_passed(seqno
, obj
->last_read_seqno
))
1337 i915_gem_object_move_to_inactive(obj
);
1340 if (unlikely(ring
->trace_irq_seqno
&&
1341 i915_seqno_passed(seqno
, ring
->trace_irq_seqno
))) {
1342 ring
->irq_put(ring
);
1343 ring
->trace_irq_seqno
= 0;
1349 i915_gem_retire_requests(struct drm_device
*dev
)
1351 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
1352 struct intel_ring_buffer
*ring
;
1355 for_each_ring(ring
, dev_priv
, i
)
1356 i915_gem_retire_requests_ring(ring
);
1360 i915_gem_retire_work_handler(struct work_struct
*work
)
1362 drm_i915_private_t
*dev_priv
;
1363 struct drm_device
*dev
;
1364 struct intel_ring_buffer
*ring
;
1368 dev_priv
= container_of(work
, drm_i915_private_t
,
1369 mm
.retire_work
.work
);
1370 dev
= dev_priv
->dev
;
1372 /* Come back later if the device is busy... */
1373 if (lockmgr(&dev
->dev_struct_lock
, LK_EXCLUSIVE
|LK_NOWAIT
)) {
1374 queue_delayed_work(dev_priv
->wq
, &dev_priv
->mm
.retire_work
,
1375 round_jiffies_up_relative(hz
));
1379 i915_gem_retire_requests(dev
);
1381 /* Send a periodic flush down the ring so we don't hold onto GEM
1382 * objects indefinitely.
1385 for_each_ring(ring
, dev_priv
, i
) {
1386 if (ring
->gpu_caches_dirty
)
1387 i915_add_request(ring
, NULL
, NULL
);
1389 idle
&= list_empty(&ring
->request_list
);
1392 if (!dev_priv
->mm
.suspended
&& !idle
)
1393 queue_delayed_work(dev_priv
->wq
, &dev_priv
->mm
.retire_work
,
1394 round_jiffies_up_relative(hz
));
1396 intel_mark_idle(dev
);
1402 * i915_gem_object_sync - sync an object to a ring.
1404 * @obj: object which may be in use on another ring.
1405 * @to: ring we wish to use the object on. May be NULL.
1407 * This code is meant to abstract object synchronization with the GPU.
1408 * Calling with NULL implies synchronizing the object with the CPU
1409 * rather than a particular GPU ring.
1411 * Returns 0 if successful, else propagates up the lower layer error.
1414 i915_gem_object_sync(struct drm_i915_gem_object
*obj
,
1415 struct intel_ring_buffer
*to
)
1417 struct intel_ring_buffer
*from
= obj
->ring
;
1421 if (from
== NULL
|| to
== from
)
1424 if (to
== NULL
|| !i915_semaphore_is_enabled(obj
->base
.dev
))
1425 return i915_gem_object_wait_rendering(obj
, false);
1427 idx
= intel_ring_sync_index(from
, to
);
1429 seqno
= obj
->last_read_seqno
;
1430 if (seqno
<= from
->sync_seqno
[idx
])
1433 ret
= i915_gem_check_olr(obj
->ring
, seqno
);
1437 ret
= to
->sync_to(to
, from
, seqno
);
1439 from
->sync_seqno
[idx
] = seqno
;
1444 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object
*obj
)
1446 u32 old_write_domain
, old_read_domains
;
1448 /* Act a barrier for all accesses through the GTT */
1451 /* Force a pagefault for domain tracking on next user access */
1452 i915_gem_release_mmap(obj
);
1454 if ((obj
->base
.read_domains
& I915_GEM_DOMAIN_GTT
) == 0)
1457 old_read_domains
= obj
->base
.read_domains
;
1458 old_write_domain
= obj
->base
.write_domain
;
1460 obj
->base
.read_domains
&= ~I915_GEM_DOMAIN_GTT
;
1461 obj
->base
.write_domain
&= ~I915_GEM_DOMAIN_GTT
;
1466 * Unbinds an object from the GTT aperture.
1469 i915_gem_object_unbind(struct drm_i915_gem_object
*obj
)
1471 drm_i915_private_t
*dev_priv
= obj
->base
.dev
->dev_private
;
1474 if (obj
->gtt_space
== NULL
)
1477 if (obj
->pin_count
!= 0) {
1478 DRM_ERROR("Attempting to unbind pinned buffer\n");
1482 ret
= i915_gem_object_finish_gpu(obj
);
1485 /* Continue on if we fail due to EIO, the GPU is hung so we
1486 * should be safe and we need to cleanup or else we might
1487 * cause memory corruption through use-after-free.
1490 i915_gem_object_finish_gtt(obj
);
1492 /* Move the object to the CPU domain to ensure that
1493 * any possible CPU writes while it's not in the GTT
1494 * are flushed when we go to remap it.
1497 ret
= i915_gem_object_set_to_cpu_domain(obj
, 1);
1498 if (ret
== -ERESTART
|| ret
== -EINTR
)
1501 /* In the event of a disaster, abandon all caches and
1502 * hope for the best.
1504 i915_gem_clflush_object(obj
);
1505 obj
->base
.read_domains
= obj
->base
.write_domain
= I915_GEM_DOMAIN_CPU
;
1508 /* release the fence reg _after_ flushing */
1509 ret
= i915_gem_object_put_fence(obj
);
1513 if (obj
->has_global_gtt_mapping
)
1514 i915_gem_gtt_unbind_object(obj
);
1515 if (obj
->has_aliasing_ppgtt_mapping
) {
1516 i915_ppgtt_unbind_object(dev_priv
->mm
.aliasing_ppgtt
, obj
);
1517 obj
->has_aliasing_ppgtt_mapping
= 0;
1519 i915_gem_gtt_finish_object(obj
);
1521 i915_gem_object_put_pages_gtt(obj
);
1523 list_del_init(&obj
->gtt_list
);
1524 list_del_init(&obj
->mm_list
);
1525 /* Avoid an unnecessary call to unbind on rebind. */
1526 obj
->map_and_fenceable
= true;
1528 drm_mm_put_block(obj
->gtt_space
);
1529 obj
->gtt_space
= NULL
;
1530 obj
->gtt_offset
= 0;
1532 if (i915_gem_object_is_purgeable(obj
))
1533 i915_gem_object_truncate(obj
);
1538 int i915_gpu_idle(struct drm_device
*dev
)
1540 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
1541 struct intel_ring_buffer
*ring
;
1544 /* Flush everything onto the inactive list. */
1545 for_each_ring(ring
, dev_priv
, i
) {
1546 ret
= intel_ring_idle(ring
);
1554 static void sandybridge_write_fence_reg(struct drm_device
*dev
, int reg
,
1555 struct drm_i915_gem_object
*obj
)
1557 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
1561 u32 size
= obj
->gtt_space
->size
;
1563 val
= (uint64_t)((obj
->gtt_offset
+ size
- 4096) &
1565 val
|= obj
->gtt_offset
& 0xfffff000;
1566 val
|= (uint64_t)((obj
->stride
/ 128) - 1) <<
1567 SANDYBRIDGE_FENCE_PITCH_SHIFT
;
1569 if (obj
->tiling_mode
== I915_TILING_Y
)
1570 val
|= 1 << I965_FENCE_TILING_Y_SHIFT
;
1571 val
|= I965_FENCE_REG_VALID
;
1575 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0
+ reg
* 8, val
);
1576 POSTING_READ(FENCE_REG_SANDYBRIDGE_0
+ reg
* 8);
1579 static void i965_write_fence_reg(struct drm_device
*dev
, int reg
,
1580 struct drm_i915_gem_object
*obj
)
1582 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
1586 u32 size
= obj
->gtt_space
->size
;
1588 val
= (uint64_t)((obj
->gtt_offset
+ size
- 4096) &
1590 val
|= obj
->gtt_offset
& 0xfffff000;
1591 val
|= ((obj
->stride
/ 128) - 1) << I965_FENCE_PITCH_SHIFT
;
1592 if (obj
->tiling_mode
== I915_TILING_Y
)
1593 val
|= 1 << I965_FENCE_TILING_Y_SHIFT
;
1594 val
|= I965_FENCE_REG_VALID
;
1598 I915_WRITE64(FENCE_REG_965_0
+ reg
* 8, val
);
1599 POSTING_READ(FENCE_REG_965_0
+ reg
* 8);
1602 static void i915_write_fence_reg(struct drm_device
*dev
, int reg
,
1603 struct drm_i915_gem_object
*obj
)
1605 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
1609 u32 size
= obj
->gtt_space
->size
;
1613 WARN((obj
->gtt_offset
& ~I915_FENCE_START_MASK
) ||
1614 (size
& -size
) != size
||
1615 (obj
->gtt_offset
& (size
- 1)),
1616 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1617 obj
->gtt_offset
, obj
->map_and_fenceable
, size
);
1619 if (obj
->tiling_mode
== I915_TILING_Y
&& HAS_128_BYTE_Y_TILING(dev
))
1624 /* Note: pitch better be a power of two tile widths */
1625 pitch_val
= obj
->stride
/ tile_width
;
1626 pitch_val
= ffs(pitch_val
) - 1;
1628 val
= obj
->gtt_offset
;
1629 if (obj
->tiling_mode
== I915_TILING_Y
)
1630 val
|= 1 << I830_FENCE_TILING_Y_SHIFT
;
1631 val
|= I915_FENCE_SIZE_BITS(size
);
1632 val
|= pitch_val
<< I830_FENCE_PITCH_SHIFT
;
1633 val
|= I830_FENCE_REG_VALID
;
1638 reg
= FENCE_REG_830_0
+ reg
* 4;
1640 reg
= FENCE_REG_945_8
+ (reg
- 8) * 4;
1642 I915_WRITE(reg
, val
);
1646 static void i830_write_fence_reg(struct drm_device
*dev
, int reg
,
1647 struct drm_i915_gem_object
*obj
)
1649 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
1653 u32 size
= obj
->gtt_space
->size
;
1656 WARN((obj
->gtt_offset
& ~I830_FENCE_START_MASK
) ||
1657 (size
& -size
) != size
||
1658 (obj
->gtt_offset
& (size
- 1)),
1659 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1660 obj
->gtt_offset
, size
);
1662 pitch_val
= obj
->stride
/ 128;
1663 pitch_val
= ffs(pitch_val
) - 1;
1665 val
= obj
->gtt_offset
;
1666 if (obj
->tiling_mode
== I915_TILING_Y
)
1667 val
|= 1 << I830_FENCE_TILING_Y_SHIFT
;
1668 val
|= I830_FENCE_SIZE_BITS(size
);
1669 val
|= pitch_val
<< I830_FENCE_PITCH_SHIFT
;
1670 val
|= I830_FENCE_REG_VALID
;
1674 I915_WRITE(FENCE_REG_830_0
+ reg
* 4, val
);
1675 POSTING_READ(FENCE_REG_830_0
+ reg
* 4);
1678 static void i915_gem_write_fence(struct drm_device
*dev
, int reg
,
1679 struct drm_i915_gem_object
*obj
)
1681 switch (INTEL_INFO(dev
)->gen
) {
1683 case 6: sandybridge_write_fence_reg(dev
, reg
, obj
); break;
1685 case 4: i965_write_fence_reg(dev
, reg
, obj
); break;
1686 case 3: i915_write_fence_reg(dev
, reg
, obj
); break;
1687 case 2: i830_write_fence_reg(dev
, reg
, obj
); break;
1692 static inline int fence_number(struct drm_i915_private
*dev_priv
,
1693 struct drm_i915_fence_reg
*fence
)
1695 return fence
- dev_priv
->fence_regs
;
1698 static void i915_gem_object_update_fence(struct drm_i915_gem_object
*obj
,
1699 struct drm_i915_fence_reg
*fence
,
1702 struct drm_i915_private
*dev_priv
= obj
->base
.dev
->dev_private
;
1703 int reg
= fence_number(dev_priv
, fence
);
1705 i915_gem_write_fence(obj
->base
.dev
, reg
, enable
? obj
: NULL
);
1708 obj
->fence_reg
= reg
;
1710 list_move_tail(&fence
->lru_list
, &dev_priv
->mm
.fence_list
);
1712 obj
->fence_reg
= I915_FENCE_REG_NONE
;
1714 list_del_init(&fence
->lru_list
);
1719 i915_gem_object_flush_fence(struct drm_i915_gem_object
*obj
)
1723 if (obj
->fenced_gpu_access
) {
1724 if (obj
->base
.write_domain
& I915_GEM_GPU_DOMAINS
) {
1725 ret
= i915_gem_flush_ring(obj
->ring
,
1726 0, obj
->base
.write_domain
);
1731 obj
->fenced_gpu_access
= false;
1734 if (obj
->last_fenced_seqno
) {
1735 ret
= i915_wait_seqno(obj
->ring
,
1736 obj
->last_fenced_seqno
);
1740 obj
->last_fenced_seqno
= 0;
1743 /* Ensure that all CPU reads are completed before installing a fence
1744 * and all writes before removing the fence.
1746 if (obj
->base
.read_domains
& I915_GEM_DOMAIN_GTT
)
1753 i915_gem_object_put_fence(struct drm_i915_gem_object
*obj
)
1755 struct drm_i915_private
*dev_priv
= obj
->base
.dev
->dev_private
;
1758 ret
= i915_gem_object_flush_fence(obj
);
1762 if (obj
->fence_reg
== I915_FENCE_REG_NONE
)
1765 i915_gem_object_update_fence(obj
,
1766 &dev_priv
->fence_regs
[obj
->fence_reg
],
1768 i915_gem_object_fence_lost(obj
);
1773 static struct drm_i915_fence_reg
*
1774 i915_find_fence_reg(struct drm_device
*dev
)
1776 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1777 struct drm_i915_fence_reg
*reg
, *avail
;
1780 /* First try to find a free reg */
1782 for (i
= dev_priv
->fence_reg_start
; i
< dev_priv
->num_fence_regs
; i
++) {
1783 reg
= &dev_priv
->fence_regs
[i
];
1787 if (!reg
->pin_count
)
1794 /* None available, try to steal one or wait for a user to finish */
1795 list_for_each_entry(reg
, &dev_priv
->mm
.fence_list
, lru_list
) {
1806 * i915_gem_object_get_fence - set up fencing for an object
1807 * @obj: object to map through a fence reg
1809 * When mapping objects through the GTT, userspace wants to be able to write
1810 * to them without having to worry about swizzling if the object is tiled.
1811 * This function walks the fence regs looking for a free one for @obj,
1812 * stealing one if it can't find any.
1814 * It then sets up the reg based on the object's properties: address, pitch
1815 * and tiling format.
1817 * For an untiled surface, this removes any existing fence.
1820 i915_gem_object_get_fence(struct drm_i915_gem_object
*obj
)
1822 struct drm_device
*dev
= obj
->base
.dev
;
1823 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
1824 bool enable
= obj
->tiling_mode
!= I915_TILING_NONE
;
1825 struct drm_i915_fence_reg
*reg
;
1828 /* Have we updated the tiling parameters upon the object and so
1829 * will need to serialise the write to the associated fence register?
1831 if (obj
->fence_dirty
) {
1832 ret
= i915_gem_object_flush_fence(obj
);
1837 /* Just update our place in the LRU if our fence is getting reused. */
1838 if (obj
->fence_reg
!= I915_FENCE_REG_NONE
) {
1839 reg
= &dev_priv
->fence_regs
[obj
->fence_reg
];
1840 if (!obj
->fence_dirty
) {
1841 list_move_tail(®
->lru_list
,
1842 &dev_priv
->mm
.fence_list
);
1845 } else if (enable
) {
1846 reg
= i915_find_fence_reg(dev
);
1851 struct drm_i915_gem_object
*old
= reg
->obj
;
1853 ret
= i915_gem_object_flush_fence(old
);
1857 i915_gem_object_fence_lost(old
);
1862 i915_gem_object_update_fence(obj
, reg
, enable
);
1863 obj
->fence_dirty
= false;
1869 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object
*obj
,
1870 unsigned alignment
, bool map_and_fenceable
)
1872 struct drm_device
*dev
;
1873 struct drm_i915_private
*dev_priv
;
1874 struct drm_mm_node
*free_space
;
1875 uint32_t size
, fence_size
, fence_alignment
, unfenced_alignment
;
1876 bool mappable
, fenceable
;
1878 bool nonblocking
= false;
1880 dev
= obj
->base
.dev
;
1881 dev_priv
= dev
->dev_private
;
1883 if (obj
->madv
!= I915_MADV_WILLNEED
) {
1884 DRM_ERROR("Attempting to bind a purgeable object\n");
1888 fence_size
= i915_gem_get_gtt_size(dev
, obj
->base
.size
,
1890 fence_alignment
= i915_gem_get_gtt_alignment(dev
, obj
->base
.size
,
1892 unfenced_alignment
= i915_gem_get_unfenced_gtt_alignment(dev
,
1893 obj
->base
.size
, obj
->tiling_mode
);
1895 alignment
= map_and_fenceable
? fence_alignment
:
1897 if (map_and_fenceable
&& (alignment
& (fence_alignment
- 1)) != 0) {
1898 DRM_ERROR("Invalid object alignment requested %u\n", alignment
);
1902 size
= map_and_fenceable
? fence_size
: obj
->base
.size
;
1904 /* If the object is bigger than the entire aperture, reject it early
1905 * before evicting everything in a vain attempt to find space.
1907 if (obj
->base
.size
> (map_and_fenceable
?
1908 dev_priv
->mm
.gtt_mappable_end
: dev_priv
->mm
.gtt_total
)) {
1910 "Attempting to bind an object larger than the aperture\n");
1915 if (map_and_fenceable
)
1916 free_space
= drm_mm_search_free_in_range(
1917 &dev_priv
->mm
.gtt_space
, size
, alignment
, 0,
1918 dev_priv
->mm
.gtt_mappable_end
, 0);
1920 free_space
= drm_mm_search_free(&dev_priv
->mm
.gtt_space
,
1921 size
, alignment
, 0);
1922 if (free_space
!= NULL
) {
1924 if (map_and_fenceable
)
1925 obj
->gtt_space
= drm_mm_get_block_range_generic(
1926 free_space
, size
, alignment
, color
, 0,
1927 dev_priv
->mm
.gtt_mappable_end
, 1);
1929 obj
->gtt_space
= drm_mm_get_block_generic(free_space
,
1930 size
, alignment
, color
, 1);
1932 if (obj
->gtt_space
== NULL
) {
1933 ret
= i915_gem_evict_something(dev
, size
, alignment
,
1943 * NOTE: i915_gem_object_get_pages_gtt() cannot
1944 * return ENOMEM, since we used VM_ALLOC_RETRY.
1946 ret
= i915_gem_object_get_pages_gtt(obj
, 0);
1948 drm_mm_put_block(obj
->gtt_space
);
1949 obj
->gtt_space
= NULL
;
1953 i915_gem_gtt_bind_object(obj
, obj
->cache_level
);
1955 i915_gem_object_put_pages_gtt(obj
);
1956 drm_mm_put_block(obj
->gtt_space
);
1957 obj
->gtt_space
= NULL
;
1958 if (i915_gem_evict_everything(dev
))
1963 list_add_tail(&obj
->gtt_list
, &dev_priv
->mm
.gtt_list
);
1964 list_add_tail(&obj
->mm_list
, &dev_priv
->mm
.inactive_list
);
1966 obj
->gtt_offset
= obj
->gtt_space
->start
;
1969 obj
->gtt_space
->size
== fence_size
&&
1970 (obj
->gtt_space
->start
& (fence_alignment
- 1)) == 0;
1973 obj
->gtt_offset
+ obj
->base
.size
<= dev_priv
->mm
.gtt_mappable_end
;
1974 obj
->map_and_fenceable
= mappable
&& fenceable
;
1980 i915_gem_clflush_object(struct drm_i915_gem_object
*obj
)
1983 /* If we don't have a page list set up, then we're not pinned
1984 * to GPU, and we can ignore the cache flush because it'll happen
1985 * again at bind time.
1987 if (obj
->pages
== NULL
)
1990 /* If the GPU is snooping the contents of the CPU cache,
1991 * we do not need to manually clear the CPU cache lines. However,
1992 * the caches are only snooped when the render cache is
1993 * flushed/invalidated. As we always have to emit invalidations
1994 * and flushes when moving into and out of the RENDER domain, correct
1995 * snooping behaviour occurs naturally as the result of our domain
1998 if (obj
->cache_level
!= I915_CACHE_NONE
)
2001 drm_clflush_pages(obj
->pages
, obj
->base
.size
/ PAGE_SIZE
);
2004 /** Flushes the GTT write domain for the object if it's dirty. */
2006 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object
*obj
)
2008 uint32_t old_write_domain
;
2010 if (obj
->base
.write_domain
!= I915_GEM_DOMAIN_GTT
)
2013 /* No actual flushing is required for the GTT write domain. Writes
2014 * to it immediately go to main memory as far as we know, so there's
2015 * no chipset flush. It also doesn't land in render cache.
2017 * However, we do have to enforce the order so that all writes through
2018 * the GTT land before any writes to the device, such as updates to
2023 old_write_domain
= obj
->base
.write_domain
;
2024 obj
->base
.write_domain
= 0;
2027 /** Flushes the CPU write domain for the object if it's dirty. */
2029 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object
*obj
)
2031 uint32_t old_write_domain
;
2033 if (obj
->base
.write_domain
!= I915_GEM_DOMAIN_CPU
)
2036 i915_gem_clflush_object(obj
);
2037 intel_gtt_chipset_flush();
2038 old_write_domain
= obj
->base
.write_domain
;
2039 obj
->base
.write_domain
= 0;
2043 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object
*obj
)
2046 if ((obj
->base
.write_domain
& I915_GEM_GPU_DOMAINS
) == 0)
2048 return (i915_gem_flush_ring(obj
->ring
, 0, obj
->base
.write_domain
));
2052 * Moves a single object to the GTT read, and possibly write domain.
2054 * This function returns when the move is complete, including waiting on
2058 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object
*obj
, bool write
)
2060 drm_i915_private_t
*dev_priv
= obj
->base
.dev
->dev_private
;
2061 uint32_t old_write_domain
, old_read_domains
;
2064 /* Not valid to be called on unbound objects. */
2065 if (obj
->gtt_space
== NULL
)
2068 if (obj
->base
.write_domain
== I915_GEM_DOMAIN_GTT
)
2071 ret
= i915_gem_object_flush_gpu_write_domain(obj
);
2075 ret
= i915_gem_object_wait_rendering(obj
, !write
);
2079 i915_gem_object_flush_cpu_write_domain(obj
);
2081 old_write_domain
= obj
->base
.write_domain
;
2082 old_read_domains
= obj
->base
.read_domains
;
2084 /* It should now be out of any other write domains, and we can update
2085 * the domain values for our changes.
2087 BUG_ON((obj
->base
.write_domain
& ~I915_GEM_DOMAIN_GTT
) != 0);
2088 obj
->base
.read_domains
|= I915_GEM_DOMAIN_GTT
;
2090 obj
->base
.read_domains
= I915_GEM_DOMAIN_GTT
;
2091 obj
->base
.write_domain
= I915_GEM_DOMAIN_GTT
;
2095 /* And bump the LRU for this access */
2096 if (i915_gem_object_is_inactive(obj
))
2097 list_move_tail(&obj
->mm_list
, &dev_priv
->mm
.inactive_list
);
2102 int i915_gem_object_set_cache_level(struct drm_i915_gem_object
*obj
,
2103 enum i915_cache_level cache_level
)
2105 struct drm_device
*dev
= obj
->base
.dev
;
2106 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
2109 if (obj
->cache_level
== cache_level
)
2112 if (obj
->pin_count
) {
2113 DRM_DEBUG("can not change the cache level of pinned objects\n");
2117 if (obj
->gtt_space
) {
2118 ret
= i915_gem_object_finish_gpu(obj
);
2122 i915_gem_object_finish_gtt(obj
);
2124 /* Before SandyBridge, you could not use tiling or fence
2125 * registers with snooped memory, so relinquish any fences
2126 * currently pointing to our region in the aperture.
2128 if (INTEL_INFO(obj
->base
.dev
)->gen
< 6) {
2129 ret
= i915_gem_object_put_fence(obj
);
2134 if (obj
->has_global_gtt_mapping
)
2135 i915_gem_gtt_bind_object(obj
, cache_level
);
2136 if (obj
->has_aliasing_ppgtt_mapping
)
2137 i915_ppgtt_bind_object(dev_priv
->mm
.aliasing_ppgtt
,
2141 if (cache_level
== I915_CACHE_NONE
) {
2142 u32 old_read_domains
, old_write_domain
;
2144 /* If we're coming from LLC cached, then we haven't
2145 * actually been tracking whether the data is in the
2146 * CPU cache or not, since we only allow one bit set
2147 * in obj->write_domain and have been skipping the clflushes.
2148 * Just set it to the CPU cache for now.
2150 KASSERT((obj
->base
.write_domain
& ~I915_GEM_DOMAIN_CPU
) == 0,
2151 ("obj %p in CPU write domain", obj
));
2152 KASSERT((obj
->base
.read_domains
& ~I915_GEM_DOMAIN_CPU
) == 0,
2153 ("obj %p in CPU read domain", obj
));
2155 old_read_domains
= obj
->base
.read_domains
;
2156 old_write_domain
= obj
->base
.write_domain
;
2158 obj
->base
.read_domains
= I915_GEM_DOMAIN_CPU
;
2159 obj
->base
.write_domain
= I915_GEM_DOMAIN_CPU
;
2163 obj
->cache_level
= cache_level
;
2168 * Prepare buffer for display plane (scanout, cursors, etc).
2169 * Can be called from an uninterruptible phase (modesetting) and allows
2170 * any flushes to be pipelined (for pageflips).
2173 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object
*obj
,
2175 struct intel_ring_buffer
*pipelined
)
2177 u32 old_read_domains
, old_write_domain
;
2180 ret
= i915_gem_object_flush_gpu_write_domain(obj
);
2184 if (pipelined
!= obj
->ring
) {
2185 ret
= i915_gem_object_sync(obj
, pipelined
);
2190 /* The display engine is not coherent with the LLC cache on gen6. As
2191 * a result, we make sure that the pinning that is about to occur is
2192 * done with uncached PTEs. This is lowest common denominator for all
2195 * However for gen6+, we could do better by using the GFDT bit instead
2196 * of uncaching, which would allow us to flush all the LLC-cached data
2197 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2199 ret
= i915_gem_object_set_cache_level(obj
, I915_CACHE_NONE
);
2203 /* As the user may map the buffer once pinned in the display plane
2204 * (e.g. libkms for the bootup splash), we have to ensure that we
2205 * always use map_and_fenceable for all scanout buffers.
2207 ret
= i915_gem_object_pin(obj
, alignment
, true);
2211 i915_gem_object_flush_cpu_write_domain(obj
);
2213 old_write_domain
= obj
->base
.write_domain
;
2214 old_read_domains
= obj
->base
.read_domains
;
2216 /* It should now be out of any other write domains, and we can update
2217 * the domain values for our changes.
2219 obj
->base
.write_domain
= 0;
2220 obj
->base
.read_domains
|= I915_GEM_DOMAIN_GTT
;
2226 i915_gem_object_finish_gpu(struct drm_i915_gem_object
*obj
)
2230 if ((obj
->base
.read_domains
& I915_GEM_GPU_DOMAINS
) == 0)
2233 if (obj
->base
.write_domain
& I915_GEM_GPU_DOMAINS
) {
2234 ret
= i915_gem_flush_ring(obj
->ring
, 0, obj
->base
.write_domain
);
2239 ret
= i915_gem_object_wait_rendering(obj
, false);
2243 /* Ensure that we invalidate the GPU's caches and TLBs. */
2244 obj
->base
.read_domains
&= ~I915_GEM_GPU_DOMAINS
;
2249 * Moves a single object to the CPU read, and possibly write domain.
2251 * This function returns when the move is complete, including waiting on
2255 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object
*obj
, bool write
)
2257 uint32_t old_write_domain
, old_read_domains
;
2260 if (obj
->base
.write_domain
== I915_GEM_DOMAIN_CPU
)
2263 ret
= i915_gem_object_flush_gpu_write_domain(obj
);
2267 ret
= i915_gem_object_wait_rendering(obj
, !write
);
2271 i915_gem_object_flush_gtt_write_domain(obj
);
2273 old_write_domain
= obj
->base
.write_domain
;
2274 old_read_domains
= obj
->base
.read_domains
;
2276 /* Flush the CPU cache if it's still invalid. */
2277 if ((obj
->base
.read_domains
& I915_GEM_DOMAIN_CPU
) == 0) {
2278 i915_gem_clflush_object(obj
);
2280 obj
->base
.read_domains
|= I915_GEM_DOMAIN_CPU
;
2283 /* It should now be out of any other write domains, and we can update
2284 * the domain values for our changes.
2286 BUG_ON((obj
->base
.write_domain
& ~I915_GEM_DOMAIN_CPU
) != 0);
2288 /* If we're writing through the CPU, then the GPU read domains will
2289 * need to be invalidated at next use.
2292 obj
->base
.read_domains
= I915_GEM_DOMAIN_CPU
;
2293 obj
->base
.write_domain
= I915_GEM_DOMAIN_CPU
;
2299 /* Throttle our rendering by waiting until the ring has completed our requests
2300 * emitted over 20 msec ago.
2302 * Note that if we were to use the current jiffies each time around the loop,
2303 * we wouldn't escape the function with any frames outstanding if the time to
2304 * render a frame was over 20ms.
2306 * This should get us reasonable parallelism between CPU and GPU but also
2307 * relatively low latency when blocking on a particular request to finish.
2310 i915_gem_ring_throttle(struct drm_device
*dev
, struct drm_file
*file
)
2312 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2313 struct drm_i915_file_private
*file_priv
= file
->driver_priv
;
2314 unsigned long recent_enough
= ticks
- (20 * hz
/ 1000);
2315 struct drm_i915_gem_request
*request
;
2316 struct intel_ring_buffer
*ring
= NULL
;
2320 if (atomic_read(&dev_priv
->mm
.wedged
))
2323 spin_lock(&file_priv
->mm
.lock
);
2324 list_for_each_entry(request
, &file_priv
->mm
.request_list
, client_list
) {
2325 if (time_after_eq(request
->emitted_jiffies
, recent_enough
))
2328 ring
= request
->ring
;
2329 seqno
= request
->seqno
;
2331 spin_unlock(&file_priv
->mm
.lock
);
2336 ret
= __wait_seqno(ring
, seqno
, true, NULL
);
2339 queue_delayed_work(dev_priv
->wq
, &dev_priv
->mm
.retire_work
, 0);
2345 i915_gem_object_pin(struct drm_i915_gem_object
*obj
,
2347 bool map_and_fenceable
)
2351 if (WARN_ON(obj
->pin_count
== DRM_I915_GEM_OBJECT_MAX_PIN_COUNT
))
2354 if (obj
->gtt_space
!= NULL
) {
2355 if ((alignment
&& obj
->gtt_offset
& (alignment
- 1)) ||
2356 (map_and_fenceable
&& !obj
->map_and_fenceable
)) {
2357 WARN(obj
->pin_count
,
2358 "bo is already pinned with incorrect alignment:"
2359 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2360 " obj->map_and_fenceable=%d\n",
2361 obj
->gtt_offset
, alignment
,
2363 obj
->map_and_fenceable
);
2364 ret
= i915_gem_object_unbind(obj
);
2370 if (obj
->gtt_space
== NULL
) {
2371 ret
= i915_gem_object_bind_to_gtt(obj
, alignment
,
2377 if (!obj
->has_global_gtt_mapping
&& map_and_fenceable
)
2378 i915_gem_gtt_bind_object(obj
, obj
->cache_level
);
2381 obj
->pin_mappable
|= map_and_fenceable
;
2387 i915_gem_object_unpin(struct drm_i915_gem_object
*obj
)
2389 BUG_ON(obj
->pin_count
== 0);
2390 BUG_ON(obj
->gtt_space
== NULL
);
2392 if (--obj
->pin_count
== 0)
2393 obj
->pin_mappable
= false;
2397 i915_gem_pin_ioctl(struct drm_device
*dev
, void *data
,
2398 struct drm_file
*file
)
2400 struct drm_i915_gem_pin
*args
;
2401 struct drm_i915_gem_object
*obj
;
2402 struct drm_gem_object
*gobj
;
2407 ret
= i915_mutex_lock_interruptible(dev
);
2411 gobj
= drm_gem_object_lookup(dev
, file
, args
->handle
);
2416 obj
= to_intel_bo(gobj
);
2418 if (obj
->madv
!= I915_MADV_WILLNEED
) {
2419 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2424 if (obj
->pin_filp
!= NULL
&& obj
->pin_filp
!= file
) {
2425 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2431 obj
->user_pin_count
++;
2432 obj
->pin_filp
= file
;
2433 if (obj
->user_pin_count
== 1) {
2434 ret
= i915_gem_object_pin(obj
, args
->alignment
, true);
2439 /* XXX - flush the CPU caches for pinned objects
2440 * as the X server doesn't manage domains yet
2442 i915_gem_object_flush_cpu_write_domain(obj
);
2443 args
->offset
= obj
->gtt_offset
;
2445 drm_gem_object_unreference(&obj
->base
);
2452 i915_gem_unpin_ioctl(struct drm_device
*dev
, void *data
,
2453 struct drm_file
*file
)
2455 struct drm_i915_gem_pin
*args
;
2456 struct drm_i915_gem_object
*obj
;
2460 ret
= i915_mutex_lock_interruptible(dev
);
2464 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file
, args
->handle
));
2465 if (&obj
->base
== NULL
) {
2470 if (obj
->pin_filp
!= file
) {
2471 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2476 obj
->user_pin_count
--;
2477 if (obj
->user_pin_count
== 0) {
2478 obj
->pin_filp
= NULL
;
2479 i915_gem_object_unpin(obj
);
2483 drm_gem_object_unreference(&obj
->base
);
2490 i915_gem_busy_ioctl(struct drm_device
*dev
, void *data
,
2491 struct drm_file
*file
)
2493 struct drm_i915_gem_busy
*args
= data
;
2494 struct drm_i915_gem_object
*obj
;
2497 ret
= i915_mutex_lock_interruptible(dev
);
2501 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file
, args
->handle
));
2502 if (&obj
->base
== NULL
) {
2507 /* Count all active objects as busy, even if they are currently not used
2508 * by the gpu. Users of this interface expect objects to eventually
2509 * become non-busy without any further actions, therefore emit any
2510 * necessary flushes here.
2512 ret
= i915_gem_object_flush_active(obj
);
2514 args
->busy
= obj
->active
;
2516 args
->busy
|= intel_ring_flag(obj
->ring
) << 17;
2519 drm_gem_object_unreference(&obj
->base
);
2526 i915_gem_throttle_ioctl(struct drm_device
*dev
, void *data
,
2527 struct drm_file
*file_priv
)
2530 return (i915_gem_ring_throttle(dev
, file_priv
));
2534 i915_gem_madvise_ioctl(struct drm_device
*dev
, void *data
,
2535 struct drm_file
*file_priv
)
2537 struct drm_i915_gem_madvise
*args
= data
;
2538 struct drm_i915_gem_object
*obj
;
2541 switch (args
->madv
) {
2542 case I915_MADV_DONTNEED
:
2543 case I915_MADV_WILLNEED
:
2549 ret
= i915_mutex_lock_interruptible(dev
);
2553 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file_priv
, args
->handle
));
2554 if (&obj
->base
== NULL
) {
2559 if (obj
->pin_count
) {
2564 if (obj
->madv
!= __I915_MADV_PURGED
)
2565 obj
->madv
= args
->madv
;
2567 /* if the object is no longer attached, discard its backing storage */
2568 if (i915_gem_object_is_purgeable(obj
) && obj
->pages
== NULL
)
2569 i915_gem_object_truncate(obj
);
2571 args
->retained
= obj
->madv
!= __I915_MADV_PURGED
;
2574 drm_gem_object_unreference(&obj
->base
);
2580 struct drm_i915_gem_object
*i915_gem_alloc_object(struct drm_device
*dev
,
2583 struct drm_i915_private
*dev_priv
;
2584 struct drm_i915_gem_object
*obj
;
2586 dev_priv
= dev
->dev_private
;
2588 obj
= kmalloc(sizeof(*obj
), DRM_I915_GEM
, M_WAITOK
| M_ZERO
);
2590 if (drm_gem_object_init(dev
, &obj
->base
, size
) != 0) {
2591 drm_free(obj
, DRM_I915_GEM
);
2595 obj
->base
.write_domain
= I915_GEM_DOMAIN_CPU
;
2596 obj
->base
.read_domains
= I915_GEM_DOMAIN_CPU
;
2599 obj
->cache_level
= I915_CACHE_LLC
;
2601 obj
->cache_level
= I915_CACHE_NONE
;
2602 obj
->base
.driver_private
= NULL
;
2603 obj
->fence_reg
= I915_FENCE_REG_NONE
;
2604 INIT_LIST_HEAD(&obj
->mm_list
);
2605 INIT_LIST_HEAD(&obj
->gtt_list
);
2606 INIT_LIST_HEAD(&obj
->ring_list
);
2607 INIT_LIST_HEAD(&obj
->exec_list
);
2608 INIT_LIST_HEAD(&obj
->gpu_write_list
);
2609 obj
->madv
= I915_MADV_WILLNEED
;
2610 /* Avoid an unnecessary call to unbind on the first bind. */
2611 obj
->map_and_fenceable
= true;
2613 i915_gem_info_add_obj(dev_priv
, size
);
2618 int i915_gem_init_object(struct drm_gem_object
*obj
)
2625 void i915_gem_free_object(struct drm_gem_object
*gem_obj
)
2627 struct drm_i915_gem_object
*obj
= to_intel_bo(gem_obj
);
2628 struct drm_device
*dev
= obj
->base
.dev
;
2629 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
2632 i915_gem_detach_phys_object(dev
, obj
);
2635 if (WARN_ON(i915_gem_object_unbind(obj
) == -ERESTARTSYS
)) {
2636 bool was_interruptible
;
2638 was_interruptible
= dev_priv
->mm
.interruptible
;
2639 dev_priv
->mm
.interruptible
= false;
2641 WARN_ON(i915_gem_object_unbind(obj
));
2643 dev_priv
->mm
.interruptible
= was_interruptible
;
2646 drm_gem_free_mmap_offset(&obj
->base
);
2648 drm_gem_object_release(&obj
->base
);
2649 i915_gem_info_remove_obj(dev_priv
, obj
->base
.size
);
2651 drm_free(obj
->bit_17
, DRM_I915_GEM
);
2652 drm_free(obj
, DRM_I915_GEM
);
2656 i915_gem_do_init(struct drm_device
*dev
, unsigned long start
,
2657 unsigned long mappable_end
, unsigned long end
)
2659 drm_i915_private_t
*dev_priv
;
2660 unsigned long mappable
;
2663 dev_priv
= dev
->dev_private
;
2664 mappable
= min(end
, mappable_end
) - start
;
2666 drm_mm_init(&dev_priv
->mm
.gtt_space
, start
, end
- start
);
2668 dev_priv
->mm
.gtt_start
= start
;
2669 dev_priv
->mm
.gtt_mappable_end
= mappable_end
;
2670 dev_priv
->mm
.gtt_end
= end
;
2671 dev_priv
->mm
.gtt_total
= end
- start
;
2672 dev_priv
->mm
.mappable_gtt_total
= mappable
;
2674 /* Take over this portion of the GTT */
2675 intel_gtt_clear_range(start
/ PAGE_SIZE
, (end
-start
) / PAGE_SIZE
);
2676 device_printf(dev
->dev
,
2677 "taking over the fictitious range 0x%lx-0x%lx\n",
2678 dev
->agp
->base
+ start
, dev
->agp
->base
+ start
+ mappable
);
2679 error
= -vm_phys_fictitious_reg_range(dev
->agp
->base
+ start
,
2680 dev
->agp
->base
+ start
+ mappable
, VM_MEMATTR_WRITE_COMBINING
);
2685 i915_gem_idle(struct drm_device
*dev
)
2687 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
2692 if (dev_priv
->mm
.suspended
) {
2697 ret
= i915_gpu_idle(dev
);
2702 i915_gem_retire_requests(dev
);
2704 /* Under UMS, be paranoid and evict. */
2705 if (!drm_core_check_feature(dev
, DRIVER_MODESET
))
2706 i915_gem_evict_everything(dev
);
2708 i915_gem_reset_fences(dev
);
2710 /* Hack! Don't let anybody do execbuf while we don't control the chip.
2711 * We need to replace this with a semaphore, or something.
2712 * And not confound mm.suspended!
2714 dev_priv
->mm
.suspended
= 1;
2715 del_timer_sync(&dev_priv
->hangcheck_timer
);
2717 i915_kernel_lost_context(dev
);
2718 i915_gem_cleanup_ringbuffer(dev
);
2722 /* Cancel the retire work handler, which should be idle now. */
2723 cancel_delayed_work_sync(&dev_priv
->mm
.retire_work
);
2728 void i915_gem_l3_remap(struct drm_device
*dev
)
2730 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
2734 if (!HAS_L3_GPU_CACHE(dev
))
2737 if (!dev_priv
->l3_parity
.remap_info
)
2740 misccpctl
= I915_READ(GEN7_MISCCPCTL
);
2741 I915_WRITE(GEN7_MISCCPCTL
, misccpctl
& ~GEN7_DOP_CLOCK_GATE_ENABLE
);
2742 POSTING_READ(GEN7_MISCCPCTL
);
2744 for (i
= 0; i
< GEN7_L3LOG_SIZE
; i
+= 4) {
2745 u32 remap
= I915_READ(GEN7_L3LOG_BASE
+ i
);
2746 if (remap
&& remap
!= dev_priv
->l3_parity
.remap_info
[i
/4])
2747 DRM_DEBUG("0x%x was already programmed to %x\n",
2748 GEN7_L3LOG_BASE
+ i
, remap
);
2749 if (remap
&& !dev_priv
->l3_parity
.remap_info
[i
/4])
2750 DRM_DEBUG_DRIVER("Clearing remapped register\n");
2751 I915_WRITE(GEN7_L3LOG_BASE
+ i
, dev_priv
->l3_parity
.remap_info
[i
/4]);
2754 /* Make sure all the writes land before disabling dop clock gating */
2755 POSTING_READ(GEN7_L3LOG_BASE
);
2757 I915_WRITE(GEN7_MISCCPCTL
, misccpctl
);
2761 i915_gem_init_swizzling(struct drm_device
*dev
)
2763 drm_i915_private_t
*dev_priv
;
2765 dev_priv
= dev
->dev_private
;
2767 if (INTEL_INFO(dev
)->gen
< 5 ||
2768 dev_priv
->mm
.bit_6_swizzle_x
== I915_BIT_6_SWIZZLE_NONE
)
2771 I915_WRITE(DISP_ARB_CTL
, I915_READ(DISP_ARB_CTL
) |
2772 DISP_TILE_SURFACE_SWIZZLING
);
2777 I915_WRITE(TILECTL
, I915_READ(TILECTL
) | TILECTL_SWZCTL
);
2779 I915_WRITE(ARB_MODE
, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB
));
2781 I915_WRITE(ARB_MODE
, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB
));
2785 intel_enable_blt(struct drm_device
*dev
)
2792 /* The blitter was dysfunctional on early prototypes */
2793 revision
= pci_read_config(dev
->dev
, PCIR_REVID
, 1);
2794 if (IS_GEN6(dev
) && revision
< 8) {
2795 DRM_INFO("BLT not supported on this pre-production hardware;"
2796 " graphics performance will be degraded.\n");
2804 i915_gem_init_hw(struct drm_device
*dev
)
2806 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
2809 if (IS_HASWELL(dev
) && (I915_READ(0x120010) == 1))
2810 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
2812 i915_gem_l3_remap(dev
);
2814 i915_gem_init_swizzling(dev
);
2816 ret
= intel_init_render_ring_buffer(dev
);
2821 ret
= intel_init_bsd_ring_buffer(dev
);
2823 goto cleanup_render_ring
;
2826 if (intel_enable_blt(dev
)) {
2827 ret
= intel_init_blt_ring_buffer(dev
);
2829 goto cleanup_bsd_ring
;
2832 dev_priv
->next_seqno
= 1;
2835 * XXX: There was some w/a described somewhere suggesting loading
2836 * contexts before PPGTT.
2838 #if 0 /* XXX: HW context support */
2839 i915_gem_context_init(dev
);
2841 i915_gem_init_ppgtt(dev
);
2846 intel_cleanup_ring_buffer(&dev_priv
->ring
[VCS
]);
2847 cleanup_render_ring
:
2848 intel_cleanup_ring_buffer(&dev_priv
->ring
[RCS
]);
2853 intel_enable_ppgtt(struct drm_device
*dev
)
2855 if (i915_enable_ppgtt
>= 0)
2856 return i915_enable_ppgtt
;
2858 /* Disable ppgtt on SNB if VT-d is on. */
2859 if (INTEL_INFO(dev
)->gen
== 6 && intel_iommu_enabled
)
2865 int i915_gem_init(struct drm_device
*dev
)
2867 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2868 unsigned long prealloc_size
, gtt_size
, mappable_size
;
2871 prealloc_size
= dev_priv
->mm
.gtt
->stolen_size
;
2872 gtt_size
= dev_priv
->mm
.gtt
->gtt_total_entries
<< PAGE_SHIFT
;
2873 mappable_size
= dev_priv
->mm
.gtt
->gtt_mappable_entries
<< PAGE_SHIFT
;
2875 /* Basic memrange allocator for stolen space */
2876 drm_mm_init(&dev_priv
->mm
.stolen
, 0, prealloc_size
);
2879 if (intel_enable_ppgtt(dev
) && HAS_ALIASING_PPGTT(dev
)) {
2880 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
2881 * aperture accordingly when using aliasing ppgtt. */
2882 gtt_size
-= I915_PPGTT_PD_ENTRIES
*PAGE_SIZE
;
2883 /* For paranoia keep the guard page in between. */
2884 gtt_size
-= PAGE_SIZE
;
2886 i915_gem_do_init(dev
, 0, mappable_size
, gtt_size
);
2888 ret
= i915_gem_init_aliasing_ppgtt(dev
);
2894 /* Let GEM Manage all of the aperture.
2896 * However, leave one page at the end still bound to the scratch
2897 * page. There are a number of places where the hardware
2898 * apparently prefetches past the end of the object, and we've
2899 * seen multiple hangs with the GPU head pointer stuck in a
2900 * batchbuffer bound at the last page of the aperture. One page
2901 * should be enough to keep any prefetching inside of the
2904 i915_gem_do_init(dev
, 0, mappable_size
, gtt_size
- PAGE_SIZE
);
2907 ret
= i915_gem_init_hw(dev
);
2910 i915_gem_cleanup_aliasing_ppgtt(dev
);
2915 /* Try to set up FBC with a reasonable compressed buffer size */
2916 if (I915_HAS_FBC(dev
) && i915_powersave
) {
2919 /* Leave 1M for line length buffer & misc. */
2921 /* Try to get a 32M buffer... */
2922 if (prealloc_size
> (36*1024*1024))
2923 cfb_size
= 32*1024*1024;
2924 else /* fall back to 7/8 of the stolen space */
2925 cfb_size
= prealloc_size
* 7 / 8;
2926 i915_setup_compression(dev
, cfb_size
);
2930 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
2931 if (!drm_core_check_feature(dev
, DRIVER_MODESET
))
2932 dev_priv
->dri1
.allow_batchbuffer
= 1;
2937 i915_gem_cleanup_ringbuffer(struct drm_device
*dev
)
2939 drm_i915_private_t
*dev_priv
;
2942 dev_priv
= dev
->dev_private
;
2943 for (i
= 0; i
< I915_NUM_RINGS
; i
++)
2944 intel_cleanup_ring_buffer(&dev_priv
->ring
[i
]);
2948 i915_gem_entervt_ioctl(struct drm_device
*dev
, void *data
,
2949 struct drm_file
*file_priv
)
2951 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
2954 if (drm_core_check_feature(dev
, DRIVER_MODESET
))
2957 if (atomic_read(&dev_priv
->mm
.wedged
)) {
2958 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2959 atomic_set(&dev_priv
->mm
.wedged
, 0);
2963 dev_priv
->mm
.suspended
= 0;
2965 ret
= i915_gem_init_hw(dev
);
2971 KASSERT(list_empty(&dev_priv
->mm
.active_list
), ("active list"));
2974 ret
= drm_irq_install(dev
);
2976 goto cleanup_ringbuffer
;
2982 i915_gem_cleanup_ringbuffer(dev
);
2983 dev_priv
->mm
.suspended
= 1;
2990 i915_gem_leavevt_ioctl(struct drm_device
*dev
, void *data
,
2991 struct drm_file
*file_priv
)
2994 if (drm_core_check_feature(dev
, DRIVER_MODESET
))
2997 drm_irq_uninstall(dev
);
2998 return (i915_gem_idle(dev
));
3002 i915_gem_lastclose(struct drm_device
*dev
)
3006 if (drm_core_check_feature(dev
, DRIVER_MODESET
))
3009 ret
= i915_gem_idle(dev
);
3011 DRM_ERROR("failed to idle hardware: %d\n", ret
);
3015 init_ring_lists(struct intel_ring_buffer
*ring
)
3018 INIT_LIST_HEAD(&ring
->active_list
);
3019 INIT_LIST_HEAD(&ring
->request_list
);
3020 INIT_LIST_HEAD(&ring
->gpu_write_list
);
3024 i915_gem_load(struct drm_device
*dev
)
3027 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
3029 INIT_LIST_HEAD(&dev_priv
->mm
.active_list
);
3030 INIT_LIST_HEAD(&dev_priv
->mm
.inactive_list
);
3031 INIT_LIST_HEAD(&dev_priv
->mm
.fence_list
);
3032 INIT_LIST_HEAD(&dev_priv
->mm
.gtt_list
);
3033 for (i
= 0; i
< I915_NUM_RINGS
; i
++)
3034 init_ring_lists(&dev_priv
->ring
[i
]);
3035 for (i
= 0; i
< I915_MAX_NUM_FENCES
; i
++)
3036 INIT_LIST_HEAD(&dev_priv
->fence_regs
[i
].lru_list
);
3037 INIT_DELAYED_WORK(&dev_priv
->mm
.retire_work
,
3038 i915_gem_retire_work_handler
);
3039 init_completion(&dev_priv
->error_completion
);
3041 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3043 I915_WRITE(MI_ARB_STATE
,
3044 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE
));
3047 dev_priv
->relative_constants_mode
= I915_EXEC_CONSTANTS_REL_GENERAL
;
3049 /* Old X drivers will take 0-2 for front, back, depth buffers */
3050 if (!drm_core_check_feature(dev
, DRIVER_MODESET
))
3051 dev_priv
->fence_reg_start
= 3;
3053 if (INTEL_INFO(dev
)->gen
>= 4 || IS_I945G(dev
) || IS_I945GM(dev
) || IS_G33(dev
))
3054 dev_priv
->num_fence_regs
= 16;
3056 dev_priv
->num_fence_regs
= 8;
3058 /* Initialize fence registers to zero */
3059 i915_gem_reset_fences(dev
);
3061 i915_gem_detect_bit_6_swizzle(dev
);
3062 init_waitqueue_head(&dev_priv
->pending_flip_queue
);
3064 dev_priv
->mm
.interruptible
= true;
3067 dev_priv
->mm
.inactive_shrinker
.shrink
= i915_gem_inactive_shrink
;
3068 dev_priv
->mm
.inactive_shrinker
.seeks
= DEFAULT_SEEKS
;
3069 register_shrinker(&dev_priv
->mm
.inactive_shrinker
);
3071 dev_priv
->mm
.i915_lowmem
= EVENTHANDLER_REGISTER(vm_lowmem
,
3072 i915_gem_lowmem
, dev
, EVENTHANDLER_PRI_ANY
);
3077 i915_gem_init_phys_object(struct drm_device
*dev
, int id
, int size
, int align
)
3079 drm_i915_private_t
*dev_priv
;
3080 struct drm_i915_gem_phys_object
*phys_obj
;
3083 dev_priv
= dev
->dev_private
;
3084 if (dev_priv
->mm
.phys_objs
[id
- 1] != NULL
|| size
== 0)
3087 phys_obj
= kmalloc(sizeof(struct drm_i915_gem_phys_object
), DRM_I915_GEM
,
3092 phys_obj
->handle
= drm_pci_alloc(dev
, size
, align
, ~0);
3093 if (phys_obj
->handle
== NULL
) {
3097 pmap_change_attr((vm_offset_t
)phys_obj
->handle
->vaddr
,
3098 size
/ PAGE_SIZE
, PAT_WRITE_COMBINING
);
3100 dev_priv
->mm
.phys_objs
[id
- 1] = phys_obj
;
3105 drm_free(phys_obj
, DRM_I915_GEM
);
3110 i915_gem_free_phys_object(struct drm_device
*dev
, int id
)
3112 drm_i915_private_t
*dev_priv
;
3113 struct drm_i915_gem_phys_object
*phys_obj
;
3115 dev_priv
= dev
->dev_private
;
3116 if (dev_priv
->mm
.phys_objs
[id
- 1] == NULL
)
3119 phys_obj
= dev_priv
->mm
.phys_objs
[id
- 1];
3120 if (phys_obj
->cur_obj
!= NULL
)
3121 i915_gem_detach_phys_object(dev
, phys_obj
->cur_obj
);
3123 drm_pci_free(dev
, phys_obj
->handle
);
3124 drm_free(phys_obj
, DRM_I915_GEM
);
3125 dev_priv
->mm
.phys_objs
[id
- 1] = NULL
;
3129 i915_gem_free_all_phys_object(struct drm_device
*dev
)
3133 for (i
= I915_GEM_PHYS_CURSOR_0
; i
<= I915_MAX_PHYS_OBJECT
; i
++)
3134 i915_gem_free_phys_object(dev
, i
);
3138 i915_gem_detach_phys_object(struct drm_device
*dev
,
3139 struct drm_i915_gem_object
*obj
)
3146 if (obj
->phys_obj
== NULL
)
3148 vaddr
= obj
->phys_obj
->handle
->vaddr
;
3150 page_count
= obj
->base
.size
/ PAGE_SIZE
;
3151 VM_OBJECT_LOCK(obj
->base
.vm_obj
);
3152 for (i
= 0; i
< page_count
; i
++) {
3153 m
= i915_gem_wire_page(obj
->base
.vm_obj
, i
);
3157 VM_OBJECT_UNLOCK(obj
->base
.vm_obj
);
3158 sf
= sf_buf_alloc(m
);
3160 dst
= (char *)sf_buf_kva(sf
);
3161 memcpy(dst
, vaddr
+ IDX_TO_OFF(i
), PAGE_SIZE
);
3164 drm_clflush_pages(&m
, 1);
3166 VM_OBJECT_LOCK(obj
->base
.vm_obj
);
3167 vm_page_reference(m
);
3169 vm_page_busy_wait(m
, FALSE
, "i915gem");
3170 vm_page_unwire(m
, 0);
3172 atomic_add_long(&i915_gem_wired_pages_cnt
, -1);
3174 VM_OBJECT_UNLOCK(obj
->base
.vm_obj
);
3175 intel_gtt_chipset_flush();
3177 obj
->phys_obj
->cur_obj
= NULL
;
3178 obj
->phys_obj
= NULL
;
3182 i915_gem_attach_phys_object(struct drm_device
*dev
,
3183 struct drm_i915_gem_object
*obj
,
3187 drm_i915_private_t
*dev_priv
;
3191 int i
, page_count
, ret
;
3193 if (id
> I915_MAX_PHYS_OBJECT
)
3196 if (obj
->phys_obj
!= NULL
) {
3197 if (obj
->phys_obj
->id
== id
)
3199 i915_gem_detach_phys_object(dev
, obj
);
3202 dev_priv
= dev
->dev_private
;
3203 if (dev_priv
->mm
.phys_objs
[id
- 1] == NULL
) {
3204 ret
= i915_gem_init_phys_object(dev
, id
, obj
->base
.size
, align
);
3206 DRM_ERROR("failed to init phys object %d size: %zu\n",
3207 id
, obj
->base
.size
);
3212 /* bind to the object */
3213 obj
->phys_obj
= dev_priv
->mm
.phys_objs
[id
- 1];
3214 obj
->phys_obj
->cur_obj
= obj
;
3216 page_count
= obj
->base
.size
/ PAGE_SIZE
;
3218 VM_OBJECT_LOCK(obj
->base
.vm_obj
);
3220 for (i
= 0; i
< page_count
; i
++) {
3221 m
= i915_gem_wire_page(obj
->base
.vm_obj
, i
);
3226 VM_OBJECT_UNLOCK(obj
->base
.vm_obj
);
3227 sf
= sf_buf_alloc(m
);
3228 src
= (char *)sf_buf_kva(sf
);
3229 dst
= (char *)obj
->phys_obj
->handle
->vaddr
+ IDX_TO_OFF(i
);
3230 memcpy(dst
, src
, PAGE_SIZE
);
3233 VM_OBJECT_LOCK(obj
->base
.vm_obj
);
3235 vm_page_reference(m
);
3236 vm_page_busy_wait(m
, FALSE
, "i915gem");
3237 vm_page_unwire(m
, 0);
3239 atomic_add_long(&i915_gem_wired_pages_cnt
, -1);
3241 VM_OBJECT_UNLOCK(obj
->base
.vm_obj
);
3247 i915_gem_phys_pwrite(struct drm_device
*dev
, struct drm_i915_gem_object
*obj
,
3248 uint64_t data_ptr
, uint64_t offset
, uint64_t size
,
3249 struct drm_file
*file_priv
)
3251 char *user_data
, *vaddr
;
3254 vaddr
= (char *)obj
->phys_obj
->handle
->vaddr
+ offset
;
3255 user_data
= (char *)(uintptr_t)data_ptr
;
3257 if (copyin_nofault(user_data
, vaddr
, size
) != 0) {
3258 /* The physical object once assigned is fixed for the lifetime
3259 * of the obj, so we can safely drop the lock and continue
3263 ret
= -copyin(user_data
, vaddr
, size
);
3269 intel_gtt_chipset_flush();
3274 i915_gem_release(struct drm_device
*dev
, struct drm_file
*file
)
3276 struct drm_i915_file_private
*file_priv
;
3277 struct drm_i915_gem_request
*request
;
3279 file_priv
= file
->driver_priv
;
3281 /* Clean up our request list when the client is going away, so that
3282 * later retire_requests won't dereference our soon-to-be-gone
3285 spin_lock(&file_priv
->mm
.lock
);
3286 while (!list_empty(&file_priv
->mm
.request_list
)) {
3287 request
= list_first_entry(&file_priv
->mm
.request_list
,
3288 struct drm_i915_gem_request
,
3290 list_del(&request
->client_list
);
3291 request
->file_priv
= NULL
;
3293 spin_unlock(&file_priv
->mm
.lock
);
3297 i915_gem_swap_io(struct drm_device
*dev
, struct drm_i915_gem_object
*obj
,
3298 uint64_t data_ptr
, uint64_t size
, uint64_t offset
, enum uio_rw rw
,
3299 struct drm_file
*file
)
3306 int cnt
, do_bit17_swizzling
, length
, obj_po
, ret
, swizzled_po
;
3308 if (obj
->gtt_offset
!= 0 && rw
== UIO_READ
)
3309 do_bit17_swizzling
= i915_gem_object_needs_bit17_swizzle(obj
);
3311 do_bit17_swizzling
= 0;
3314 vm_obj
= obj
->base
.vm_obj
;
3317 VM_OBJECT_LOCK(vm_obj
);
3318 vm_object_pip_add(vm_obj
, 1);
3320 obj_pi
= OFF_TO_IDX(offset
);
3321 obj_po
= offset
& PAGE_MASK
;
3323 m
= i915_gem_wire_page(vm_obj
, obj_pi
);
3324 VM_OBJECT_UNLOCK(vm_obj
);
3326 sf
= sf_buf_alloc(m
);
3327 mkva
= sf_buf_kva(sf
);
3328 length
= min(size
, PAGE_SIZE
- obj_po
);
3329 while (length
> 0) {
3330 if (do_bit17_swizzling
&&
3331 (VM_PAGE_TO_PHYS(m
) & (1 << 17)) != 0) {
3332 cnt
= roundup2(obj_po
+ 1, 64);
3333 cnt
= min(cnt
- obj_po
, length
);
3334 swizzled_po
= obj_po
^ 64;
3337 swizzled_po
= obj_po
;
3340 ret
= -copyout_nofault(
3341 (char *)mkva
+ swizzled_po
,
3342 (void *)(uintptr_t)data_ptr
, cnt
);
3344 ret
= -copyin_nofault(
3345 (void *)(uintptr_t)data_ptr
,
3346 (char *)mkva
+ swizzled_po
, cnt
);
3356 VM_OBJECT_LOCK(vm_obj
);
3357 if (rw
== UIO_WRITE
)
3359 vm_page_reference(m
);
3360 vm_page_busy_wait(m
, FALSE
, "i915gem");
3361 vm_page_unwire(m
, 1);
3363 atomic_add_long(&i915_gem_wired_pages_cnt
, -1);
3368 vm_object_pip_wakeup(vm_obj
);
3369 VM_OBJECT_UNLOCK(vm_obj
);
3375 i915_gem_gtt_write(struct drm_device
*dev
, struct drm_i915_gem_object
*obj
,
3376 uint64_t data_ptr
, uint64_t size
, uint64_t offset
, struct drm_file
*file
)
3382 * Pass the unaligned physical address and size to pmap_mapdev_attr()
3383 * so it can properly calculate whether an extra page needs to be
3384 * mapped or not to cover the requested range. The function will
3385 * add the page offset into the returned mkva for us.
3387 mkva
= (vm_offset_t
)pmap_mapdev_attr(dev
->agp
->base
+ obj
->gtt_offset
+
3388 offset
, size
, PAT_WRITE_COMBINING
);
3389 ret
= -copyin_nofault((void *)(uintptr_t)data_ptr
, (char *)mkva
, size
);
3390 pmap_unmapdev(mkva
, size
);
3395 i915_gem_obj_io(struct drm_device
*dev
, uint32_t handle
, uint64_t data_ptr
,
3396 uint64_t size
, uint64_t offset
, enum uio_rw rw
, struct drm_file
*file
)
3398 struct drm_i915_gem_object
*obj
;
3400 vm_offset_t start
, end
;
3405 start
= trunc_page(data_ptr
);
3406 end
= round_page(data_ptr
+ size
);
3407 npages
= howmany(end
- start
, PAGE_SIZE
);
3408 ma
= kmalloc(npages
* sizeof(vm_page_t
), DRM_I915_GEM
, M_WAITOK
|
3410 npages
= vm_fault_quick_hold_pages(&curproc
->p_vmspace
->vm_map
,
3411 (vm_offset_t
)data_ptr
, size
,
3412 (rw
== UIO_READ
? VM_PROT_WRITE
: 0 ) | VM_PROT_READ
, ma
, npages
);
3418 ret
= i915_mutex_lock_interruptible(dev
);
3422 obj
= to_intel_bo(drm_gem_object_lookup(dev
, file
, handle
));
3423 if (&obj
->base
== NULL
) {
3427 if (offset
> obj
->base
.size
|| size
> obj
->base
.size
- offset
) {
3432 if (rw
== UIO_READ
) {
3433 ret
= i915_gem_swap_io(dev
, obj
, data_ptr
, size
, offset
,
3436 if (obj
->phys_obj
) {
3437 ret
= i915_gem_phys_pwrite(dev
, obj
, data_ptr
, offset
,
3439 } else if (obj
->gtt_space
&&
3440 obj
->base
.write_domain
!= I915_GEM_DOMAIN_CPU
) {
3441 ret
= i915_gem_object_pin(obj
, 0, true);
3444 ret
= i915_gem_object_set_to_gtt_domain(obj
, true);
3447 ret
= i915_gem_object_put_fence(obj
);
3450 ret
= i915_gem_gtt_write(dev
, obj
, data_ptr
, size
,
3453 i915_gem_object_unpin(obj
);
3455 ret
= i915_gem_object_set_to_cpu_domain(obj
, true);
3458 ret
= i915_gem_swap_io(dev
, obj
, data_ptr
, size
, offset
,
3463 drm_gem_object_unreference(&obj
->base
);
3467 vm_page_unhold_pages(ma
, npages
);
3469 drm_free(ma
, DRM_I915_GEM
);
3474 i915_gem_pager_ctor(void *handle
, vm_ooffset_t size
, vm_prot_t prot
,
3475 vm_ooffset_t foff
, struct ucred
*cred
, u_short
*color
)
3478 *color
= 0; /* XXXKIB */
3485 i915_gem_pager_fault(vm_object_t vm_obj
, vm_ooffset_t offset
, int prot
,
3488 struct drm_gem_object
*gem_obj
;
3489 struct drm_i915_gem_object
*obj
;
3490 struct drm_device
*dev
;
3491 drm_i915_private_t
*dev_priv
;
3496 gem_obj
= vm_obj
->handle
;
3497 obj
= to_intel_bo(gem_obj
);
3498 dev
= obj
->base
.dev
;
3499 dev_priv
= dev
->dev_private
;
3501 write
= (prot
& VM_PROT_WRITE
) != 0;
3505 vm_object_pip_add(vm_obj
, 1);
3508 * Remove the placeholder page inserted by vm_fault() from the
3509 * object before dropping the object lock. If
3510 * i915_gem_release_mmap() is active in parallel on this gem
3511 * object, then it owns the drm device sx and might find the
3512 * placeholder already. Then, since the page is busy,
3513 * i915_gem_release_mmap() sleeps waiting for the busy state
3514 * of the page cleared. We will be not able to acquire drm
3515 * device lock until i915_gem_release_mmap() is able to make a
3518 if (*mres
!= NULL
) {
3520 vm_page_remove(oldm
);
3525 VM_OBJECT_UNLOCK(vm_obj
);
3531 ret
= i915_mutex_lock_interruptible(dev
);
3540 * Since the object lock was dropped, other thread might have
3541 * faulted on the same GTT address and instantiated the
3542 * mapping for the page. Recheck.
3544 VM_OBJECT_LOCK(vm_obj
);
3545 m
= vm_page_lookup(vm_obj
, OFF_TO_IDX(offset
));
3547 if ((m
->flags
& PG_BUSY
) != 0) {
3550 vm_page_sleep(m
, "915pee");
3556 VM_OBJECT_UNLOCK(vm_obj
);
3558 /* Now bind it into the GTT if needed */
3559 if (!obj
->map_and_fenceable
) {
3560 ret
= i915_gem_object_unbind(obj
);
3566 if (!obj
->gtt_space
) {
3567 ret
= i915_gem_object_bind_to_gtt(obj
, 0, true);
3573 ret
= i915_gem_object_set_to_gtt_domain(obj
, write
);
3580 if (obj
->tiling_mode
== I915_TILING_NONE
)
3581 ret
= i915_gem_object_put_fence(obj
);
3583 ret
= i915_gem_object_get_fence(obj
);
3589 if (i915_gem_object_is_inactive(obj
))
3590 list_move_tail(&obj
->mm_list
, &dev_priv
->mm
.inactive_list
);
3592 obj
->fault_mappable
= true;
3593 VM_OBJECT_LOCK(vm_obj
);
3594 m
= vm_phys_fictitious_to_vm_page(dev
->agp
->base
+ obj
->gtt_offset
+
3601 KASSERT((m
->flags
& PG_FICTITIOUS
) != 0,
3602 ("not fictitious %p", m
));
3603 KASSERT(m
->wire_count
== 1, ("wire_count not 1 %p", m
));
3605 if ((m
->flags
& PG_BUSY
) != 0) {
3608 vm_page_sleep(m
, "915pbs");
3612 m
->valid
= VM_PAGE_BITS_ALL
;
3613 vm_page_insert(m
, vm_obj
, OFF_TO_IDX(offset
));
3616 vm_page_busy_try(m
, false);
3622 vm_object_pip_wakeup(vm_obj
);
3623 return (VM_PAGER_OK
);
3628 KASSERT(ret
!= 0, ("i915_gem_pager_fault: wrong return"));
3629 if (ret
== -EAGAIN
|| ret
== -EIO
|| ret
== -EINTR
) {
3630 goto unlocked_vmobj
;
3632 VM_OBJECT_LOCK(vm_obj
);
3633 vm_object_pip_wakeup(vm_obj
);
3634 return (VM_PAGER_ERROR
);
3638 i915_gem_pager_dtor(void *handle
)
3640 struct drm_gem_object
*obj
;
3641 struct drm_device
*dev
;
3647 drm_gem_free_mmap_offset(obj
);
3648 i915_gem_release_mmap(to_intel_bo(obj
));
3649 drm_gem_object_unreference(obj
);
3653 struct cdev_pager_ops i915_gem_pager_ops
= {
3654 .cdev_pg_fault
= i915_gem_pager_fault
,
3655 .cdev_pg_ctor
= i915_gem_pager_ctor
,
3656 .cdev_pg_dtor
= i915_gem_pager_dtor
3659 #define GEM_PARANOID_CHECK_GTT 0
3660 #if GEM_PARANOID_CHECK_GTT
3662 i915_gem_assert_pages_not_mapped(struct drm_device
*dev
, vm_page_t
*ma
,
3665 struct drm_i915_private
*dev_priv
;
3667 unsigned long start
, end
;
3671 dev_priv
= dev
->dev_private
;
3672 start
= OFF_TO_IDX(dev_priv
->mm
.gtt_start
);
3673 end
= OFF_TO_IDX(dev_priv
->mm
.gtt_end
);
3674 for (i
= start
; i
< end
; i
++) {
3675 pa
= intel_gtt_read_pte_paddr(i
);
3676 for (j
= 0; j
< page_count
; j
++) {
3677 if (pa
== VM_PAGE_TO_PHYS(ma
[j
])) {
3678 panic("Page %p in GTT pte index %d pte %x",
3679 ma
[i
], i
, intel_gtt_read_pte(i
));
3687 i915_gem_process_flushing_list(struct intel_ring_buffer
*ring
,
3688 uint32_t flush_domains
)
3690 struct drm_i915_gem_object
*obj
, *next
;
3691 uint32_t old_write_domain
;
3693 list_for_each_entry_safe(obj
, next
, &ring
->gpu_write_list
,
3695 if (obj
->base
.write_domain
& flush_domains
) {
3696 old_write_domain
= obj
->base
.write_domain
;
3697 obj
->base
.write_domain
= 0;
3698 list_del_init(&obj
->gpu_write_list
);
3699 i915_gem_object_move_to_active(obj
, ring
);
3704 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3707 i915_gem_wire_page(vm_object_t object
, vm_pindex_t pindex
)
3712 VM_OBJECT_LOCK_ASSERT_OWNED(object
);
3713 m
= vm_page_grab(object
, pindex
, VM_ALLOC_NORMAL
| VM_ALLOC_RETRY
);
3714 if (m
->valid
!= VM_PAGE_BITS_ALL
) {
3715 if (vm_pager_has_page(object
, pindex
)) {
3716 rv
= vm_pager_get_page(object
, &m
, 1);
3717 m
= vm_page_lookup(object
, pindex
);
3720 if (rv
!= VM_PAGER_OK
) {
3725 pmap_zero_page(VM_PAGE_TO_PHYS(m
));
3726 m
->valid
= VM_PAGE_BITS_ALL
;
3732 atomic_add_long(&i915_gem_wired_pages_cnt
, 1);
3737 i915_gem_flush_ring(struct intel_ring_buffer
*ring
, uint32_t invalidate_domains
,
3738 uint32_t flush_domains
)
3742 if (((invalidate_domains
| flush_domains
) & I915_GEM_GPU_DOMAINS
) == 0)
3745 ret
= ring
->flush(ring
, invalidate_domains
, flush_domains
);
3749 if (flush_domains
& I915_GEM_GPU_DOMAINS
)
3750 i915_gem_process_flushing_list(ring
, flush_domains
);
3755 i915_gpu_is_active(struct drm_device
*dev
)
3757 drm_i915_private_t
*dev_priv
= dev
->dev_private
;
3759 return !list_empty(&dev_priv
->mm
.active_list
);
3763 i915_gem_lowmem(void *arg
)
3765 struct drm_device
*dev
;
3766 struct drm_i915_private
*dev_priv
;
3767 struct drm_i915_gem_object
*obj
, *next
;
3768 int cnt
, cnt_fail
, cnt_total
;
3771 dev_priv
= dev
->dev_private
;
3773 if (lockmgr(&dev
->dev_struct_lock
, LK_EXCLUSIVE
|LK_NOWAIT
))
3777 /* first scan for clean buffers */
3778 i915_gem_retire_requests(dev
);
3780 cnt_total
= cnt_fail
= cnt
= 0;
3782 list_for_each_entry_safe(obj
, next
, &dev_priv
->mm
.inactive_list
,
3784 if (i915_gem_object_is_purgeable(obj
)) {
3785 if (i915_gem_object_unbind(obj
) != 0)
3791 /* second pass, evict/count anything still on the inactive list */
3792 list_for_each_entry_safe(obj
, next
, &dev_priv
->mm
.inactive_list
,
3794 if (i915_gem_object_unbind(obj
) == 0)
3800 if (cnt_fail
> cnt_total
/ 100 && i915_gpu_is_active(dev
)) {
3802 * We are desperate for pages, so as a last resort, wait
3803 * for the GPU to finish and discard whatever we can.
3804 * This has a dramatic impact to reduce the number of
3805 * OOM-killer events whilst running the GPU aggressively.
3807 if (i915_gpu_idle(dev
) == 0)
3814 i915_gem_unload(struct drm_device
*dev
)
3816 struct drm_i915_private
*dev_priv
;
3818 dev_priv
= dev
->dev_private
;
3819 EVENTHANDLER_DEREGISTER(vm_lowmem
, dev_priv
->mm
.i915_lowmem
);