2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/list.h>
35 #include <linux/kref.h>
36 #include <linux/slab.h>
39 #include "radeon_reg.h"
41 #include "radeon_trace.h"
43 static void radeon_fence_write(struct radeon_device
*rdev
, u32 seq
, int ring
)
45 if (rdev
->wb
.enabled
) {
46 *rdev
->fence_drv
[ring
].cpu_addr
= cpu_to_le32(seq
);
48 WREG32(rdev
->fence_drv
[ring
].scratch_reg
, seq
);
52 static u32
radeon_fence_read(struct radeon_device
*rdev
, int ring
)
56 if (rdev
->wb
.enabled
) {
57 seq
= le32_to_cpu(*rdev
->fence_drv
[ring
].cpu_addr
);
59 seq
= RREG32(rdev
->fence_drv
[ring
].scratch_reg
);
64 int radeon_fence_emit(struct radeon_device
*rdev
, struct radeon_fence
*fence
)
66 unsigned long irq_flags
;
68 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
70 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
73 fence
->seq
= atomic_add_return(1, &rdev
->fence_drv
[fence
->ring
].seq
);
74 if (!rdev
->ring
[fence
->ring
].ready
)
75 /* FIXME: cp is not running assume everythings is done right
78 radeon_fence_write(rdev
, fence
->seq
, fence
->ring
);
80 radeon_fence_ring_emit(rdev
, fence
->ring
, fence
);
82 trace_radeon_fence_emit(rdev
->ddev
, fence
->seq
);
83 fence
->emitted
= true;
84 list_move_tail(&fence
->list
, &rdev
->fence_drv
[fence
->ring
].emitted
);
85 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
89 static bool radeon_fence_poll_locked(struct radeon_device
*rdev
, int ring
)
91 struct radeon_fence
*fence
;
92 struct list_head
*i
, *n
;
95 unsigned long cjiffies
;
97 seq
= radeon_fence_read(rdev
, ring
);
98 if (seq
!= rdev
->fence_drv
[ring
].last_seq
) {
99 rdev
->fence_drv
[ring
].last_seq
= seq
;
100 rdev
->fence_drv
[ring
].last_jiffies
= jiffies
;
101 rdev
->fence_drv
[ring
].last_timeout
= RADEON_FENCE_JIFFIES_TIMEOUT
;
104 if (time_after(cjiffies
, rdev
->fence_drv
[ring
].last_jiffies
)) {
105 cjiffies
-= rdev
->fence_drv
[ring
].last_jiffies
;
106 if (time_after(rdev
->fence_drv
[ring
].last_timeout
, cjiffies
)) {
107 /* update the timeout */
108 rdev
->fence_drv
[ring
].last_timeout
-= cjiffies
;
110 /* the 500ms timeout is elapsed we should test
113 rdev
->fence_drv
[ring
].last_timeout
= 1;
116 /* wrap around update last jiffies, we will just wait
119 rdev
->fence_drv
[ring
].last_jiffies
= cjiffies
;
124 list_for_each(i
, &rdev
->fence_drv
[ring
].emitted
) {
125 fence
= list_entry(i
, struct radeon_fence
, list
);
126 if (fence
->seq
== seq
) {
131 /* all fence previous to this one are considered as signaled */
136 list_move_tail(i
, &rdev
->fence_drv
[ring
].signaled
);
137 fence
= list_entry(i
, struct radeon_fence
, list
);
138 fence
->signaled
= true;
140 } while (i
!= &rdev
->fence_drv
[ring
].emitted
);
146 static void radeon_fence_destroy(struct kref
*kref
)
148 unsigned long irq_flags
;
149 struct radeon_fence
*fence
;
151 fence
= container_of(kref
, struct radeon_fence
, kref
);
152 write_lock_irqsave(&fence
->rdev
->fence_lock
, irq_flags
);
153 list_del(&fence
->list
);
154 fence
->emitted
= false;
155 write_unlock_irqrestore(&fence
->rdev
->fence_lock
, irq_flags
);
159 int radeon_fence_create(struct radeon_device
*rdev
,
160 struct radeon_fence
**fence
,
163 unsigned long irq_flags
;
165 *fence
= kmalloc(sizeof(struct radeon_fence
), GFP_KERNEL
);
166 if ((*fence
) == NULL
) {
169 kref_init(&((*fence
)->kref
));
170 (*fence
)->rdev
= rdev
;
171 (*fence
)->emitted
= false;
172 (*fence
)->signaled
= false;
174 (*fence
)->ring
= ring
;
175 INIT_LIST_HEAD(&(*fence
)->list
);
177 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
178 list_add_tail(&(*fence
)->list
, &rdev
->fence_drv
[ring
].created
);
179 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
183 bool radeon_fence_signaled(struct radeon_fence
*fence
)
185 unsigned long irq_flags
;
186 bool signaled
= false;
191 if (fence
->rdev
->gpu_lockup
)
194 write_lock_irqsave(&fence
->rdev
->fence_lock
, irq_flags
);
195 signaled
= fence
->signaled
;
196 /* if we are shuting down report all fence as signaled */
197 if (fence
->rdev
->shutdown
) {
200 if (!fence
->emitted
) {
201 WARN(1, "Querying an unemitted fence : %p !\n", fence
);
205 radeon_fence_poll_locked(fence
->rdev
, fence
->ring
);
206 signaled
= fence
->signaled
;
208 write_unlock_irqrestore(&fence
->rdev
->fence_lock
, irq_flags
);
212 int radeon_fence_wait(struct radeon_fence
*fence
, bool intr
)
214 struct radeon_device
*rdev
;
215 unsigned long irq_flags
, timeout
;
220 WARN(1, "Querying an invalid fence : %p !\n", fence
);
224 if (radeon_fence_signaled(fence
)) {
227 timeout
= rdev
->fence_drv
[fence
->ring
].last_timeout
;
229 /* save current sequence used to check for GPU lockup */
230 seq
= rdev
->fence_drv
[fence
->ring
].last_seq
;
231 trace_radeon_fence_wait_begin(rdev
->ddev
, seq
);
233 radeon_irq_kms_sw_irq_get(rdev
, fence
->ring
);
234 r
= wait_event_interruptible_timeout(rdev
->fence_drv
[fence
->ring
].queue
,
235 radeon_fence_signaled(fence
), timeout
);
236 radeon_irq_kms_sw_irq_put(rdev
, fence
->ring
);
237 if (unlikely(r
< 0)) {
241 radeon_irq_kms_sw_irq_get(rdev
, fence
->ring
);
242 r
= wait_event_timeout(rdev
->fence_drv
[fence
->ring
].queue
,
243 radeon_fence_signaled(fence
), timeout
);
244 radeon_irq_kms_sw_irq_put(rdev
, fence
->ring
);
246 trace_radeon_fence_wait_end(rdev
->ddev
, seq
);
247 if (unlikely(!radeon_fence_signaled(fence
))) {
248 /* we were interrupted for some reason and fence isn't
249 * isn't signaled yet, resume wait
255 /* don't protect read access to rdev->fence_drv[t].last_seq
256 * if we experiencing a lockup the value doesn't change
258 if (seq
== rdev
->fence_drv
[fence
->ring
].last_seq
&&
259 radeon_gpu_is_lockup(rdev
, &rdev
->ring
[fence
->ring
])) {
260 /* good news we believe it's a lockup */
261 printk(KERN_WARNING
"GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
263 /* FIXME: what should we do ? marking everyone
264 * as signaled for now
266 rdev
->gpu_lockup
= true;
267 r
= radeon_gpu_reset(rdev
);
270 radeon_fence_write(rdev
, fence
->seq
, fence
->ring
);
271 rdev
->gpu_lockup
= false;
273 timeout
= RADEON_FENCE_JIFFIES_TIMEOUT
;
274 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
275 rdev
->fence_drv
[fence
->ring
].last_timeout
= RADEON_FENCE_JIFFIES_TIMEOUT
;
276 rdev
->fence_drv
[fence
->ring
].last_jiffies
= jiffies
;
277 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
283 int radeon_fence_wait_next(struct radeon_device
*rdev
, int ring
)
285 unsigned long irq_flags
;
286 struct radeon_fence
*fence
;
289 if (rdev
->gpu_lockup
) {
292 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
293 if (list_empty(&rdev
->fence_drv
[ring
].emitted
)) {
294 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
297 fence
= list_entry(rdev
->fence_drv
[ring
].emitted
.next
,
298 struct radeon_fence
, list
);
299 radeon_fence_ref(fence
);
300 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
301 r
= radeon_fence_wait(fence
, false);
302 radeon_fence_unref(&fence
);
306 int radeon_fence_wait_last(struct radeon_device
*rdev
, int ring
)
308 unsigned long irq_flags
;
309 struct radeon_fence
*fence
;
312 if (rdev
->gpu_lockup
) {
315 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
316 if (list_empty(&rdev
->fence_drv
[ring
].emitted
)) {
317 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
320 fence
= list_entry(rdev
->fence_drv
[ring
].emitted
.prev
,
321 struct radeon_fence
, list
);
322 radeon_fence_ref(fence
);
323 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
324 r
= radeon_fence_wait(fence
, false);
325 radeon_fence_unref(&fence
);
329 struct radeon_fence
*radeon_fence_ref(struct radeon_fence
*fence
)
331 kref_get(&fence
->kref
);
335 void radeon_fence_unref(struct radeon_fence
**fence
)
337 struct radeon_fence
*tmp
= *fence
;
341 kref_put(&tmp
->kref
, radeon_fence_destroy
);
345 void radeon_fence_process(struct radeon_device
*rdev
, int ring
)
347 unsigned long irq_flags
;
350 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
351 wake
= radeon_fence_poll_locked(rdev
, ring
);
352 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
354 wake_up_all(&rdev
->fence_drv
[ring
].queue
);
358 int radeon_fence_count_emitted(struct radeon_device
*rdev
, int ring
)
360 unsigned long irq_flags
;
361 int not_processed
= 0;
363 read_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
364 if (!rdev
->fence_drv
[ring
].initialized
)
367 if (!list_empty(&rdev
->fence_drv
[ring
].emitted
)) {
368 struct list_head
*ptr
;
369 list_for_each(ptr
, &rdev
->fence_drv
[ring
].emitted
) {
370 /* count up to 3, that's enought info */
371 if (++not_processed
>= 3)
375 read_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
376 return not_processed
;
379 int radeon_fence_driver_start_ring(struct radeon_device
*rdev
, int ring
)
381 unsigned long irq_flags
;
385 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
386 radeon_scratch_free(rdev
, rdev
->fence_drv
[ring
].scratch_reg
);
387 if (rdev
->wb
.use_event
) {
388 rdev
->fence_drv
[ring
].scratch_reg
= 0;
389 index
= R600_WB_EVENT_OFFSET
+ ring
* 4;
391 r
= radeon_scratch_get(rdev
, &rdev
->fence_drv
[ring
].scratch_reg
);
393 dev_err(rdev
->dev
, "fence failed to get scratch register\n");
394 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
397 index
= RADEON_WB_SCRATCH_OFFSET
+
398 rdev
->fence_drv
[ring
].scratch_reg
-
399 rdev
->scratch
.reg_base
;
401 rdev
->fence_drv
[ring
].cpu_addr
= &rdev
->wb
.wb
[index
/4];
402 rdev
->fence_drv
[ring
].gpu_addr
= rdev
->wb
.gpu_addr
+ index
;
403 radeon_fence_write(rdev
, atomic_read(&rdev
->fence_drv
[ring
].seq
), ring
);
404 rdev
->fence_drv
[ring
].initialized
= true;
405 DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
406 ring
, rdev
->fence_drv
[ring
].gpu_addr
, rdev
->fence_drv
[ring
].cpu_addr
);
407 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
411 static void radeon_fence_driver_init_ring(struct radeon_device
*rdev
, int ring
)
413 rdev
->fence_drv
[ring
].scratch_reg
= -1;
414 rdev
->fence_drv
[ring
].cpu_addr
= NULL
;
415 rdev
->fence_drv
[ring
].gpu_addr
= 0;
416 atomic_set(&rdev
->fence_drv
[ring
].seq
, 0);
417 INIT_LIST_HEAD(&rdev
->fence_drv
[ring
].created
);
418 INIT_LIST_HEAD(&rdev
->fence_drv
[ring
].emitted
);
419 INIT_LIST_HEAD(&rdev
->fence_drv
[ring
].signaled
);
420 init_waitqueue_head(&rdev
->fence_drv
[ring
].queue
);
421 rdev
->fence_drv
[ring
].initialized
= false;
424 int radeon_fence_driver_init(struct radeon_device
*rdev
)
426 unsigned long irq_flags
;
429 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
430 for (ring
= 0; ring
< RADEON_NUM_RINGS
; ring
++) {
431 radeon_fence_driver_init_ring(rdev
, ring
);
433 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
434 if (radeon_debugfs_fence_init(rdev
)) {
435 dev_err(rdev
->dev
, "fence debugfs file creation failed\n");
440 void radeon_fence_driver_fini(struct radeon_device
*rdev
)
442 unsigned long irq_flags
;
445 for (ring
= 0; ring
< RADEON_NUM_RINGS
; ring
++) {
446 if (!rdev
->fence_drv
[ring
].initialized
)
448 radeon_fence_wait_last(rdev
, ring
);
449 wake_up_all(&rdev
->fence_drv
[ring
].queue
);
450 write_lock_irqsave(&rdev
->fence_lock
, irq_flags
);
451 radeon_scratch_free(rdev
, rdev
->fence_drv
[ring
].scratch_reg
);
452 write_unlock_irqrestore(&rdev
->fence_lock
, irq_flags
);
453 rdev
->fence_drv
[ring
].initialized
= false;
461 #if defined(CONFIG_DEBUG_FS)
462 static int radeon_debugfs_fence_info(struct seq_file
*m
, void *data
)
464 struct drm_info_node
*node
= (struct drm_info_node
*)m
->private;
465 struct drm_device
*dev
= node
->minor
->dev
;
466 struct radeon_device
*rdev
= dev
->dev_private
;
467 struct radeon_fence
*fence
;
470 for (i
= 0; i
< RADEON_NUM_RINGS
; ++i
) {
471 if (!rdev
->fence_drv
[i
].initialized
)
474 seq_printf(m
, "--- ring %d ---\n", i
);
475 seq_printf(m
, "Last signaled fence 0x%08X\n",
476 radeon_fence_read(rdev
, i
));
477 if (!list_empty(&rdev
->fence_drv
[i
].emitted
)) {
478 fence
= list_entry(rdev
->fence_drv
[i
].emitted
.prev
,
479 struct radeon_fence
, list
);
480 seq_printf(m
, "Last emitted fence %p with 0x%08X\n",
487 static struct drm_info_list radeon_debugfs_fence_list
[] = {
488 {"radeon_fence_info", &radeon_debugfs_fence_info
, 0, NULL
},
492 int radeon_debugfs_fence_init(struct radeon_device
*rdev
)
494 #if defined(CONFIG_DEBUG_FS)
495 return radeon_debugfs_add_files(rdev
, radeon_debugfs_fence_list
, 1);