2 * Copyright 2013 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
26 #include "radeon_asic.h"
28 #include "radeon_trace.h"
33 * si_dma_is_lockup - Check if the DMA engine is locked up
35 * @rdev: radeon_device pointer
36 * @ring: radeon_ring structure holding ring information
38 * Check if the async DMA engine is locked up.
39 * Returns true if the engine appears to be locked up, false if not.
41 bool si_dma_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
43 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
46 if (ring
->idx
== R600_RING_TYPE_DMA_INDEX
)
47 mask
= RADEON_RESET_DMA
;
49 mask
= RADEON_RESET_DMA1
;
51 if (!(reset_mask
& mask
)) {
52 radeon_ring_lockup_update(rdev
, ring
);
55 return radeon_ring_test_lockup(rdev
, ring
);
59 * si_dma_vm_copy_pages - update PTEs by copying them from the GART
61 * @rdev: radeon_device pointer
62 * @ib: indirect buffer to fill with commands
63 * @pe: addr of the page entry
64 * @src: src addr where to copy from
65 * @count: number of page entries to update
67 * Update PTEs by copying them from the GART using the DMA (SI).
69 void si_dma_vm_copy_pages(struct radeon_device
*rdev
,
71 uint64_t pe
, uint64_t src
,
75 unsigned bytes
= count
* 8;
79 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_COPY
,
81 ib
->ptr
[ib
->length_dw
++] = lower_32_bits(pe
);
82 ib
->ptr
[ib
->length_dw
++] = lower_32_bits(src
);
83 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
84 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(src
) & 0xff;
93 * si_dma_vm_write_pages - update PTEs by writing them manually
95 * @rdev: radeon_device pointer
96 * @ib: indirect buffer to fill with commands
97 * @pe: addr of the page entry
98 * @addr: dst addr to write into pe
99 * @count: number of page entries to update
100 * @incr: increase next addr by incr bytes
101 * @flags: access flags
103 * Update PTEs by writing them manually using the DMA (SI).
105 void si_dma_vm_write_pages(struct radeon_device
*rdev
,
106 struct radeon_ib
*ib
,
108 uint64_t addr
, unsigned count
,
109 uint32_t incr
, uint32_t flags
)
119 /* for non-physically contiguous pages (system) */
120 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_WRITE
, 0, 0, 0, ndw
);
121 ib
->ptr
[ib
->length_dw
++] = pe
;
122 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
123 for (; ndw
> 0; ndw
-= 2, --count
, pe
+= 8) {
124 if (flags
& R600_PTE_SYSTEM
) {
125 value
= radeon_vm_map_gart(rdev
, addr
);
126 } else if (flags
& R600_PTE_VALID
) {
133 ib
->ptr
[ib
->length_dw
++] = value
;
134 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
140 * si_dma_vm_set_pages - update the page tables using the DMA
142 * @rdev: radeon_device pointer
143 * @ib: indirect buffer to fill with commands
144 * @pe: addr of the page entry
145 * @addr: dst addr to write into pe
146 * @count: number of page entries to update
147 * @incr: increase next addr by incr bytes
148 * @flags: access flags
150 * Update the page tables using the DMA (SI).
152 void si_dma_vm_set_pages(struct radeon_device
*rdev
,
153 struct radeon_ib
*ib
,
155 uint64_t addr
, unsigned count
,
156 uint32_t incr
, uint32_t flags
)
166 if (flags
& R600_PTE_VALID
)
171 /* for physically contiguous pages (vram) */
172 ib
->ptr
[ib
->length_dw
++] = DMA_PTE_PDE_PACKET(ndw
);
173 ib
->ptr
[ib
->length_dw
++] = pe
; /* dst addr */
174 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
175 ib
->ptr
[ib
->length_dw
++] = flags
; /* mask */
176 ib
->ptr
[ib
->length_dw
++] = 0;
177 ib
->ptr
[ib
->length_dw
++] = value
; /* value */
178 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
179 ib
->ptr
[ib
->length_dw
++] = incr
; /* increment size */
180 ib
->ptr
[ib
->length_dw
++] = 0;
182 addr
+= (ndw
/ 2) * incr
;
187 void si_dma_vm_flush(struct radeon_device
*rdev
, struct radeon_ring
*ring
,
188 unsigned vm_id
, uint64_t pd_addr
)
191 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
194 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm_id
<< 2)) >> 2));
196 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm_id
- 8) << 2)) >> 2));
198 radeon_ring_write(ring
, pd_addr
>> 12);
200 /* flush hdp cache */
201 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
202 radeon_ring_write(ring
, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2));
203 radeon_ring_write(ring
, 1);
205 /* bits 0-7 are the VM contexts0-7 */
206 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
207 radeon_ring_write(ring
, (0xf << 16) | (VM_INVALIDATE_REQUEST
>> 2));
208 radeon_ring_write(ring
, 1 << vm_id
);
210 /* wait for invalidate to complete */
211 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_POLL_REG_MEM
, 0, 0, 0, 0));
212 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
);
213 radeon_ring_write(ring
, 0xff << 16); /* retry */
214 radeon_ring_write(ring
, 1 << vm_id
); /* mask */
215 radeon_ring_write(ring
, 0); /* value */
216 radeon_ring_write(ring
, (0 << 28) | 0x20); /* func(always) | poll interval */
220 * si_copy_dma - copy pages using the DMA engine
222 * @rdev: radeon_device pointer
223 * @src_offset: src GPU address
224 * @dst_offset: dst GPU address
225 * @num_gpu_pages: number of GPU pages to xfer
226 * @fence: radeon fence object
228 * Copy GPU paging using the DMA engine (SI).
229 * Used by the radeon ttm implementation to move pages if
230 * registered as the asic copy callback.
232 int si_copy_dma(struct radeon_device
*rdev
,
233 uint64_t src_offset
, uint64_t dst_offset
,
234 unsigned num_gpu_pages
,
235 struct radeon_fence
**fence
)
237 struct radeon_semaphore
*sem
= NULL
;
238 int ring_index
= rdev
->asic
->copy
.dma_ring_index
;
239 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
240 u32 size_in_bytes
, cur_size_in_bytes
;
244 r
= radeon_semaphore_create(rdev
, &sem
);
246 DRM_ERROR("radeon: moving bo (%d).\n", r
);
250 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
251 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0xfffff);
252 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 5 + 11);
254 DRM_ERROR("radeon: moving bo (%d).\n", r
);
255 radeon_semaphore_free(rdev
, &sem
, NULL
);
259 radeon_semaphore_sync_to(sem
, *fence
);
260 radeon_semaphore_sync_rings(rdev
, sem
, ring
->idx
);
262 for (i
= 0; i
< num_loops
; i
++) {
263 cur_size_in_bytes
= size_in_bytes
;
264 if (cur_size_in_bytes
> 0xFFFFF)
265 cur_size_in_bytes
= 0xFFFFF;
266 size_in_bytes
-= cur_size_in_bytes
;
267 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_COPY
, 1, 0, 0, cur_size_in_bytes
));
268 radeon_ring_write(ring
, lower_32_bits(dst_offset
));
269 radeon_ring_write(ring
, lower_32_bits(src_offset
));
270 radeon_ring_write(ring
, upper_32_bits(dst_offset
) & 0xff);
271 radeon_ring_write(ring
, upper_32_bits(src_offset
) & 0xff);
272 src_offset
+= cur_size_in_bytes
;
273 dst_offset
+= cur_size_in_bytes
;
276 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
278 radeon_ring_unlock_undo(rdev
, ring
);
279 radeon_semaphore_free(rdev
, &sem
, NULL
);
283 radeon_ring_unlock_commit(rdev
, ring
, false);
284 radeon_semaphore_free(rdev
, &sem
, *fence
);