?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
radeon: sync to radeon 3.10
[dragonfly.git] / sys / dev / drm / radeon / r600.c
blob8b04605655bfde7b315089145da6eaf441c49b77
1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 *
28 * $FreeBSD: head/sys/dev/drm2/radeon/r600.c 254885 2013-08-25 19:37:15Z dumbbell $
29 */
30
31 #include <drm/drmP.h>
32 #include <uapi_drm/radeon_drm.h>
33 #include "radeon.h"
34 #include "radeon_asic.h"
35 #include "radeon_mode.h"
36 #include "r600d.h"
37 #include "atom.h"
38 #include "avivod.h"
39
40 #define PFP_UCODE_SIZE 576
41 #define PM4_UCODE_SIZE 1792
42 #define RLC_UCODE_SIZE 768
43 #define R700_PFP_UCODE_SIZE 848
44 #define R700_PM4_UCODE_SIZE 1360
45 #define R700_RLC_UCODE_SIZE 1024
46 #define EVERGREEN_PFP_UCODE_SIZE 1120
47 #define EVERGREEN_PM4_UCODE_SIZE 1376
48 #define EVERGREEN_RLC_UCODE_SIZE 768
49 #define CAYMAN_RLC_UCODE_SIZE 1024
50 #define ARUBA_RLC_UCODE_SIZE 1536
51
52 #ifdef DUMBBELL_WIP
53 /* Firmware Names */
54 MODULE_FIRMWARE("radeon/R600_pfp.bin");
55 MODULE_FIRMWARE("radeon/R600_me.bin");
56 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
57 MODULE_FIRMWARE("radeon/RV610_me.bin");
58 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
59 MODULE_FIRMWARE("radeon/RV630_me.bin");
60 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
61 MODULE_FIRMWARE("radeon/RV620_me.bin");
62 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
63 MODULE_FIRMWARE("radeon/RV635_me.bin");
64 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
65 MODULE_FIRMWARE("radeon/RV670_me.bin");
66 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
67 MODULE_FIRMWARE("radeon/RS780_me.bin");
68 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
69 MODULE_FIRMWARE("radeon/RV770_me.bin");
70 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
71 MODULE_FIRMWARE("radeon/RV730_me.bin");
72 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
73 MODULE_FIRMWARE("radeon/RV710_me.bin");
74 MODULE_FIRMWARE("radeon/R600_rlc.bin");
75 MODULE_FIRMWARE("radeon/R700_rlc.bin");
76 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
77 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
78 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
79 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
80 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
81 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
82 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
83 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
84 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
85 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
86 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
87 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
88 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
89 MODULE_FIRMWARE("radeon/PALM_me.bin");
90 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
91 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
92 MODULE_FIRMWARE("radeon/SUMO_me.bin");
93 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
94 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
95 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
96 MODULE_FIRMWARE("radeon/OLAND_me.bin");
97 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
98 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
99 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
100 #endif /* DUMBBELL_WIP */
101
102 static const u32 crtc_offsets[2] =
103 {
104 0,
105 AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
106 };
107
108 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
109
110 /* r600,rv610,rv630,rv620,rv635,rv670 */
111 static void r600_gpu_init(struct radeon_device *rdev);
112 void r600_irq_disable(struct radeon_device *rdev);
113 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
114
115 /**
116 * r600_get_xclk - get the xclk
117 *
118 * @rdev: radeon_device pointer
119 *
120 * Returns the reference clock used by the gfx engine
121 * (r6xx, IGPs, APUs).
122 */
123 u32 r600_get_xclk(struct radeon_device *rdev)
124 {
125 return rdev->clock.spll.reference_freq;
126 }
127
128 /* get temperature in millidegrees */
129 int rv6xx_get_temp(struct radeon_device *rdev)
130 {
131 u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
132 ASIC_T_SHIFT;
133 int actual_temp = temp & 0xff;
134
135 if (temp & 0x100)
136 actual_temp -= 256;
137
138 return actual_temp * 1000;
139 }
140
141 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
142 {
143 int i;
144
145 rdev->pm.dynpm_can_upclock = true;
146 rdev->pm.dynpm_can_downclock = true;
147
148 /* power state array is low to high, default is first */
149 if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
150 int min_power_state_index = 0;
151
152 if (rdev->pm.num_power_states > 2)
153 min_power_state_index = 1;
154
155 switch (rdev->pm.dynpm_planned_action) {
156 case DYNPM_ACTION_MINIMUM:
157 rdev->pm.requested_power_state_index = min_power_state_index;
158 rdev->pm.requested_clock_mode_index = 0;
159 rdev->pm.dynpm_can_downclock = false;
160 break;
161 case DYNPM_ACTION_DOWNCLOCK:
162 if (rdev->pm.current_power_state_index == min_power_state_index) {
163 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
164 rdev->pm.dynpm_can_downclock = false;
165 } else {
166 if (rdev->pm.active_crtc_count > 1) {
167 for (i = 0; i < rdev->pm.num_power_states; i++) {
168 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
169 continue;
170 else if (i >= rdev->pm.current_power_state_index) {
171 rdev->pm.requested_power_state_index =
172 rdev->pm.current_power_state_index;
173 break;
174 } else {
175 rdev->pm.requested_power_state_index = i;
176 break;
177 }
178 }
179 } else {
180 if (rdev->pm.current_power_state_index == 0)
181 rdev->pm.requested_power_state_index =
182 rdev->pm.num_power_states - 1;
183 else
184 rdev->pm.requested_power_state_index =
185 rdev->pm.current_power_state_index - 1;
186 }
187 }
188 rdev->pm.requested_clock_mode_index = 0;
189 /* don't use the power state if crtcs are active and no display flag is set */
190 if ((rdev->pm.active_crtc_count > 0) &&
191 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
192 clock_info[rdev->pm.requested_clock_mode_index].flags &
193 RADEON_PM_MODE_NO_DISPLAY)) {
194 rdev->pm.requested_power_state_index++;
195 }
196 break;
197 case DYNPM_ACTION_UPCLOCK:
198 if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
199 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
200 rdev->pm.dynpm_can_upclock = false;
201 } else {
202 if (rdev->pm.active_crtc_count > 1) {
203 for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
204 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
205 continue;
206 else if (i <= rdev->pm.current_power_state_index) {
207 rdev->pm.requested_power_state_index =
208 rdev->pm.current_power_state_index;
209 break;
210 } else {
211 rdev->pm.requested_power_state_index = i;
212 break;
213 }
214 }
215 } else
216 rdev->pm.requested_power_state_index =
217 rdev->pm.current_power_state_index + 1;
218 }
219 rdev->pm.requested_clock_mode_index = 0;
220 break;
221 case DYNPM_ACTION_DEFAULT:
222 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
223 rdev->pm.requested_clock_mode_index = 0;
224 rdev->pm.dynpm_can_upclock = false;
225 break;
226 case DYNPM_ACTION_NONE:
227 default:
228 DRM_ERROR("Requested mode for not defined action\n");
229 return;
230 }
231 } else {
232 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
233 /* for now just select the first power state and switch between clock modes */
234 /* power state array is low to high, default is first (0) */
235 if (rdev->pm.active_crtc_count > 1) {
236 rdev->pm.requested_power_state_index = -1;
237 /* start at 1 as we don't want the default mode */
238 for (i = 1; i < rdev->pm.num_power_states; i++) {
239 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
240 continue;
241 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
242 (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
243 rdev->pm.requested_power_state_index = i;
244 break;
245 }
246 }
247 /* if nothing selected, grab the default state. */
248 if (rdev->pm.requested_power_state_index == -1)
249 rdev->pm.requested_power_state_index = 0;
250 } else
251 rdev->pm.requested_power_state_index = 1;
252
253 switch (rdev->pm.dynpm_planned_action) {
254 case DYNPM_ACTION_MINIMUM:
255 rdev->pm.requested_clock_mode_index = 0;
256 rdev->pm.dynpm_can_downclock = false;
257 break;
258 case DYNPM_ACTION_DOWNCLOCK:
259 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
260 if (rdev->pm.current_clock_mode_index == 0) {
261 rdev->pm.requested_clock_mode_index = 0;
262 rdev->pm.dynpm_can_downclock = false;
263 } else
264 rdev->pm.requested_clock_mode_index =
265 rdev->pm.current_clock_mode_index - 1;
266 } else {
267 rdev->pm.requested_clock_mode_index = 0;
268 rdev->pm.dynpm_can_downclock = false;
269 }
270 /* don't use the power state if crtcs are active and no display flag is set */
271 if ((rdev->pm.active_crtc_count > 0) &&
272 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
273 clock_info[rdev->pm.requested_clock_mode_index].flags &
274 RADEON_PM_MODE_NO_DISPLAY)) {
275 rdev->pm.requested_clock_mode_index++;
276 }
277 break;
278 case DYNPM_ACTION_UPCLOCK:
279 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
280 if (rdev->pm.current_clock_mode_index ==
281 (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
282 rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
283 rdev->pm.dynpm_can_upclock = false;
284 } else
285 rdev->pm.requested_clock_mode_index =
286 rdev->pm.current_clock_mode_index + 1;
287 } else {
288 rdev->pm.requested_clock_mode_index =
289 rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
290 rdev->pm.dynpm_can_upclock = false;
291 }
292 break;
293 case DYNPM_ACTION_DEFAULT:
294 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
295 rdev->pm.requested_clock_mode_index = 0;
296 rdev->pm.dynpm_can_upclock = false;
297 break;
298 case DYNPM_ACTION_NONE:
299 default:
300 DRM_ERROR("Requested mode for not defined action\n");
301 return;
302 }
303 }
304
305 DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
306 rdev->pm.power_state[rdev->pm.requested_power_state_index].
307 clock_info[rdev->pm.requested_clock_mode_index].sclk,
308 rdev->pm.power_state[rdev->pm.requested_power_state_index].
309 clock_info[rdev->pm.requested_clock_mode_index].mclk,
310 rdev->pm.power_state[rdev->pm.requested_power_state_index].
311 pcie_lanes);
312 }
313
314 void rs780_pm_init_profile(struct radeon_device *rdev)
315 {
316 if (rdev->pm.num_power_states == 2) {
317 /* default */
318 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
319 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
320 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
321 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
322 /* low sh */
323 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
324 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
325 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
326 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
327 /* mid sh */
328 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
329 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
330 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
331 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
332 /* high sh */
333 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
334 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
335 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
336 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
337 /* low mh */
338 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
339 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
340 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
341 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
342 /* mid mh */
343 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
344 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
345 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
346 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
347 /* high mh */
348 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
349 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
350 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
351 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
352 } else if (rdev->pm.num_power_states == 3) {
353 /* default */
354 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
355 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
356 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
357 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
358 /* low sh */
359 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
360 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
361 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
362 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
363 /* mid sh */
364 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
365 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
366 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
367 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
368 /* high sh */
369 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
370 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
371 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
372 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
373 /* low mh */
374 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
375 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
376 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
377 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
378 /* mid mh */
379 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
380 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
381 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
382 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
383 /* high mh */
384 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
385 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
386 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
387 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
388 } else {
389 /* default */
390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
391 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
392 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
393 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
394 /* low sh */
395 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
396 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
397 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
399 /* mid sh */
400 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
401 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
402 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
403 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
404 /* high sh */
405 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
406 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
407 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
408 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
409 /* low mh */
410 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
411 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
412 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
413 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
414 /* mid mh */
415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
417 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
418 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
419 /* high mh */
420 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
421 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
422 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
423 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
424 }
425 }
426
427 void r600_pm_init_profile(struct radeon_device *rdev)
428 {
429 int idx;
430
431 if (rdev->family == CHIP_R600) {
432 /* XXX */
433 /* default */
434 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
435 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
436 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
437 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
438 /* low sh */
439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
442 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
443 /* mid sh */
444 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
445 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
446 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
447 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
448 /* high sh */
449 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
450 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
451 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
452 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
453 /* low mh */
454 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
455 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
456 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
457 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
458 /* mid mh */
459 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
460 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
461 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
462 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
463 /* high mh */
464 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
465 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
466 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
468 } else {
469 if (rdev->pm.num_power_states < 4) {
470 /* default */
471 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
472 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
473 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
474 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
475 /* low sh */
476 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
477 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
478 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
479 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
480 /* mid sh */
481 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
482 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
483 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
484 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
485 /* high sh */
486 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
487 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
488 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
489 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
490 /* low mh */
491 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
492 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
493 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
494 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
495 /* low mh */
496 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
497 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
498 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
499 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
500 /* high mh */
501 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
502 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
503 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
504 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
505 } else {
506 /* default */
507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
508 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
509 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
510 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
511 /* low sh */
512 if (rdev->flags & RADEON_IS_MOBILITY)
513 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
514 else
515 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
516 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
517 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
519 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
520 /* mid sh */
521 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
525 /* high sh */
526 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
527 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
528 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
529 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
530 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
531 /* low mh */
532 if (rdev->flags & RADEON_IS_MOBILITY)
533 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
534 else
535 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
536 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
537 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
538 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
539 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
540 /* mid mh */
541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
544 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
545 /* high mh */
546 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
547 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
548 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
549 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
550 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
551 }
552 }
553 }
554
555 void r600_pm_misc(struct radeon_device *rdev)
556 {
557 int req_ps_idx = rdev->pm.requested_power_state_index;
558 int req_cm_idx = rdev->pm.requested_clock_mode_index;
559 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
560 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
561
562 if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
563 /* 0xff01 is a flag rather then an actual voltage */
564 if (voltage->voltage == 0xff01)
565 return;
566 if (voltage->voltage != rdev->pm.current_vddc) {
567 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
568 rdev->pm.current_vddc = voltage->voltage;
569 DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
570 }
571 }
572 }
573
574 bool r600_gui_idle(struct radeon_device *rdev)
575 {
576 if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
577 return false;
578 else
579 return true;
580 }
581
582 /* hpd for digital panel detect/disconnect */
583 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
584 {
585 bool connected = false;
586
587 if (ASIC_IS_DCE3(rdev)) {
588 switch (hpd) {
589 case RADEON_HPD_1:
590 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
591 connected = true;
592 break;
593 case RADEON_HPD_2:
594 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
595 connected = true;
596 break;
597 case RADEON_HPD_3:
598 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
599 connected = true;
600 break;
601 case RADEON_HPD_4:
602 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
603 connected = true;
604 break;
605 /* DCE 3.2 */
606 case RADEON_HPD_5:
607 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
608 connected = true;
609 break;
610 case RADEON_HPD_6:
611 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
612 connected = true;
613 break;
614 default:
615 break;
616 }
617 } else {
618 switch (hpd) {
619 case RADEON_HPD_1:
620 if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
621 connected = true;
622 break;
623 case RADEON_HPD_2:
624 if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
625 connected = true;
626 break;
627 case RADEON_HPD_3:
628 if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
629 connected = true;
630 break;
631 default:
632 break;
633 }
634 }
635 return connected;
636 }
637
638 void r600_hpd_set_polarity(struct radeon_device *rdev,
639 enum radeon_hpd_id hpd)
640 {
641 u32 tmp;
642 bool connected = r600_hpd_sense(rdev, hpd);
643
644 if (ASIC_IS_DCE3(rdev)) {
645 switch (hpd) {
646 case RADEON_HPD_1:
647 tmp = RREG32(DC_HPD1_INT_CONTROL);
648 if (connected)
649 tmp &= ~DC_HPDx_INT_POLARITY;
650 else
651 tmp |= DC_HPDx_INT_POLARITY;
652 WREG32(DC_HPD1_INT_CONTROL, tmp);
653 break;
654 case RADEON_HPD_2:
655 tmp = RREG32(DC_HPD2_INT_CONTROL);
656 if (connected)
657 tmp &= ~DC_HPDx_INT_POLARITY;
658 else
659 tmp |= DC_HPDx_INT_POLARITY;
660 WREG32(DC_HPD2_INT_CONTROL, tmp);
661 break;
662 case RADEON_HPD_3:
663 tmp = RREG32(DC_HPD3_INT_CONTROL);
664 if (connected)
665 tmp &= ~DC_HPDx_INT_POLARITY;
666 else
667 tmp |= DC_HPDx_INT_POLARITY;
668 WREG32(DC_HPD3_INT_CONTROL, tmp);
669 break;
670 case RADEON_HPD_4:
671 tmp = RREG32(DC_HPD4_INT_CONTROL);
672 if (connected)
673 tmp &= ~DC_HPDx_INT_POLARITY;
674 else
675 tmp |= DC_HPDx_INT_POLARITY;
676 WREG32(DC_HPD4_INT_CONTROL, tmp);
677 break;
678 case RADEON_HPD_5:
679 tmp = RREG32(DC_HPD5_INT_CONTROL);
680 if (connected)
681 tmp &= ~DC_HPDx_INT_POLARITY;
682 else
683 tmp |= DC_HPDx_INT_POLARITY;
684 WREG32(DC_HPD5_INT_CONTROL, tmp);
685 break;
686 /* DCE 3.2 */
687 case RADEON_HPD_6:
688 tmp = RREG32(DC_HPD6_INT_CONTROL);
689 if (connected)
690 tmp &= ~DC_HPDx_INT_POLARITY;
691 else
692 tmp |= DC_HPDx_INT_POLARITY;
693 WREG32(DC_HPD6_INT_CONTROL, tmp);
694 break;
695 default:
696 break;
697 }
698 } else {
699 switch (hpd) {
700 case RADEON_HPD_1:
701 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
702 if (connected)
703 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
704 else
705 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
706 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
707 break;
708 case RADEON_HPD_2:
709 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
710 if (connected)
711 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
712 else
713 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
714 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
715 break;
716 case RADEON_HPD_3:
717 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
718 if (connected)
719 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
720 else
721 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
722 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
723 break;
724 default:
725 break;
726 }
727 }
728 }
729
730 void r600_hpd_init(struct radeon_device *rdev)
731 {
732 struct drm_device *dev = rdev->ddev;
733 struct drm_connector *connector;
734 unsigned enable = 0;
735
736 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
737 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
738
739 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
740 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
741 /* don't try to enable hpd on eDP or LVDS avoid breaking the
742 * aux dp channel on imac and help (but not completely fix)
743 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
744 */
745 continue;
746 }
747 if (ASIC_IS_DCE3(rdev)) {
748 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
749 if (ASIC_IS_DCE32(rdev))
750 tmp |= DC_HPDx_EN;
751
752 switch (radeon_connector->hpd.hpd) {
753 case RADEON_HPD_1:
754 WREG32(DC_HPD1_CONTROL, tmp);
755 break;
756 case RADEON_HPD_2:
757 WREG32(DC_HPD2_CONTROL, tmp);
758 break;
759 case RADEON_HPD_3:
760 WREG32(DC_HPD3_CONTROL, tmp);
761 break;
762 case RADEON_HPD_4:
763 WREG32(DC_HPD4_CONTROL, tmp);
764 break;
765 /* DCE 3.2 */
766 case RADEON_HPD_5:
767 WREG32(DC_HPD5_CONTROL, tmp);
768 break;
769 case RADEON_HPD_6:
770 WREG32(DC_HPD6_CONTROL, tmp);
771 break;
772 default:
773 break;
774 }
775 } else {
776 switch (radeon_connector->hpd.hpd) {
777 case RADEON_HPD_1:
778 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
779 break;
780 case RADEON_HPD_2:
781 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
782 break;
783 case RADEON_HPD_3:
784 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
785 break;
786 default:
787 break;
788 }
789 }
790 enable |= 1 << radeon_connector->hpd.hpd;
791 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
792 }
793 radeon_irq_kms_enable_hpd(rdev, enable);
794 }
795
796 void r600_hpd_fini(struct radeon_device *rdev)
797 {
798 struct drm_device *dev = rdev->ddev;
799 struct drm_connector *connector;
800 unsigned disable = 0;
801
802 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
803 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
804 if (ASIC_IS_DCE3(rdev)) {
805 switch (radeon_connector->hpd.hpd) {
806 case RADEON_HPD_1:
807 WREG32(DC_HPD1_CONTROL, 0);
808 break;
809 case RADEON_HPD_2:
810 WREG32(DC_HPD2_CONTROL, 0);
811 break;
812 case RADEON_HPD_3:
813 WREG32(DC_HPD3_CONTROL, 0);
814 break;
815 case RADEON_HPD_4:
816 WREG32(DC_HPD4_CONTROL, 0);
817 break;
818 /* DCE 3.2 */
819 case RADEON_HPD_5:
820 WREG32(DC_HPD5_CONTROL, 0);
821 break;
822 case RADEON_HPD_6:
823 WREG32(DC_HPD6_CONTROL, 0);
824 break;
825 default:
826 break;
827 }
828 } else {
829 switch (radeon_connector->hpd.hpd) {
830 case RADEON_HPD_1:
831 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
832 break;
833 case RADEON_HPD_2:
834 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
835 break;
836 case RADEON_HPD_3:
837 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
838 break;
839 default:
840 break;
841 }
842 }
843 disable |= 1 << radeon_connector->hpd.hpd;
844 }
845 radeon_irq_kms_disable_hpd(rdev, disable);
846 }
847
848 /*
849 * R600 PCIE GART
850 */
851 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
852 {
853 unsigned i;
854 u32 tmp;
855
856 /* flush hdp cache so updates hit vram */
857 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
858 !(rdev->flags & RADEON_IS_AGP)) {
859 volatile uint32_t *ptr = rdev->gart.ptr;
860 u32 tmp;
861
862 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
863 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
864 * This seems to cause problems on some AGP cards. Just use the old
865 * method for them.
866 */
867 WREG32(HDP_DEBUG1, 0);
868 tmp = *ptr;
869 } else
870 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
871
872 WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
873 WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
874 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
875 for (i = 0; i < rdev->usec_timeout; i++) {
876 /* read MC_STATUS */
877 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
878 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
879 if (tmp == 2) {
880 DRM_ERROR("[drm] r600 flush TLB failed\n");
881 return;
882 }
883 if (tmp) {
884 return;
885 }
886 DRM_UDELAY(1);
887 }
888 }
889
890 int r600_pcie_gart_init(struct radeon_device *rdev)
891 {
892 int r;
893
894 if (rdev->gart.robj) {
895 DRM_ERROR("R600 PCIE GART already initialized\n");
896 return 0;
897 }
898 /* Initialize common gart structure */
899 r = radeon_gart_init(rdev);
900 if (r)
901 return r;
902 rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
903 return radeon_gart_table_vram_alloc(rdev);
904 }
905
906 static int r600_pcie_gart_enable(struct radeon_device *rdev)
907 {
908 u32 tmp;
909 int r, i;
910
911 if (rdev->gart.robj == NULL) {
912 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
913 return -EINVAL;
914 }
915 r = radeon_gart_table_vram_pin(rdev);
916 if (r)
917 return r;
918 radeon_gart_restore(rdev);
919
920 /* Setup L2 cache */
921 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
922 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
923 EFFECTIVE_L2_QUEUE_SIZE(7));
924 WREG32(VM_L2_CNTL2, 0);
925 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
926 /* Setup TLB control */
927 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
928 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
929 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
930 ENABLE_WAIT_L2_QUERY;
931 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
932 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
933 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
934 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
935 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
936 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
937 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
938 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
939 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
940 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
941 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
942 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
943 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
944 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
945 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
946 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
947 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
948 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
949 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
950 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
951 (u32)(rdev->dummy_page.addr >> 12));
952 for (i = 1; i < 7; i++)
953 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
954
955 r600_pcie_gart_tlb_flush(rdev);
956 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
957 (unsigned)(rdev->mc.gtt_size >> 20),
958 (unsigned long long)rdev->gart.table_addr);
959 rdev->gart.ready = true;
960 return 0;
961 }
962
963 static void r600_pcie_gart_disable(struct radeon_device *rdev)
964 {
965 u32 tmp;
966 int i;
967
968 /* Disable all tables */
969 for (i = 0; i < 7; i++)
970 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
971
972 /* Disable L2 cache */
973 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
974 EFFECTIVE_L2_QUEUE_SIZE(7));
975 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
976 /* Setup L1 TLB control */
977 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
978 ENABLE_WAIT_L2_QUERY;
979 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
980 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
981 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
982 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
983 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
984 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
985 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
986 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
987 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
988 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
989 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
990 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
991 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
992 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
993 radeon_gart_table_vram_unpin(rdev);
994 }
995
996 static void r600_pcie_gart_fini(struct radeon_device *rdev)
997 {
998 radeon_gart_fini(rdev);
999 r600_pcie_gart_disable(rdev);
1000 radeon_gart_table_vram_free(rdev);
1001 }
1002
1003 static void r600_agp_enable(struct radeon_device *rdev)
1004 {
1005 u32 tmp;
1006 int i;
1007
1008 /* Setup L2 cache */
1009 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1010 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1011 EFFECTIVE_L2_QUEUE_SIZE(7));
1012 WREG32(VM_L2_CNTL2, 0);
1013 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1014 /* Setup TLB control */
1015 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1016 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1017 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1018 ENABLE_WAIT_L2_QUERY;
1019 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1020 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1021 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1022 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1023 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1024 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1025 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1026 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1027 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1028 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1029 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1030 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1031 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1032 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1033 for (i = 0; i < 7; i++)
1034 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1035 }
1036
1037 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1038 {
1039 unsigned i;
1040 u32 tmp;
1041
1042 for (i = 0; i < rdev->usec_timeout; i++) {
1043 /* read MC_STATUS */
1044 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1045 if (!tmp)
1046 return 0;
1047 DRM_UDELAY(1);
1048 }
1049 return -1;
1050 }
1051
1052 uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
1053 {
1054 uint32_t r;
1055
1056 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
1057 r = RREG32(R_0028FC_MC_DATA);
1058 WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
1059 return r;
1060 }
1061
1062 void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
1063 {
1064 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) |
1065 S_0028F8_MC_IND_WR_EN(1));
1066 WREG32(R_0028FC_MC_DATA, v);
1067 WREG32(R_0028F8_MC_INDEX, 0x7F);
1068 }
1069
1070 static void r600_mc_program(struct radeon_device *rdev)
1071 {
1072 struct rv515_mc_save save;
1073 u32 tmp;
1074 int i, j;
1075
1076 /* Initialize HDP */
1077 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1078 WREG32((0x2c14 + j), 0x00000000);
1079 WREG32((0x2c18 + j), 0x00000000);
1080 WREG32((0x2c1c + j), 0x00000000);
1081 WREG32((0x2c20 + j), 0x00000000);
1082 WREG32((0x2c24 + j), 0x00000000);
1083 }
1084 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1085
1086 rv515_mc_stop(rdev, &save);
1087 if (r600_mc_wait_for_idle(rdev)) {
1088 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1089 }
1090 /* Lockout access through VGA aperture (doesn't exist before R600) */
1091 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1092 /* Update configuration */
1093 if (rdev->flags & RADEON_IS_AGP) {
1094 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1095 /* VRAM before AGP */
1096 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1097 rdev->mc.vram_start >> 12);
1098 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1099 rdev->mc.gtt_end >> 12);
1100 } else {
1101 /* VRAM after AGP */
1102 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1103 rdev->mc.gtt_start >> 12);
1104 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1105 rdev->mc.vram_end >> 12);
1106 }
1107 } else {
1108 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1109 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1110 }
1111 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1112 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1113 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1114 WREG32(MC_VM_FB_LOCATION, tmp);
1115 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1116 WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1117 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1118 if (rdev->flags & RADEON_IS_AGP) {
1119 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1120 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1121 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1122 } else {
1123 WREG32(MC_VM_AGP_BASE, 0);
1124 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1125 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1126 }
1127 if (r600_mc_wait_for_idle(rdev)) {
1128 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1129 }
1130 rv515_mc_resume(rdev, &save);
1131 /* we need to own VRAM, so turn off the VGA renderer here
1132 * to stop it overwriting our objects */
1133 rv515_vga_render_disable(rdev);
1134 }
1135
1136 /**
1137 * r600_vram_gtt_location - try to find VRAM & GTT location
1138 * @rdev: radeon device structure holding all necessary informations
1139 * @mc: memory controller structure holding memory informations
1140 *
1141 * Function will place try to place VRAM at same place as in CPU (PCI)
1142 * address space as some GPU seems to have issue when we reprogram at
1143 * different address space.
1144 *
1145 * If there is not enough space to fit the unvisible VRAM after the
1146 * aperture then we limit the VRAM size to the aperture.
1147 *
1148 * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1149 * them to be in one from GPU point of view so that we can program GPU to
1150 * catch access outside them (weird GPU policy see ??).
1151 *
1152 * This function will never fails, worst case are limiting VRAM or GTT.
1153 *
1154 * Note: GTT start, end, size should be initialized before calling this
1155 * function on AGP platform.
1156 */
1157 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1158 {
1159 u64 size_bf, size_af;
1160
1161 if (mc->mc_vram_size > 0xE0000000) {
1162 /* leave room for at least 512M GTT */
1163 dev_warn(rdev->dev, "limiting VRAM\n");
1164 mc->real_vram_size = 0xE0000000;
1165 mc->mc_vram_size = 0xE0000000;
1166 }
1167 if (rdev->flags & RADEON_IS_AGP) {
1168 size_bf = mc->gtt_start;
1169 size_af = mc->mc_mask - mc->gtt_end;
1170 if (size_bf > size_af) {
1171 if (mc->mc_vram_size > size_bf) {
1172 dev_warn(rdev->dev, "limiting VRAM\n");
1173 mc->real_vram_size = size_bf;
1174 mc->mc_vram_size = size_bf;
1175 }
1176 mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1177 } else {
1178 if (mc->mc_vram_size > size_af) {
1179 dev_warn(rdev->dev, "limiting VRAM\n");
1180 mc->real_vram_size = size_af;
1181 mc->mc_vram_size = size_af;
1182 }
1183 mc->vram_start = mc->gtt_end + 1;
1184 }
1185 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1186 dev_info(rdev->dev, "VRAM: %juM 0x%08jX - 0x%08jX (%juM used)\n",
1187 (uintmax_t)mc->mc_vram_size >> 20, (uintmax_t)mc->vram_start,
1188 (uintmax_t)mc->vram_end, (uintmax_t)mc->real_vram_size >> 20);
1189 } else {
1190 u64 base = 0;
1191 if (rdev->flags & RADEON_IS_IGP) {
1192 base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1193 base <<= 24;
1194 }
1195 radeon_vram_location(rdev, &rdev->mc, base);
1196 rdev->mc.gtt_base_align = 0;
1197 radeon_gtt_location(rdev, mc);
1198 }
1199 }
1200
1201 static int r600_mc_init(struct radeon_device *rdev)
1202 {
1203 u32 tmp;
1204 int chansize, numchan;
1205 uint32_t h_addr, l_addr;
1206 unsigned long long k8_addr;
1207
1208 /* Get VRAM informations */
1209 rdev->mc.vram_is_ddr = true;
1210 tmp = RREG32(RAMCFG);
1211 if (tmp & CHANSIZE_OVERRIDE) {
1212 chansize = 16;
1213 } else if (tmp & CHANSIZE_MASK) {
1214 chansize = 64;
1215 } else {
1216 chansize = 32;
1217 }
1218 tmp = RREG32(CHMAP);
1219 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1220 case 0:
1221 default:
1222 numchan = 1;
1223 break;
1224 case 1:
1225 numchan = 2;
1226 break;
1227 case 2:
1228 numchan = 4;
1229 break;
1230 case 3:
1231 numchan = 8;
1232 break;
1233 }
1234 rdev->mc.vram_width = numchan * chansize;
1235 /* Could aper size report 0 ? */
1236 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
1237 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
1238 /* Setup GPU memory space */
1239 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1240 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1241 rdev->mc.visible_vram_size = rdev->mc.aper_size;
1242 r600_vram_gtt_location(rdev, &rdev->mc);
1243
1244 if (rdev->flags & RADEON_IS_IGP) {
1245 rs690_pm_info(rdev);
1246 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1247
1248 if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {
1249 /* Use K8 direct mapping for fast fb access. */
1250 rdev->fastfb_working = false;
1251 h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
1252 l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
1253 k8_addr = ((unsigned long long)h_addr) << 32 | l_addr;
1254 #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1255 if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
1256 #endif
1257 {
1258 /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
1259 * memory is present.
1260 */
1261 if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
1262 DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
1263 (unsigned long long)rdev->mc.aper_base, k8_addr);
1264 rdev->mc.aper_base = (resource_size_t)k8_addr;
1265 rdev->fastfb_working = true;
1266 }
1267 }
1268 }
1269 }
1270
1271 radeon_update_bandwidth_info(rdev);
1272 return 0;
1273 }
1274
1275 int r600_vram_scratch_init(struct radeon_device *rdev)
1276 {
1277 int r;
1278 void *vram_scratch_ptr_ptr;
1279
1280 if (rdev->vram_scratch.robj == NULL) {
1281 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1282 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1283 NULL, &rdev->vram_scratch.robj);
1284 if (r) {
1285 return r;
1286 }
1287 }
1288
1289 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1290 if (unlikely(r != 0)) {
1291 radeon_bo_unref(&rdev->vram_scratch.robj);
1292 return r;
1293 }
1294 r = radeon_bo_pin(rdev->vram_scratch.robj,
1295 RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1296 if (r) {
1297 radeon_bo_unreserve(rdev->vram_scratch.robj);
1298 radeon_bo_unref(&rdev->vram_scratch.robj);
1299 return r;
1300 }
1301 vram_scratch_ptr_ptr = &rdev->vram_scratch.ptr;
1302 r = radeon_bo_kmap(rdev->vram_scratch.robj,
1303 vram_scratch_ptr_ptr);
1304 if (r)
1305 radeon_bo_unpin(rdev->vram_scratch.robj);
1306 radeon_bo_unreserve(rdev->vram_scratch.robj);
1307 if (r)
1308 radeon_bo_unref(&rdev->vram_scratch.robj);
1309
1310 return r;
1311 }
1312
1313 void r600_vram_scratch_fini(struct radeon_device *rdev)
1314 {
1315 int r;
1316
1317 if (rdev->vram_scratch.robj == NULL) {
1318 return;
1319 }
1320 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1321 if (likely(r == 0)) {
1322 radeon_bo_kunmap(rdev->vram_scratch.robj);
1323 radeon_bo_unpin(rdev->vram_scratch.robj);
1324 radeon_bo_unreserve(rdev->vram_scratch.robj);
1325 }
1326 radeon_bo_unref(&rdev->vram_scratch.robj);
1327 }
1328
1329 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1330 {
1331 u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1332
1333 if (hung)
1334 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1335 else
1336 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1337
1338 WREG32(R600_BIOS_3_SCRATCH, tmp);
1339 }
1340
1341 static void r600_print_gpu_status_regs(struct radeon_device *rdev)
1342 {
1343 dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
1344 RREG32(R_008010_GRBM_STATUS));
1345 dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
1346 RREG32(R_008014_GRBM_STATUS2));
1347 dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
1348 RREG32(R_000E50_SRBM_STATUS));
1349 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1350 RREG32(CP_STALLED_STAT1));
1351 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1352 RREG32(CP_STALLED_STAT2));
1353 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
1354 RREG32(CP_BUSY_STAT));
1355 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
1356 RREG32(CP_STAT));
1357 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
1358 RREG32(DMA_STATUS_REG));
1359 }
1360
1361 static bool r600_is_display_hung(struct radeon_device *rdev)
1362 {
1363 u32 crtc_hung = 0;
1364 u32 crtc_status[2];
1365 u32 i, j, tmp;
1366
1367 for (i = 0; i < rdev->num_crtc; i++) {
1368 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1369 crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1370 crtc_hung |= (1 << i);
1371 }
1372 }
1373
1374 for (j = 0; j < 10; j++) {
1375 for (i = 0; i < rdev->num_crtc; i++) {
1376 if (crtc_hung & (1 << i)) {
1377 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1378 if (tmp != crtc_status[i])
1379 crtc_hung &= ~(1 << i);
1380 }
1381 }
1382 if (crtc_hung == 0)
1383 return false;
1384 DRM_UDELAY(100);
1385 }
1386
1387 return true;
1388 }
1389
1390 static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1391 {
1392 u32 reset_mask = 0;
1393 u32 tmp;
1394
1395 /* GRBM_STATUS */
1396 tmp = RREG32(R_008010_GRBM_STATUS);
1397 if (rdev->family >= CHIP_RV770) {
1398 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1399 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1400 G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1401 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1402 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1403 reset_mask |= RADEON_RESET_GFX;
1404 } else {
1405 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1406 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1407 G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1408 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1409 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1410 reset_mask |= RADEON_RESET_GFX;
1411 }
1412
1413 if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1414 G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1415 reset_mask |= RADEON_RESET_CP;
1416
1417 if (G_008010_GRBM_EE_BUSY(tmp))
1418 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1419
1420 /* DMA_STATUS_REG */
1421 tmp = RREG32(DMA_STATUS_REG);
1422 if (!(tmp & DMA_IDLE))
1423 reset_mask |= RADEON_RESET_DMA;
1424
1425 /* SRBM_STATUS */
1426 tmp = RREG32(R_000E50_SRBM_STATUS);
1427 if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1428 reset_mask |= RADEON_RESET_RLC;
1429
1430 if (G_000E50_IH_BUSY(tmp))
1431 reset_mask |= RADEON_RESET_IH;
1432
1433 if (G_000E50_SEM_BUSY(tmp))
1434 reset_mask |= RADEON_RESET_SEM;
1435
1436 if (G_000E50_GRBM_RQ_PENDING(tmp))
1437 reset_mask |= RADEON_RESET_GRBM;
1438
1439 if (G_000E50_VMC_BUSY(tmp))
1440 reset_mask |= RADEON_RESET_VMC;
1441
1442 if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1443 G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1444 G_000E50_MCDW_BUSY(tmp))
1445 reset_mask |= RADEON_RESET_MC;
1446
1447 if (r600_is_display_hung(rdev))
1448 reset_mask |= RADEON_RESET_DISPLAY;
1449
1450 /* Skip MC reset as it's mostly likely not hung, just busy */
1451 if (reset_mask & RADEON_RESET_MC) {
1452 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1453 reset_mask &= ~RADEON_RESET_MC;
1454 }
1455
1456 return reset_mask;
1457 }
1458
1459 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1460 {
1461 struct rv515_mc_save save;
1462 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1463 u32 tmp;
1464
1465 if (reset_mask == 0)
1466 return;
1467
1468 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1469
1470 r600_print_gpu_status_regs(rdev);
1471
1472 /* Disable CP parsing/prefetching */
1473 if (rdev->family >= CHIP_RV770)
1474 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1475 else
1476 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1477
1478 /* disable the RLC */
1479 WREG32(RLC_CNTL, 0);
1480
1481 if (reset_mask & RADEON_RESET_DMA) {
1482 /* Disable DMA */
1483 tmp = RREG32(DMA_RB_CNTL);
1484 tmp &= ~DMA_RB_ENABLE;
1485 WREG32(DMA_RB_CNTL, tmp);
1486 }
1487
1488 DRM_MDELAY(50);
1489
1490 rv515_mc_stop(rdev, &save);
1491 if (r600_mc_wait_for_idle(rdev)) {
1492 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1493 }
1494
1495 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1496 if (rdev->family >= CHIP_RV770)
1497 grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1498 S_008020_SOFT_RESET_CB(1) |
1499 S_008020_SOFT_RESET_PA(1) |
1500 S_008020_SOFT_RESET_SC(1) |
1501 S_008020_SOFT_RESET_SPI(1) |
1502 S_008020_SOFT_RESET_SX(1) |
1503 S_008020_SOFT_RESET_SH(1) |
1504 S_008020_SOFT_RESET_TC(1) |
1505 S_008020_SOFT_RESET_TA(1) |
1506 S_008020_SOFT_RESET_VC(1) |
1507 S_008020_SOFT_RESET_VGT(1);
1508 else
1509 grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1510 S_008020_SOFT_RESET_DB(1) |
1511 S_008020_SOFT_RESET_CB(1) |
1512 S_008020_SOFT_RESET_PA(1) |
1513 S_008020_SOFT_RESET_SC(1) |
1514 S_008020_SOFT_RESET_SMX(1) |
1515 S_008020_SOFT_RESET_SPI(1) |
1516 S_008020_SOFT_RESET_SX(1) |
1517 S_008020_SOFT_RESET_SH(1) |
1518 S_008020_SOFT_RESET_TC(1) |
1519 S_008020_SOFT_RESET_TA(1) |
1520 S_008020_SOFT_RESET_VC(1) |
1521 S_008020_SOFT_RESET_VGT(1);
1522 }
1523
1524 if (reset_mask & RADEON_RESET_CP) {
1525 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1526 S_008020_SOFT_RESET_VGT(1);
1527
1528 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1529 }
1530
1531 if (reset_mask & RADEON_RESET_DMA) {
1532 if (rdev->family >= CHIP_RV770)
1533 srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1534 else
1535 srbm_soft_reset |= SOFT_RESET_DMA;
1536 }
1537
1538 if (reset_mask & RADEON_RESET_RLC)
1539 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1540
1541 if (reset_mask & RADEON_RESET_SEM)
1542 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1543
1544 if (reset_mask & RADEON_RESET_IH)
1545 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1546
1547 if (reset_mask & RADEON_RESET_GRBM)
1548 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1549
1550 if (!(rdev->flags & RADEON_IS_IGP)) {
1551 if (reset_mask & RADEON_RESET_MC)
1552 srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1553 }
1554
1555 if (reset_mask & RADEON_RESET_VMC)
1556 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1557
1558 if (grbm_soft_reset) {
1559 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1560 tmp |= grbm_soft_reset;
1561 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1562 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1563 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1564
1565 DRM_UDELAY(50);
1566
1567 tmp &= ~grbm_soft_reset;
1568 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1569 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1570 }
1571
1572 if (srbm_soft_reset) {
1573 tmp = RREG32(SRBM_SOFT_RESET);
1574 tmp |= srbm_soft_reset;
1575 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1576 WREG32(SRBM_SOFT_RESET, tmp);
1577 tmp = RREG32(SRBM_SOFT_RESET);
1578
1579 DRM_UDELAY(50);
1580
1581 tmp &= ~srbm_soft_reset;
1582 WREG32(SRBM_SOFT_RESET, tmp);
1583 tmp = RREG32(SRBM_SOFT_RESET);
1584 }
1585
1586 /* Wait a little for things to settle down */
1587 DRM_MDELAY(1);
1588
1589 rv515_mc_resume(rdev, &save);
1590 DRM_UDELAY(50);
1591
1592 r600_print_gpu_status_regs(rdev);
1593 }
1594
1595 int r600_asic_reset(struct radeon_device *rdev)
1596 {
1597 u32 reset_mask;
1598
1599 reset_mask = r600_gpu_check_soft_reset(rdev);
1600
1601 if (reset_mask)
1602 r600_set_bios_scratch_engine_hung(rdev, true);
1603
1604 r600_gpu_soft_reset(rdev, reset_mask);
1605
1606 reset_mask = r600_gpu_check_soft_reset(rdev);
1607
1608 if (!reset_mask)
1609 r600_set_bios_scratch_engine_hung(rdev, false);
1610
1611 return 0;
1612 }
1613
1614 /**
1615 * r600_gfx_is_lockup - Check if the GFX engine is locked up
1616 *
1617 * @rdev: radeon_device pointer
1618 * @ring: radeon_ring structure holding ring information
1619 *
1620 * Check if the GFX engine is locked up.
1621 * Returns true if the engine appears to be locked up, false if not.
1622 */
1623 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1624 {
1625 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1626
1627 if (!(reset_mask & (RADEON_RESET_GFX |
1628 RADEON_RESET_COMPUTE |
1629 RADEON_RESET_CP))) {
1630 radeon_ring_lockup_update(ring);
1631 return false;
1632 }
1633 /* force CP activities */
1634 radeon_ring_force_activity(rdev, ring);
1635 return radeon_ring_test_lockup(rdev, ring);
1636 }
1637
1638 /**
1639 * r600_dma_is_lockup - Check if the DMA engine is locked up
1640 *
1641 * @rdev: radeon_device pointer
1642 * @ring: radeon_ring structure holding ring information
1643 *
1644 * Check if the async DMA engine is locked up.
1645 * Returns true if the engine appears to be locked up, false if not.
1646 */
1647 bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1648 {
1649 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1650
1651 if (!(reset_mask & RADEON_RESET_DMA)) {
1652 radeon_ring_lockup_update(ring);
1653 return false;
1654 }
1655 /* force ring activities */
1656 radeon_ring_force_activity(rdev, ring);
1657 return radeon_ring_test_lockup(rdev, ring);
1658 }
1659
1660 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1661 u32 tiling_pipe_num,
1662 u32 max_rb_num,
1663 u32 total_max_rb_num,
1664 u32 disabled_rb_mask)
1665 {
1666 u32 rendering_pipe_num, rb_num_width, req_rb_num;
1667 u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1668 u32 data = 0, mask = 1 << (max_rb_num - 1);
1669 unsigned i, j;
1670
1671 /* mask out the RBs that don't exist on that asic */
1672 tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1673 /* make sure at least one RB is available */
1674 if ((tmp & 0xff) != 0xff)
1675 disabled_rb_mask = tmp;
1676
1677 rendering_pipe_num = 1 << tiling_pipe_num;
1678 req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1679 KASSERT(rendering_pipe_num >= req_rb_num, ("rendering_pipe_num < req_rb_num"));
1680
1681 pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1682 pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1683
1684 if (rdev->family <= CHIP_RV740) {
1685 /* r6xx/r7xx */
1686 rb_num_width = 2;
1687 } else {
1688 /* eg+ */
1689 rb_num_width = 4;
1690 }
1691
1692 for (i = 0; i < max_rb_num; i++) {
1693 if (!(mask & disabled_rb_mask)) {
1694 for (j = 0; j < pipe_rb_ratio; j++) {
1695 data <<= rb_num_width;
1696 data |= max_rb_num - i - 1;
1697 }
1698 if (pipe_rb_remain) {
1699 data <<= rb_num_width;
1700 data |= max_rb_num - i - 1;
1701 pipe_rb_remain--;
1702 }
1703 }
1704 mask >>= 1;
1705 }
1706
1707 return data;
1708 }
1709
1710 int r600_count_pipe_bits(uint32_t val)
1711 {
1712 return hweight32(val);
1713 }
1714
1715 static void r600_gpu_init(struct radeon_device *rdev)
1716 {
1717 u32 tiling_config;
1718 u32 ramcfg;
1719 u32 cc_rb_backend_disable;
1720 u32 cc_gc_shader_pipe_config;
1721 u32 tmp;
1722 int i, j;
1723 u32 sq_config;
1724 u32 sq_gpr_resource_mgmt_1 = 0;
1725 u32 sq_gpr_resource_mgmt_2 = 0;
1726 u32 sq_thread_resource_mgmt = 0;
1727 u32 sq_stack_resource_mgmt_1 = 0;
1728 u32 sq_stack_resource_mgmt_2 = 0;
1729 u32 disabled_rb_mask;
1730
1731 rdev->config.r600.tiling_group_size = 256;
1732 switch (rdev->family) {
1733 case CHIP_R600:
1734 rdev->config.r600.max_pipes = 4;
1735 rdev->config.r600.max_tile_pipes = 8;
1736 rdev->config.r600.max_simds = 4;
1737 rdev->config.r600.max_backends = 4;
1738 rdev->config.r600.max_gprs = 256;
1739 rdev->config.r600.max_threads = 192;
1740 rdev->config.r600.max_stack_entries = 256;
1741 rdev->config.r600.max_hw_contexts = 8;
1742 rdev->config.r600.max_gs_threads = 16;
1743 rdev->config.r600.sx_max_export_size = 128;
1744 rdev->config.r600.sx_max_export_pos_size = 16;
1745 rdev->config.r600.sx_max_export_smx_size = 128;
1746 rdev->config.r600.sq_num_cf_insts = 2;
1747 break;
1748 case CHIP_RV630:
1749 case CHIP_RV635:
1750 rdev->config.r600.max_pipes = 2;
1751 rdev->config.r600.max_tile_pipes = 2;
1752 rdev->config.r600.max_simds = 3;
1753 rdev->config.r600.max_backends = 1;
1754 rdev->config.r600.max_gprs = 128;
1755 rdev->config.r600.max_threads = 192;
1756 rdev->config.r600.max_stack_entries = 128;
1757 rdev->config.r600.max_hw_contexts = 8;
1758 rdev->config.r600.max_gs_threads = 4;
1759 rdev->config.r600.sx_max_export_size = 128;
1760 rdev->config.r600.sx_max_export_pos_size = 16;
1761 rdev->config.r600.sx_max_export_smx_size = 128;
1762 rdev->config.r600.sq_num_cf_insts = 2;
1763 break;
1764 case CHIP_RV610:
1765 case CHIP_RV620:
1766 case CHIP_RS780:
1767 case CHIP_RS880:
1768 rdev->config.r600.max_pipes = 1;
1769 rdev->config.r600.max_tile_pipes = 1;
1770 rdev->config.r600.max_simds = 2;
1771 rdev->config.r600.max_backends = 1;
1772 rdev->config.r600.max_gprs = 128;
1773 rdev->config.r600.max_threads = 192;
1774 rdev->config.r600.max_stack_entries = 128;
1775 rdev->config.r600.max_hw_contexts = 4;
1776 rdev->config.r600.max_gs_threads = 4;
1777 rdev->config.r600.sx_max_export_size = 128;
1778 rdev->config.r600.sx_max_export_pos_size = 16;
1779 rdev->config.r600.sx_max_export_smx_size = 128;
1780 rdev->config.r600.sq_num_cf_insts = 1;
1781 break;
1782 case CHIP_RV670:
1783 rdev->config.r600.max_pipes = 4;
1784 rdev->config.r600.max_tile_pipes = 4;
1785 rdev->config.r600.max_simds = 4;
1786 rdev->config.r600.max_backends = 4;
1787 rdev->config.r600.max_gprs = 192;
1788 rdev->config.r600.max_threads = 192;
1789 rdev->config.r600.max_stack_entries = 256;
1790 rdev->config.r600.max_hw_contexts = 8;
1791 rdev->config.r600.max_gs_threads = 16;
1792 rdev->config.r600.sx_max_export_size = 128;
1793 rdev->config.r600.sx_max_export_pos_size = 16;
1794 rdev->config.r600.sx_max_export_smx_size = 128;
1795 rdev->config.r600.sq_num_cf_insts = 2;
1796 break;
1797 default:
1798 break;
1799 }
1800
1801 /* Initialize HDP */
1802 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1803 WREG32((0x2c14 + j), 0x00000000);
1804 WREG32((0x2c18 + j), 0x00000000);
1805 WREG32((0x2c1c + j), 0x00000000);
1806 WREG32((0x2c20 + j), 0x00000000);
1807 WREG32((0x2c24 + j), 0x00000000);
1808 }
1809
1810 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1811
1812 /* Setup tiling */
1813 tiling_config = 0;
1814 ramcfg = RREG32(RAMCFG);
1815 switch (rdev->config.r600.max_tile_pipes) {
1816 case 1:
1817 tiling_config |= PIPE_TILING(0);
1818 break;
1819 case 2:
1820 tiling_config |= PIPE_TILING(1);
1821 break;
1822 case 4:
1823 tiling_config |= PIPE_TILING(2);
1824 break;
1825 case 8:
1826 tiling_config |= PIPE_TILING(3);
1827 break;
1828 default:
1829 break;
1830 }
1831 rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1832 rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1833 tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1834 tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1835
1836 tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1837 if (tmp > 3) {
1838 tiling_config |= ROW_TILING(3);
1839 tiling_config |= SAMPLE_SPLIT(3);
1840 } else {
1841 tiling_config |= ROW_TILING(tmp);
1842 tiling_config |= SAMPLE_SPLIT(tmp);
1843 }
1844 tiling_config |= BANK_SWAPS(1);
1845
1846 cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1847 tmp = R6XX_MAX_BACKENDS -
1848 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1849 if (tmp < rdev->config.r600.max_backends) {
1850 rdev->config.r600.max_backends = tmp;
1851 }
1852
1853 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1854 tmp = R6XX_MAX_PIPES -
1855 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1856 if (tmp < rdev->config.r600.max_pipes) {
1857 rdev->config.r600.max_pipes = tmp;
1858 }
1859 tmp = R6XX_MAX_SIMDS -
1860 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1861 if (tmp < rdev->config.r600.max_simds) {
1862 rdev->config.r600.max_simds = tmp;
1863 }
1864
1865 disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1866 tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1867 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1868 R6XX_MAX_BACKENDS, disabled_rb_mask);
1869 tiling_config |= tmp << 16;
1870 rdev->config.r600.backend_map = tmp;
1871
1872 rdev->config.r600.tile_config = tiling_config;
1873 WREG32(GB_TILING_CONFIG, tiling_config);
1874 WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1875 WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1876 WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1877
1878 tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1879 WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1880 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1881
1882 /* Setup some CP states */
1883 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1884 WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1885
1886 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1887 SYNC_WALKER | SYNC_ALIGNER));
1888 /* Setup various GPU states */
1889 if (rdev->family == CHIP_RV670)
1890 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1891
1892 tmp = RREG32(SX_DEBUG_1);
1893 tmp |= SMX_EVENT_RELEASE;
1894 if ((rdev->family > CHIP_R600))
1895 tmp |= ENABLE_NEW_SMX_ADDRESS;
1896 WREG32(SX_DEBUG_1, tmp);
1897
1898 if (((rdev->family) == CHIP_R600) ||
1899 ((rdev->family) == CHIP_RV630) ||
1900 ((rdev->family) == CHIP_RV610) ||
1901 ((rdev->family) == CHIP_RV620) ||
1902 ((rdev->family) == CHIP_RS780) ||
1903 ((rdev->family) == CHIP_RS880)) {
1904 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1905 } else {
1906 WREG32(DB_DEBUG, 0);
1907 }
1908 WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1909 DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1910
1911 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1912 WREG32(VGT_NUM_INSTANCES, 0);
1913
1914 WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1915 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1916
1917 tmp = RREG32(SQ_MS_FIFO_SIZES);
1918 if (((rdev->family) == CHIP_RV610) ||
1919 ((rdev->family) == CHIP_RV620) ||
1920 ((rdev->family) == CHIP_RS780) ||
1921 ((rdev->family) == CHIP_RS880)) {
1922 tmp = (CACHE_FIFO_SIZE(0xa) |
1923 FETCH_FIFO_HIWATER(0xa) |
1924 DONE_FIFO_HIWATER(0xe0) |
1925 ALU_UPDATE_FIFO_HIWATER(0x8));
1926 } else if (((rdev->family) == CHIP_R600) ||
1927 ((rdev->family) == CHIP_RV630)) {
1928 tmp &= ~DONE_FIFO_HIWATER(0xff);
1929 tmp |= DONE_FIFO_HIWATER(0x4);
1930 }
1931 WREG32(SQ_MS_FIFO_SIZES, tmp);
1932
1933 /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1934 * should be adjusted as needed by the 2D/3D drivers. This just sets default values
1935 */
1936 sq_config = RREG32(SQ_CONFIG);
1937 sq_config &= ~(PS_PRIO(3) |
1938 VS_PRIO(3) |
1939 GS_PRIO(3) |
1940 ES_PRIO(3));
1941 sq_config |= (DX9_CONSTS |
1942 VC_ENABLE |
1943 PS_PRIO(0) |
1944 VS_PRIO(1) |
1945 GS_PRIO(2) |
1946 ES_PRIO(3));
1947
1948 if ((rdev->family) == CHIP_R600) {
1949 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1950 NUM_VS_GPRS(124) |
1951 NUM_CLAUSE_TEMP_GPRS(4));
1952 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1953 NUM_ES_GPRS(0));
1954 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1955 NUM_VS_THREADS(48) |
1956 NUM_GS_THREADS(4) |
1957 NUM_ES_THREADS(4));
1958 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1959 NUM_VS_STACK_ENTRIES(128));
1960 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1961 NUM_ES_STACK_ENTRIES(0));
1962 } else if (((rdev->family) == CHIP_RV610) ||
1963 ((rdev->family) == CHIP_RV620) ||
1964 ((rdev->family) == CHIP_RS780) ||
1965 ((rdev->family) == CHIP_RS880)) {
1966 /* no vertex cache */
1967 sq_config &= ~VC_ENABLE;
1968
1969 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1970 NUM_VS_GPRS(44) |
1971 NUM_CLAUSE_TEMP_GPRS(2));
1972 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1973 NUM_ES_GPRS(17));
1974 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1975 NUM_VS_THREADS(78) |
1976 NUM_GS_THREADS(4) |
1977 NUM_ES_THREADS(31));
1978 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1979 NUM_VS_STACK_ENTRIES(40));
1980 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1981 NUM_ES_STACK_ENTRIES(16));
1982 } else if (((rdev->family) == CHIP_RV630) ||
1983 ((rdev->family) == CHIP_RV635)) {
1984 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1985 NUM_VS_GPRS(44) |
1986 NUM_CLAUSE_TEMP_GPRS(2));
1987 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1988 NUM_ES_GPRS(18));
1989 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1990 NUM_VS_THREADS(78) |
1991 NUM_GS_THREADS(4) |
1992 NUM_ES_THREADS(31));
1993 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1994 NUM_VS_STACK_ENTRIES(40));
1995 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1996 NUM_ES_STACK_ENTRIES(16));
1997 } else if ((rdev->family) == CHIP_RV670) {
1998 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1999 NUM_VS_GPRS(44) |
2000 NUM_CLAUSE_TEMP_GPRS(2));
2001 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
2002 NUM_ES_GPRS(17));
2003 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2004 NUM_VS_THREADS(78) |
2005 NUM_GS_THREADS(4) |
2006 NUM_ES_THREADS(31));
2007 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
2008 NUM_VS_STACK_ENTRIES(64));
2009 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
2010 NUM_ES_STACK_ENTRIES(64));
2011 }
2012
2013 WREG32(SQ_CONFIG, sq_config);
2014 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2015 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2016 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2017 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2018 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2019
2020 if (((rdev->family) == CHIP_RV610) ||
2021 ((rdev->family) == CHIP_RV620) ||
2022 ((rdev->family) == CHIP_RS780) ||
2023 ((rdev->family) == CHIP_RS880)) {
2024 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
2025 } else {
2026 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
2027 }
2028
2029 /* More default values. 2D/3D driver should adjust as needed */
2030 WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
2031 S1_X(0x4) | S1_Y(0xc)));
2032 WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
2033 S1_X(0x2) | S1_Y(0x2) |
2034 S2_X(0xa) | S2_Y(0x6) |
2035 S3_X(0x6) | S3_Y(0xa)));
2036 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
2037 S1_X(0x4) | S1_Y(0xc) |
2038 S2_X(0x1) | S2_Y(0x6) |
2039 S3_X(0xa) | S3_Y(0xe)));
2040 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
2041 S5_X(0x0) | S5_Y(0x0) |
2042 S6_X(0xb) | S6_Y(0x4) |
2043 S7_X(0x7) | S7_Y(0x8)));
2044
2045 WREG32(VGT_STRMOUT_EN, 0);
2046 tmp = rdev->config.r600.max_pipes * 16;
2047 switch (rdev->family) {
2048 case CHIP_RV610:
2049 case CHIP_RV620:
2050 case CHIP_RS780:
2051 case CHIP_RS880:
2052 tmp += 32;
2053 break;
2054 case CHIP_RV670:
2055 tmp += 128;
2056 break;
2057 default:
2058 break;
2059 }
2060 if (tmp > 256) {
2061 tmp = 256;
2062 }
2063 WREG32(VGT_ES_PER_GS, 128);
2064 WREG32(VGT_GS_PER_ES, tmp);
2065 WREG32(VGT_GS_PER_VS, 2);
2066 WREG32(VGT_GS_VERTEX_REUSE, 16);
2067
2068 /* more default values. 2D/3D driver should adjust as needed */
2069 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2070 WREG32(VGT_STRMOUT_EN, 0);
2071 WREG32(SX_MISC, 0);
2072 WREG32(PA_SC_MODE_CNTL, 0);
2073 WREG32(PA_SC_AA_CONFIG, 0);
2074 WREG32(PA_SC_LINE_STIPPLE, 0);
2075 WREG32(SPI_INPUT_Z, 0);
2076 WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2077 WREG32(CB_COLOR7_FRAG, 0);
2078
2079 /* Clear render buffer base addresses */
2080 WREG32(CB_COLOR0_BASE, 0);
2081 WREG32(CB_COLOR1_BASE, 0);
2082 WREG32(CB_COLOR2_BASE, 0);
2083 WREG32(CB_COLOR3_BASE, 0);
2084 WREG32(CB_COLOR4_BASE, 0);
2085 WREG32(CB_COLOR5_BASE, 0);
2086 WREG32(CB_COLOR6_BASE, 0);
2087 WREG32(CB_COLOR7_BASE, 0);
2088 WREG32(CB_COLOR7_FRAG, 0);
2089
2090 switch (rdev->family) {
2091 case CHIP_RV610:
2092 case CHIP_RV620:
2093 case CHIP_RS780:
2094 case CHIP_RS880:
2095 tmp = TC_L2_SIZE(8);
2096 break;
2097 case CHIP_RV630:
2098 case CHIP_RV635:
2099 tmp = TC_L2_SIZE(4);
2100 break;
2101 case CHIP_R600:
2102 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2103 break;
2104 default:
2105 tmp = TC_L2_SIZE(0);
2106 break;
2107 }
2108 WREG32(TC_CNTL, tmp);
2109
2110 tmp = RREG32(HDP_HOST_PATH_CNTL);
2111 WREG32(HDP_HOST_PATH_CNTL, tmp);
2112
2113 tmp = RREG32(ARB_POP);
2114 tmp |= ENABLE_TC128;
2115 WREG32(ARB_POP, tmp);
2116
2117 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2118 WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2119 NUM_CLIP_SEQ(3)));
2120 WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2121 WREG32(VC_ENHANCE, 0);
2122 }
2123
2124
2125 /*
2126 * Indirect registers accessor
2127 */
2128 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2129 {
2130 u32 r;
2131
2132 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2133 (void)RREG32(PCIE_PORT_INDEX);
2134 r = RREG32(PCIE_PORT_DATA);
2135 return r;
2136 }
2137
2138 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2139 {
2140 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2141 (void)RREG32(PCIE_PORT_INDEX);
2142 WREG32(PCIE_PORT_DATA, (v));
2143 (void)RREG32(PCIE_PORT_DATA);
2144 }
2145
2146 /*
2147 * CP & Ring
2148 */
2149 void r600_cp_stop(struct radeon_device *rdev)
2150 {
2151 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2152 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
2153 WREG32(SCRATCH_UMSK, 0);
2154 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2155 }
2156
2157 int r600_init_microcode(struct radeon_device *rdev)
2158 {
2159 const char *chip_name;
2160 const char *rlc_chip_name;
2161 size_t pfp_req_size, me_req_size, rlc_req_size;
2162 char fw_name[30];
2163 int err;
2164
2165 DRM_DEBUG("\n");
2166
2167 switch (rdev->family) {
2168 case CHIP_R600:
2169 chip_name = "R600";
2170 rlc_chip_name = "R600";
2171 break;
2172 case CHIP_RV610:
2173 chip_name = "RV610";
2174 rlc_chip_name = "R600";
2175 break;
2176 case CHIP_RV630:
2177 chip_name = "RV630";
2178 rlc_chip_name = "R600";
2179 break;
2180 case CHIP_RV620:
2181 chip_name = "RV620";
2182 rlc_chip_name = "R600";
2183 break;
2184 case CHIP_RV635:
2185 chip_name = "RV635";
2186 rlc_chip_name = "R600";
2187 break;
2188 case CHIP_RV670:
2189 chip_name = "RV670";
2190 rlc_chip_name = "R600";
2191 break;
2192 case CHIP_RS780:
2193 case CHIP_RS880:
2194 chip_name = "RS780";
2195 rlc_chip_name = "R600";
2196 break;
2197 case CHIP_RV770:
2198 chip_name = "RV770";
2199 rlc_chip_name = "R700";
2200 break;
2201 case CHIP_RV730:
2202 case CHIP_RV740:
2203 chip_name = "RV730";
2204 rlc_chip_name = "R700";
2205 break;
2206 case CHIP_RV710:
2207 chip_name = "RV710";
2208 rlc_chip_name = "R700";
2209 break;
2210 case CHIP_CEDAR:
2211 chip_name = "CEDAR";
2212 rlc_chip_name = "CEDAR";
2213 break;
2214 case CHIP_REDWOOD:
2215 chip_name = "REDWOOD";
2216 rlc_chip_name = "REDWOOD";
2217 break;
2218 case CHIP_JUNIPER:
2219 chip_name = "JUNIPER";
2220 rlc_chip_name = "JUNIPER";
2221 break;
2222 case CHIP_CYPRESS:
2223 case CHIP_HEMLOCK:
2224 chip_name = "CYPRESS";
2225 rlc_chip_name = "CYPRESS";
2226 break;
2227 case CHIP_PALM:
2228 chip_name = "PALM";
2229 rlc_chip_name = "SUMO";
2230 break;
2231 case CHIP_SUMO:
2232 chip_name = "SUMO";
2233 rlc_chip_name = "SUMO";
2234 break;
2235 case CHIP_SUMO2:
2236 chip_name = "SUMO2";
2237 rlc_chip_name = "SUMO";
2238 break;
2239 default: panic("%s: Unsupported family %d", __func__, rdev->family);
2240 }
2241
2242 if (rdev->family >= CHIP_CEDAR) {
2243 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2244 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2245 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2246 } else if (rdev->family >= CHIP_RV770) {
2247 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2248 me_req_size = R700_PM4_UCODE_SIZE * 4;
2249 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2250 } else {
2251 pfp_req_size = PFP_UCODE_SIZE * 4;
2252 me_req_size = PM4_UCODE_SIZE * 12;
2253 rlc_req_size = RLC_UCODE_SIZE * 4;
2254 }
2255
2256 DRM_INFO("Loading %s Microcode\n", chip_name);
2257 err = 0;
2258
2259 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
2260 rdev->pfp_fw = firmware_get(fw_name);
2261 if (rdev->pfp_fw == NULL) {
2262 err = -ENOENT;
2263 goto out;
2264 }
2265 if (rdev->pfp_fw->datasize != pfp_req_size) {
2266 DRM_ERROR(
2267 "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2268 rdev->pfp_fw->datasize, fw_name);
2269 err = -EINVAL;
2270 goto out;
2271 }
2272
2273 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
2274 rdev->me_fw = firmware_get(fw_name);
2275 if (rdev->me_fw == NULL) {
2276 err = -ENOENT;
2277 goto out;
2278 }
2279 if (rdev->me_fw->datasize != me_req_size) {
2280 DRM_ERROR(
2281 "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2282 rdev->me_fw->datasize, fw_name);
2283 err = -EINVAL;
2284 }
2285
2286 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc",
2287 rlc_chip_name);
2288 rdev->rlc_fw = firmware_get(fw_name);
2289 if (rdev->rlc_fw == NULL) {
2290 err = -ENOENT;
2291 goto out;
2292 }
2293 if (rdev->rlc_fw->datasize != rlc_req_size) {
2294 DRM_ERROR(
2295 "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2296 rdev->rlc_fw->datasize, fw_name);
2297 err = -EINVAL;
2298 }
2299
2300 out:
2301 if (err) {
2302 if (err != -EINVAL)
2303 DRM_ERROR(
2304 "r600_cp: Failed to load firmware \"%s\"\n",
2305 fw_name);
2306 if (rdev->pfp_fw != NULL) {
2307 firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
2308 rdev->pfp_fw = NULL;
2309 }
2310 if (rdev->me_fw != NULL) {
2311 firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
2312 rdev->me_fw = NULL;
2313 }
2314 if (rdev->rlc_fw != NULL) {
2315 firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
2316 rdev->rlc_fw = NULL;
2317 }
2318 }
2319 return err;
2320 }
2321
2322 /**
2323 * r600_fini_microcode - drop the firmwares image references
2324 *
2325 * @rdev: radeon_device pointer
2326 *
2327 * Drop the pfp, me and rlc firmwares image references.
2328 * Called at driver shutdown.
2329 */
2330 void r600_fini_microcode(struct radeon_device *rdev)
2331 {
2332
2333 if (rdev->pfp_fw != NULL) {
2334 firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
2335 rdev->pfp_fw = NULL;
2336 }
2337
2338 if (rdev->me_fw != NULL) {
2339 firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
2340 rdev->me_fw = NULL;
2341 }
2342
2343 if (rdev->rlc_fw != NULL) {
2344 firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
2345 rdev->rlc_fw = NULL;
2346 }
2347 }
2348
2349 static int r600_cp_load_microcode(struct radeon_device *rdev)
2350 {
2351 const __be32 *fw_data;
2352 int i;
2353
2354 if (!rdev->me_fw || !rdev->pfp_fw)
2355 return -EINVAL;
2356
2357 r600_cp_stop(rdev);
2358
2359 WREG32(CP_RB_CNTL,
2360 #ifdef __BIG_ENDIAN
2361 BUF_SWAP_32BIT |
2362 #endif
2363 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2364
2365 /* Reset cp */
2366 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2367 RREG32(GRBM_SOFT_RESET);
2368 DRM_MDELAY(15);
2369 WREG32(GRBM_SOFT_RESET, 0);
2370
2371 WREG32(CP_ME_RAM_WADDR, 0);
2372
2373 fw_data = (const __be32 *)rdev->me_fw->data;
2374 WREG32(CP_ME_RAM_WADDR, 0);
2375 for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
2376 WREG32(CP_ME_RAM_DATA,
2377 be32_to_cpup(fw_data++));
2378
2379 fw_data = (const __be32 *)rdev->pfp_fw->data;
2380 WREG32(CP_PFP_UCODE_ADDR, 0);
2381 for (i = 0; i < PFP_UCODE_SIZE; i++)
2382 WREG32(CP_PFP_UCODE_DATA,
2383 be32_to_cpup(fw_data++));
2384
2385 WREG32(CP_PFP_UCODE_ADDR, 0);
2386 WREG32(CP_ME_RAM_WADDR, 0);
2387 WREG32(CP_ME_RAM_RADDR, 0);
2388 return 0;
2389 }
2390
2391 int r600_cp_start(struct radeon_device *rdev)
2392 {
2393 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2394 int r;
2395 uint32_t cp_me;
2396
2397 r = radeon_ring_lock(rdev, ring, 7);
2398 if (r) {
2399 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2400 return r;
2401 }
2402 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2403 radeon_ring_write(ring, 0x1);
2404 if (rdev->family >= CHIP_RV770) {
2405 radeon_ring_write(ring, 0x0);
2406 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2407 } else {
2408 radeon_ring_write(ring, 0x3);
2409 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2410 }
2411 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2412 radeon_ring_write(ring, 0);
2413 radeon_ring_write(ring, 0);
2414 radeon_ring_unlock_commit(rdev, ring);
2415
2416 cp_me = 0xff;
2417 WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2418 return 0;
2419 }
2420
2421 int r600_cp_resume(struct radeon_device *rdev)
2422 {
2423 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2424 u32 tmp;
2425 u32 rb_bufsz;
2426 int r;
2427
2428 /* Reset cp */
2429 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2430 RREG32(GRBM_SOFT_RESET);
2431 DRM_MDELAY(15);
2432 WREG32(GRBM_SOFT_RESET, 0);
2433
2434 /* Set ring buffer size */
2435 rb_bufsz = drm_order(ring->ring_size / 8);
2436 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2437 #ifdef __BIG_ENDIAN
2438 tmp |= BUF_SWAP_32BIT;
2439 #endif
2440 WREG32(CP_RB_CNTL, tmp);
2441 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2442
2443 /* Set the write pointer delay */
2444 WREG32(CP_RB_WPTR_DELAY, 0);
2445
2446 /* Initialize the ring buffer's read and write pointers */
2447 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2448 WREG32(CP_RB_RPTR_WR, 0);
2449 ring->wptr = 0;
2450 WREG32(CP_RB_WPTR, ring->wptr);
2451
2452 /* set the wb address whether it's enabled or not */
2453 WREG32(CP_RB_RPTR_ADDR,
2454 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2455 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2456 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2457
2458 if (rdev->wb.enabled)
2459 WREG32(SCRATCH_UMSK, 0xff);
2460 else {
2461 tmp |= RB_NO_UPDATE;
2462 WREG32(SCRATCH_UMSK, 0);
2463 }
2464
2465 DRM_MDELAY(1);
2466 WREG32(CP_RB_CNTL, tmp);
2467
2468 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2469 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2470
2471 ring->rptr = RREG32(CP_RB_RPTR);
2472
2473 r600_cp_start(rdev);
2474 ring->ready = true;
2475 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2476 if (r) {
2477 ring->ready = false;
2478 return r;
2479 }
2480 return 0;
2481 }
2482
2483 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2484 {
2485 u32 rb_bufsz;
2486 int r;
2487
2488 /* Align ring size */
2489 rb_bufsz = drm_order(ring_size / 8);
2490 ring_size = (1 << (rb_bufsz + 1)) * 4;
2491 ring->ring_size = ring_size;
2492 ring->align_mask = 16 - 1;
2493
2494 if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2495 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2496 if (r) {
2497 DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2498 ring->rptr_save_reg = 0;
2499 }
2500 }
2501 }
2502
2503 void r600_cp_fini(struct radeon_device *rdev)
2504 {
2505 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2506 r600_cp_stop(rdev);
2507 radeon_ring_fini(rdev, ring);
2508 radeon_scratch_free(rdev, ring->rptr_save_reg);
2509 }
2510
2511 /*
2512 * DMA
2513 * Starting with R600, the GPU has an asynchronous
2514 * DMA engine. The programming model is very similar
2515 * to the 3D engine (ring buffer, IBs, etc.), but the
2516 * DMA controller has it's own packet format that is
2517 * different form the PM4 format used by the 3D engine.
2518 * It supports copying data, writing embedded data,
2519 * solid fills, and a number of other things. It also
2520 * has support for tiling/detiling of buffers.
2521 */
2522 /**
2523 * r600_dma_stop - stop the async dma engine
2524 *
2525 * @rdev: radeon_device pointer
2526 *
2527 * Stop the async dma engine (r6xx-evergreen).
2528 */
2529 void r600_dma_stop(struct radeon_device *rdev)
2530 {
2531 u32 rb_cntl = RREG32(DMA_RB_CNTL);
2532
2533 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2534
2535 rb_cntl &= ~DMA_RB_ENABLE;
2536 WREG32(DMA_RB_CNTL, rb_cntl);
2537
2538 rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2539 }
2540
2541 /**
2542 * r600_dma_resume - setup and start the async dma engine
2543 *
2544 * @rdev: radeon_device pointer
2545 *
2546 * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2547 * Returns 0 for success, error for failure.
2548 */
2549 int r600_dma_resume(struct radeon_device *rdev)
2550 {
2551 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2552 u32 rb_cntl, dma_cntl, ib_cntl;
2553 u32 rb_bufsz;
2554 int r;
2555
2556 /* Reset dma */
2557 if (rdev->family >= CHIP_RV770)
2558 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2559 else
2560 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2561 RREG32(SRBM_SOFT_RESET);
2562 DRM_UDELAY(50);
2563 WREG32(SRBM_SOFT_RESET, 0);
2564
2565 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2566 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2567
2568 /* Set ring buffer size in dwords */
2569 rb_bufsz = drm_order(ring->ring_size / 4);
2570 rb_cntl = rb_bufsz << 1;
2571 #ifdef __BIG_ENDIAN
2572 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2573 #endif
2574 WREG32(DMA_RB_CNTL, rb_cntl);
2575
2576 /* Initialize the ring buffer's read and write pointers */
2577 WREG32(DMA_RB_RPTR, 0);
2578 WREG32(DMA_RB_WPTR, 0);
2579
2580 /* set the wb address whether it's enabled or not */
2581 WREG32(DMA_RB_RPTR_ADDR_HI,
2582 upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2583 WREG32(DMA_RB_RPTR_ADDR_LO,
2584 ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2585
2586 if (rdev->wb.enabled)
2587 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2588
2589 WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2590
2591 /* enable DMA IBs */
2592 ib_cntl = DMA_IB_ENABLE;
2593 #ifdef __BIG_ENDIAN
2594 ib_cntl |= DMA_IB_SWAP_ENABLE;
2595 #endif
2596 WREG32(DMA_IB_CNTL, ib_cntl);
2597
2598 dma_cntl = RREG32(DMA_CNTL);
2599 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2600 WREG32(DMA_CNTL, dma_cntl);
2601
2602 if (rdev->family >= CHIP_RV770)
2603 WREG32(DMA_MODE, 1);
2604
2605 ring->wptr = 0;
2606 WREG32(DMA_RB_WPTR, ring->wptr << 2);
2607
2608 ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2609
2610 WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2611
2612 ring->ready = true;
2613
2614 r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2615 if (r) {
2616 ring->ready = false;
2617 return r;
2618 }
2619
2620 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2621
2622 return 0;
2623 }
2624
2625 /**
2626 * r600_dma_fini - tear down the async dma engine
2627 *
2628 * @rdev: radeon_device pointer
2629 *
2630 * Stop the async dma engine and free the ring (r6xx-evergreen).
2631 */
2632 void r600_dma_fini(struct radeon_device *rdev)
2633 {
2634 r600_dma_stop(rdev);
2635 radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2636 }
2637
2638 /*
2639 * UVD
2640 */
2641 int r600_uvd_rbc_start(struct radeon_device *rdev)
2642 {
2643 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2644 uint64_t rptr_addr;
2645 uint32_t rb_bufsz, tmp;
2646 int r;
2647
2648 rptr_addr = rdev->wb.gpu_addr + R600_WB_UVD_RPTR_OFFSET;
2649
2650 if (upper_32_bits(rptr_addr) != upper_32_bits(ring->gpu_addr)) {
2651 DRM_ERROR("UVD ring and rptr not in the same 4GB segment!\n");
2652 return -EINVAL;
2653 }
2654
2655 /* force RBC into idle state */
2656 WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2657
2658 /* Set the write pointer delay */
2659 WREG32(UVD_RBC_RB_WPTR_CNTL, 0);
2660
2661 /* set the wb address */
2662 WREG32(UVD_RBC_RB_RPTR_ADDR, rptr_addr >> 2);
2663
2664 /* programm the 4GB memory segment for rptr and ring buffer */
2665 WREG32(UVD_LMI_EXT40_ADDR, upper_32_bits(rptr_addr) |
2666 (0x7 << 16) | (0x1 << 31));
2667
2668 /* Initialize the ring buffer's read and write pointers */
2669 WREG32(UVD_RBC_RB_RPTR, 0x0);
2670
2671 ring->wptr = ring->rptr = RREG32(UVD_RBC_RB_RPTR);
2672 WREG32(UVD_RBC_RB_WPTR, ring->wptr);
2673
2674 /* set the ring address */
2675 WREG32(UVD_RBC_RB_BASE, ring->gpu_addr);
2676
2677 /* Set ring buffer size */
2678 rb_bufsz = drm_order(ring->ring_size);
2679 rb_bufsz = (0x1 << 8) | rb_bufsz;
2680 WREG32(UVD_RBC_RB_CNTL, rb_bufsz);
2681
2682 ring->ready = true;
2683 r = radeon_ring_test(rdev, R600_RING_TYPE_UVD_INDEX, ring);
2684 if (r) {
2685 ring->ready = false;
2686 return r;
2687 }
2688
2689 r = radeon_ring_lock(rdev, ring, 10);
2690 if (r) {
2691 DRM_ERROR("radeon: ring failed to lock UVD ring (%d).\n", r);
2692 return r;
2693 }
2694
2695 tmp = PACKET0(UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0);
2696 radeon_ring_write(ring, tmp);
2697 radeon_ring_write(ring, 0xFFFFF);
2698
2699 tmp = PACKET0(UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0);
2700 radeon_ring_write(ring, tmp);
2701 radeon_ring_write(ring, 0xFFFFF);
2702
2703 tmp = PACKET0(UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0);
2704 radeon_ring_write(ring, tmp);
2705 radeon_ring_write(ring, 0xFFFFF);
2706
2707 /* Clear timeout status bits */
2708 radeon_ring_write(ring, PACKET0(UVD_SEMA_TIMEOUT_STATUS, 0));
2709 radeon_ring_write(ring, 0x8);
2710
2711 radeon_ring_write(ring, PACKET0(UVD_SEMA_CNTL, 0));
2712 radeon_ring_write(ring, 3);
2713
2714 radeon_ring_unlock_commit(rdev, ring);
2715
2716 return 0;
2717 }
2718
2719 void r600_uvd_rbc_stop(struct radeon_device *rdev)
2720 {
2721 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2722
2723 /* force RBC into idle state */
2724 WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2725 ring->ready = false;
2726 }
2727
2728 int r600_uvd_init(struct radeon_device *rdev)
2729 {
2730 int i, j, r;
2731 /* disable byte swapping */
2732 u32 lmi_swap_cntl = 0;
2733 u32 mp_swap_cntl = 0;
2734
2735 /* raise clocks while booting up the VCPU */
2736 radeon_set_uvd_clocks(rdev, 53300, 40000);
2737
2738 /* disable clock gating */
2739 WREG32(UVD_CGC_GATE, 0);
2740
2741 /* disable interupt */
2742 WREG32_P(UVD_MASTINT_EN, 0, ~(1 << 1));
2743
2744 /* put LMI, VCPU, RBC etc... into reset */
2745 WREG32(UVD_SOFT_RESET, LMI_SOFT_RESET | VCPU_SOFT_RESET |
2746 LBSI_SOFT_RESET | RBC_SOFT_RESET | CSM_SOFT_RESET |
2747 CXW_SOFT_RESET | TAP_SOFT_RESET | LMI_UMC_SOFT_RESET);
2748 DRM_MDELAY(5);
2749
2750 /* take UVD block out of reset */
2751 WREG32_P(SRBM_SOFT_RESET, 0, ~SOFT_RESET_UVD);
2752 DRM_MDELAY(5);
2753
2754 /* initialize UVD memory controller */
2755 WREG32(UVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) |
2756 (1 << 21) | (1 << 9) | (1 << 20));
2757
2758 #ifdef __BIG_ENDIAN
2759 /* swap (8 in 32) RB and IB */
2760 lmi_swap_cntl = 0xa;
2761 mp_swap_cntl = 0;
2762 #endif
2763 WREG32(UVD_LMI_SWAP_CNTL, lmi_swap_cntl);
2764 WREG32(UVD_MP_SWAP_CNTL, mp_swap_cntl);
2765
2766 WREG32(UVD_MPC_SET_MUXA0, 0x40c2040);
2767 WREG32(UVD_MPC_SET_MUXA1, 0x0);
2768 WREG32(UVD_MPC_SET_MUXB0, 0x40c2040);
2769 WREG32(UVD_MPC_SET_MUXB1, 0x0);
2770 WREG32(UVD_MPC_SET_ALU, 0);
2771 WREG32(UVD_MPC_SET_MUX, 0x88);
2772
2773 /* Stall UMC */
2774 WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
2775 WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3));
2776
2777 /* take all subblocks out of reset, except VCPU */
2778 WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET);
2779 DRM_MDELAY(5);
2780
2781 /* enable VCPU clock */
2782 WREG32(UVD_VCPU_CNTL, 1 << 9);
2783
2784 /* enable UMC */
2785 WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8));
2786
2787 /* boot up the VCPU */
2788 WREG32(UVD_SOFT_RESET, 0);
2789 DRM_MDELAY(10);
2790
2791 WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3));
2792
2793 for (i = 0; i < 10; ++i) {
2794 uint32_t status;
2795 for (j = 0; j < 100; ++j) {
2796 status = RREG32(UVD_STATUS);
2797 if (status & 2)
2798 break;
2799 DRM_MDELAY(10);
2800 }
2801 r = 0;
2802 if (status & 2)
2803 break;
2804
2805 DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
2806 WREG32_P(UVD_SOFT_RESET, VCPU_SOFT_RESET, ~VCPU_SOFT_RESET);
2807 DRM_MDELAY(10);
2808 WREG32_P(UVD_SOFT_RESET, 0, ~VCPU_SOFT_RESET);
2809 DRM_MDELAY(10);
2810 r = -1;
2811 }
2812
2813 if (r) {
2814 DRM_ERROR("UVD not responding, giving up!!!\n");
2815 radeon_set_uvd_clocks(rdev, 0, 0);
2816 return r;
2817 }
2818
2819 /* enable interupt */
2820 WREG32_P(UVD_MASTINT_EN, 3<<1, ~(3 << 1));
2821
2822 r = r600_uvd_rbc_start(rdev);
2823 if (!r)
2824 DRM_INFO("UVD initialized successfully.\n");
2825
2826 /* lower clocks again */
2827 radeon_set_uvd_clocks(rdev, 0, 0);
2828
2829 return r;
2830 }
2831
2832 /*
2833 * GPU scratch registers helpers function.
2834 */
2835 void r600_scratch_init(struct radeon_device *rdev)
2836 {
2837 int i;
2838
2839 rdev->scratch.num_reg = 7;
2840 rdev->scratch.reg_base = SCRATCH_REG0;
2841 for (i = 0; i < rdev->scratch.num_reg; i++) {
2842 rdev->scratch.free[i] = true;
2843 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2844 }
2845 }
2846
2847 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2848 {
2849 uint32_t scratch;
2850 uint32_t tmp = 0;
2851 unsigned i;
2852 int r;
2853
2854 r = radeon_scratch_get(rdev, &scratch);
2855 if (r) {
2856 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2857 return r;
2858 }
2859 WREG32(scratch, 0xCAFEDEAD);
2860 r = radeon_ring_lock(rdev, ring, 3);
2861 if (r) {
2862 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2863 radeon_scratch_free(rdev, scratch);
2864 return r;
2865 }
2866 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2867 radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2868 radeon_ring_write(ring, 0xDEADBEEF);
2869 radeon_ring_unlock_commit(rdev, ring);
2870 for (i = 0; i < rdev->usec_timeout; i++) {
2871 tmp = RREG32(scratch);
2872 if (tmp == 0xDEADBEEF)
2873 break;
2874 DRM_UDELAY(1);
2875 }
2876 if (i < rdev->usec_timeout) {
2877 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2878 } else {
2879 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2880 ring->idx, scratch, tmp);
2881 r = -EINVAL;
2882 }
2883 radeon_scratch_free(rdev, scratch);
2884 return r;
2885 }
2886
2887 /**
2888 * r600_dma_ring_test - simple async dma engine test
2889 *
2890 * @rdev: radeon_device pointer
2891 * @ring: radeon_ring structure holding ring information
2892 *
2893 * Test the DMA engine by writing using it to write an
2894 * value to memory. (r6xx-SI).
2895 * Returns 0 for success, error for failure.
2896 */
2897 int r600_dma_ring_test(struct radeon_device *rdev,
2898 struct radeon_ring *ring)
2899 {
2900 unsigned i;
2901 int r;
2902 volatile uint32_t *ptr = rdev->vram_scratch.ptr;
2903 u32 tmp;
2904
2905 if (!ptr) {
2906 DRM_ERROR("invalid vram scratch pointer\n");
2907 return -EINVAL;
2908 }
2909
2910 tmp = 0xCAFEDEAD;
2911 *ptr = tmp;
2912
2913 r = radeon_ring_lock(rdev, ring, 4);
2914 if (r) {
2915 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2916 return r;
2917 }
2918 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2919 radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2920 radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2921 radeon_ring_write(ring, 0xDEADBEEF);
2922 radeon_ring_unlock_commit(rdev, ring);
2923
2924 for (i = 0; i < rdev->usec_timeout; i++) {
2925 tmp = *ptr;
2926 if (tmp == 0xDEADBEEF)
2927 break;
2928 DRM_UDELAY(1);
2929 }
2930
2931 if (i < rdev->usec_timeout) {
2932 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2933 } else {
2934 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2935 ring->idx, tmp);
2936 r = -EINVAL;
2937 }
2938 return r;
2939 }
2940
2941 int r600_uvd_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2942 {
2943 uint32_t tmp = 0;
2944 unsigned i;
2945 int r;
2946
2947 WREG32(UVD_CONTEXT_ID, 0xCAFEDEAD);
2948 r = radeon_ring_lock(rdev, ring, 3);
2949 if (r) {
2950 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n",
2951 ring->idx, r);
2952 return r;
2953 }
2954 radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
2955 radeon_ring_write(ring, 0xDEADBEEF);
2956 radeon_ring_unlock_commit(rdev, ring);
2957 for (i = 0; i < rdev->usec_timeout; i++) {
2958 tmp = RREG32(UVD_CONTEXT_ID);
2959 if (tmp == 0xDEADBEEF)
2960 break;
2961 DRM_UDELAY(1);
2962 }
2963
2964 if (i < rdev->usec_timeout) {
2965 DRM_INFO("ring test on %d succeeded in %d usecs\n",
2966 ring->idx, i);
2967 } else {
2968 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2969 ring->idx, tmp);
2970 r = -EINVAL;
2971 }
2972 return r;
2973 }
2974
2975 /*
2976 * CP fences/semaphores
2977 */
2978
2979 void r600_fence_ring_emit(struct radeon_device *rdev,
2980 struct radeon_fence *fence)
2981 {
2982 struct radeon_ring *ring = &rdev->ring[fence->ring];
2983
2984 if (rdev->wb.use_event) {
2985 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2986 /* flush read cache over gart */
2987 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2988 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2989 PACKET3_VC_ACTION_ENA |
2990 PACKET3_SH_ACTION_ENA);
2991 radeon_ring_write(ring, 0xFFFFFFFF);
2992 radeon_ring_write(ring, 0);
2993 radeon_ring_write(ring, 10); /* poll interval */
2994 /* EVENT_WRITE_EOP - flush caches, send int */
2995 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2996 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2997 radeon_ring_write(ring, addr & 0xffffffff);
2998 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2999 radeon_ring_write(ring, fence->seq);
3000 radeon_ring_write(ring, 0);
3001 } else {
3002 /* flush read cache over gart */
3003 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3004 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
3005 PACKET3_VC_ACTION_ENA |
3006 PACKET3_SH_ACTION_ENA);
3007 radeon_ring_write(ring, 0xFFFFFFFF);
3008 radeon_ring_write(ring, 0);
3009 radeon_ring_write(ring, 10); /* poll interval */
3010 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
3011 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
3012 /* wait for 3D idle clean */
3013 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3014 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3015 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
3016 /* Emit fence sequence & fire IRQ */
3017 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3018 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3019 radeon_ring_write(ring, fence->seq);
3020 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
3021 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
3022 radeon_ring_write(ring, RB_INT_STAT);
3023 }
3024 }
3025
3026 void r600_uvd_fence_emit(struct radeon_device *rdev,
3027 struct radeon_fence *fence)
3028 {
3029 struct radeon_ring *ring = &rdev->ring[fence->ring];
3030 uint32_t addr = rdev->fence_drv[fence->ring].gpu_addr;
3031
3032 radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
3033 radeon_ring_write(ring, fence->seq);
3034 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
3035 radeon_ring_write(ring, addr & 0xffffffff);
3036 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
3037 radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
3038 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
3039 radeon_ring_write(ring, 0);
3040
3041 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
3042 radeon_ring_write(ring, 0);
3043 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
3044 radeon_ring_write(ring, 0);
3045 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
3046 radeon_ring_write(ring, 2);
3047 return;
3048 }
3049
3050 void r600_semaphore_ring_emit(struct radeon_device *rdev,
3051 struct radeon_ring *ring,
3052 struct radeon_semaphore *semaphore,
3053 bool emit_wait)
3054 {
3055 uint64_t addr = semaphore->gpu_addr;
3056 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3057
3058 if (rdev->family < CHIP_CAYMAN)
3059 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
3060
3061 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3062 radeon_ring_write(ring, addr & 0xffffffff);
3063 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
3064 }
3065
3066 /*
3067 * DMA fences/semaphores
3068 */
3069
3070 /**
3071 * r600_dma_fence_ring_emit - emit a fence on the DMA ring
3072 *
3073 * @rdev: radeon_device pointer
3074 * @fence: radeon fence object
3075 *
3076 * Add a DMA fence packet to the ring to write
3077 * the fence seq number and DMA trap packet to generate
3078 * an interrupt if needed (r6xx-r7xx).
3079 */
3080 void r600_dma_fence_ring_emit(struct radeon_device *rdev,
3081 struct radeon_fence *fence)
3082 {
3083 struct radeon_ring *ring = &rdev->ring[fence->ring];
3084 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3085
3086 /* write the fence */
3087 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
3088 radeon_ring_write(ring, addr & 0xfffffffc);
3089 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
3090 radeon_ring_write(ring, lower_32_bits(fence->seq));
3091 /* generate an interrupt */
3092 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
3093 }
3094
3095 /**
3096 * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
3097 *
3098 * @rdev: radeon_device pointer
3099 * @ring: radeon_ring structure holding ring information
3100 * @semaphore: radeon semaphore object
3101 * @emit_wait: wait or signal semaphore
3102 *
3103 * Add a DMA semaphore packet to the ring wait on or signal
3104 * other rings (r6xx-SI).
3105 */
3106 void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
3107 struct radeon_ring *ring,
3108 struct radeon_semaphore *semaphore,
3109 bool emit_wait)
3110 {
3111 u64 addr = semaphore->gpu_addr;
3112 u32 s = emit_wait ? 0 : 1;
3113
3114 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
3115 radeon_ring_write(ring, addr & 0xfffffffc);
3116 radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
3117 }
3118
3119 void r600_uvd_semaphore_emit(struct radeon_device *rdev,
3120 struct radeon_ring *ring,
3121 struct radeon_semaphore *semaphore,
3122 bool emit_wait)
3123 {
3124 uint64_t addr = semaphore->gpu_addr;
3125
3126 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
3127 radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
3128
3129 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
3130 radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
3131
3132 radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
3133 radeon_ring_write(ring, emit_wait ? 1 : 0);
3134 }
3135
3136 int r600_copy_blit(struct radeon_device *rdev,
3137 uint64_t src_offset,
3138 uint64_t dst_offset,
3139 unsigned num_gpu_pages,
3140 struct radeon_fence **fence)
3141 {
3142 struct radeon_semaphore *sem = NULL;
3143 struct radeon_sa_bo *vb = NULL;
3144 int r;
3145
3146 r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
3147 if (r) {
3148 return r;
3149 }
3150 r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
3151 r600_blit_done_copy(rdev, fence, vb, sem);
3152 return 0;
3153 }
3154
3155 /**
3156 * r600_copy_dma - copy pages using the DMA engine
3157 *
3158 * @rdev: radeon_device pointer
3159 * @src_offset: src GPU address
3160 * @dst_offset: dst GPU address
3161 * @num_gpu_pages: number of GPU pages to xfer
3162 * @fence: radeon fence object
3163 *
3164 * Copy GPU paging using the DMA engine (r6xx).
3165 * Used by the radeon ttm implementation to move pages if
3166 * registered as the asic copy callback.
3167 */
3168 int r600_copy_dma(struct radeon_device *rdev,
3169 uint64_t src_offset, uint64_t dst_offset,
3170 unsigned num_gpu_pages,
3171 struct radeon_fence **fence)
3172 {
3173 struct radeon_semaphore *sem = NULL;
3174 int ring_index = rdev->asic->copy.dma_ring_index;
3175 struct radeon_ring *ring = &rdev->ring[ring_index];
3176 u32 size_in_dw, cur_size_in_dw;
3177 int i, num_loops;
3178 int r = 0;
3179
3180 r = radeon_semaphore_create(rdev, &sem);
3181 if (r) {
3182 DRM_ERROR("radeon: moving bo (%d).\n", r);
3183 return r;
3184 }
3185
3186 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
3187 num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
3188 r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
3189 if (r) {
3190 DRM_ERROR("radeon: moving bo (%d).\n", r);
3191 radeon_semaphore_free(rdev, &sem, NULL);
3192 return r;
3193 }
3194
3195 if (radeon_fence_need_sync(*fence, ring->idx)) {
3196 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3197 ring->idx);
3198 radeon_fence_note_sync(*fence, ring->idx);
3199 } else {
3200 radeon_semaphore_free(rdev, &sem, NULL);
3201 }
3202
3203 for (i = 0; i < num_loops; i++) {
3204 cur_size_in_dw = size_in_dw;
3205 if (cur_size_in_dw > 0xFFFE)
3206 cur_size_in_dw = 0xFFFE;
3207 size_in_dw -= cur_size_in_dw;
3208 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
3209 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3210 radeon_ring_write(ring, src_offset & 0xfffffffc);
3211 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
3212 (upper_32_bits(src_offset) & 0xff)));
3213 src_offset += cur_size_in_dw * 4;
3214 dst_offset += cur_size_in_dw * 4;
3215 }
3216
3217 r = radeon_fence_emit(rdev, fence, ring->idx);
3218 if (r) {
3219 radeon_ring_unlock_undo(rdev, ring);
3220 return r;
3221 }
3222
3223 radeon_ring_unlock_commit(rdev, ring);
3224 radeon_semaphore_free(rdev, &sem, *fence);
3225
3226 return r;
3227 }
3228
3229 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
3230 uint32_t tiling_flags, uint32_t pitch,
3231 uint32_t offset, uint32_t obj_size)
3232 {
3233 /* FIXME: implement */
3234 return 0;
3235 }
3236
3237 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
3238 {
3239 /* FIXME: implement */
3240 }
3241
3242 static int r600_startup(struct radeon_device *rdev)
3243 {
3244 struct radeon_ring *ring;
3245 int r;
3246
3247 /* enable pcie gen2 link */
3248 r600_pcie_gen2_enable(rdev);
3249
3250 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3251 r = r600_init_microcode(rdev);
3252 if (r) {
3253 DRM_ERROR("Failed to load firmware!\n");
3254 return r;
3255 }
3256 }
3257
3258 r = r600_vram_scratch_init(rdev);
3259 if (r)
3260 return r;
3261
3262 r600_mc_program(rdev);
3263 if (rdev->flags & RADEON_IS_AGP) {
3264 r600_agp_enable(rdev);
3265 } else {
3266 r = r600_pcie_gart_enable(rdev);
3267 if (r)
3268 return r;
3269 }
3270 r600_gpu_init(rdev);
3271 r = r600_blit_init(rdev);
3272 if (r) {
3273 r600_blit_fini(rdev);
3274 rdev->asic->copy.copy = NULL;
3275 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3276 }
3277
3278 /* allocate wb buffer */
3279 r = radeon_wb_init(rdev);
3280 if (r)
3281 return r;
3282
3283 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3284 if (r) {
3285 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3286 return r;
3287 }
3288
3289 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3290 if (r) {
3291 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3292 return r;
3293 }
3294
3295 /* Enable IRQ */
3296 if (!rdev->irq.installed) {
3297 r = radeon_irq_kms_init(rdev);
3298 if (r)
3299 return r;
3300 }
3301
3302 r = r600_irq_init(rdev);
3303 if (r) {
3304 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3305 radeon_irq_kms_fini(rdev);
3306 return r;
3307 }
3308 r600_irq_set(rdev);
3309
3310 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3311 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3312 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3313 0, 0xfffff, RADEON_CP_PACKET2);
3314 if (r)
3315 return r;
3316
3317 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3318 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3319 DMA_RB_RPTR, DMA_RB_WPTR,
3320 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3321 if (r)
3322 return r;
3323
3324 r = r600_cp_load_microcode(rdev);
3325 if (r)
3326 return r;
3327 r = r600_cp_resume(rdev);
3328 if (r)
3329 return r;
3330
3331 r = r600_dma_resume(rdev);
3332 if (r)
3333 return r;
3334
3335 r = radeon_ib_pool_init(rdev);
3336 if (r) {
3337 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
3338 return r;
3339 }
3340
3341 r = r600_audio_init(rdev);
3342 if (r) {
3343 DRM_ERROR("radeon: audio init failed\n");
3344 return r;
3345 }
3346
3347 return 0;
3348 }
3349
3350 void r600_vga_set_state(struct radeon_device *rdev, bool state)
3351 {
3352 uint32_t temp;
3353
3354 temp = RREG32(CONFIG_CNTL);
3355 if (state == false) {
3356 temp &= ~(1<<0);
3357 temp |= (1<<1);
3358 } else {
3359 temp &= ~(1<<1);
3360 }
3361 WREG32(CONFIG_CNTL, temp);
3362 }
3363
3364 int r600_resume(struct radeon_device *rdev)
3365 {
3366 int r;
3367
3368 /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
3369 * posting will perform necessary task to bring back GPU into good
3370 * shape.
3371 */
3372 /* post card */
3373 atom_asic_init(rdev->mode_info.atom_context);
3374
3375 rdev->accel_working = true;
3376 r = r600_startup(rdev);
3377 if (r) {
3378 DRM_ERROR("r600 startup failed on resume\n");
3379 rdev->accel_working = false;
3380 return r;
3381 }
3382
3383 return r;
3384 }
3385
3386 int r600_suspend(struct radeon_device *rdev)
3387 {
3388 r600_audio_fini(rdev);
3389 r600_cp_stop(rdev);
3390 r600_dma_stop(rdev);
3391 r600_irq_suspend(rdev);
3392 radeon_wb_disable(rdev);
3393 r600_pcie_gart_disable(rdev);
3394
3395 return 0;
3396 }
3397
3398 /* Plan is to move initialization in that function and use
3399 * helper function so that radeon_device_init pretty much
3400 * do nothing more than calling asic specific function. This
3401 * should also allow to remove a bunch of callback function
3402 * like vram_info.
3403 */
3404 int r600_init(struct radeon_device *rdev)
3405 {
3406 int r;
3407
3408 if (r600_debugfs_mc_info_init(rdev)) {
3409 DRM_ERROR("Failed to register debugfs file for mc !\n");
3410 }
3411 /* Read BIOS */
3412 if (!radeon_get_bios(rdev)) {
3413 if (ASIC_IS_AVIVO(rdev))
3414 return -EINVAL;
3415 }
3416 /* Must be an ATOMBIOS */
3417 if (!rdev->is_atom_bios) {
3418 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3419 return -EINVAL;
3420 }
3421 r = radeon_atombios_init(rdev);
3422 if (r)
3423 return r;
3424 /* Post card if necessary */
3425 if (!radeon_card_posted(rdev)) {
3426 if (!rdev->bios) {
3427 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3428 return -EINVAL;
3429 }
3430 DRM_INFO("GPU not posted. posting now...\n");
3431 atom_asic_init(rdev->mode_info.atom_context);
3432 }
3433 /* Initialize scratch registers */
3434 r600_scratch_init(rdev);
3435 /* Initialize surface registers */
3436 radeon_surface_init(rdev);
3437 /* Initialize clocks */
3438 radeon_get_clock_info(rdev->ddev);
3439 /* Fence driver */
3440 r = radeon_fence_driver_init(rdev);
3441 if (r)
3442 return r;
3443 if (rdev->flags & RADEON_IS_AGP) {
3444 r = radeon_agp_init(rdev);
3445 if (r)
3446 radeon_agp_disable(rdev);
3447 }
3448 r = r600_mc_init(rdev);
3449 if (r)
3450 return r;
3451 /* Memory manager */
3452 r = radeon_bo_init(rdev);
3453 if (r)
3454 return r;
3455
3456 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3457 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3458
3459 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3460 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3461
3462 rdev->ih.ring_obj = NULL;
3463 r600_ih_ring_init(rdev, 64 * 1024);
3464
3465 r = r600_pcie_gart_init(rdev);
3466 if (r)
3467 return r;
3468
3469 rdev->accel_working = true;
3470 r = r600_startup(rdev);
3471 if (r) {
3472 dev_err(rdev->dev, "disabling GPU acceleration\n");
3473 r600_cp_fini(rdev);
3474 r600_dma_fini(rdev);
3475 r600_irq_fini(rdev);
3476 radeon_wb_fini(rdev);
3477 radeon_ib_pool_fini(rdev);
3478 radeon_irq_kms_fini(rdev);
3479 r600_pcie_gart_fini(rdev);
3480 rdev->accel_working = false;
3481 }
3482
3483 return 0;
3484 }
3485
3486 void r600_fini(struct radeon_device *rdev)
3487 {
3488 r600_audio_fini(rdev);
3489 r600_blit_fini(rdev);
3490 r600_cp_fini(rdev);
3491 r600_dma_fini(rdev);
3492 r600_irq_fini(rdev);
3493 radeon_wb_fini(rdev);
3494 radeon_ib_pool_fini(rdev);
3495 radeon_irq_kms_fini(rdev);
3496 r600_pcie_gart_fini(rdev);
3497 r600_vram_scratch_fini(rdev);
3498 radeon_agp_fini(rdev);
3499 radeon_gem_fini(rdev);
3500 radeon_fence_driver_fini(rdev);
3501 radeon_bo_fini(rdev);
3502 radeon_atombios_fini(rdev);
3503 r600_fini_microcode(rdev);
3504 drm_free(rdev->bios, M_DRM);
3505 rdev->bios = NULL;
3506 }
3507
3508
3509 /*
3510 * CS stuff
3511 */
3512 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3513 {
3514 struct radeon_ring *ring = &rdev->ring[ib->ring];
3515 u32 next_rptr;
3516
3517 if (ring->rptr_save_reg) {
3518 next_rptr = ring->wptr + 3 + 4;
3519 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3520 radeon_ring_write(ring, ((ring->rptr_save_reg -
3521 PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3522 radeon_ring_write(ring, next_rptr);
3523 } else if (rdev->wb.enabled) {
3524 next_rptr = ring->wptr + 5 + 4;
3525 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3526 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3527 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3528 radeon_ring_write(ring, next_rptr);
3529 radeon_ring_write(ring, 0);
3530 }
3531
3532 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3533 radeon_ring_write(ring,
3534 #ifdef __BIG_ENDIAN
3535 (2 << 0) |
3536 #endif
3537 (ib->gpu_addr & 0xFFFFFFFC));
3538 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3539 radeon_ring_write(ring, ib->length_dw);
3540 }
3541
3542 void r600_uvd_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3543 {
3544 struct radeon_ring *ring = &rdev->ring[ib->ring];
3545
3546 radeon_ring_write(ring, PACKET0(UVD_RBC_IB_BASE, 0));
3547 radeon_ring_write(ring, ib->gpu_addr);
3548 radeon_ring_write(ring, PACKET0(UVD_RBC_IB_SIZE, 0));
3549 radeon_ring_write(ring, ib->length_dw);
3550 }
3551
3552 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3553 {
3554 struct radeon_ib ib;
3555 uint32_t scratch;
3556 uint32_t tmp = 0;
3557 unsigned i;
3558 int r;
3559
3560 r = radeon_scratch_get(rdev, &scratch);
3561 if (r) {
3562 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3563 return r;
3564 }
3565 WREG32(scratch, 0xCAFEDEAD);
3566 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3567 if (r) {
3568 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3569 goto free_scratch;
3570 }
3571 ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3572 ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3573 ib.ptr[2] = 0xDEADBEEF;
3574 ib.length_dw = 3;
3575 r = radeon_ib_schedule(rdev, &ib, NULL);
3576 if (r) {
3577 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3578 goto free_ib;
3579 }
3580 r = radeon_fence_wait(ib.fence, false);
3581 if (r) {
3582 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3583 goto free_ib;
3584 }
3585 for (i = 0; i < rdev->usec_timeout; i++) {
3586 tmp = RREG32(scratch);
3587 if (tmp == 0xDEADBEEF)
3588 break;
3589 DRM_UDELAY(1);
3590 }
3591 if (i < rdev->usec_timeout) {
3592 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3593 } else {
3594 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3595 scratch, tmp);
3596 r = -EINVAL;
3597 }
3598 free_ib:
3599 radeon_ib_free(rdev, &ib);
3600 free_scratch:
3601 radeon_scratch_free(rdev, scratch);
3602 return r;
3603 }
3604
3605 /**
3606 * r600_dma_ib_test - test an IB on the DMA engine
3607 *
3608 * @rdev: radeon_device pointer
3609 * @ring: radeon_ring structure holding ring information
3610 *
3611 * Test a simple IB in the DMA ring (r6xx-SI).
3612 * Returns 0 on success, error on failure.
3613 */
3614 int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3615 {
3616 struct radeon_ib ib;
3617 unsigned i;
3618 int r;
3619 volatile uint32_t *ptr = rdev->vram_scratch.ptr;
3620 u32 tmp = 0;
3621
3622 if (!ptr) {
3623 DRM_ERROR("invalid vram scratch pointer\n");
3624 return -EINVAL;
3625 }
3626
3627 tmp = 0xCAFEDEAD;
3628 *ptr = tmp;
3629
3630 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3631 if (r) {
3632 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3633 return r;
3634 }
3635
3636 ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3637 ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3638 ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3639 ib.ptr[3] = 0xDEADBEEF;
3640 ib.length_dw = 4;
3641
3642 r = radeon_ib_schedule(rdev, &ib, NULL);
3643 if (r) {
3644 radeon_ib_free(rdev, &ib);
3645 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3646 return r;
3647 }
3648 r = radeon_fence_wait(ib.fence, false);
3649 if (r) {
3650 radeon_ib_free(rdev, &ib);
3651 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3652 return r;
3653 }
3654 for (i = 0; i < rdev->usec_timeout; i++) {
3655 tmp = *ptr;
3656 if (tmp == 0xDEADBEEF)
3657 break;
3658 DRM_UDELAY(1);
3659 }
3660 if (i < rdev->usec_timeout) {
3661 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3662 } else {
3663 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3664 r = -EINVAL;
3665 }
3666 radeon_ib_free(rdev, &ib);
3667 return r;
3668 }
3669
3670 int r600_uvd_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3671 {
3672 struct radeon_fence *fence = NULL;
3673 int r;
3674
3675 r = radeon_set_uvd_clocks(rdev, 53300, 40000);
3676 if (r) {
3677 DRM_ERROR("radeon: failed to raise UVD clocks (%d).\n", r);
3678 return r;
3679 }
3680
3681 r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL);
3682 if (r) {
3683 DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
3684 goto error;
3685 }
3686
3687 r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, &fence);
3688 if (r) {
3689 DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
3690 goto error;
3691 }
3692
3693 r = radeon_fence_wait(fence, false);
3694 if (r) {
3695 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3696 goto error;
3697 }
3698 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
3699 error:
3700 radeon_fence_unref(&fence);
3701 radeon_set_uvd_clocks(rdev, 0, 0);
3702 return r;
3703 }
3704
3705 /**
3706 * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3707 *
3708 * @rdev: radeon_device pointer
3709 * @ib: IB object to schedule
3710 *
3711 * Schedule an IB in the DMA ring (r6xx-r7xx).
3712 */
3713 void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3714 {
3715 struct radeon_ring *ring = &rdev->ring[ib->ring];
3716
3717 if (rdev->wb.enabled) {
3718 u32 next_rptr = ring->wptr + 4;
3719 while ((next_rptr & 7) != 5)
3720 next_rptr++;
3721 next_rptr += 3;
3722 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3723 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3724 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3725 radeon_ring_write(ring, next_rptr);
3726 }
3727
3728 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3729 * Pad as necessary with NOPs.
3730 */
3731 while ((ring->wptr & 7) != 5)
3732 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3733 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3734 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3735 radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3736
3737 }
3738
3739 /*
3740 * Interrupts
3741 *
3742 * Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
3743 * the same as the CP ring buffer, but in reverse. Rather than the CPU
3744 * writing to the ring and the GPU consuming, the GPU writes to the ring
3745 * and host consumes. As the host irq handler processes interrupts, it
3746 * increments the rptr. When the rptr catches up with the wptr, all the
3747 * current interrupts have been processed.
3748 */
3749
3750 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3751 {
3752 u32 rb_bufsz;
3753
3754 /* Align ring size */
3755 rb_bufsz = drm_order(ring_size / 4);
3756 ring_size = (1 << rb_bufsz) * 4;
3757 rdev->ih.ring_size = ring_size;
3758 rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3759 rdev->ih.rptr = 0;
3760 }
3761
3762 int r600_ih_ring_alloc(struct radeon_device *rdev)
3763 {
3764 int r;
3765 void *ring_ptr;
3766
3767 /* Allocate ring buffer */
3768 if (rdev->ih.ring_obj == NULL) {
3769 r = radeon_bo_create(rdev, rdev->ih.ring_size,
3770 PAGE_SIZE, true,
3771 RADEON_GEM_DOMAIN_GTT,
3772 NULL, &rdev->ih.ring_obj);
3773 if (r) {
3774 DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3775 return r;
3776 }
3777 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3778 if (unlikely(r != 0)) {
3779 radeon_bo_unref(&rdev->ih.ring_obj);
3780 return r;
3781 }
3782 r = radeon_bo_pin(rdev->ih.ring_obj,
3783 RADEON_GEM_DOMAIN_GTT,
3784 &rdev->ih.gpu_addr);
3785 if (r) {
3786 radeon_bo_unreserve(rdev->ih.ring_obj);
3787 radeon_bo_unref(&rdev->ih.ring_obj);
3788 DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3789 return r;
3790 }
3791 ring_ptr = &rdev->ih.ring;
3792 r = radeon_bo_kmap(rdev->ih.ring_obj,
3793 ring_ptr);
3794 if (r)
3795 radeon_bo_unpin(rdev->ih.ring_obj);
3796 radeon_bo_unreserve(rdev->ih.ring_obj);
3797 if (r) {
3798 DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3799 radeon_bo_unref(&rdev->ih.ring_obj);
3800 return r;
3801 }
3802 }
3803 return 0;
3804 }
3805
3806 void r600_ih_ring_fini(struct radeon_device *rdev)
3807 {
3808 int r;
3809 if (rdev->ih.ring_obj) {
3810 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3811 if (likely(r == 0)) {
3812 radeon_bo_kunmap(rdev->ih.ring_obj);
3813 radeon_bo_unpin(rdev->ih.ring_obj);
3814 radeon_bo_unreserve(rdev->ih.ring_obj);
3815 }
3816 radeon_bo_unref(&rdev->ih.ring_obj);
3817 rdev->ih.ring = NULL;
3818 rdev->ih.ring_obj = NULL;
3819 }
3820 }
3821
3822 void r600_rlc_stop(struct radeon_device *rdev)
3823 {
3824
3825 if ((rdev->family >= CHIP_RV770) &&
3826 (rdev->family <= CHIP_RV740)) {
3827 /* r7xx asics need to soft reset RLC before halting */
3828 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3829 RREG32(SRBM_SOFT_RESET);
3830 DRM_MDELAY(15);
3831 WREG32(SRBM_SOFT_RESET, 0);
3832 RREG32(SRBM_SOFT_RESET);
3833 }
3834
3835 WREG32(RLC_CNTL, 0);
3836 }
3837
3838 static void r600_rlc_start(struct radeon_device *rdev)
3839 {
3840 WREG32(RLC_CNTL, RLC_ENABLE);
3841 }
3842
3843 static int r600_rlc_init(struct radeon_device *rdev)
3844 {
3845 u32 i;
3846 const __be32 *fw_data;
3847
3848 if (!rdev->rlc_fw)
3849 return -EINVAL;
3850
3851 r600_rlc_stop(rdev);
3852
3853 WREG32(RLC_HB_CNTL, 0);
3854
3855 if (rdev->family == CHIP_ARUBA) {
3856 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3857 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3858 }
3859 if (rdev->family <= CHIP_CAYMAN) {
3860 WREG32(RLC_HB_BASE, 0);
3861 WREG32(RLC_HB_RPTR, 0);
3862 WREG32(RLC_HB_WPTR, 0);
3863 }
3864 if (rdev->family <= CHIP_CAICOS) {
3865 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3866 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3867 }
3868 WREG32(RLC_MC_CNTL, 0);
3869 WREG32(RLC_UCODE_CNTL, 0);
3870
3871 fw_data = (const __be32 *)rdev->rlc_fw->data;
3872 if (rdev->family >= CHIP_ARUBA) {
3873 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
3874 WREG32(RLC_UCODE_ADDR, i);
3875 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3876 }
3877 } else if (rdev->family >= CHIP_CAYMAN) {
3878 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
3879 WREG32(RLC_UCODE_ADDR, i);
3880 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3881 }
3882 } else if (rdev->family >= CHIP_CEDAR) {
3883 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
3884 WREG32(RLC_UCODE_ADDR, i);
3885 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3886 }
3887 } else if (rdev->family >= CHIP_RV770) {
3888 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3889 WREG32(RLC_UCODE_ADDR, i);
3890 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3891 }
3892 } else {
3893 for (i = 0; i < RLC_UCODE_SIZE; i++) {
3894 WREG32(RLC_UCODE_ADDR, i);
3895 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3896 }
3897 }
3898 WREG32(RLC_UCODE_ADDR, 0);
3899
3900 r600_rlc_start(rdev);
3901
3902 return 0;
3903 }
3904
3905 static void r600_enable_interrupts(struct radeon_device *rdev)
3906 {
3907 u32 ih_cntl = RREG32(IH_CNTL);
3908 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3909
3910 ih_cntl |= ENABLE_INTR;
3911 ih_rb_cntl |= IH_RB_ENABLE;
3912 WREG32(IH_CNTL, ih_cntl);
3913 WREG32(IH_RB_CNTL, ih_rb_cntl);
3914 rdev->ih.enabled = true;
3915 }
3916
3917 void r600_disable_interrupts(struct radeon_device *rdev)
3918 {
3919 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3920 u32 ih_cntl = RREG32(IH_CNTL);
3921
3922 ih_rb_cntl &= ~IH_RB_ENABLE;
3923 ih_cntl &= ~ENABLE_INTR;
3924 WREG32(IH_RB_CNTL, ih_rb_cntl);
3925 WREG32(IH_CNTL, ih_cntl);
3926 /* set rptr, wptr to 0 */
3927 WREG32(IH_RB_RPTR, 0);
3928 WREG32(IH_RB_WPTR, 0);
3929 rdev->ih.enabled = false;
3930 rdev->ih.rptr = 0;
3931 }
3932
3933 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3934 {
3935 u32 tmp;
3936
3937 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3938 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3939 WREG32(DMA_CNTL, tmp);
3940 WREG32(GRBM_INT_CNTL, 0);
3941 WREG32(DxMODE_INT_MASK, 0);
3942 WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3943 WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3944 if (ASIC_IS_DCE3(rdev)) {
3945 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3946 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3947 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3948 WREG32(DC_HPD1_INT_CONTROL, tmp);
3949 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3950 WREG32(DC_HPD2_INT_CONTROL, tmp);
3951 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3952 WREG32(DC_HPD3_INT_CONTROL, tmp);
3953 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3954 WREG32(DC_HPD4_INT_CONTROL, tmp);
3955 if (ASIC_IS_DCE32(rdev)) {
3956 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3957 WREG32(DC_HPD5_INT_CONTROL, tmp);
3958 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3959 WREG32(DC_HPD6_INT_CONTROL, tmp);
3960 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3961 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3962 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3963 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3964 } else {
3965 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3966 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3967 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3968 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3969 }
3970 } else {
3971 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3972 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3973 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3974 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3975 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3976 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3977 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3978 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3979 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3980 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3981 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3982 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3983 }
3984 }
3985
3986 int r600_irq_init(struct radeon_device *rdev)
3987 {
3988 int ret = 0;
3989 int rb_bufsz;
3990 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3991
3992 /* allocate ring */
3993 ret = r600_ih_ring_alloc(rdev);
3994 if (ret)
3995 return ret;
3996
3997 /* disable irqs */
3998 r600_disable_interrupts(rdev);
3999
4000 /* init rlc */
4001 ret = r600_rlc_init(rdev);
4002 if (ret) {
4003 r600_ih_ring_fini(rdev);
4004 return ret;
4005 }
4006
4007 /* setup interrupt control */
4008 /* set dummy read address to ring address */
4009 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4010 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4011 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4012 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4013 */
4014 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4015 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4016 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4017 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4018
4019 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4020 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4021
4022 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4023 IH_WPTR_OVERFLOW_CLEAR |
4024 (rb_bufsz << 1));
4025
4026 if (rdev->wb.enabled)
4027 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4028
4029 /* set the writeback address whether it's enabled or not */
4030 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4031 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4032
4033 WREG32(IH_RB_CNTL, ih_rb_cntl);
4034
4035 /* set rptr, wptr to 0 */
4036 WREG32(IH_RB_RPTR, 0);
4037 WREG32(IH_RB_WPTR, 0);
4038
4039 /* Default settings for IH_CNTL (disabled at first) */
4040 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
4041 /* RPTR_REARM only works if msi's are enabled */
4042 if (rdev->msi_enabled)
4043 ih_cntl |= RPTR_REARM;
4044 WREG32(IH_CNTL, ih_cntl);
4045
4046 /* force the active interrupt state to all disabled */
4047 if (rdev->family >= CHIP_CEDAR)
4048 evergreen_disable_interrupt_state(rdev);
4049 else
4050 r600_disable_interrupt_state(rdev);
4051
4052 /* at this point everything should be setup correctly to enable master */
4053 pci_enable_busmaster(rdev->dev);
4054
4055 /* enable irqs */
4056 r600_enable_interrupts(rdev);
4057
4058 return ret;
4059 }
4060
4061 void r600_irq_suspend(struct radeon_device *rdev)
4062 {
4063 r600_irq_disable(rdev);
4064 r600_rlc_stop(rdev);
4065 }
4066
4067 void r600_irq_fini(struct radeon_device *rdev)
4068 {
4069 r600_irq_suspend(rdev);
4070 r600_ih_ring_fini(rdev);
4071 }
4072
4073 int r600_irq_set(struct radeon_device *rdev)
4074 {
4075 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4076 u32 mode_int = 0;
4077 u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
4078 u32 grbm_int_cntl = 0;
4079 u32 hdmi0, hdmi1;
4080 u32 d1grph = 0, d2grph = 0;
4081 u32 dma_cntl;
4082
4083 if (!rdev->irq.installed) {
4084 DRM_ERROR("Can't enable IRQ/MSI because no handler is installed\n");
4085 return -EINVAL;
4086 }
4087 /* don't enable anything if the ih is disabled */
4088 if (!rdev->ih.enabled) {
4089 r600_disable_interrupts(rdev);
4090 /* force the active interrupt state to all disabled */
4091 r600_disable_interrupt_state(rdev);
4092 return 0;
4093 }
4094
4095 if (ASIC_IS_DCE3(rdev)) {
4096 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4097 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4098 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4099 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4100 if (ASIC_IS_DCE32(rdev)) {
4101 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4102 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4103 hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4104 hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4105 } else {
4106 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4107 hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4108 }
4109 } else {
4110 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4111 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4112 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4113 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4114 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4115 }
4116 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4117
4118 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4119 DRM_DEBUG("r600_irq_set: sw int\n");
4120 cp_int_cntl |= RB_INT_ENABLE;
4121 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4122 }
4123
4124 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4125 DRM_DEBUG("r600_irq_set: sw int dma\n");
4126 dma_cntl |= TRAP_ENABLE;
4127 }
4128
4129 if (rdev->irq.crtc_vblank_int[0] ||
4130 atomic_read(&rdev->irq.pflip[0])) {
4131 DRM_DEBUG("r600_irq_set: vblank 0\n");
4132 mode_int |= D1MODE_VBLANK_INT_MASK;
4133 }
4134 if (rdev->irq.crtc_vblank_int[1] ||
4135 atomic_read(&rdev->irq.pflip[1])) {
4136 DRM_DEBUG("r600_irq_set: vblank 1\n");
4137 mode_int |= D2MODE_VBLANK_INT_MASK;
4138 }
4139 if (rdev->irq.hpd[0]) {
4140 DRM_DEBUG("r600_irq_set: hpd 1\n");
4141 hpd1 |= DC_HPDx_INT_EN;
4142 }
4143 if (rdev->irq.hpd[1]) {
4144 DRM_DEBUG("r600_irq_set: hpd 2\n");
4145 hpd2 |= DC_HPDx_INT_EN;
4146 }
4147 if (rdev->irq.hpd[2]) {
4148 DRM_DEBUG("r600_irq_set: hpd 3\n");
4149 hpd3 |= DC_HPDx_INT_EN;
4150 }
4151 if (rdev->irq.hpd[3]) {
4152 DRM_DEBUG("r600_irq_set: hpd 4\n");
4153 hpd4 |= DC_HPDx_INT_EN;
4154 }
4155 if (rdev->irq.hpd[4]) {
4156 DRM_DEBUG("r600_irq_set: hpd 5\n");
4157 hpd5 |= DC_HPDx_INT_EN;
4158 }
4159 if (rdev->irq.hpd[5]) {
4160 DRM_DEBUG("r600_irq_set: hpd 6\n");
4161 hpd6 |= DC_HPDx_INT_EN;
4162 }
4163 if (rdev->irq.afmt[0]) {
4164 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4165 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
4166 }
4167 if (rdev->irq.afmt[1]) {
4168 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4169 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
4170 }
4171
4172 WREG32(CP_INT_CNTL, cp_int_cntl);
4173 WREG32(DMA_CNTL, dma_cntl);
4174 WREG32(DxMODE_INT_MASK, mode_int);
4175 WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
4176 WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
4177 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4178 if (ASIC_IS_DCE3(rdev)) {
4179 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4180 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4181 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4182 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4183 if (ASIC_IS_DCE32(rdev)) {
4184 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4185 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4186 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
4187 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
4188 } else {
4189 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4190 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
4191 }
4192 } else {
4193 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
4194 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
4195 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
4196 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4197 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
4198 }
4199
4200 return 0;
4201 }
4202
4203 static void r600_irq_ack(struct radeon_device *rdev)
4204 {
4205 u32 tmp;
4206
4207 if (ASIC_IS_DCE3(rdev)) {
4208 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
4209 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
4210 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
4211 if (ASIC_IS_DCE32(rdev)) {
4212 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
4213 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
4214 } else {
4215 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4216 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
4217 }
4218 } else {
4219 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4220 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4221 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
4222 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4223 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
4224 }
4225 rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
4226 rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
4227
4228 if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4229 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4230 if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4231 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4232 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
4233 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
4234 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
4235 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
4236 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
4237 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
4238 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
4239 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
4240 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4241 if (ASIC_IS_DCE3(rdev)) {
4242 tmp = RREG32(DC_HPD1_INT_CONTROL);
4243 tmp |= DC_HPDx_INT_ACK;
4244 WREG32(DC_HPD1_INT_CONTROL, tmp);
4245 } else {
4246 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
4247 tmp |= DC_HPDx_INT_ACK;
4248 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
4249 }
4250 }
4251 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4252 if (ASIC_IS_DCE3(rdev)) {
4253 tmp = RREG32(DC_HPD2_INT_CONTROL);
4254 tmp |= DC_HPDx_INT_ACK;
4255 WREG32(DC_HPD2_INT_CONTROL, tmp);
4256 } else {
4257 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
4258 tmp |= DC_HPDx_INT_ACK;
4259 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
4260 }
4261 }
4262 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4263 if (ASIC_IS_DCE3(rdev)) {
4264 tmp = RREG32(DC_HPD3_INT_CONTROL);
4265 tmp |= DC_HPDx_INT_ACK;
4266 WREG32(DC_HPD3_INT_CONTROL, tmp);
4267 } else {
4268 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
4269 tmp |= DC_HPDx_INT_ACK;
4270 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
4271 }
4272 }
4273 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4274 tmp = RREG32(DC_HPD4_INT_CONTROL);
4275 tmp |= DC_HPDx_INT_ACK;
4276 WREG32(DC_HPD4_INT_CONTROL, tmp);
4277 }
4278 if (ASIC_IS_DCE32(rdev)) {
4279 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4280 tmp = RREG32(DC_HPD5_INT_CONTROL);
4281 tmp |= DC_HPDx_INT_ACK;
4282 WREG32(DC_HPD5_INT_CONTROL, tmp);
4283 }
4284 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4285 tmp = RREG32(DC_HPD5_INT_CONTROL);
4286 tmp |= DC_HPDx_INT_ACK;
4287 WREG32(DC_HPD6_INT_CONTROL, tmp);
4288 }
4289 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
4290 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
4291 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4292 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
4293 }
4294 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
4295 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
4296 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4297 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
4298 }
4299 } else {
4300 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4301 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
4302 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4303 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4304 }
4305 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4306 if (ASIC_IS_DCE3(rdev)) {
4307 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
4308 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4309 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
4310 } else {
4311 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
4312 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4313 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
4314 }
4315 }
4316 }
4317 }
4318
4319 void r600_irq_disable(struct radeon_device *rdev)
4320 {
4321 r600_disable_interrupts(rdev);
4322 /* Wait and acknowledge irq */
4323 DRM_MDELAY(1);
4324 r600_irq_ack(rdev);
4325 r600_disable_interrupt_state(rdev);
4326 }
4327
4328 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
4329 {
4330 u32 wptr, tmp;
4331
4332 if (rdev->wb.enabled)
4333 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4334 else
4335 wptr = RREG32(IH_RB_WPTR);
4336
4337 if (wptr & RB_OVERFLOW) {
4338 /* When a ring buffer overflow happen start parsing interrupt
4339 * from the last not overwritten vector (wptr + 16). Hopefully
4340 * this should allow us to catchup.
4341 */
4342 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4343 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4344 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4345 tmp = RREG32(IH_RB_CNTL);
4346 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4347 WREG32(IH_RB_CNTL, tmp);
4348 }
4349 return (wptr & rdev->ih.ptr_mask);
4350 }
4351
4352 /* r600 IV Ring
4353 * Each IV ring entry is 128 bits:
4354 * [7:0] - interrupt source id
4355 * [31:8] - reserved
4356 * [59:32] - interrupt source data
4357 * [127:60] - reserved
4358 *
4359 * The basic interrupt vector entries
4360 * are decoded as follows:
4361 * src_id src_data description
4362 * 1 0 D1 Vblank
4363 * 1 1 D1 Vline
4364 * 5 0 D2 Vblank
4365 * 5 1 D2 Vline
4366 * 19 0 FP Hot plug detection A
4367 * 19 1 FP Hot plug detection B
4368 * 19 2 DAC A auto-detection
4369 * 19 3 DAC B auto-detection
4370 * 21 4 HDMI block A
4371 * 21 5 HDMI block B
4372 * 176 - CP_INT RB
4373 * 177 - CP_INT IB1
4374 * 178 - CP_INT IB2
4375 * 181 - EOP Interrupt
4376 * 233 - GUI Idle
4377 *
4378 * Note, these are based on r600 and may need to be
4379 * adjusted or added to on newer asics
4380 */
4381
4382 irqreturn_t r600_irq_process(struct radeon_device *rdev)
4383 {
4384 u32 wptr;
4385 u32 rptr;
4386 u32 src_id, src_data;
4387 u32 ring_index;
4388 bool queue_hotplug = false;
4389 bool queue_hdmi = false;
4390
4391 if (!rdev->ih.enabled || rdev->shutdown)
4392 return IRQ_NONE;
4393
4394 /* No MSIs, need a dummy read to flush PCI DMAs */
4395 if (!rdev->msi_enabled)
4396 RREG32(IH_RB_WPTR);
4397
4398 wptr = r600_get_ih_wptr(rdev);
4399
4400 restart_ih:
4401 /* is somebody else already processing irqs? */
4402 if (atomic_xchg(&rdev->ih.lock, 1))
4403 return IRQ_NONE;
4404
4405 rptr = rdev->ih.rptr;
4406 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4407
4408 /* Order reading of wptr vs. reading of IH ring data */
4409 cpu_lfence();
4410
4411 /* display interrupts */
4412 r600_irq_ack(rdev);
4413
4414 while (rptr != wptr) {
4415 /* wptr/rptr are in bytes! */
4416 ring_index = rptr / 4;
4417 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4418 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4419
4420 switch (src_id) {
4421 case 1: /* D1 vblank/vline */
4422 switch (src_data) {
4423 case 0: /* D1 vblank */
4424 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
4425 if (rdev->irq.crtc_vblank_int[0]) {
4426 drm_handle_vblank(rdev->ddev, 0);
4427 rdev->pm.vblank_sync = true;
4428 DRM_WAKEUP(&rdev->irq.vblank_queue);
4429 }
4430 if (atomic_read(&rdev->irq.pflip[0]))
4431 radeon_crtc_handle_flip(rdev, 0);
4432 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4433 DRM_DEBUG("IH: D1 vblank\n");
4434 }
4435 break;
4436 case 1: /* D1 vline */
4437 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
4438 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4439 DRM_DEBUG("IH: D1 vline\n");
4440 }
4441 break;
4442 default:
4443 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4444 break;
4445 }
4446 break;
4447 case 5: /* D2 vblank/vline */
4448 switch (src_data) {
4449 case 0: /* D2 vblank */
4450 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
4451 if (rdev->irq.crtc_vblank_int[1]) {
4452 drm_handle_vblank(rdev->ddev, 1);
4453 rdev->pm.vblank_sync = true;
4454 DRM_WAKEUP(&rdev->irq.vblank_queue);
4455 }
4456 if (atomic_read(&rdev->irq.pflip[1]))
4457 radeon_crtc_handle_flip(rdev, 1);
4458 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
4459 DRM_DEBUG("IH: D2 vblank\n");
4460 }
4461 break;
4462 case 1: /* D1 vline */
4463 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
4464 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
4465 DRM_DEBUG("IH: D2 vline\n");
4466 }
4467 break;
4468 default:
4469 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4470 break;
4471 }
4472 break;
4473 case 19: /* HPD/DAC hotplug */
4474 switch (src_data) {
4475 case 0:
4476 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4477 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
4478 queue_hotplug = true;
4479 DRM_DEBUG("IH: HPD1\n");
4480 }
4481 break;
4482 case 1:
4483 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4484 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
4485 queue_hotplug = true;
4486 DRM_DEBUG("IH: HPD2\n");
4487 }
4488 break;
4489 case 4:
4490 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4491 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
4492 queue_hotplug = true;
4493 DRM_DEBUG("IH: HPD3\n");
4494 }
4495 break;
4496 case 5:
4497 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4498 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
4499 queue_hotplug = true;
4500 DRM_DEBUG("IH: HPD4\n");
4501 }
4502 break;
4503 case 10:
4504 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4505 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
4506 queue_hotplug = true;
4507 DRM_DEBUG("IH: HPD5\n");
4508 }
4509 break;
4510 case 12:
4511 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4512 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
4513 queue_hotplug = true;
4514 DRM_DEBUG("IH: HPD6\n");
4515 }
4516 break;
4517 default:
4518 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4519 break;
4520 }
4521 break;
4522 case 21: /* hdmi */
4523 switch (src_data) {
4524 case 4:
4525 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4526 rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4527 queue_hdmi = true;
4528 DRM_DEBUG("IH: HDMI0\n");
4529 }
4530 break;
4531 case 5:
4532 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4533 rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4534 queue_hdmi = true;
4535 DRM_DEBUG("IH: HDMI1\n");
4536 }
4537 break;
4538 default:
4539 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4540 break;
4541 }
4542 break;
4543 case 176: /* CP_INT in ring buffer */
4544 case 177: /* CP_INT in IB1 */
4545 case 178: /* CP_INT in IB2 */
4546 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4547 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4548 break;
4549 case 181: /* CP EOP event */
4550 DRM_DEBUG("IH: CP EOP\n");
4551 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4552 break;
4553 case 224: /* DMA trap event */
4554 DRM_DEBUG("IH: DMA trap\n");
4555 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4556 break;
4557 case 233: /* GUI IDLE */
4558 DRM_DEBUG("IH: GUI idle\n");
4559 break;
4560 default:
4561 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4562 break;
4563 }
4564
4565 /* wptr/rptr are in bytes! */
4566 rptr += 16;
4567 rptr &= rdev->ih.ptr_mask;
4568 }
4569 if (queue_hotplug)
4570 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
4571 if (queue_hdmi)
4572 taskqueue_enqueue(rdev->tq, &rdev->audio_work);
4573 rdev->ih.rptr = rptr;
4574 WREG32(IH_RB_RPTR, rdev->ih.rptr);
4575 atomic_set(&rdev->ih.lock, 0);
4576
4577 /* make sure wptr hasn't changed while processing */
4578 wptr = r600_get_ih_wptr(rdev);
4579 if (wptr != rptr)
4580 goto restart_ih;
4581
4582 return IRQ_HANDLED;
4583 }
4584
4585 /*
4586 * Debugfs info
4587 */
4588 #if defined(CONFIG_DEBUG_FS)
4589
4590 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4591 {
4592 struct drm_info_node *node = (struct drm_info_node *) m->private;
4593 struct drm_device *dev = node->minor->dev;
4594 struct radeon_device *rdev = dev->dev_private;
4595
4596 DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4597 DREG32_SYS(m, rdev, VM_L2_STATUS);
4598 return 0;
4599 }
4600
4601 static struct drm_info_list r600_mc_info_list[] = {
4602 {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
4603 };
4604 #endif
4605
4606 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4607 {
4608 #if defined(CONFIG_DEBUG_FS)
4609 return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4610 #else
4611 return 0;
4612 #endif
4613 }
4614
4615 /**
4616 * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4617 * rdev: radeon device structure
4618 * bo: buffer object struct which userspace is waiting for idle
4619 *
4620 * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4621 * through ring buffer, this leads to corruption in rendering, see
4622 * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4623 * directly perform HDP flush by writing register through MMIO.
4624 */
4625 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4626 {
4627 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
4628 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4629 * This seems to cause problems on some AGP cards. Just use the old
4630 * method for them.
4631 */
4632 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4633 rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4634 volatile uint32_t *ptr = rdev->vram_scratch.ptr;
4635 u32 tmp;
4636
4637 WREG32(HDP_DEBUG1, 0);
4638 tmp = *ptr;
4639 } else
4640 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4641 }
4642
4643 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4644 {
4645 u32 link_width_cntl, mask;
4646
4647 if (rdev->flags & RADEON_IS_IGP)
4648 return;
4649
4650 if (!(rdev->flags & RADEON_IS_PCIE))
4651 return;
4652
4653 /* x2 cards have a special sequence */
4654 if (ASIC_IS_X2(rdev))
4655 return;
4656
4657 radeon_gui_idle(rdev);
4658
4659 switch (lanes) {
4660 case 0:
4661 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4662 break;
4663 case 1:
4664 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4665 break;
4666 case 2:
4667 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4668 break;
4669 case 4:
4670 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4671 break;
4672 case 8:
4673 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4674 break;
4675 case 12:
4676 /* not actually supported */
4677 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4678 break;
4679 case 16:
4680 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4681 break;
4682 default:
4683 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
4684 return;
4685 }
4686
4687 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4688 link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
4689 link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
4690 link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW |
4691 R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4692
4693 WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4694 }
4695
4696 int r600_get_pcie_lanes(struct radeon_device *rdev)
4697 {
4698 u32 link_width_cntl;
4699
4700 if (rdev->flags & RADEON_IS_IGP)
4701 return 0;
4702
4703 if (!(rdev->flags & RADEON_IS_PCIE))
4704 return 0;
4705
4706 /* x2 cards have a special sequence */
4707 if (ASIC_IS_X2(rdev))
4708 return 0;
4709
4710 radeon_gui_idle(rdev);
4711
4712 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4713
4714 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4715 case RADEON_PCIE_LC_LINK_WIDTH_X1:
4716 return 1;
4717 case RADEON_PCIE_LC_LINK_WIDTH_X2:
4718 return 2;
4719 case RADEON_PCIE_LC_LINK_WIDTH_X4:
4720 return 4;
4721 case RADEON_PCIE_LC_LINK_WIDTH_X8:
4722 return 8;
4723 case RADEON_PCIE_LC_LINK_WIDTH_X12:
4724 /* not actually supported */
4725 return 12;
4726 case RADEON_PCIE_LC_LINK_WIDTH_X0:
4727 case RADEON_PCIE_LC_LINK_WIDTH_X16:
4728 default:
4729 return 16;
4730 }
4731 }
4732
4733 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4734 {
4735 u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4736 u16 link_cntl2;
4737 u32 mask;
4738 int ret;
4739
4740 if (radeon_pcie_gen2 == 0)
4741 return;
4742
4743 if (rdev->flags & RADEON_IS_IGP)
4744 return;
4745
4746 if (!(rdev->flags & RADEON_IS_PCIE))
4747 return;
4748
4749 /* x2 cards have a special sequence */
4750 if (ASIC_IS_X2(rdev))
4751 return;
4752
4753 /* only RV6xx+ chips are supported */
4754 if (rdev->family <= CHIP_R600)
4755 return;
4756
4757 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
4758 if (ret != 0)
4759 return;
4760
4761 if (!(mask & DRM_PCIE_SPEED_50))
4762 return;
4763
4764 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4765 if (speed_cntl & LC_CURRENT_DATA_RATE) {
4766 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4767 return;
4768 }
4769
4770 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4771
4772 /* 55 nm r6xx asics */
4773 if ((rdev->family == CHIP_RV670) ||
4774 (rdev->family == CHIP_RV620) ||
4775 (rdev->family == CHIP_RV635)) {
4776 /* advertise upconfig capability */
4777 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4778 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4779 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4780 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4781 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4782 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4783 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4784 LC_RECONFIG_ARC_MISSING_ESCAPE);
4785 link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4786 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4787 } else {
4788 link_width_cntl |= LC_UPCONFIGURE_DIS;
4789 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4790 }
4791 }
4792
4793 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4794 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4795 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4796
4797 /* 55 nm r6xx asics */
4798 if ((rdev->family == CHIP_RV670) ||
4799 (rdev->family == CHIP_RV620) ||
4800 (rdev->family == CHIP_RV635)) {
4801 WREG32(MM_CFGREGS_CNTL, 0x8);
4802 link_cntl2 = RREG32(0x4088);
4803 WREG32(MM_CFGREGS_CNTL, 0);
4804 /* not supported yet */
4805 if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4806 return;
4807 }
4808
4809 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4810 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4811 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4812 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4813 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4814 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4815
4816 tmp = RREG32(0x541c);
4817 WREG32(0x541c, tmp | 0x8);
4818 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4819 link_cntl2 = RREG16(0x4088);
4820 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4821 link_cntl2 |= 0x2;
4822 WREG16(0x4088, link_cntl2);
4823 WREG32(MM_CFGREGS_CNTL, 0);
4824
4825 if ((rdev->family == CHIP_RV670) ||
4826 (rdev->family == CHIP_RV620) ||
4827 (rdev->family == CHIP_RV635)) {
4828 training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
4829 training_cntl &= ~LC_POINT_7_PLUS_EN;
4830 WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
4831 } else {
4832 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4833 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4834 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4835 }
4836
4837 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4838 speed_cntl |= LC_GEN2_EN_STRAP;
4839 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4840
4841 } else {
4842 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4843 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4844 if (1)
4845 link_width_cntl |= LC_UPCONFIGURE_DIS;
4846 else
4847 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4848 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4849 }
4850 }
4851
4852 /**
4853 * r600_get_gpu_clock_counter - return GPU clock counter snapshot
4854 *
4855 * @rdev: radeon_device pointer
4856 *
4857 * Fetches a GPU clock counter snapshot (R6xx-cayman).
4858 * Returns the 64 bit clock counter snapshot.
4859 */
4860 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
4861 {
4862 uint64_t clock;
4863
4864 spin_lock(&rdev->gpu_clock_mutex);
4865 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4866 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4867 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4868 spin_unlock(&rdev->gpu_clock_mutex);
4869 return clock;
4870 }
DragonFly BSD