?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/radeon: add UVD tiling addr config v2
[linux-2.6.git] / drivers / gpu / drm / radeon / evergreen.c
blobc6d80175d18bcc31bf788fdf42b9275dc30b47d2
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36
37 #define EVERGREEN_PFP_UCODE_SIZE 1120
38 #define EVERGREEN_PM4_UCODE_SIZE 1376
39
40 static const u32 crtc_offsets[6] =
41 {
42 EVERGREEN_CRTC0_REGISTER_OFFSET,
43 EVERGREEN_CRTC1_REGISTER_OFFSET,
44 EVERGREEN_CRTC2_REGISTER_OFFSET,
45 EVERGREEN_CRTC3_REGISTER_OFFSET,
46 EVERGREEN_CRTC4_REGISTER_OFFSET,
47 EVERGREEN_CRTC5_REGISTER_OFFSET
48 };
49
50 static void evergreen_gpu_init(struct radeon_device *rdev);
51 void evergreen_fini(struct radeon_device *rdev);
52 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
53 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
54 int ring, u32 cp_int_cntl);
55
56 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
57 unsigned *bankh, unsigned *mtaspect,
58 unsigned *tile_split)
59 {
60 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
61 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
62 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
63 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
64 switch (*bankw) {
65 default:
66 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
67 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
68 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
69 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
70 }
71 switch (*bankh) {
72 default:
73 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
74 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
75 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
76 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
77 }
78 switch (*mtaspect) {
79 default:
80 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
81 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
82 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
83 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
84 }
85 }
86
87 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
88 u32 cntl_reg, u32 status_reg)
89 {
90 int r, i;
91 struct atom_clock_dividers dividers;
92
93 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
94 clock, false, &dividers);
95 if (r)
96 return r;
97
98 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
99
100 for (i = 0; i < 100; i++) {
101 if (RREG32(status_reg) & DCLK_STATUS)
102 break;
103 mdelay(10);
104 }
105 if (i == 100)
106 return -ETIMEDOUT;
107
108 return 0;
109 }
110
111 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
112 {
113 int r = 0;
114 u32 cg_scratch = RREG32(CG_SCRATCH1);
115
116 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
117 if (r)
118 goto done;
119 cg_scratch &= 0xffff0000;
120 cg_scratch |= vclk / 100; /* Mhz */
121
122 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
123 if (r)
124 goto done;
125 cg_scratch &= 0x0000ffff;
126 cg_scratch |= (dclk / 100) << 16; /* Mhz */
127
128 done:
129 WREG32(CG_SCRATCH1, cg_scratch);
130
131 return r;
132 }
133
134 static int evergreen_uvd_calc_post_div(unsigned target_freq,
135 unsigned vco_freq,
136 unsigned *div)
137 {
138 /* target larger than vco frequency ? */
139 if (vco_freq < target_freq)
140 return -1; /* forget it */
141
142 /* Fclk = Fvco / PDIV */
143 *div = vco_freq / target_freq;
144
145 /* we alway need a frequency less than or equal the target */
146 if ((vco_freq / *div) > target_freq)
147 *div += 1;
148
149 /* dividers above 5 must be even */
150 if (*div > 5 && *div % 2)
151 *div += 1;
152
153 /* out of range ? */
154 if (*div >= 128)
155 return -1; /* forget it */
156
157 return vco_freq / *div;
158 }
159
160 static int evergreen_uvd_send_upll_ctlreq(struct radeon_device *rdev)
161 {
162 unsigned i;
163
164 /* assert UPLL_CTLREQ */
165 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
166
167 /* wait for CTLACK and CTLACK2 to get asserted */
168 for (i = 0; i < 100; ++i) {
169 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
170 if ((RREG32(CG_UPLL_FUNC_CNTL) & mask) == mask)
171 break;
172 mdelay(10);
173 }
174 if (i == 100)
175 return -ETIMEDOUT;
176
177 /* deassert UPLL_CTLREQ */
178 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
179
180 return 0;
181 }
182
183 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
184 {
185 /* start off with something large */
186 int optimal_diff_score = 0x7FFFFFF;
187 unsigned optimal_fb_div = 0, optimal_vclk_div = 0;
188 unsigned optimal_dclk_div = 0, optimal_vco_freq = 0;
189 unsigned vco_freq;
190 int r;
191
192 /* loop through vco from low to high */
193 for (vco_freq = 125000; vco_freq <= 250000; vco_freq += 100) {
194 unsigned fb_div = vco_freq / rdev->clock.spll.reference_freq * 16384;
195 int calc_clk, diff_score, diff_vclk, diff_dclk;
196 unsigned vclk_div, dclk_div;
197
198 /* fb div out of range ? */
199 if (fb_div > 0x03FFFFFF)
200 break; /* it can oly get worse */
201
202 /* calc vclk with current vco freq. */
203 calc_clk = evergreen_uvd_calc_post_div(vclk, vco_freq, &vclk_div);
204 if (calc_clk == -1)
205 break; /* vco is too big, it has to stop. */
206 diff_vclk = vclk - calc_clk;
207
208 /* calc dclk with current vco freq. */
209 calc_clk = evergreen_uvd_calc_post_div(dclk, vco_freq, &dclk_div);
210 if (calc_clk == -1)
211 break; /* vco is too big, it has to stop. */
212 diff_dclk = dclk - calc_clk;
213
214 /* determine if this vco setting is better than current optimal settings */
215 diff_score = abs(diff_vclk) + abs(diff_dclk);
216 if (diff_score < optimal_diff_score) {
217 optimal_fb_div = fb_div;
218 optimal_vclk_div = vclk_div;
219 optimal_dclk_div = dclk_div;
220 optimal_vco_freq = vco_freq;
221 optimal_diff_score = diff_score;
222 if (optimal_diff_score == 0)
223 break; /* it can't get better than this */
224 }
225 }
226
227 /* set VCO_MODE to 1 */
228 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
229
230 /* toggle UPLL_SLEEP to 1 then back to 0 */
231 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
232 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
233
234 /* deassert UPLL_RESET */
235 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
236
237 mdelay(1);
238
239 /* bypass vclk and dclk with bclk */
240 WREG32_P(CG_UPLL_FUNC_CNTL_2,
241 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
242 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
243
244 /* put PLL in bypass mode */
245 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
246
247 r = evergreen_uvd_send_upll_ctlreq(rdev);
248 if (r)
249 return r;
250
251 /* assert UPLL_RESET again */
252 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
253
254 /* disable spread spectrum. */
255 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
256
257 /* set feedback divider */
258 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(optimal_fb_div), ~UPLL_FB_DIV_MASK);
259
260 /* set ref divider to 0 */
261 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
262
263 if (optimal_vco_freq < 187500)
264 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
265 else
266 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
267
268 /* set PDIV_A and PDIV_B */
269 WREG32_P(CG_UPLL_FUNC_CNTL_2,
270 UPLL_PDIV_A(optimal_vclk_div) | UPLL_PDIV_B(optimal_dclk_div),
271 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
272
273 /* give the PLL some time to settle */
274 mdelay(15);
275
276 /* deassert PLL_RESET */
277 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
278
279 mdelay(15);
280
281 /* switch from bypass mode to normal mode */
282 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
283
284 r = evergreen_uvd_send_upll_ctlreq(rdev);
285 if (r)
286 return r;
287
288 /* switch VCLK and DCLK selection */
289 WREG32_P(CG_UPLL_FUNC_CNTL_2,
290 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
291 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
292
293 mdelay(100);
294
295 return 0;
296 }
297
298 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
299 {
300 u16 ctl, v;
301 int err;
302
303 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
304 if (err)
305 return;
306
307 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
308
309 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
310 * to avoid hangs or perfomance issues
311 */
312 if ((v == 0) || (v == 6) || (v == 7)) {
313 ctl &= ~PCI_EXP_DEVCTL_READRQ;
314 ctl |= (2 << 12);
315 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
316 }
317 }
318
319 /**
320 * dce4_wait_for_vblank - vblank wait asic callback.
321 *
322 * @rdev: radeon_device pointer
323 * @crtc: crtc to wait for vblank on
324 *
325 * Wait for vblank on the requested crtc (evergreen+).
326 */
327 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
328 {
329 int i;
330
331 if (crtc >= rdev->num_crtc)
332 return;
333
334 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
335 for (i = 0; i < rdev->usec_timeout; i++) {
336 if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
337 break;
338 udelay(1);
339 }
340 for (i = 0; i < rdev->usec_timeout; i++) {
341 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
342 break;
343 udelay(1);
344 }
345 }
346 }
347
348 /**
349 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
350 *
351 * @rdev: radeon_device pointer
352 * @crtc: crtc to prepare for pageflip on
353 *
354 * Pre-pageflip callback (evergreen+).
355 * Enables the pageflip irq (vblank irq).
356 */
357 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
358 {
359 /* enable the pflip int */
360 radeon_irq_kms_pflip_irq_get(rdev, crtc);
361 }
362
363 /**
364 * evergreen_post_page_flip - pos-pageflip callback.
365 *
366 * @rdev: radeon_device pointer
367 * @crtc: crtc to cleanup pageflip on
368 *
369 * Post-pageflip callback (evergreen+).
370 * Disables the pageflip irq (vblank irq).
371 */
372 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
373 {
374 /* disable the pflip int */
375 radeon_irq_kms_pflip_irq_put(rdev, crtc);
376 }
377
378 /**
379 * evergreen_page_flip - pageflip callback.
380 *
381 * @rdev: radeon_device pointer
382 * @crtc_id: crtc to cleanup pageflip on
383 * @crtc_base: new address of the crtc (GPU MC address)
384 *
385 * Does the actual pageflip (evergreen+).
386 * During vblank we take the crtc lock and wait for the update_pending
387 * bit to go high, when it does, we release the lock, and allow the
388 * double buffered update to take place.
389 * Returns the current update pending status.
390 */
391 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
392 {
393 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
394 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
395 int i;
396
397 /* Lock the graphics update lock */
398 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
399 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
400
401 /* update the scanout addresses */
402 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
403 upper_32_bits(crtc_base));
404 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
405 (u32)crtc_base);
406
407 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
408 upper_32_bits(crtc_base));
409 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
410 (u32)crtc_base);
411
412 /* Wait for update_pending to go high. */
413 for (i = 0; i < rdev->usec_timeout; i++) {
414 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
415 break;
416 udelay(1);
417 }
418 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
419
420 /* Unlock the lock, so double-buffering can take place inside vblank */
421 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
422 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
423
424 /* Return current update_pending status: */
425 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
426 }
427
428 /* get temperature in millidegrees */
429 int evergreen_get_temp(struct radeon_device *rdev)
430 {
431 u32 temp, toffset;
432 int actual_temp = 0;
433
434 if (rdev->family == CHIP_JUNIPER) {
435 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
436 TOFFSET_SHIFT;
437 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
438 TS0_ADC_DOUT_SHIFT;
439
440 if (toffset & 0x100)
441 actual_temp = temp / 2 - (0x200 - toffset);
442 else
443 actual_temp = temp / 2 + toffset;
444
445 actual_temp = actual_temp * 1000;
446
447 } else {
448 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
449 ASIC_T_SHIFT;
450
451 if (temp & 0x400)
452 actual_temp = -256;
453 else if (temp & 0x200)
454 actual_temp = 255;
455 else if (temp & 0x100) {
456 actual_temp = temp & 0x1ff;
457 actual_temp |= ~0x1ff;
458 } else
459 actual_temp = temp & 0xff;
460
461 actual_temp = (actual_temp * 1000) / 2;
462 }
463
464 return actual_temp;
465 }
466
467 int sumo_get_temp(struct radeon_device *rdev)
468 {
469 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
470 int actual_temp = temp - 49;
471
472 return actual_temp * 1000;
473 }
474
475 /**
476 * sumo_pm_init_profile - Initialize power profiles callback.
477 *
478 * @rdev: radeon_device pointer
479 *
480 * Initialize the power states used in profile mode
481 * (sumo, trinity, SI).
482 * Used for profile mode only.
483 */
484 void sumo_pm_init_profile(struct radeon_device *rdev)
485 {
486 int idx;
487
488 /* default */
489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
491 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
492 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
493
494 /* low,mid sh/mh */
495 if (rdev->flags & RADEON_IS_MOBILITY)
496 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
497 else
498 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
499
500 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
504
505 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
506 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
507 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
508 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
509
510 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
511 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
512 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
513 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
514
515 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
516 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
517 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
518 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
519
520 /* high sh/mh */
521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
522 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
523 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
524 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
525 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
526 rdev->pm.power_state[idx].num_clock_modes - 1;
527
528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
529 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
530 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
531 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
532 rdev->pm.power_state[idx].num_clock_modes - 1;
533 }
534
535 /**
536 * btc_pm_init_profile - Initialize power profiles callback.
537 *
538 * @rdev: radeon_device pointer
539 *
540 * Initialize the power states used in profile mode
541 * (BTC, cayman).
542 * Used for profile mode only.
543 */
544 void btc_pm_init_profile(struct radeon_device *rdev)
545 {
546 int idx;
547
548 /* default */
549 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
550 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
551 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
552 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
553 /* starting with BTC, there is one state that is used for both
554 * MH and SH. Difference is that we always use the high clock index for
555 * mclk.
556 */
557 if (rdev->flags & RADEON_IS_MOBILITY)
558 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
559 else
560 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
561 /* low sh */
562 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
563 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
564 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
565 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
566 /* mid sh */
567 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
568 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
569 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
570 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
571 /* high sh */
572 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
573 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
574 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
575 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
576 /* low mh */
577 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
578 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
579 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
580 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
581 /* mid mh */
582 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
583 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
584 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
585 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
586 /* high mh */
587 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
588 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
589 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
590 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
591 }
592
593 /**
594 * evergreen_pm_misc - set additional pm hw parameters callback.
595 *
596 * @rdev: radeon_device pointer
597 *
598 * Set non-clock parameters associated with a power state
599 * (voltage, etc.) (evergreen+).
600 */
601 void evergreen_pm_misc(struct radeon_device *rdev)
602 {
603 int req_ps_idx = rdev->pm.requested_power_state_index;
604 int req_cm_idx = rdev->pm.requested_clock_mode_index;
605 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
606 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
607
608 if (voltage->type == VOLTAGE_SW) {
609 /* 0xff01 is a flag rather then an actual voltage */
610 if (voltage->voltage == 0xff01)
611 return;
612 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
613 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
614 rdev->pm.current_vddc = voltage->voltage;
615 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
616 }
617
618 /* starting with BTC, there is one state that is used for both
619 * MH and SH. Difference is that we always use the high clock index for
620 * mclk and vddci.
621 */
622 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
623 (rdev->family >= CHIP_BARTS) &&
624 rdev->pm.active_crtc_count &&
625 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
626 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
627 voltage = &rdev->pm.power_state[req_ps_idx].
628 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
629
630 /* 0xff01 is a flag rather then an actual voltage */
631 if (voltage->vddci == 0xff01)
632 return;
633 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
634 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
635 rdev->pm.current_vddci = voltage->vddci;
636 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
637 }
638 }
639 }
640
641 /**
642 * evergreen_pm_prepare - pre-power state change callback.
643 *
644 * @rdev: radeon_device pointer
645 *
646 * Prepare for a power state change (evergreen+).
647 */
648 void evergreen_pm_prepare(struct radeon_device *rdev)
649 {
650 struct drm_device *ddev = rdev->ddev;
651 struct drm_crtc *crtc;
652 struct radeon_crtc *radeon_crtc;
653 u32 tmp;
654
655 /* disable any active CRTCs */
656 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
657 radeon_crtc = to_radeon_crtc(crtc);
658 if (radeon_crtc->enabled) {
659 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
660 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
661 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
662 }
663 }
664 }
665
666 /**
667 * evergreen_pm_finish - post-power state change callback.
668 *
669 * @rdev: radeon_device pointer
670 *
671 * Clean up after a power state change (evergreen+).
672 */
673 void evergreen_pm_finish(struct radeon_device *rdev)
674 {
675 struct drm_device *ddev = rdev->ddev;
676 struct drm_crtc *crtc;
677 struct radeon_crtc *radeon_crtc;
678 u32 tmp;
679
680 /* enable any active CRTCs */
681 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
682 radeon_crtc = to_radeon_crtc(crtc);
683 if (radeon_crtc->enabled) {
684 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
685 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
686 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
687 }
688 }
689 }
690
691 /**
692 * evergreen_hpd_sense - hpd sense callback.
693 *
694 * @rdev: radeon_device pointer
695 * @hpd: hpd (hotplug detect) pin
696 *
697 * Checks if a digital monitor is connected (evergreen+).
698 * Returns true if connected, false if not connected.
699 */
700 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
701 {
702 bool connected = false;
703
704 switch (hpd) {
705 case RADEON_HPD_1:
706 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
707 connected = true;
708 break;
709 case RADEON_HPD_2:
710 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
711 connected = true;
712 break;
713 case RADEON_HPD_3:
714 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
715 connected = true;
716 break;
717 case RADEON_HPD_4:
718 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
719 connected = true;
720 break;
721 case RADEON_HPD_5:
722 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
723 connected = true;
724 break;
725 case RADEON_HPD_6:
726 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
727 connected = true;
728 break;
729 default:
730 break;
731 }
732
733 return connected;
734 }
735
736 /**
737 * evergreen_hpd_set_polarity - hpd set polarity callback.
738 *
739 * @rdev: radeon_device pointer
740 * @hpd: hpd (hotplug detect) pin
741 *
742 * Set the polarity of the hpd pin (evergreen+).
743 */
744 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
745 enum radeon_hpd_id hpd)
746 {
747 u32 tmp;
748 bool connected = evergreen_hpd_sense(rdev, hpd);
749
750 switch (hpd) {
751 case RADEON_HPD_1:
752 tmp = RREG32(DC_HPD1_INT_CONTROL);
753 if (connected)
754 tmp &= ~DC_HPDx_INT_POLARITY;
755 else
756 tmp |= DC_HPDx_INT_POLARITY;
757 WREG32(DC_HPD1_INT_CONTROL, tmp);
758 break;
759 case RADEON_HPD_2:
760 tmp = RREG32(DC_HPD2_INT_CONTROL);
761 if (connected)
762 tmp &= ~DC_HPDx_INT_POLARITY;
763 else
764 tmp |= DC_HPDx_INT_POLARITY;
765 WREG32(DC_HPD2_INT_CONTROL, tmp);
766 break;
767 case RADEON_HPD_3:
768 tmp = RREG32(DC_HPD3_INT_CONTROL);
769 if (connected)
770 tmp &= ~DC_HPDx_INT_POLARITY;
771 else
772 tmp |= DC_HPDx_INT_POLARITY;
773 WREG32(DC_HPD3_INT_CONTROL, tmp);
774 break;
775 case RADEON_HPD_4:
776 tmp = RREG32(DC_HPD4_INT_CONTROL);
777 if (connected)
778 tmp &= ~DC_HPDx_INT_POLARITY;
779 else
780 tmp |= DC_HPDx_INT_POLARITY;
781 WREG32(DC_HPD4_INT_CONTROL, tmp);
782 break;
783 case RADEON_HPD_5:
784 tmp = RREG32(DC_HPD5_INT_CONTROL);
785 if (connected)
786 tmp &= ~DC_HPDx_INT_POLARITY;
787 else
788 tmp |= DC_HPDx_INT_POLARITY;
789 WREG32(DC_HPD5_INT_CONTROL, tmp);
790 break;
791 case RADEON_HPD_6:
792 tmp = RREG32(DC_HPD6_INT_CONTROL);
793 if (connected)
794 tmp &= ~DC_HPDx_INT_POLARITY;
795 else
796 tmp |= DC_HPDx_INT_POLARITY;
797 WREG32(DC_HPD6_INT_CONTROL, tmp);
798 break;
799 default:
800 break;
801 }
802 }
803
804 /**
805 * evergreen_hpd_init - hpd setup callback.
806 *
807 * @rdev: radeon_device pointer
808 *
809 * Setup the hpd pins used by the card (evergreen+).
810 * Enable the pin, set the polarity, and enable the hpd interrupts.
811 */
812 void evergreen_hpd_init(struct radeon_device *rdev)
813 {
814 struct drm_device *dev = rdev->ddev;
815 struct drm_connector *connector;
816 unsigned enabled = 0;
817 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
818 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
819
820 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
821 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
822 switch (radeon_connector->hpd.hpd) {
823 case RADEON_HPD_1:
824 WREG32(DC_HPD1_CONTROL, tmp);
825 break;
826 case RADEON_HPD_2:
827 WREG32(DC_HPD2_CONTROL, tmp);
828 break;
829 case RADEON_HPD_3:
830 WREG32(DC_HPD3_CONTROL, tmp);
831 break;
832 case RADEON_HPD_4:
833 WREG32(DC_HPD4_CONTROL, tmp);
834 break;
835 case RADEON_HPD_5:
836 WREG32(DC_HPD5_CONTROL, tmp);
837 break;
838 case RADEON_HPD_6:
839 WREG32(DC_HPD6_CONTROL, tmp);
840 break;
841 default:
842 break;
843 }
844 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
845 enabled |= 1 << radeon_connector->hpd.hpd;
846 }
847 radeon_irq_kms_enable_hpd(rdev, enabled);
848 }
849
850 /**
851 * evergreen_hpd_fini - hpd tear down callback.
852 *
853 * @rdev: radeon_device pointer
854 *
855 * Tear down the hpd pins used by the card (evergreen+).
856 * Disable the hpd interrupts.
857 */
858 void evergreen_hpd_fini(struct radeon_device *rdev)
859 {
860 struct drm_device *dev = rdev->ddev;
861 struct drm_connector *connector;
862 unsigned disabled = 0;
863
864 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
865 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
866 switch (radeon_connector->hpd.hpd) {
867 case RADEON_HPD_1:
868 WREG32(DC_HPD1_CONTROL, 0);
869 break;
870 case RADEON_HPD_2:
871 WREG32(DC_HPD2_CONTROL, 0);
872 break;
873 case RADEON_HPD_3:
874 WREG32(DC_HPD3_CONTROL, 0);
875 break;
876 case RADEON_HPD_4:
877 WREG32(DC_HPD4_CONTROL, 0);
878 break;
879 case RADEON_HPD_5:
880 WREG32(DC_HPD5_CONTROL, 0);
881 break;
882 case RADEON_HPD_6:
883 WREG32(DC_HPD6_CONTROL, 0);
884 break;
885 default:
886 break;
887 }
888 disabled |= 1 << radeon_connector->hpd.hpd;
889 }
890 radeon_irq_kms_disable_hpd(rdev, disabled);
891 }
892
893 /* watermark setup */
894
895 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
896 struct radeon_crtc *radeon_crtc,
897 struct drm_display_mode *mode,
898 struct drm_display_mode *other_mode)
899 {
900 u32 tmp;
901 /*
902 * Line Buffer Setup
903 * There are 3 line buffers, each one shared by 2 display controllers.
904 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
905 * the display controllers. The paritioning is done via one of four
906 * preset allocations specified in bits 2:0:
907 * first display controller
908 * 0 - first half of lb (3840 * 2)
909 * 1 - first 3/4 of lb (5760 * 2)
910 * 2 - whole lb (7680 * 2), other crtc must be disabled
911 * 3 - first 1/4 of lb (1920 * 2)
912 * second display controller
913 * 4 - second half of lb (3840 * 2)
914 * 5 - second 3/4 of lb (5760 * 2)
915 * 6 - whole lb (7680 * 2), other crtc must be disabled
916 * 7 - last 1/4 of lb (1920 * 2)
917 */
918 /* this can get tricky if we have two large displays on a paired group
919 * of crtcs. Ideally for multiple large displays we'd assign them to
920 * non-linked crtcs for maximum line buffer allocation.
921 */
922 if (radeon_crtc->base.enabled && mode) {
923 if (other_mode)
924 tmp = 0; /* 1/2 */
925 else
926 tmp = 2; /* whole */
927 } else
928 tmp = 0;
929
930 /* second controller of the pair uses second half of the lb */
931 if (radeon_crtc->crtc_id % 2)
932 tmp += 4;
933 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
934
935 if (radeon_crtc->base.enabled && mode) {
936 switch (tmp) {
937 case 0:
938 case 4:
939 default:
940 if (ASIC_IS_DCE5(rdev))
941 return 4096 * 2;
942 else
943 return 3840 * 2;
944 case 1:
945 case 5:
946 if (ASIC_IS_DCE5(rdev))
947 return 6144 * 2;
948 else
949 return 5760 * 2;
950 case 2:
951 case 6:
952 if (ASIC_IS_DCE5(rdev))
953 return 8192 * 2;
954 else
955 return 7680 * 2;
956 case 3:
957 case 7:
958 if (ASIC_IS_DCE5(rdev))
959 return 2048 * 2;
960 else
961 return 1920 * 2;
962 }
963 }
964
965 /* controller not enabled, so no lb used */
966 return 0;
967 }
968
969 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
970 {
971 u32 tmp = RREG32(MC_SHARED_CHMAP);
972
973 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
974 case 0:
975 default:
976 return 1;
977 case 1:
978 return 2;
979 case 2:
980 return 4;
981 case 3:
982 return 8;
983 }
984 }
985
986 struct evergreen_wm_params {
987 u32 dram_channels; /* number of dram channels */
988 u32 yclk; /* bandwidth per dram data pin in kHz */
989 u32 sclk; /* engine clock in kHz */
990 u32 disp_clk; /* display clock in kHz */
991 u32 src_width; /* viewport width */
992 u32 active_time; /* active display time in ns */
993 u32 blank_time; /* blank time in ns */
994 bool interlaced; /* mode is interlaced */
995 fixed20_12 vsc; /* vertical scale ratio */
996 u32 num_heads; /* number of active crtcs */
997 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
998 u32 lb_size; /* line buffer allocated to pipe */
999 u32 vtaps; /* vertical scaler taps */
1000 };
1001
1002 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1003 {
1004 /* Calculate DRAM Bandwidth and the part allocated to display. */
1005 fixed20_12 dram_efficiency; /* 0.7 */
1006 fixed20_12 yclk, dram_channels, bandwidth;
1007 fixed20_12 a;
1008
1009 a.full = dfixed_const(1000);
1010 yclk.full = dfixed_const(wm->yclk);
1011 yclk.full = dfixed_div(yclk, a);
1012 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1013 a.full = dfixed_const(10);
1014 dram_efficiency.full = dfixed_const(7);
1015 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1016 bandwidth.full = dfixed_mul(dram_channels, yclk);
1017 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1018
1019 return dfixed_trunc(bandwidth);
1020 }
1021
1022 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1023 {
1024 /* Calculate DRAM Bandwidth and the part allocated to display. */
1025 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1026 fixed20_12 yclk, dram_channels, bandwidth;
1027 fixed20_12 a;
1028
1029 a.full = dfixed_const(1000);
1030 yclk.full = dfixed_const(wm->yclk);
1031 yclk.full = dfixed_div(yclk, a);
1032 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1033 a.full = dfixed_const(10);
1034 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1035 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1036 bandwidth.full = dfixed_mul(dram_channels, yclk);
1037 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1038
1039 return dfixed_trunc(bandwidth);
1040 }
1041
1042 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1043 {
1044 /* Calculate the display Data return Bandwidth */
1045 fixed20_12 return_efficiency; /* 0.8 */
1046 fixed20_12 sclk, bandwidth;
1047 fixed20_12 a;
1048
1049 a.full = dfixed_const(1000);
1050 sclk.full = dfixed_const(wm->sclk);
1051 sclk.full = dfixed_div(sclk, a);
1052 a.full = dfixed_const(10);
1053 return_efficiency.full = dfixed_const(8);
1054 return_efficiency.full = dfixed_div(return_efficiency, a);
1055 a.full = dfixed_const(32);
1056 bandwidth.full = dfixed_mul(a, sclk);
1057 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1058
1059 return dfixed_trunc(bandwidth);
1060 }
1061
1062 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1063 {
1064 /* Calculate the DMIF Request Bandwidth */
1065 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1066 fixed20_12 disp_clk, bandwidth;
1067 fixed20_12 a;
1068
1069 a.full = dfixed_const(1000);
1070 disp_clk.full = dfixed_const(wm->disp_clk);
1071 disp_clk.full = dfixed_div(disp_clk, a);
1072 a.full = dfixed_const(10);
1073 disp_clk_request_efficiency.full = dfixed_const(8);
1074 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1075 a.full = dfixed_const(32);
1076 bandwidth.full = dfixed_mul(a, disp_clk);
1077 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1078
1079 return dfixed_trunc(bandwidth);
1080 }
1081
1082 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1083 {
1084 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1085 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1086 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1087 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1088
1089 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1090 }
1091
1092 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
1093 {
1094 /* Calculate the display mode Average Bandwidth
1095 * DisplayMode should contain the source and destination dimensions,
1096 * timing, etc.
1097 */
1098 fixed20_12 bpp;
1099 fixed20_12 line_time;
1100 fixed20_12 src_width;
1101 fixed20_12 bandwidth;
1102 fixed20_12 a;
1103
1104 a.full = dfixed_const(1000);
1105 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1106 line_time.full = dfixed_div(line_time, a);
1107 bpp.full = dfixed_const(wm->bytes_per_pixel);
1108 src_width.full = dfixed_const(wm->src_width);
1109 bandwidth.full = dfixed_mul(src_width, bpp);
1110 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1111 bandwidth.full = dfixed_div(bandwidth, line_time);
1112
1113 return dfixed_trunc(bandwidth);
1114 }
1115
1116 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
1117 {
1118 /* First calcualte the latency in ns */
1119 u32 mc_latency = 2000; /* 2000 ns. */
1120 u32 available_bandwidth = evergreen_available_bandwidth(wm);
1121 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1122 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1123 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1124 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1125 (wm->num_heads * cursor_line_pair_return_time);
1126 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1127 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1128 fixed20_12 a, b, c;
1129
1130 if (wm->num_heads == 0)
1131 return 0;
1132
1133 a.full = dfixed_const(2);
1134 b.full = dfixed_const(1);
1135 if ((wm->vsc.full > a.full) ||
1136 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1137 (wm->vtaps >= 5) ||
1138 ((wm->vsc.full >= a.full) && wm->interlaced))
1139 max_src_lines_per_dst_line = 4;
1140 else
1141 max_src_lines_per_dst_line = 2;
1142
1143 a.full = dfixed_const(available_bandwidth);
1144 b.full = dfixed_const(wm->num_heads);
1145 a.full = dfixed_div(a, b);
1146
1147 b.full = dfixed_const(1000);
1148 c.full = dfixed_const(wm->disp_clk);
1149 b.full = dfixed_div(c, b);
1150 c.full = dfixed_const(wm->bytes_per_pixel);
1151 b.full = dfixed_mul(b, c);
1152
1153 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
1154
1155 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1156 b.full = dfixed_const(1000);
1157 c.full = dfixed_const(lb_fill_bw);
1158 b.full = dfixed_div(c, b);
1159 a.full = dfixed_div(a, b);
1160 line_fill_time = dfixed_trunc(a);
1161
1162 if (line_fill_time < wm->active_time)
1163 return latency;
1164 else
1165 return latency + (line_fill_time - wm->active_time);
1166
1167 }
1168
1169 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1170 {
1171 if (evergreen_average_bandwidth(wm) <=
1172 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
1173 return true;
1174 else
1175 return false;
1176 };
1177
1178 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
1179 {
1180 if (evergreen_average_bandwidth(wm) <=
1181 (evergreen_available_bandwidth(wm) / wm->num_heads))
1182 return true;
1183 else
1184 return false;
1185 };
1186
1187 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
1188 {
1189 u32 lb_partitions = wm->lb_size / wm->src_width;
1190 u32 line_time = wm->active_time + wm->blank_time;
1191 u32 latency_tolerant_lines;
1192 u32 latency_hiding;
1193 fixed20_12 a;
1194
1195 a.full = dfixed_const(1);
1196 if (wm->vsc.full > a.full)
1197 latency_tolerant_lines = 1;
1198 else {
1199 if (lb_partitions <= (wm->vtaps + 1))
1200 latency_tolerant_lines = 1;
1201 else
1202 latency_tolerant_lines = 2;
1203 }
1204
1205 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1206
1207 if (evergreen_latency_watermark(wm) <= latency_hiding)
1208 return true;
1209 else
1210 return false;
1211 }
1212
1213 static void evergreen_program_watermarks(struct radeon_device *rdev,
1214 struct radeon_crtc *radeon_crtc,
1215 u32 lb_size, u32 num_heads)
1216 {
1217 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1218 struct evergreen_wm_params wm;
1219 u32 pixel_period;
1220 u32 line_time = 0;
1221 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1222 u32 priority_a_mark = 0, priority_b_mark = 0;
1223 u32 priority_a_cnt = PRIORITY_OFF;
1224 u32 priority_b_cnt = PRIORITY_OFF;
1225 u32 pipe_offset = radeon_crtc->crtc_id * 16;
1226 u32 tmp, arb_control3;
1227 fixed20_12 a, b, c;
1228
1229 if (radeon_crtc->base.enabled && num_heads && mode) {
1230 pixel_period = 1000000 / (u32)mode->clock;
1231 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1232 priority_a_cnt = 0;
1233 priority_b_cnt = 0;
1234
1235 wm.yclk = rdev->pm.current_mclk * 10;
1236 wm.sclk = rdev->pm.current_sclk * 10;
1237 wm.disp_clk = mode->clock;
1238 wm.src_width = mode->crtc_hdisplay;
1239 wm.active_time = mode->crtc_hdisplay * pixel_period;
1240 wm.blank_time = line_time - wm.active_time;
1241 wm.interlaced = false;
1242 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1243 wm.interlaced = true;
1244 wm.vsc = radeon_crtc->vsc;
1245 wm.vtaps = 1;
1246 if (radeon_crtc->rmx_type != RMX_OFF)
1247 wm.vtaps = 2;
1248 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1249 wm.lb_size = lb_size;
1250 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1251 wm.num_heads = num_heads;
1252
1253 /* set for high clocks */
1254 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
1255 /* set for low clocks */
1256 /* wm.yclk = low clk; wm.sclk = low clk */
1257 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
1258
1259 /* possibly force display priority to high */
1260 /* should really do this at mode validation time... */
1261 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1262 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
1263 !evergreen_check_latency_hiding(&wm) ||
1264 (rdev->disp_priority == 2)) {
1265 DRM_DEBUG_KMS("force priority to high\n");
1266 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1267 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1268 }
1269
1270 a.full = dfixed_const(1000);
1271 b.full = dfixed_const(mode->clock);
1272 b.full = dfixed_div(b, a);
1273 c.full = dfixed_const(latency_watermark_a);
1274 c.full = dfixed_mul(c, b);
1275 c.full = dfixed_mul(c, radeon_crtc->hsc);
1276 c.full = dfixed_div(c, a);
1277 a.full = dfixed_const(16);
1278 c.full = dfixed_div(c, a);
1279 priority_a_mark = dfixed_trunc(c);
1280 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1281
1282 a.full = dfixed_const(1000);
1283 b.full = dfixed_const(mode->clock);
1284 b.full = dfixed_div(b, a);
1285 c.full = dfixed_const(latency_watermark_b);
1286 c.full = dfixed_mul(c, b);
1287 c.full = dfixed_mul(c, radeon_crtc->hsc);
1288 c.full = dfixed_div(c, a);
1289 a.full = dfixed_const(16);
1290 c.full = dfixed_div(c, a);
1291 priority_b_mark = dfixed_trunc(c);
1292 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1293 }
1294
1295 /* select wm A */
1296 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
1297 tmp = arb_control3;
1298 tmp &= ~LATENCY_WATERMARK_MASK(3);
1299 tmp |= LATENCY_WATERMARK_MASK(1);
1300 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
1301 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
1302 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1303 LATENCY_HIGH_WATERMARK(line_time)));
1304 /* select wm B */
1305 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
1306 tmp &= ~LATENCY_WATERMARK_MASK(3);
1307 tmp |= LATENCY_WATERMARK_MASK(2);
1308 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
1309 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
1310 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1311 LATENCY_HIGH_WATERMARK(line_time)));
1312 /* restore original selection */
1313 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
1314
1315 /* write the priority marks */
1316 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1317 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1318
1319 }
1320
1321 /**
1322 * evergreen_bandwidth_update - update display watermarks callback.
1323 *
1324 * @rdev: radeon_device pointer
1325 *
1326 * Update the display watermarks based on the requested mode(s)
1327 * (evergreen+).
1328 */
1329 void evergreen_bandwidth_update(struct radeon_device *rdev)
1330 {
1331 struct drm_display_mode *mode0 = NULL;
1332 struct drm_display_mode *mode1 = NULL;
1333 u32 num_heads = 0, lb_size;
1334 int i;
1335
1336 radeon_update_display_priority(rdev);
1337
1338 for (i = 0; i < rdev->num_crtc; i++) {
1339 if (rdev->mode_info.crtcs[i]->base.enabled)
1340 num_heads++;
1341 }
1342 for (i = 0; i < rdev->num_crtc; i += 2) {
1343 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1344 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1345 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1346 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1347 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1348 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1349 }
1350 }
1351
1352 /**
1353 * evergreen_mc_wait_for_idle - wait for MC idle callback.
1354 *
1355 * @rdev: radeon_device pointer
1356 *
1357 * Wait for the MC (memory controller) to be idle.
1358 * (evergreen+).
1359 * Returns 0 if the MC is idle, -1 if not.
1360 */
1361 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1362 {
1363 unsigned i;
1364 u32 tmp;
1365
1366 for (i = 0; i < rdev->usec_timeout; i++) {
1367 /* read MC_STATUS */
1368 tmp = RREG32(SRBM_STATUS) & 0x1F00;
1369 if (!tmp)
1370 return 0;
1371 udelay(1);
1372 }
1373 return -1;
1374 }
1375
1376 /*
1377 * GART
1378 */
1379 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
1380 {
1381 unsigned i;
1382 u32 tmp;
1383
1384 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1385
1386 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
1387 for (i = 0; i < rdev->usec_timeout; i++) {
1388 /* read MC_STATUS */
1389 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
1390 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
1391 if (tmp == 2) {
1392 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
1393 return;
1394 }
1395 if (tmp) {
1396 return;
1397 }
1398 udelay(1);
1399 }
1400 }
1401
1402 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
1403 {
1404 u32 tmp;
1405 int r;
1406
1407 if (rdev->gart.robj == NULL) {
1408 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1409 return -EINVAL;
1410 }
1411 r = radeon_gart_table_vram_pin(rdev);
1412 if (r)
1413 return r;
1414 radeon_gart_restore(rdev);
1415 /* Setup L2 cache */
1416 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1417 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1418 EFFECTIVE_L2_QUEUE_SIZE(7));
1419 WREG32(VM_L2_CNTL2, 0);
1420 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1421 /* Setup TLB control */
1422 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1423 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1424 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1425 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1426 if (rdev->flags & RADEON_IS_IGP) {
1427 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
1428 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
1429 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
1430 } else {
1431 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1432 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1433 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1434 if ((rdev->family == CHIP_JUNIPER) ||
1435 (rdev->family == CHIP_CYPRESS) ||
1436 (rdev->family == CHIP_HEMLOCK) ||
1437 (rdev->family == CHIP_BARTS))
1438 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
1439 }
1440 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1441 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1442 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1443 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1444 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1445 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1446 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1447 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1448 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1449 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1450 (u32)(rdev->dummy_page.addr >> 12));
1451 WREG32(VM_CONTEXT1_CNTL, 0);
1452
1453 evergreen_pcie_gart_tlb_flush(rdev);
1454 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1455 (unsigned)(rdev->mc.gtt_size >> 20),
1456 (unsigned long long)rdev->gart.table_addr);
1457 rdev->gart.ready = true;
1458 return 0;
1459 }
1460
1461 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
1462 {
1463 u32 tmp;
1464
1465 /* Disable all tables */
1466 WREG32(VM_CONTEXT0_CNTL, 0);
1467 WREG32(VM_CONTEXT1_CNTL, 0);
1468
1469 /* Setup L2 cache */
1470 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
1471 EFFECTIVE_L2_QUEUE_SIZE(7));
1472 WREG32(VM_L2_CNTL2, 0);
1473 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1474 /* Setup TLB control */
1475 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1476 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1477 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1478 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1479 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1480 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1481 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1482 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1483 radeon_gart_table_vram_unpin(rdev);
1484 }
1485
1486 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
1487 {
1488 evergreen_pcie_gart_disable(rdev);
1489 radeon_gart_table_vram_free(rdev);
1490 radeon_gart_fini(rdev);
1491 }
1492
1493
1494 static void evergreen_agp_enable(struct radeon_device *rdev)
1495 {
1496 u32 tmp;
1497
1498 /* Setup L2 cache */
1499 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1500 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1501 EFFECTIVE_L2_QUEUE_SIZE(7));
1502 WREG32(VM_L2_CNTL2, 0);
1503 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1504 /* Setup TLB control */
1505 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1506 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1507 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1508 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1509 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1510 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1511 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1512 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1513 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1514 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1515 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1516 WREG32(VM_CONTEXT0_CNTL, 0);
1517 WREG32(VM_CONTEXT1_CNTL, 0);
1518 }
1519
1520 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1521 {
1522 u32 crtc_enabled, tmp, frame_count, blackout;
1523 int i, j;
1524
1525 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
1526 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
1527
1528 /* disable VGA render */
1529 WREG32(VGA_RENDER_CONTROL, 0);
1530 /* blank the display controllers */
1531 for (i = 0; i < rdev->num_crtc; i++) {
1532 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
1533 if (crtc_enabled) {
1534 save->crtc_enabled[i] = true;
1535 if (ASIC_IS_DCE6(rdev)) {
1536 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
1537 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
1538 radeon_wait_for_vblank(rdev, i);
1539 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
1540 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1541 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
1542 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1543 }
1544 } else {
1545 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
1546 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
1547 radeon_wait_for_vblank(rdev, i);
1548 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1549 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1550 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
1551 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1552 }
1553 }
1554 /* wait for the next frame */
1555 frame_count = radeon_get_vblank_counter(rdev, i);
1556 for (j = 0; j < rdev->usec_timeout; j++) {
1557 if (radeon_get_vblank_counter(rdev, i) != frame_count)
1558 break;
1559 udelay(1);
1560 }
1561 } else {
1562 save->crtc_enabled[i] = false;
1563 }
1564 }
1565
1566 radeon_mc_wait_for_idle(rdev);
1567
1568 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1569 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
1570 /* Block CPU access */
1571 WREG32(BIF_FB_EN, 0);
1572 /* blackout the MC */
1573 blackout &= ~BLACKOUT_MODE_MASK;
1574 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1575 }
1576 /* wait for the MC to settle */
1577 udelay(100);
1578 }
1579
1580 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1581 {
1582 u32 tmp, frame_count;
1583 int i, j;
1584
1585 /* update crtc base addresses */
1586 for (i = 0; i < rdev->num_crtc; i++) {
1587 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
1588 upper_32_bits(rdev->mc.vram_start));
1589 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
1590 upper_32_bits(rdev->mc.vram_start));
1591 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
1592 (u32)rdev->mc.vram_start);
1593 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
1594 (u32)rdev->mc.vram_start);
1595 }
1596 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1597 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1598
1599 /* unblackout the MC */
1600 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
1601 tmp &= ~BLACKOUT_MODE_MASK;
1602 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
1603 /* allow CPU access */
1604 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
1605
1606 for (i = 0; i < rdev->num_crtc; i++) {
1607 if (save->crtc_enabled[i]) {
1608 if (ASIC_IS_DCE6(rdev)) {
1609 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
1610 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
1611 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1612 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
1613 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1614 } else {
1615 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
1616 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1617 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1618 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
1619 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1620 }
1621 /* wait for the next frame */
1622 frame_count = radeon_get_vblank_counter(rdev, i);
1623 for (j = 0; j < rdev->usec_timeout; j++) {
1624 if (radeon_get_vblank_counter(rdev, i) != frame_count)
1625 break;
1626 udelay(1);
1627 }
1628 }
1629 }
1630 /* Unlock vga access */
1631 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1632 mdelay(1);
1633 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1634 }
1635
1636 void evergreen_mc_program(struct radeon_device *rdev)
1637 {
1638 struct evergreen_mc_save save;
1639 u32 tmp;
1640 int i, j;
1641
1642 /* Initialize HDP */
1643 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1644 WREG32((0x2c14 + j), 0x00000000);
1645 WREG32((0x2c18 + j), 0x00000000);
1646 WREG32((0x2c1c + j), 0x00000000);
1647 WREG32((0x2c20 + j), 0x00000000);
1648 WREG32((0x2c24 + j), 0x00000000);
1649 }
1650 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1651
1652 evergreen_mc_stop(rdev, &save);
1653 if (evergreen_mc_wait_for_idle(rdev)) {
1654 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1655 }
1656 /* Lockout access through VGA aperture*/
1657 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1658 /* Update configuration */
1659 if (rdev->flags & RADEON_IS_AGP) {
1660 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1661 /* VRAM before AGP */
1662 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1663 rdev->mc.vram_start >> 12);
1664 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1665 rdev->mc.gtt_end >> 12);
1666 } else {
1667 /* VRAM after AGP */
1668 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1669 rdev->mc.gtt_start >> 12);
1670 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1671 rdev->mc.vram_end >> 12);
1672 }
1673 } else {
1674 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1675 rdev->mc.vram_start >> 12);
1676 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1677 rdev->mc.vram_end >> 12);
1678 }
1679 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1680 /* llano/ontario only */
1681 if ((rdev->family == CHIP_PALM) ||
1682 (rdev->family == CHIP_SUMO) ||
1683 (rdev->family == CHIP_SUMO2)) {
1684 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1685 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1686 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1687 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1688 }
1689 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1690 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1691 WREG32(MC_VM_FB_LOCATION, tmp);
1692 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1693 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1694 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1695 if (rdev->flags & RADEON_IS_AGP) {
1696 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1697 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1698 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1699 } else {
1700 WREG32(MC_VM_AGP_BASE, 0);
1701 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1702 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1703 }
1704 if (evergreen_mc_wait_for_idle(rdev)) {
1705 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1706 }
1707 evergreen_mc_resume(rdev, &save);
1708 /* we need to own VRAM, so turn off the VGA renderer here
1709 * to stop it overwriting our objects */
1710 rv515_vga_render_disable(rdev);
1711 }
1712
1713 /*
1714 * CP.
1715 */
1716 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1717 {
1718 struct radeon_ring *ring = &rdev->ring[ib->ring];
1719 u32 next_rptr;
1720
1721 /* set to DX10/11 mode */
1722 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1723 radeon_ring_write(ring, 1);
1724
1725 if (ring->rptr_save_reg) {
1726 next_rptr = ring->wptr + 3 + 4;
1727 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1728 radeon_ring_write(ring, ((ring->rptr_save_reg -
1729 PACKET3_SET_CONFIG_REG_START) >> 2));
1730 radeon_ring_write(ring, next_rptr);
1731 } else if (rdev->wb.enabled) {
1732 next_rptr = ring->wptr + 5 + 4;
1733 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
1734 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1735 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
1736 radeon_ring_write(ring, next_rptr);
1737 radeon_ring_write(ring, 0);
1738 }
1739
1740 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1741 radeon_ring_write(ring,
1742 #ifdef __BIG_ENDIAN
1743 (2 << 0) |
1744 #endif
1745 (ib->gpu_addr & 0xFFFFFFFC));
1746 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1747 radeon_ring_write(ring, ib->length_dw);
1748 }
1749
1750
1751 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1752 {
1753 const __be32 *fw_data;
1754 int i;
1755
1756 if (!rdev->me_fw || !rdev->pfp_fw)
1757 return -EINVAL;
1758
1759 r700_cp_stop(rdev);
1760 WREG32(CP_RB_CNTL,
1761 #ifdef __BIG_ENDIAN
1762 BUF_SWAP_32BIT |
1763 #endif
1764 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1765
1766 fw_data = (const __be32 *)rdev->pfp_fw->data;
1767 WREG32(CP_PFP_UCODE_ADDR, 0);
1768 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1769 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1770 WREG32(CP_PFP_UCODE_ADDR, 0);
1771
1772 fw_data = (const __be32 *)rdev->me_fw->data;
1773 WREG32(CP_ME_RAM_WADDR, 0);
1774 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1775 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1776
1777 WREG32(CP_PFP_UCODE_ADDR, 0);
1778 WREG32(CP_ME_RAM_WADDR, 0);
1779 WREG32(CP_ME_RAM_RADDR, 0);
1780 return 0;
1781 }
1782
1783 static int evergreen_cp_start(struct radeon_device *rdev)
1784 {
1785 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1786 int r, i;
1787 uint32_t cp_me;
1788
1789 r = radeon_ring_lock(rdev, ring, 7);
1790 if (r) {
1791 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1792 return r;
1793 }
1794 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1795 radeon_ring_write(ring, 0x1);
1796 radeon_ring_write(ring, 0x0);
1797 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
1798 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1799 radeon_ring_write(ring, 0);
1800 radeon_ring_write(ring, 0);
1801 radeon_ring_unlock_commit(rdev, ring);
1802
1803 cp_me = 0xff;
1804 WREG32(CP_ME_CNTL, cp_me);
1805
1806 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1807 if (r) {
1808 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1809 return r;
1810 }
1811
1812 /* setup clear context state */
1813 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1814 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1815
1816 for (i = 0; i < evergreen_default_size; i++)
1817 radeon_ring_write(ring, evergreen_default_state[i]);
1818
1819 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1820 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1821
1822 /* set clear context state */
1823 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1824 radeon_ring_write(ring, 0);
1825
1826 /* SQ_VTX_BASE_VTX_LOC */
1827 radeon_ring_write(ring, 0xc0026f00);
1828 radeon_ring_write(ring, 0x00000000);
1829 radeon_ring_write(ring, 0x00000000);
1830 radeon_ring_write(ring, 0x00000000);
1831
1832 /* Clear consts */
1833 radeon_ring_write(ring, 0xc0036f00);
1834 radeon_ring_write(ring, 0x00000bc4);
1835 radeon_ring_write(ring, 0xffffffff);
1836 radeon_ring_write(ring, 0xffffffff);
1837 radeon_ring_write(ring, 0xffffffff);
1838
1839 radeon_ring_write(ring, 0xc0026900);
1840 radeon_ring_write(ring, 0x00000316);
1841 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1842 radeon_ring_write(ring, 0x00000010); /* */
1843
1844 radeon_ring_unlock_commit(rdev, ring);
1845
1846 return 0;
1847 }
1848
1849 static int evergreen_cp_resume(struct radeon_device *rdev)
1850 {
1851 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1852 u32 tmp;
1853 u32 rb_bufsz;
1854 int r;
1855
1856 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1857 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1858 SOFT_RESET_PA |
1859 SOFT_RESET_SH |
1860 SOFT_RESET_VGT |
1861 SOFT_RESET_SPI |
1862 SOFT_RESET_SX));
1863 RREG32(GRBM_SOFT_RESET);
1864 mdelay(15);
1865 WREG32(GRBM_SOFT_RESET, 0);
1866 RREG32(GRBM_SOFT_RESET);
1867
1868 /* Set ring buffer size */
1869 rb_bufsz = drm_order(ring->ring_size / 8);
1870 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1871 #ifdef __BIG_ENDIAN
1872 tmp |= BUF_SWAP_32BIT;
1873 #endif
1874 WREG32(CP_RB_CNTL, tmp);
1875 WREG32(CP_SEM_WAIT_TIMER, 0x0);
1876 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1877
1878 /* Set the write pointer delay */
1879 WREG32(CP_RB_WPTR_DELAY, 0);
1880
1881 /* Initialize the ring buffer's read and write pointers */
1882 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1883 WREG32(CP_RB_RPTR_WR, 0);
1884 ring->wptr = 0;
1885 WREG32(CP_RB_WPTR, ring->wptr);
1886
1887 /* set the wb address whether it's enabled or not */
1888 WREG32(CP_RB_RPTR_ADDR,
1889 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1890 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1891 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1892
1893 if (rdev->wb.enabled)
1894 WREG32(SCRATCH_UMSK, 0xff);
1895 else {
1896 tmp |= RB_NO_UPDATE;
1897 WREG32(SCRATCH_UMSK, 0);
1898 }
1899
1900 mdelay(1);
1901 WREG32(CP_RB_CNTL, tmp);
1902
1903 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1904 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1905
1906 ring->rptr = RREG32(CP_RB_RPTR);
1907
1908 evergreen_cp_start(rdev);
1909 ring->ready = true;
1910 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1911 if (r) {
1912 ring->ready = false;
1913 return r;
1914 }
1915 return 0;
1916 }
1917
1918 /*
1919 * Core functions
1920 */
1921 static void evergreen_gpu_init(struct radeon_device *rdev)
1922 {
1923 u32 gb_addr_config;
1924 u32 mc_shared_chmap, mc_arb_ramcfg;
1925 u32 sx_debug_1;
1926 u32 smx_dc_ctl0;
1927 u32 sq_config;
1928 u32 sq_lds_resource_mgmt;
1929 u32 sq_gpr_resource_mgmt_1;
1930 u32 sq_gpr_resource_mgmt_2;
1931 u32 sq_gpr_resource_mgmt_3;
1932 u32 sq_thread_resource_mgmt;
1933 u32 sq_thread_resource_mgmt_2;
1934 u32 sq_stack_resource_mgmt_1;
1935 u32 sq_stack_resource_mgmt_2;
1936 u32 sq_stack_resource_mgmt_3;
1937 u32 vgt_cache_invalidation;
1938 u32 hdp_host_path_cntl, tmp;
1939 u32 disabled_rb_mask;
1940 int i, j, num_shader_engines, ps_thread_count;
1941
1942 switch (rdev->family) {
1943 case CHIP_CYPRESS:
1944 case CHIP_HEMLOCK:
1945 rdev->config.evergreen.num_ses = 2;
1946 rdev->config.evergreen.max_pipes = 4;
1947 rdev->config.evergreen.max_tile_pipes = 8;
1948 rdev->config.evergreen.max_simds = 10;
1949 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1950 rdev->config.evergreen.max_gprs = 256;
1951 rdev->config.evergreen.max_threads = 248;
1952 rdev->config.evergreen.max_gs_threads = 32;
1953 rdev->config.evergreen.max_stack_entries = 512;
1954 rdev->config.evergreen.sx_num_of_sets = 4;
1955 rdev->config.evergreen.sx_max_export_size = 256;
1956 rdev->config.evergreen.sx_max_export_pos_size = 64;
1957 rdev->config.evergreen.sx_max_export_smx_size = 192;
1958 rdev->config.evergreen.max_hw_contexts = 8;
1959 rdev->config.evergreen.sq_num_cf_insts = 2;
1960
1961 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1962 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1963 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1964 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
1965 break;
1966 case CHIP_JUNIPER:
1967 rdev->config.evergreen.num_ses = 1;
1968 rdev->config.evergreen.max_pipes = 4;
1969 rdev->config.evergreen.max_tile_pipes = 4;
1970 rdev->config.evergreen.max_simds = 10;
1971 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1972 rdev->config.evergreen.max_gprs = 256;
1973 rdev->config.evergreen.max_threads = 248;
1974 rdev->config.evergreen.max_gs_threads = 32;
1975 rdev->config.evergreen.max_stack_entries = 512;
1976 rdev->config.evergreen.sx_num_of_sets = 4;
1977 rdev->config.evergreen.sx_max_export_size = 256;
1978 rdev->config.evergreen.sx_max_export_pos_size = 64;
1979 rdev->config.evergreen.sx_max_export_smx_size = 192;
1980 rdev->config.evergreen.max_hw_contexts = 8;
1981 rdev->config.evergreen.sq_num_cf_insts = 2;
1982
1983 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1984 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1985 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1986 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
1987 break;
1988 case CHIP_REDWOOD:
1989 rdev->config.evergreen.num_ses = 1;
1990 rdev->config.evergreen.max_pipes = 4;
1991 rdev->config.evergreen.max_tile_pipes = 4;
1992 rdev->config.evergreen.max_simds = 5;
1993 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1994 rdev->config.evergreen.max_gprs = 256;
1995 rdev->config.evergreen.max_threads = 248;
1996 rdev->config.evergreen.max_gs_threads = 32;
1997 rdev->config.evergreen.max_stack_entries = 256;
1998 rdev->config.evergreen.sx_num_of_sets = 4;
1999 rdev->config.evergreen.sx_max_export_size = 256;
2000 rdev->config.evergreen.sx_max_export_pos_size = 64;
2001 rdev->config.evergreen.sx_max_export_smx_size = 192;
2002 rdev->config.evergreen.max_hw_contexts = 8;
2003 rdev->config.evergreen.sq_num_cf_insts = 2;
2004
2005 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2006 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2007 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2008 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
2009 break;
2010 case CHIP_CEDAR:
2011 default:
2012 rdev->config.evergreen.num_ses = 1;
2013 rdev->config.evergreen.max_pipes = 2;
2014 rdev->config.evergreen.max_tile_pipes = 2;
2015 rdev->config.evergreen.max_simds = 2;
2016 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2017 rdev->config.evergreen.max_gprs = 256;
2018 rdev->config.evergreen.max_threads = 192;
2019 rdev->config.evergreen.max_gs_threads = 16;
2020 rdev->config.evergreen.max_stack_entries = 256;
2021 rdev->config.evergreen.sx_num_of_sets = 4;
2022 rdev->config.evergreen.sx_max_export_size = 128;
2023 rdev->config.evergreen.sx_max_export_pos_size = 32;
2024 rdev->config.evergreen.sx_max_export_smx_size = 96;
2025 rdev->config.evergreen.max_hw_contexts = 4;
2026 rdev->config.evergreen.sq_num_cf_insts = 1;
2027
2028 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2029 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2030 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2031 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
2032 break;
2033 case CHIP_PALM:
2034 rdev->config.evergreen.num_ses = 1;
2035 rdev->config.evergreen.max_pipes = 2;
2036 rdev->config.evergreen.max_tile_pipes = 2;
2037 rdev->config.evergreen.max_simds = 2;
2038 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2039 rdev->config.evergreen.max_gprs = 256;
2040 rdev->config.evergreen.max_threads = 192;
2041 rdev->config.evergreen.max_gs_threads = 16;
2042 rdev->config.evergreen.max_stack_entries = 256;
2043 rdev->config.evergreen.sx_num_of_sets = 4;
2044 rdev->config.evergreen.sx_max_export_size = 128;
2045 rdev->config.evergreen.sx_max_export_pos_size = 32;
2046 rdev->config.evergreen.sx_max_export_smx_size = 96;
2047 rdev->config.evergreen.max_hw_contexts = 4;
2048 rdev->config.evergreen.sq_num_cf_insts = 1;
2049
2050 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2051 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2052 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2053 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
2054 break;
2055 case CHIP_SUMO:
2056 rdev->config.evergreen.num_ses = 1;
2057 rdev->config.evergreen.max_pipes = 4;
2058 rdev->config.evergreen.max_tile_pipes = 4;
2059 if (rdev->pdev->device == 0x9648)
2060 rdev->config.evergreen.max_simds = 3;
2061 else if ((rdev->pdev->device == 0x9647) ||
2062 (rdev->pdev->device == 0x964a))
2063 rdev->config.evergreen.max_simds = 4;
2064 else
2065 rdev->config.evergreen.max_simds = 5;
2066 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2067 rdev->config.evergreen.max_gprs = 256;
2068 rdev->config.evergreen.max_threads = 248;
2069 rdev->config.evergreen.max_gs_threads = 32;
2070 rdev->config.evergreen.max_stack_entries = 256;
2071 rdev->config.evergreen.sx_num_of_sets = 4;
2072 rdev->config.evergreen.sx_max_export_size = 256;
2073 rdev->config.evergreen.sx_max_export_pos_size = 64;
2074 rdev->config.evergreen.sx_max_export_smx_size = 192;
2075 rdev->config.evergreen.max_hw_contexts = 8;
2076 rdev->config.evergreen.sq_num_cf_insts = 2;
2077
2078 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2079 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2080 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2081 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
2082 break;
2083 case CHIP_SUMO2:
2084 rdev->config.evergreen.num_ses = 1;
2085 rdev->config.evergreen.max_pipes = 4;
2086 rdev->config.evergreen.max_tile_pipes = 4;
2087 rdev->config.evergreen.max_simds = 2;
2088 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2089 rdev->config.evergreen.max_gprs = 256;
2090 rdev->config.evergreen.max_threads = 248;
2091 rdev->config.evergreen.max_gs_threads = 32;
2092 rdev->config.evergreen.max_stack_entries = 512;
2093 rdev->config.evergreen.sx_num_of_sets = 4;
2094 rdev->config.evergreen.sx_max_export_size = 256;
2095 rdev->config.evergreen.sx_max_export_pos_size = 64;
2096 rdev->config.evergreen.sx_max_export_smx_size = 192;
2097 rdev->config.evergreen.max_hw_contexts = 8;
2098 rdev->config.evergreen.sq_num_cf_insts = 2;
2099
2100 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2101 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2102 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2103 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
2104 break;
2105 case CHIP_BARTS:
2106 rdev->config.evergreen.num_ses = 2;
2107 rdev->config.evergreen.max_pipes = 4;
2108 rdev->config.evergreen.max_tile_pipes = 8;
2109 rdev->config.evergreen.max_simds = 7;
2110 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2111 rdev->config.evergreen.max_gprs = 256;
2112 rdev->config.evergreen.max_threads = 248;
2113 rdev->config.evergreen.max_gs_threads = 32;
2114 rdev->config.evergreen.max_stack_entries = 512;
2115 rdev->config.evergreen.sx_num_of_sets = 4;
2116 rdev->config.evergreen.sx_max_export_size = 256;
2117 rdev->config.evergreen.sx_max_export_pos_size = 64;
2118 rdev->config.evergreen.sx_max_export_smx_size = 192;
2119 rdev->config.evergreen.max_hw_contexts = 8;
2120 rdev->config.evergreen.sq_num_cf_insts = 2;
2121
2122 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2123 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2124 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2125 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
2126 break;
2127 case CHIP_TURKS:
2128 rdev->config.evergreen.num_ses = 1;
2129 rdev->config.evergreen.max_pipes = 4;
2130 rdev->config.evergreen.max_tile_pipes = 4;
2131 rdev->config.evergreen.max_simds = 6;
2132 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2133 rdev->config.evergreen.max_gprs = 256;
2134 rdev->config.evergreen.max_threads = 248;
2135 rdev->config.evergreen.max_gs_threads = 32;
2136 rdev->config.evergreen.max_stack_entries = 256;
2137 rdev->config.evergreen.sx_num_of_sets = 4;
2138 rdev->config.evergreen.sx_max_export_size = 256;
2139 rdev->config.evergreen.sx_max_export_pos_size = 64;
2140 rdev->config.evergreen.sx_max_export_smx_size = 192;
2141 rdev->config.evergreen.max_hw_contexts = 8;
2142 rdev->config.evergreen.sq_num_cf_insts = 2;
2143
2144 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2145 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2146 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2147 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
2148 break;
2149 case CHIP_CAICOS:
2150 rdev->config.evergreen.num_ses = 1;
2151 rdev->config.evergreen.max_pipes = 2;
2152 rdev->config.evergreen.max_tile_pipes = 2;
2153 rdev->config.evergreen.max_simds = 2;
2154 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2155 rdev->config.evergreen.max_gprs = 256;
2156 rdev->config.evergreen.max_threads = 192;
2157 rdev->config.evergreen.max_gs_threads = 16;
2158 rdev->config.evergreen.max_stack_entries = 256;
2159 rdev->config.evergreen.sx_num_of_sets = 4;
2160 rdev->config.evergreen.sx_max_export_size = 128;
2161 rdev->config.evergreen.sx_max_export_pos_size = 32;
2162 rdev->config.evergreen.sx_max_export_smx_size = 96;
2163 rdev->config.evergreen.max_hw_contexts = 4;
2164 rdev->config.evergreen.sq_num_cf_insts = 1;
2165
2166 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2167 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2168 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2169 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
2170 break;
2171 }
2172
2173 /* Initialize HDP */
2174 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2175 WREG32((0x2c14 + j), 0x00000000);
2176 WREG32((0x2c18 + j), 0x00000000);
2177 WREG32((0x2c1c + j), 0x00000000);
2178 WREG32((0x2c20 + j), 0x00000000);
2179 WREG32((0x2c24 + j), 0x00000000);
2180 }
2181
2182 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2183
2184 evergreen_fix_pci_max_read_req_size(rdev);
2185
2186 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2187 if ((rdev->family == CHIP_PALM) ||
2188 (rdev->family == CHIP_SUMO) ||
2189 (rdev->family == CHIP_SUMO2))
2190 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
2191 else
2192 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2193
2194 /* setup tiling info dword. gb_addr_config is not adequate since it does
2195 * not have bank info, so create a custom tiling dword.
2196 * bits 3:0 num_pipes
2197 * bits 7:4 num_banks
2198 * bits 11:8 group_size
2199 * bits 15:12 row_size
2200 */
2201 rdev->config.evergreen.tile_config = 0;
2202 switch (rdev->config.evergreen.max_tile_pipes) {
2203 case 1:
2204 default:
2205 rdev->config.evergreen.tile_config |= (0 << 0);
2206 break;
2207 case 2:
2208 rdev->config.evergreen.tile_config |= (1 << 0);
2209 break;
2210 case 4:
2211 rdev->config.evergreen.tile_config |= (2 << 0);
2212 break;
2213 case 8:
2214 rdev->config.evergreen.tile_config |= (3 << 0);
2215 break;
2216 }
2217 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
2218 if (rdev->flags & RADEON_IS_IGP)
2219 rdev->config.evergreen.tile_config |= 1 << 4;
2220 else {
2221 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2222 case 0: /* four banks */
2223 rdev->config.evergreen.tile_config |= 0 << 4;
2224 break;
2225 case 1: /* eight banks */
2226 rdev->config.evergreen.tile_config |= 1 << 4;
2227 break;
2228 case 2: /* sixteen banks */
2229 default:
2230 rdev->config.evergreen.tile_config |= 2 << 4;
2231 break;
2232 }
2233 }
2234 rdev->config.evergreen.tile_config |= 0 << 8;
2235 rdev->config.evergreen.tile_config |=
2236 ((gb_addr_config & 0x30000000) >> 28) << 12;
2237
2238 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
2239
2240 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
2241 u32 efuse_straps_4;
2242 u32 efuse_straps_3;
2243
2244 WREG32(RCU_IND_INDEX, 0x204);
2245 efuse_straps_4 = RREG32(RCU_IND_DATA);
2246 WREG32(RCU_IND_INDEX, 0x203);
2247 efuse_straps_3 = RREG32(RCU_IND_DATA);
2248 tmp = (((efuse_straps_4 & 0xf) << 4) |
2249 ((efuse_straps_3 & 0xf0000000) >> 28));
2250 } else {
2251 tmp = 0;
2252 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
2253 u32 rb_disable_bitmap;
2254
2255 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
2256 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
2257 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
2258 tmp <<= 4;
2259 tmp |= rb_disable_bitmap;
2260 }
2261 }
2262 /* enabled rb are just the one not disabled :) */
2263 disabled_rb_mask = tmp;
2264
2265 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
2266 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
2267
2268 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2269 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2270 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2271 WREG32(DMA_TILING_CONFIG, gb_addr_config);
2272 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2273 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2274 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2275
2276 if ((rdev->config.evergreen.max_backends == 1) &&
2277 (rdev->flags & RADEON_IS_IGP)) {
2278 if ((disabled_rb_mask & 3) == 1) {
2279 /* RB0 disabled, RB1 enabled */
2280 tmp = 0x11111111;
2281 } else {
2282 /* RB1 disabled, RB0 enabled */
2283 tmp = 0x00000000;
2284 }
2285 } else {
2286 tmp = gb_addr_config & NUM_PIPES_MASK;
2287 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
2288 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
2289 }
2290 WREG32(GB_BACKEND_MAP, tmp);
2291
2292 WREG32(CGTS_SYS_TCC_DISABLE, 0);
2293 WREG32(CGTS_TCC_DISABLE, 0);
2294 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2295 WREG32(CGTS_USER_TCC_DISABLE, 0);
2296
2297 /* set HW defaults for 3D engine */
2298 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2299 ROQ_IB2_START(0x2b)));
2300
2301 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2302
2303 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2304 SYNC_GRADIENT |
2305 SYNC_WALKER |
2306 SYNC_ALIGNER));
2307
2308 sx_debug_1 = RREG32(SX_DEBUG_1);
2309 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2310 WREG32(SX_DEBUG_1, sx_debug_1);
2311
2312
2313 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2314 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2315 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2316 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2317
2318 if (rdev->family <= CHIP_SUMO2)
2319 WREG32(SMX_SAR_CTL0, 0x00010000);
2320
2321 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2322 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2323 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2324
2325 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2326 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2327 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2328
2329 WREG32(VGT_NUM_INSTANCES, 1);
2330 WREG32(SPI_CONFIG_CNTL, 0);
2331 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2332 WREG32(CP_PERFMON_CNTL, 0);
2333
2334 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2335 FETCH_FIFO_HIWATER(0x4) |
2336 DONE_FIFO_HIWATER(0xe0) |
2337 ALU_UPDATE_FIFO_HIWATER(0x8)));
2338
2339 sq_config = RREG32(SQ_CONFIG);
2340 sq_config &= ~(PS_PRIO(3) |
2341 VS_PRIO(3) |
2342 GS_PRIO(3) |
2343 ES_PRIO(3));
2344 sq_config |= (VC_ENABLE |
2345 EXPORT_SRC_C |
2346 PS_PRIO(0) |
2347 VS_PRIO(1) |
2348 GS_PRIO(2) |
2349 ES_PRIO(3));
2350
2351 switch (rdev->family) {
2352 case CHIP_CEDAR:
2353 case CHIP_PALM:
2354 case CHIP_SUMO:
2355 case CHIP_SUMO2:
2356 case CHIP_CAICOS:
2357 /* no vertex cache */
2358 sq_config &= ~VC_ENABLE;
2359 break;
2360 default:
2361 break;
2362 }
2363
2364 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2365
2366 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2367 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2368 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2369 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2370 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2371 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2372 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2373
2374 switch (rdev->family) {
2375 case CHIP_CEDAR:
2376 case CHIP_PALM:
2377 case CHIP_SUMO:
2378 case CHIP_SUMO2:
2379 ps_thread_count = 96;
2380 break;
2381 default:
2382 ps_thread_count = 128;
2383 break;
2384 }
2385
2386 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2387 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2388 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2389 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2390 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2391 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2392
2393 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2394 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2395 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2396 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2397 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2398 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2399
2400 WREG32(SQ_CONFIG, sq_config);
2401 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2402 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2403 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2404 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2405 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2406 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2407 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2408 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2409 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2410 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2411
2412 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2413 FORCE_EOV_MAX_REZ_CNT(255)));
2414
2415 switch (rdev->family) {
2416 case CHIP_CEDAR:
2417 case CHIP_PALM:
2418 case CHIP_SUMO:
2419 case CHIP_SUMO2:
2420 case CHIP_CAICOS:
2421 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2422 break;
2423 default:
2424 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2425 break;
2426 }
2427 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2428 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2429
2430 WREG32(VGT_GS_VERTEX_REUSE, 16);
2431 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2432 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2433
2434 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2435 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2436
2437 WREG32(CB_PERF_CTR0_SEL_0, 0);
2438 WREG32(CB_PERF_CTR0_SEL_1, 0);
2439 WREG32(CB_PERF_CTR1_SEL_0, 0);
2440 WREG32(CB_PERF_CTR1_SEL_1, 0);
2441 WREG32(CB_PERF_CTR2_SEL_0, 0);
2442 WREG32(CB_PERF_CTR2_SEL_1, 0);
2443 WREG32(CB_PERF_CTR3_SEL_0, 0);
2444 WREG32(CB_PERF_CTR3_SEL_1, 0);
2445
2446 /* clear render buffer base addresses */
2447 WREG32(CB_COLOR0_BASE, 0);
2448 WREG32(CB_COLOR1_BASE, 0);
2449 WREG32(CB_COLOR2_BASE, 0);
2450 WREG32(CB_COLOR3_BASE, 0);
2451 WREG32(CB_COLOR4_BASE, 0);
2452 WREG32(CB_COLOR5_BASE, 0);
2453 WREG32(CB_COLOR6_BASE, 0);
2454 WREG32(CB_COLOR7_BASE, 0);
2455 WREG32(CB_COLOR8_BASE, 0);
2456 WREG32(CB_COLOR9_BASE, 0);
2457 WREG32(CB_COLOR10_BASE, 0);
2458 WREG32(CB_COLOR11_BASE, 0);
2459
2460 /* set the shader const cache sizes to 0 */
2461 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2462 WREG32(i, 0);
2463 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2464 WREG32(i, 0);
2465
2466 tmp = RREG32(HDP_MISC_CNTL);
2467 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2468 WREG32(HDP_MISC_CNTL, tmp);
2469
2470 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2471 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2472
2473 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2474
2475 udelay(50);
2476
2477 }
2478
2479 int evergreen_mc_init(struct radeon_device *rdev)
2480 {
2481 u32 tmp;
2482 int chansize, numchan;
2483
2484 /* Get VRAM informations */
2485 rdev->mc.vram_is_ddr = true;
2486 if ((rdev->family == CHIP_PALM) ||
2487 (rdev->family == CHIP_SUMO) ||
2488 (rdev->family == CHIP_SUMO2))
2489 tmp = RREG32(FUS_MC_ARB_RAMCFG);
2490 else
2491 tmp = RREG32(MC_ARB_RAMCFG);
2492 if (tmp & CHANSIZE_OVERRIDE) {
2493 chansize = 16;
2494 } else if (tmp & CHANSIZE_MASK) {
2495 chansize = 64;
2496 } else {
2497 chansize = 32;
2498 }
2499 tmp = RREG32(MC_SHARED_CHMAP);
2500 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2501 case 0:
2502 default:
2503 numchan = 1;
2504 break;
2505 case 1:
2506 numchan = 2;
2507 break;
2508 case 2:
2509 numchan = 4;
2510 break;
2511 case 3:
2512 numchan = 8;
2513 break;
2514 }
2515 rdev->mc.vram_width = numchan * chansize;
2516 /* Could aper size report 0 ? */
2517 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2518 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
2519 /* Setup GPU memory space */
2520 if ((rdev->family == CHIP_PALM) ||
2521 (rdev->family == CHIP_SUMO) ||
2522 (rdev->family == CHIP_SUMO2)) {
2523 /* size in bytes on fusion */
2524 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2525 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2526 } else {
2527 /* size in MB on evergreen/cayman/tn */
2528 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2529 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2530 }
2531 rdev->mc.visible_vram_size = rdev->mc.aper_size;
2532 r700_vram_gtt_location(rdev, &rdev->mc);
2533 radeon_update_bandwidth_info(rdev);
2534
2535 return 0;
2536 }
2537
2538 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
2539 {
2540 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
2541 RREG32(GRBM_STATUS));
2542 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
2543 RREG32(GRBM_STATUS_SE0));
2544 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
2545 RREG32(GRBM_STATUS_SE1));
2546 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
2547 RREG32(SRBM_STATUS));
2548 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
2549 RREG32(SRBM_STATUS2));
2550 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
2551 RREG32(CP_STALLED_STAT1));
2552 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
2553 RREG32(CP_STALLED_STAT2));
2554 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
2555 RREG32(CP_BUSY_STAT));
2556 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
2557 RREG32(CP_STAT));
2558 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
2559 RREG32(DMA_STATUS_REG));
2560 if (rdev->family >= CHIP_CAYMAN) {
2561 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
2562 RREG32(DMA_STATUS_REG + 0x800));
2563 }
2564 }
2565
2566 bool evergreen_is_display_hung(struct radeon_device *rdev)
2567 {
2568 u32 crtc_hung = 0;
2569 u32 crtc_status[6];
2570 u32 i, j, tmp;
2571
2572 for (i = 0; i < rdev->num_crtc; i++) {
2573 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
2574 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
2575 crtc_hung |= (1 << i);
2576 }
2577 }
2578
2579 for (j = 0; j < 10; j++) {
2580 for (i = 0; i < rdev->num_crtc; i++) {
2581 if (crtc_hung & (1 << i)) {
2582 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
2583 if (tmp != crtc_status[i])
2584 crtc_hung &= ~(1 << i);
2585 }
2586 }
2587 if (crtc_hung == 0)
2588 return false;
2589 udelay(100);
2590 }
2591
2592 return true;
2593 }
2594
2595 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
2596 {
2597 u32 reset_mask = 0;
2598 u32 tmp;
2599
2600 /* GRBM_STATUS */
2601 tmp = RREG32(GRBM_STATUS);
2602 if (tmp & (PA_BUSY | SC_BUSY |
2603 SH_BUSY | SX_BUSY |
2604 TA_BUSY | VGT_BUSY |
2605 DB_BUSY | CB_BUSY |
2606 SPI_BUSY | VGT_BUSY_NO_DMA))
2607 reset_mask |= RADEON_RESET_GFX;
2608
2609 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
2610 CP_BUSY | CP_COHERENCY_BUSY))
2611 reset_mask |= RADEON_RESET_CP;
2612
2613 if (tmp & GRBM_EE_BUSY)
2614 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
2615
2616 /* DMA_STATUS_REG */
2617 tmp = RREG32(DMA_STATUS_REG);
2618 if (!(tmp & DMA_IDLE))
2619 reset_mask |= RADEON_RESET_DMA;
2620
2621 /* SRBM_STATUS2 */
2622 tmp = RREG32(SRBM_STATUS2);
2623 if (tmp & DMA_BUSY)
2624 reset_mask |= RADEON_RESET_DMA;
2625
2626 /* SRBM_STATUS */
2627 tmp = RREG32(SRBM_STATUS);
2628 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
2629 reset_mask |= RADEON_RESET_RLC;
2630
2631 if (tmp & IH_BUSY)
2632 reset_mask |= RADEON_RESET_IH;
2633
2634 if (tmp & SEM_BUSY)
2635 reset_mask |= RADEON_RESET_SEM;
2636
2637 if (tmp & GRBM_RQ_PENDING)
2638 reset_mask |= RADEON_RESET_GRBM;
2639
2640 if (tmp & VMC_BUSY)
2641 reset_mask |= RADEON_RESET_VMC;
2642
2643 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
2644 MCC_BUSY | MCD_BUSY))
2645 reset_mask |= RADEON_RESET_MC;
2646
2647 if (evergreen_is_display_hung(rdev))
2648 reset_mask |= RADEON_RESET_DISPLAY;
2649
2650 /* VM_L2_STATUS */
2651 tmp = RREG32(VM_L2_STATUS);
2652 if (tmp & L2_BUSY)
2653 reset_mask |= RADEON_RESET_VMC;
2654
2655 /* Skip MC reset as it's mostly likely not hung, just busy */
2656 if (reset_mask & RADEON_RESET_MC) {
2657 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
2658 reset_mask &= ~RADEON_RESET_MC;
2659 }
2660
2661 return reset_mask;
2662 }
2663
2664 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
2665 {
2666 struct evergreen_mc_save save;
2667 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
2668 u32 tmp;
2669
2670 if (reset_mask == 0)
2671 return;
2672
2673 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
2674
2675 evergreen_print_gpu_status_regs(rdev);
2676
2677 /* Disable CP parsing/prefetching */
2678 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2679
2680 if (reset_mask & RADEON_RESET_DMA) {
2681 /* Disable DMA */
2682 tmp = RREG32(DMA_RB_CNTL);
2683 tmp &= ~DMA_RB_ENABLE;
2684 WREG32(DMA_RB_CNTL, tmp);
2685 }
2686
2687 udelay(50);
2688
2689 evergreen_mc_stop(rdev, &save);
2690 if (evergreen_mc_wait_for_idle(rdev)) {
2691 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2692 }
2693
2694 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
2695 grbm_soft_reset |= SOFT_RESET_DB |
2696 SOFT_RESET_CB |
2697 SOFT_RESET_PA |
2698 SOFT_RESET_SC |
2699 SOFT_RESET_SPI |
2700 SOFT_RESET_SX |
2701 SOFT_RESET_SH |
2702 SOFT_RESET_TC |
2703 SOFT_RESET_TA |
2704 SOFT_RESET_VC |
2705 SOFT_RESET_VGT;
2706 }
2707
2708 if (reset_mask & RADEON_RESET_CP) {
2709 grbm_soft_reset |= SOFT_RESET_CP |
2710 SOFT_RESET_VGT;
2711
2712 srbm_soft_reset |= SOFT_RESET_GRBM;
2713 }
2714
2715 if (reset_mask & RADEON_RESET_DMA)
2716 srbm_soft_reset |= SOFT_RESET_DMA;
2717
2718 if (reset_mask & RADEON_RESET_DISPLAY)
2719 srbm_soft_reset |= SOFT_RESET_DC;
2720
2721 if (reset_mask & RADEON_RESET_RLC)
2722 srbm_soft_reset |= SOFT_RESET_RLC;
2723
2724 if (reset_mask & RADEON_RESET_SEM)
2725 srbm_soft_reset |= SOFT_RESET_SEM;
2726
2727 if (reset_mask & RADEON_RESET_IH)
2728 srbm_soft_reset |= SOFT_RESET_IH;
2729
2730 if (reset_mask & RADEON_RESET_GRBM)
2731 srbm_soft_reset |= SOFT_RESET_GRBM;
2732
2733 if (reset_mask & RADEON_RESET_VMC)
2734 srbm_soft_reset |= SOFT_RESET_VMC;
2735
2736 if (!(rdev->flags & RADEON_IS_IGP)) {
2737 if (reset_mask & RADEON_RESET_MC)
2738 srbm_soft_reset |= SOFT_RESET_MC;
2739 }
2740
2741 if (grbm_soft_reset) {
2742 tmp = RREG32(GRBM_SOFT_RESET);
2743 tmp |= grbm_soft_reset;
2744 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2745 WREG32(GRBM_SOFT_RESET, tmp);
2746 tmp = RREG32(GRBM_SOFT_RESET);
2747
2748 udelay(50);
2749
2750 tmp &= ~grbm_soft_reset;
2751 WREG32(GRBM_SOFT_RESET, tmp);
2752 tmp = RREG32(GRBM_SOFT_RESET);
2753 }
2754
2755 if (srbm_soft_reset) {
2756 tmp = RREG32(SRBM_SOFT_RESET);
2757 tmp |= srbm_soft_reset;
2758 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
2759 WREG32(SRBM_SOFT_RESET, tmp);
2760 tmp = RREG32(SRBM_SOFT_RESET);
2761
2762 udelay(50);
2763
2764 tmp &= ~srbm_soft_reset;
2765 WREG32(SRBM_SOFT_RESET, tmp);
2766 tmp = RREG32(SRBM_SOFT_RESET);
2767 }
2768
2769 /* Wait a little for things to settle down */
2770 udelay(50);
2771
2772 evergreen_mc_resume(rdev, &save);
2773 udelay(50);
2774
2775 evergreen_print_gpu_status_regs(rdev);
2776 }
2777
2778 int evergreen_asic_reset(struct radeon_device *rdev)
2779 {
2780 u32 reset_mask;
2781
2782 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2783
2784 if (reset_mask)
2785 r600_set_bios_scratch_engine_hung(rdev, true);
2786
2787 evergreen_gpu_soft_reset(rdev, reset_mask);
2788
2789 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2790
2791 if (!reset_mask)
2792 r600_set_bios_scratch_engine_hung(rdev, false);
2793
2794 return 0;
2795 }
2796
2797 /**
2798 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
2799 *
2800 * @rdev: radeon_device pointer
2801 * @ring: radeon_ring structure holding ring information
2802 *
2803 * Check if the GFX engine is locked up.
2804 * Returns true if the engine appears to be locked up, false if not.
2805 */
2806 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2807 {
2808 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2809
2810 if (!(reset_mask & (RADEON_RESET_GFX |
2811 RADEON_RESET_COMPUTE |
2812 RADEON_RESET_CP))) {
2813 radeon_ring_lockup_update(ring);
2814 return false;
2815 }
2816 /* force CP activities */
2817 radeon_ring_force_activity(rdev, ring);
2818 return radeon_ring_test_lockup(rdev, ring);
2819 }
2820
2821 /**
2822 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
2823 *
2824 * @rdev: radeon_device pointer
2825 * @ring: radeon_ring structure holding ring information
2826 *
2827 * Check if the async DMA engine is locked up.
2828 * Returns true if the engine appears to be locked up, false if not.
2829 */
2830 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2831 {
2832 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2833
2834 if (!(reset_mask & RADEON_RESET_DMA)) {
2835 radeon_ring_lockup_update(ring);
2836 return false;
2837 }
2838 /* force ring activities */
2839 radeon_ring_force_activity(rdev, ring);
2840 return radeon_ring_test_lockup(rdev, ring);
2841 }
2842
2843 /* Interrupts */
2844
2845 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2846 {
2847 if (crtc >= rdev->num_crtc)
2848 return 0;
2849 else
2850 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
2851 }
2852
2853 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2854 {
2855 u32 tmp;
2856
2857 if (rdev->family >= CHIP_CAYMAN) {
2858 cayman_cp_int_cntl_setup(rdev, 0,
2859 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2860 cayman_cp_int_cntl_setup(rdev, 1, 0);
2861 cayman_cp_int_cntl_setup(rdev, 2, 0);
2862 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
2863 WREG32(CAYMAN_DMA1_CNTL, tmp);
2864 } else
2865 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2866 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2867 WREG32(DMA_CNTL, tmp);
2868 WREG32(GRBM_INT_CNTL, 0);
2869 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2870 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2871 if (rdev->num_crtc >= 4) {
2872 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2873 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2874 }
2875 if (rdev->num_crtc >= 6) {
2876 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2877 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2878 }
2879
2880 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2881 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2882 if (rdev->num_crtc >= 4) {
2883 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2884 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2885 }
2886 if (rdev->num_crtc >= 6) {
2887 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2888 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2889 }
2890
2891 /* only one DAC on DCE6 */
2892 if (!ASIC_IS_DCE6(rdev))
2893 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2894 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2895
2896 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2897 WREG32(DC_HPD1_INT_CONTROL, tmp);
2898 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2899 WREG32(DC_HPD2_INT_CONTROL, tmp);
2900 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2901 WREG32(DC_HPD3_INT_CONTROL, tmp);
2902 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2903 WREG32(DC_HPD4_INT_CONTROL, tmp);
2904 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2905 WREG32(DC_HPD5_INT_CONTROL, tmp);
2906 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2907 WREG32(DC_HPD6_INT_CONTROL, tmp);
2908
2909 }
2910
2911 int evergreen_irq_set(struct radeon_device *rdev)
2912 {
2913 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2914 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2915 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2916 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2917 u32 grbm_int_cntl = 0;
2918 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2919 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
2920 u32 dma_cntl, dma_cntl1 = 0;
2921
2922 if (!rdev->irq.installed) {
2923 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2924 return -EINVAL;
2925 }
2926 /* don't enable anything if the ih is disabled */
2927 if (!rdev->ih.enabled) {
2928 r600_disable_interrupts(rdev);
2929 /* force the active interrupt state to all disabled */
2930 evergreen_disable_interrupt_state(rdev);
2931 return 0;
2932 }
2933
2934 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2935 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2936 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2937 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2938 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2939 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2940
2941 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2942 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2943 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2944 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2945 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2946 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2947
2948 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2949
2950 if (rdev->family >= CHIP_CAYMAN) {
2951 /* enable CP interrupts on all rings */
2952 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2953 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2954 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2955 }
2956 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
2957 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
2958 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
2959 }
2960 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
2961 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
2962 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
2963 }
2964 } else {
2965 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2966 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2967 cp_int_cntl |= RB_INT_ENABLE;
2968 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2969 }
2970 }
2971
2972 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
2973 DRM_DEBUG("r600_irq_set: sw int dma\n");
2974 dma_cntl |= TRAP_ENABLE;
2975 }
2976
2977 if (rdev->family >= CHIP_CAYMAN) {
2978 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
2979 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
2980 DRM_DEBUG("r600_irq_set: sw int dma1\n");
2981 dma_cntl1 |= TRAP_ENABLE;
2982 }
2983 }
2984
2985 if (rdev->irq.crtc_vblank_int[0] ||
2986 atomic_read(&rdev->irq.pflip[0])) {
2987 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2988 crtc1 |= VBLANK_INT_MASK;
2989 }
2990 if (rdev->irq.crtc_vblank_int[1] ||
2991 atomic_read(&rdev->irq.pflip[1])) {
2992 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2993 crtc2 |= VBLANK_INT_MASK;
2994 }
2995 if (rdev->irq.crtc_vblank_int[2] ||
2996 atomic_read(&rdev->irq.pflip[2])) {
2997 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2998 crtc3 |= VBLANK_INT_MASK;
2999 }
3000 if (rdev->irq.crtc_vblank_int[3] ||
3001 atomic_read(&rdev->irq.pflip[3])) {
3002 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
3003 crtc4 |= VBLANK_INT_MASK;
3004 }
3005 if (rdev->irq.crtc_vblank_int[4] ||
3006 atomic_read(&rdev->irq.pflip[4])) {
3007 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
3008 crtc5 |= VBLANK_INT_MASK;
3009 }
3010 if (rdev->irq.crtc_vblank_int[5] ||
3011 atomic_read(&rdev->irq.pflip[5])) {
3012 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
3013 crtc6 |= VBLANK_INT_MASK;
3014 }
3015 if (rdev->irq.hpd[0]) {
3016 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
3017 hpd1 |= DC_HPDx_INT_EN;
3018 }
3019 if (rdev->irq.hpd[1]) {
3020 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
3021 hpd2 |= DC_HPDx_INT_EN;
3022 }
3023 if (rdev->irq.hpd[2]) {
3024 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
3025 hpd3 |= DC_HPDx_INT_EN;
3026 }
3027 if (rdev->irq.hpd[3]) {
3028 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
3029 hpd4 |= DC_HPDx_INT_EN;
3030 }
3031 if (rdev->irq.hpd[4]) {
3032 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
3033 hpd5 |= DC_HPDx_INT_EN;
3034 }
3035 if (rdev->irq.hpd[5]) {
3036 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
3037 hpd6 |= DC_HPDx_INT_EN;
3038 }
3039 if (rdev->irq.afmt[0]) {
3040 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
3041 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3042 }
3043 if (rdev->irq.afmt[1]) {
3044 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
3045 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3046 }
3047 if (rdev->irq.afmt[2]) {
3048 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
3049 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3050 }
3051 if (rdev->irq.afmt[3]) {
3052 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
3053 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3054 }
3055 if (rdev->irq.afmt[4]) {
3056 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
3057 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3058 }
3059 if (rdev->irq.afmt[5]) {
3060 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
3061 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
3062 }
3063
3064 if (rdev->family >= CHIP_CAYMAN) {
3065 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
3066 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
3067 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
3068 } else
3069 WREG32(CP_INT_CNTL, cp_int_cntl);
3070
3071 WREG32(DMA_CNTL, dma_cntl);
3072
3073 if (rdev->family >= CHIP_CAYMAN)
3074 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
3075
3076 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3077
3078 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
3079 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
3080 if (rdev->num_crtc >= 4) {
3081 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
3082 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
3083 }
3084 if (rdev->num_crtc >= 6) {
3085 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
3086 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
3087 }
3088
3089 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
3090 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
3091 if (rdev->num_crtc >= 4) {
3092 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
3093 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
3094 }
3095 if (rdev->num_crtc >= 6) {
3096 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
3097 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
3098 }
3099
3100 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3101 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3102 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3103 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3104 WREG32(DC_HPD5_INT_CONTROL, hpd5);
3105 WREG32(DC_HPD6_INT_CONTROL, hpd6);
3106
3107 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
3108 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
3109 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
3110 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
3111 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
3112 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
3113
3114 return 0;
3115 }
3116
3117 static void evergreen_irq_ack(struct radeon_device *rdev)
3118 {
3119 u32 tmp;
3120
3121 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3122 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3123 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
3124 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
3125 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
3126 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
3127 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
3128 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
3129 if (rdev->num_crtc >= 4) {
3130 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
3131 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
3132 }
3133 if (rdev->num_crtc >= 6) {
3134 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
3135 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
3136 }
3137
3138 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
3139 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
3140 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
3141 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
3142 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
3143 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
3144
3145 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
3146 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3147 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
3148 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3149 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
3150 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
3151 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
3152 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
3153 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
3154 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
3155 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
3156 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
3157
3158 if (rdev->num_crtc >= 4) {
3159 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
3160 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3161 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
3162 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3163 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
3164 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
3165 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
3166 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
3167 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
3168 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
3169 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
3170 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
3171 }
3172
3173 if (rdev->num_crtc >= 6) {
3174 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
3175 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3176 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
3177 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
3178 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
3179 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
3180 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
3181 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
3182 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
3183 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
3184 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
3185 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
3186 }
3187
3188 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3189 tmp = RREG32(DC_HPD1_INT_CONTROL);
3190 tmp |= DC_HPDx_INT_ACK;
3191 WREG32(DC_HPD1_INT_CONTROL, tmp);
3192 }
3193 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3194 tmp = RREG32(DC_HPD2_INT_CONTROL);
3195 tmp |= DC_HPDx_INT_ACK;
3196 WREG32(DC_HPD2_INT_CONTROL, tmp);
3197 }
3198 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3199 tmp = RREG32(DC_HPD3_INT_CONTROL);
3200 tmp |= DC_HPDx_INT_ACK;
3201 WREG32(DC_HPD3_INT_CONTROL, tmp);
3202 }
3203 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3204 tmp = RREG32(DC_HPD4_INT_CONTROL);
3205 tmp |= DC_HPDx_INT_ACK;
3206 WREG32(DC_HPD4_INT_CONTROL, tmp);
3207 }
3208 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3209 tmp = RREG32(DC_HPD5_INT_CONTROL);
3210 tmp |= DC_HPDx_INT_ACK;
3211 WREG32(DC_HPD5_INT_CONTROL, tmp);
3212 }
3213 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3214 tmp = RREG32(DC_HPD5_INT_CONTROL);
3215 tmp |= DC_HPDx_INT_ACK;
3216 WREG32(DC_HPD6_INT_CONTROL, tmp);
3217 }
3218 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
3219 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
3220 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3221 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
3222 }
3223 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
3224 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
3225 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3226 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
3227 }
3228 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
3229 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
3230 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3231 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
3232 }
3233 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
3234 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
3235 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3236 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
3237 }
3238 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
3239 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
3240 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3241 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
3242 }
3243 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
3244 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
3245 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3246 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
3247 }
3248 }
3249
3250 static void evergreen_irq_disable(struct radeon_device *rdev)
3251 {
3252 r600_disable_interrupts(rdev);
3253 /* Wait and acknowledge irq */
3254 mdelay(1);
3255 evergreen_irq_ack(rdev);
3256 evergreen_disable_interrupt_state(rdev);
3257 }
3258
3259 void evergreen_irq_suspend(struct radeon_device *rdev)
3260 {
3261 evergreen_irq_disable(rdev);
3262 r600_rlc_stop(rdev);
3263 }
3264
3265 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
3266 {
3267 u32 wptr, tmp;
3268
3269 if (rdev->wb.enabled)
3270 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3271 else
3272 wptr = RREG32(IH_RB_WPTR);
3273
3274 if (wptr & RB_OVERFLOW) {
3275 /* When a ring buffer overflow happen start parsing interrupt
3276 * from the last not overwritten vector (wptr + 16). Hopefully
3277 * this should allow us to catchup.
3278 */
3279 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3280 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3281 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3282 tmp = RREG32(IH_RB_CNTL);
3283 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3284 WREG32(IH_RB_CNTL, tmp);
3285 }
3286 return (wptr & rdev->ih.ptr_mask);
3287 }
3288
3289 int evergreen_irq_process(struct radeon_device *rdev)
3290 {
3291 u32 wptr;
3292 u32 rptr;
3293 u32 src_id, src_data;
3294 u32 ring_index;
3295 bool queue_hotplug = false;
3296 bool queue_hdmi = false;
3297
3298 if (!rdev->ih.enabled || rdev->shutdown)
3299 return IRQ_NONE;
3300
3301 wptr = evergreen_get_ih_wptr(rdev);
3302
3303 restart_ih:
3304 /* is somebody else already processing irqs? */
3305 if (atomic_xchg(&rdev->ih.lock, 1))
3306 return IRQ_NONE;
3307
3308 rptr = rdev->ih.rptr;
3309 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3310
3311 /* Order reading of wptr vs. reading of IH ring data */
3312 rmb();
3313
3314 /* display interrupts */
3315 evergreen_irq_ack(rdev);
3316
3317 while (rptr != wptr) {
3318 /* wptr/rptr are in bytes! */
3319 ring_index = rptr / 4;
3320 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3321 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3322
3323 switch (src_id) {
3324 case 1: /* D1 vblank/vline */
3325 switch (src_data) {
3326 case 0: /* D1 vblank */
3327 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
3328 if (rdev->irq.crtc_vblank_int[0]) {
3329 drm_handle_vblank(rdev->ddev, 0);
3330 rdev->pm.vblank_sync = true;
3331 wake_up(&rdev->irq.vblank_queue);
3332 }
3333 if (atomic_read(&rdev->irq.pflip[0]))
3334 radeon_crtc_handle_flip(rdev, 0);
3335 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3336 DRM_DEBUG("IH: D1 vblank\n");
3337 }
3338 break;
3339 case 1: /* D1 vline */
3340 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
3341 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3342 DRM_DEBUG("IH: D1 vline\n");
3343 }
3344 break;
3345 default:
3346 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3347 break;
3348 }
3349 break;
3350 case 2: /* D2 vblank/vline */
3351 switch (src_data) {
3352 case 0: /* D2 vblank */
3353 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
3354 if (rdev->irq.crtc_vblank_int[1]) {
3355 drm_handle_vblank(rdev->ddev, 1);
3356 rdev->pm.vblank_sync = true;
3357 wake_up(&rdev->irq.vblank_queue);
3358 }
3359 if (atomic_read(&rdev->irq.pflip[1]))
3360 radeon_crtc_handle_flip(rdev, 1);
3361 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
3362 DRM_DEBUG("IH: D2 vblank\n");
3363 }
3364 break;
3365 case 1: /* D2 vline */
3366 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
3367 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
3368 DRM_DEBUG("IH: D2 vline\n");
3369 }
3370 break;
3371 default:
3372 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3373 break;
3374 }
3375 break;
3376 case 3: /* D3 vblank/vline */
3377 switch (src_data) {
3378 case 0: /* D3 vblank */
3379 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
3380 if (rdev->irq.crtc_vblank_int[2]) {
3381 drm_handle_vblank(rdev->ddev, 2);
3382 rdev->pm.vblank_sync = true;
3383 wake_up(&rdev->irq.vblank_queue);
3384 }
3385 if (atomic_read(&rdev->irq.pflip[2]))
3386 radeon_crtc_handle_flip(rdev, 2);
3387 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
3388 DRM_DEBUG("IH: D3 vblank\n");
3389 }
3390 break;
3391 case 1: /* D3 vline */
3392 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
3393 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
3394 DRM_DEBUG("IH: D3 vline\n");
3395 }
3396 break;
3397 default:
3398 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3399 break;
3400 }
3401 break;
3402 case 4: /* D4 vblank/vline */
3403 switch (src_data) {
3404 case 0: /* D4 vblank */
3405 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
3406 if (rdev->irq.crtc_vblank_int[3]) {
3407 drm_handle_vblank(rdev->ddev, 3);
3408 rdev->pm.vblank_sync = true;
3409 wake_up(&rdev->irq.vblank_queue);
3410 }
3411 if (atomic_read(&rdev->irq.pflip[3]))
3412 radeon_crtc_handle_flip(rdev, 3);
3413 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
3414 DRM_DEBUG("IH: D4 vblank\n");
3415 }
3416 break;
3417 case 1: /* D4 vline */
3418 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
3419 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
3420 DRM_DEBUG("IH: D4 vline\n");
3421 }
3422 break;
3423 default:
3424 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3425 break;
3426 }
3427 break;
3428 case 5: /* D5 vblank/vline */
3429 switch (src_data) {
3430 case 0: /* D5 vblank */
3431 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
3432 if (rdev->irq.crtc_vblank_int[4]) {
3433 drm_handle_vblank(rdev->ddev, 4);
3434 rdev->pm.vblank_sync = true;
3435 wake_up(&rdev->irq.vblank_queue);
3436 }
3437 if (atomic_read(&rdev->irq.pflip[4]))
3438 radeon_crtc_handle_flip(rdev, 4);
3439 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
3440 DRM_DEBUG("IH: D5 vblank\n");
3441 }
3442 break;
3443 case 1: /* D5 vline */
3444 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
3445 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
3446 DRM_DEBUG("IH: D5 vline\n");
3447 }
3448 break;
3449 default:
3450 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3451 break;
3452 }
3453 break;
3454 case 6: /* D6 vblank/vline */
3455 switch (src_data) {
3456 case 0: /* D6 vblank */
3457 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
3458 if (rdev->irq.crtc_vblank_int[5]) {
3459 drm_handle_vblank(rdev->ddev, 5);
3460 rdev->pm.vblank_sync = true;
3461 wake_up(&rdev->irq.vblank_queue);
3462 }
3463 if (atomic_read(&rdev->irq.pflip[5]))
3464 radeon_crtc_handle_flip(rdev, 5);
3465 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
3466 DRM_DEBUG("IH: D6 vblank\n");
3467 }
3468 break;
3469 case 1: /* D6 vline */
3470 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
3471 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
3472 DRM_DEBUG("IH: D6 vline\n");
3473 }
3474 break;
3475 default:
3476 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3477 break;
3478 }
3479 break;
3480 case 42: /* HPD hotplug */
3481 switch (src_data) {
3482 case 0:
3483 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3484 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
3485 queue_hotplug = true;
3486 DRM_DEBUG("IH: HPD1\n");
3487 }
3488 break;
3489 case 1:
3490 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3491 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
3492 queue_hotplug = true;
3493 DRM_DEBUG("IH: HPD2\n");
3494 }
3495 break;
3496 case 2:
3497 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3498 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
3499 queue_hotplug = true;
3500 DRM_DEBUG("IH: HPD3\n");
3501 }
3502 break;
3503 case 3:
3504 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3505 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
3506 queue_hotplug = true;
3507 DRM_DEBUG("IH: HPD4\n");
3508 }
3509 break;
3510 case 4:
3511 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3512 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
3513 queue_hotplug = true;
3514 DRM_DEBUG("IH: HPD5\n");
3515 }
3516 break;
3517 case 5:
3518 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3519 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
3520 queue_hotplug = true;
3521 DRM_DEBUG("IH: HPD6\n");
3522 }
3523 break;
3524 default:
3525 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3526 break;
3527 }
3528 break;
3529 case 44: /* hdmi */
3530 switch (src_data) {
3531 case 0:
3532 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
3533 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
3534 queue_hdmi = true;
3535 DRM_DEBUG("IH: HDMI0\n");
3536 }
3537 break;
3538 case 1:
3539 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
3540 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
3541 queue_hdmi = true;
3542 DRM_DEBUG("IH: HDMI1\n");
3543 }
3544 break;
3545 case 2:
3546 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
3547 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
3548 queue_hdmi = true;
3549 DRM_DEBUG("IH: HDMI2\n");
3550 }
3551 break;
3552 case 3:
3553 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
3554 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
3555 queue_hdmi = true;
3556 DRM_DEBUG("IH: HDMI3\n");
3557 }
3558 break;
3559 case 4:
3560 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
3561 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
3562 queue_hdmi = true;
3563 DRM_DEBUG("IH: HDMI4\n");
3564 }
3565 break;
3566 case 5:
3567 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
3568 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
3569 queue_hdmi = true;
3570 DRM_DEBUG("IH: HDMI5\n");
3571 }
3572 break;
3573 default:
3574 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
3575 break;
3576 }
3577 case 124: /* UVD */
3578 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
3579 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
3580 break;
3581 case 146:
3582 case 147:
3583 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
3584 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3585 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3586 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3587 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3588 /* reset addr and status */
3589 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
3590 break;
3591 case 176: /* CP_INT in ring buffer */
3592 case 177: /* CP_INT in IB1 */
3593 case 178: /* CP_INT in IB2 */
3594 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3595 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3596 break;
3597 case 181: /* CP EOP event */
3598 DRM_DEBUG("IH: CP EOP\n");
3599 if (rdev->family >= CHIP_CAYMAN) {
3600 switch (src_data) {
3601 case 0:
3602 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3603 break;
3604 case 1:
3605 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3606 break;
3607 case 2:
3608 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3609 break;
3610 }
3611 } else
3612 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3613 break;
3614 case 224: /* DMA trap event */
3615 DRM_DEBUG("IH: DMA trap\n");
3616 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
3617 break;
3618 case 233: /* GUI IDLE */
3619 DRM_DEBUG("IH: GUI idle\n");
3620 break;
3621 case 244: /* DMA trap event */
3622 if (rdev->family >= CHIP_CAYMAN) {
3623 DRM_DEBUG("IH: DMA1 trap\n");
3624 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
3625 }
3626 break;
3627 default:
3628 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3629 break;
3630 }
3631
3632 /* wptr/rptr are in bytes! */
3633 rptr += 16;
3634 rptr &= rdev->ih.ptr_mask;
3635 }
3636 if (queue_hotplug)
3637 schedule_work(&rdev->hotplug_work);
3638 if (queue_hdmi)
3639 schedule_work(&rdev->audio_work);
3640 rdev->ih.rptr = rptr;
3641 WREG32(IH_RB_RPTR, rdev->ih.rptr);
3642 atomic_set(&rdev->ih.lock, 0);
3643
3644 /* make sure wptr hasn't changed while processing */
3645 wptr = evergreen_get_ih_wptr(rdev);
3646 if (wptr != rptr)
3647 goto restart_ih;
3648
3649 return IRQ_HANDLED;
3650 }
3651
3652 /**
3653 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
3654 *
3655 * @rdev: radeon_device pointer
3656 * @fence: radeon fence object
3657 *
3658 * Add a DMA fence packet to the ring to write
3659 * the fence seq number and DMA trap packet to generate
3660 * an interrupt if needed (evergreen-SI).
3661 */
3662 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
3663 struct radeon_fence *fence)
3664 {
3665 struct radeon_ring *ring = &rdev->ring[fence->ring];
3666 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3667 /* write the fence */
3668 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
3669 radeon_ring_write(ring, addr & 0xfffffffc);
3670 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
3671 radeon_ring_write(ring, fence->seq);
3672 /* generate an interrupt */
3673 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
3674 /* flush HDP */
3675 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
3676 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
3677 radeon_ring_write(ring, 1);
3678 }
3679
3680 /**
3681 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
3682 *
3683 * @rdev: radeon_device pointer
3684 * @ib: IB object to schedule
3685 *
3686 * Schedule an IB in the DMA ring (evergreen).
3687 */
3688 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
3689 struct radeon_ib *ib)
3690 {
3691 struct radeon_ring *ring = &rdev->ring[ib->ring];
3692
3693 if (rdev->wb.enabled) {
3694 u32 next_rptr = ring->wptr + 4;
3695 while ((next_rptr & 7) != 5)
3696 next_rptr++;
3697 next_rptr += 3;
3698 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
3699 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3700 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3701 radeon_ring_write(ring, next_rptr);
3702 }
3703
3704 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3705 * Pad as necessary with NOPs.
3706 */
3707 while ((ring->wptr & 7) != 5)
3708 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
3709 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
3710 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3711 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3712
3713 }
3714
3715 /**
3716 * evergreen_copy_dma - copy pages using the DMA engine
3717 *
3718 * @rdev: radeon_device pointer
3719 * @src_offset: src GPU address
3720 * @dst_offset: dst GPU address
3721 * @num_gpu_pages: number of GPU pages to xfer
3722 * @fence: radeon fence object
3723 *
3724 * Copy GPU paging using the DMA engine (evergreen-cayman).
3725 * Used by the radeon ttm implementation to move pages if
3726 * registered as the asic copy callback.
3727 */
3728 int evergreen_copy_dma(struct radeon_device *rdev,
3729 uint64_t src_offset, uint64_t dst_offset,
3730 unsigned num_gpu_pages,
3731 struct radeon_fence **fence)
3732 {
3733 struct radeon_semaphore *sem = NULL;
3734 int ring_index = rdev->asic->copy.dma_ring_index;
3735 struct radeon_ring *ring = &rdev->ring[ring_index];
3736 u32 size_in_dw, cur_size_in_dw;
3737 int i, num_loops;
3738 int r = 0;
3739
3740 r = radeon_semaphore_create(rdev, &sem);
3741 if (r) {
3742 DRM_ERROR("radeon: moving bo (%d).\n", r);
3743 return r;
3744 }
3745
3746 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
3747 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
3748 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
3749 if (r) {
3750 DRM_ERROR("radeon: moving bo (%d).\n", r);
3751 radeon_semaphore_free(rdev, &sem, NULL);
3752 return r;
3753 }
3754
3755 if (radeon_fence_need_sync(*fence, ring->idx)) {
3756 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3757 ring->idx);
3758 radeon_fence_note_sync(*fence, ring->idx);
3759 } else {
3760 radeon_semaphore_free(rdev, &sem, NULL);
3761 }
3762
3763 for (i = 0; i < num_loops; i++) {
3764 cur_size_in_dw = size_in_dw;
3765 if (cur_size_in_dw > 0xFFFFF)
3766 cur_size_in_dw = 0xFFFFF;
3767 size_in_dw -= cur_size_in_dw;
3768 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
3769 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3770 radeon_ring_write(ring, src_offset & 0xfffffffc);
3771 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
3772 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
3773 src_offset += cur_size_in_dw * 4;
3774 dst_offset += cur_size_in_dw * 4;
3775 }
3776
3777 r = radeon_fence_emit(rdev, fence, ring->idx);
3778 if (r) {
3779 radeon_ring_unlock_undo(rdev, ring);
3780 return r;
3781 }
3782
3783 radeon_ring_unlock_commit(rdev, ring);
3784 radeon_semaphore_free(rdev, &sem, *fence);
3785
3786 return r;
3787 }
3788
3789 static int evergreen_startup(struct radeon_device *rdev)
3790 {
3791 struct radeon_ring *ring;
3792 int r;
3793
3794 /* enable pcie gen2 link */
3795 evergreen_pcie_gen2_enable(rdev);
3796
3797 if (ASIC_IS_DCE5(rdev)) {
3798 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3799 r = ni_init_microcode(rdev);
3800 if (r) {
3801 DRM_ERROR("Failed to load firmware!\n");
3802 return r;
3803 }
3804 }
3805 r = ni_mc_load_microcode(rdev);
3806 if (r) {
3807 DRM_ERROR("Failed to load MC firmware!\n");
3808 return r;
3809 }
3810 } else {
3811 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3812 r = r600_init_microcode(rdev);
3813 if (r) {
3814 DRM_ERROR("Failed to load firmware!\n");
3815 return r;
3816 }
3817 }
3818 }
3819
3820 r = r600_vram_scratch_init(rdev);
3821 if (r)
3822 return r;
3823
3824 evergreen_mc_program(rdev);
3825 if (rdev->flags & RADEON_IS_AGP) {
3826 evergreen_agp_enable(rdev);
3827 } else {
3828 r = evergreen_pcie_gart_enable(rdev);
3829 if (r)
3830 return r;
3831 }
3832 evergreen_gpu_init(rdev);
3833
3834 r = evergreen_blit_init(rdev);
3835 if (r) {
3836 r600_blit_fini(rdev);
3837 rdev->asic->copy.copy = NULL;
3838 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3839 }
3840
3841 /* allocate wb buffer */
3842 r = radeon_wb_init(rdev);
3843 if (r)
3844 return r;
3845
3846 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3847 if (r) {
3848 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3849 return r;
3850 }
3851
3852 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3853 if (r) {
3854 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3855 return r;
3856 }
3857
3858 r = rv770_uvd_resume(rdev);
3859 if (!r) {
3860 r = radeon_fence_driver_start_ring(rdev,
3861 R600_RING_TYPE_UVD_INDEX);
3862 if (r)
3863 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
3864 }
3865
3866 if (r)
3867 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
3868
3869 /* Enable IRQ */
3870 r = r600_irq_init(rdev);
3871 if (r) {
3872 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3873 radeon_irq_kms_fini(rdev);
3874 return r;
3875 }
3876 evergreen_irq_set(rdev);
3877
3878 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3879 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3880 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3881 0, 0xfffff, RADEON_CP_PACKET2);
3882 if (r)
3883 return r;
3884
3885 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3886 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3887 DMA_RB_RPTR, DMA_RB_WPTR,
3888 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
3889 if (r)
3890 return r;
3891
3892 r = evergreen_cp_load_microcode(rdev);
3893 if (r)
3894 return r;
3895 r = evergreen_cp_resume(rdev);
3896 if (r)
3897 return r;
3898 r = r600_dma_resume(rdev);
3899 if (r)
3900 return r;
3901
3902 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
3903 if (ring->ring_size) {
3904 r = radeon_ring_init(rdev, ring, ring->ring_size,
3905 R600_WB_UVD_RPTR_OFFSET,
3906 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
3907 0, 0xfffff, RADEON_CP_PACKET2);
3908 if (!r)
3909 r = r600_uvd_init(rdev);
3910
3911 if (r)
3912 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
3913 }
3914
3915 r = radeon_ib_pool_init(rdev);
3916 if (r) {
3917 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
3918 return r;
3919 }
3920
3921 r = r600_audio_init(rdev);
3922 if (r) {
3923 DRM_ERROR("radeon: audio init failed\n");
3924 return r;
3925 }
3926
3927 return 0;
3928 }
3929
3930 int evergreen_resume(struct radeon_device *rdev)
3931 {
3932 int r;
3933
3934 /* reset the asic, the gfx blocks are often in a bad state
3935 * after the driver is unloaded or after a resume
3936 */
3937 if (radeon_asic_reset(rdev))
3938 dev_warn(rdev->dev, "GPU reset failed !\n");
3939 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3940 * posting will perform necessary task to bring back GPU into good
3941 * shape.
3942 */
3943 /* post card */
3944 atom_asic_init(rdev->mode_info.atom_context);
3945
3946 rdev->accel_working = true;
3947 r = evergreen_startup(rdev);
3948 if (r) {
3949 DRM_ERROR("evergreen startup failed on resume\n");
3950 rdev->accel_working = false;
3951 return r;
3952 }
3953
3954 return r;
3955
3956 }
3957
3958 int evergreen_suspend(struct radeon_device *rdev)
3959 {
3960 r600_audio_fini(rdev);
3961 radeon_uvd_suspend(rdev);
3962 r700_cp_stop(rdev);
3963 r600_dma_stop(rdev);
3964 r600_uvd_rbc_stop(rdev);
3965 evergreen_irq_suspend(rdev);
3966 radeon_wb_disable(rdev);
3967 evergreen_pcie_gart_disable(rdev);
3968
3969 return 0;
3970 }
3971
3972 /* Plan is to move initialization in that function and use
3973 * helper function so that radeon_device_init pretty much
3974 * do nothing more than calling asic specific function. This
3975 * should also allow to remove a bunch of callback function
3976 * like vram_info.
3977 */
3978 int evergreen_init(struct radeon_device *rdev)
3979 {
3980 int r;
3981
3982 /* Read BIOS */
3983 if (!radeon_get_bios(rdev)) {
3984 if (ASIC_IS_AVIVO(rdev))
3985 return -EINVAL;
3986 }
3987 /* Must be an ATOMBIOS */
3988 if (!rdev->is_atom_bios) {
3989 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
3990 return -EINVAL;
3991 }
3992 r = radeon_atombios_init(rdev);
3993 if (r)
3994 return r;
3995 /* reset the asic, the gfx blocks are often in a bad state
3996 * after the driver is unloaded or after a resume
3997 */
3998 if (radeon_asic_reset(rdev))
3999 dev_warn(rdev->dev, "GPU reset failed !\n");
4000 /* Post card if necessary */
4001 if (!radeon_card_posted(rdev)) {
4002 if (!rdev->bios) {
4003 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
4004 return -EINVAL;
4005 }
4006 DRM_INFO("GPU not posted. posting now...\n");
4007 atom_asic_init(rdev->mode_info.atom_context);
4008 }
4009 /* Initialize scratch registers */
4010 r600_scratch_init(rdev);
4011 /* Initialize surface registers */
4012 radeon_surface_init(rdev);
4013 /* Initialize clocks */
4014 radeon_get_clock_info(rdev->ddev);
4015 /* Fence driver */
4016 r = radeon_fence_driver_init(rdev);
4017 if (r)
4018 return r;
4019 /* initialize AGP */
4020 if (rdev->flags & RADEON_IS_AGP) {
4021 r = radeon_agp_init(rdev);
4022 if (r)
4023 radeon_agp_disable(rdev);
4024 }
4025 /* initialize memory controller */
4026 r = evergreen_mc_init(rdev);
4027 if (r)
4028 return r;
4029 /* Memory manager */
4030 r = radeon_bo_init(rdev);
4031 if (r)
4032 return r;
4033
4034 r = radeon_irq_kms_init(rdev);
4035 if (r)
4036 return r;
4037
4038 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
4039 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
4040
4041 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
4042 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
4043
4044 r = radeon_uvd_init(rdev);
4045 if (!r) {
4046 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4047 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
4048 4096);
4049 }
4050
4051 rdev->ih.ring_obj = NULL;
4052 r600_ih_ring_init(rdev, 64 * 1024);
4053
4054 r = r600_pcie_gart_init(rdev);
4055 if (r)
4056 return r;
4057
4058 rdev->accel_working = true;
4059 r = evergreen_startup(rdev);
4060 if (r) {
4061 dev_err(rdev->dev, "disabling GPU acceleration\n");
4062 r700_cp_fini(rdev);
4063 r600_dma_fini(rdev);
4064 r600_irq_fini(rdev);
4065 radeon_wb_fini(rdev);
4066 radeon_ib_pool_fini(rdev);
4067 radeon_irq_kms_fini(rdev);
4068 evergreen_pcie_gart_fini(rdev);
4069 rdev->accel_working = false;
4070 }
4071
4072 /* Don't start up if the MC ucode is missing on BTC parts.
4073 * The default clocks and voltages before the MC ucode
4074 * is loaded are not suffient for advanced operations.
4075 */
4076 if (ASIC_IS_DCE5(rdev)) {
4077 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
4078 DRM_ERROR("radeon: MC ucode required for NI+.\n");
4079 return -EINVAL;
4080 }
4081 }
4082
4083 return 0;
4084 }
4085
4086 void evergreen_fini(struct radeon_device *rdev)
4087 {
4088 r600_audio_fini(rdev);
4089 r600_blit_fini(rdev);
4090 r700_cp_fini(rdev);
4091 r600_dma_fini(rdev);
4092 r600_irq_fini(rdev);
4093 radeon_wb_fini(rdev);
4094 radeon_ib_pool_fini(rdev);
4095 radeon_irq_kms_fini(rdev);
4096 evergreen_pcie_gart_fini(rdev);
4097 radeon_uvd_fini(rdev);
4098 r600_vram_scratch_fini(rdev);
4099 radeon_gem_fini(rdev);
4100 radeon_fence_driver_fini(rdev);
4101 radeon_agp_fini(rdev);
4102 radeon_bo_fini(rdev);
4103 radeon_atombios_fini(rdev);
4104 kfree(rdev->bios);
4105 rdev->bios = NULL;
4106 }
4107
4108 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
4109 {
4110 u32 link_width_cntl, speed_cntl, mask;
4111 int ret;
4112
4113 if (radeon_pcie_gen2 == 0)
4114 return;
4115
4116 if (rdev->flags & RADEON_IS_IGP)
4117 return;
4118
4119 if (!(rdev->flags & RADEON_IS_PCIE))
4120 return;
4121
4122 /* x2 cards have a special sequence */
4123 if (ASIC_IS_X2(rdev))
4124 return;
4125
4126 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
4127 if (ret != 0)
4128 return;
4129
4130 if (!(mask & DRM_PCIE_SPEED_50))
4131 return;
4132
4133 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4134 if (speed_cntl & LC_CURRENT_DATA_RATE) {
4135 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4136 return;
4137 }
4138
4139 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4140
4141 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
4142 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4143
4144 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4145 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4146 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4147
4148 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4149 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4150 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4151
4152 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4153 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
4154 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4155
4156 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4157 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
4158 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4159
4160 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4161 speed_cntl |= LC_GEN2_EN_STRAP;
4162 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4163
4164 } else {
4165 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4166 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4167 if (1)
4168 link_width_cntl |= LC_UPCONFIGURE_DIS;
4169 else
4170 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4171 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4172 }
4173 }
Linus' kernel tree