?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/linux: Improve request_firmware() compatibility
[dragonfly.git] / sys / dev / drm / radeon / cik.c
blobd8947f45045deeec83d51e5ebd003ece02bf8502
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/module.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include "cikd.h"
30 #include "atom.h"
31 #include "cik_blit_shaders.h"
32 #include "radeon_ucode.h"
33 #include "clearstate_ci.h"
34
35 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
36 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44
45 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
46 MODULE_FIRMWARE("radeon/bonaire_me.bin");
47 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
48 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
49 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
50 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
51 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
52 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
53
54 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
55 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
56 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
57 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
63
64 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
65 MODULE_FIRMWARE("radeon/hawaii_me.bin");
66 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
67 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
68 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
69 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
70 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
71 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
72
73 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
74 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
75 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
76 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
79
80 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
81 MODULE_FIRMWARE("radeon/kaveri_me.bin");
82 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
83 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
84 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
85 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
86 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
87
88 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
89 MODULE_FIRMWARE("radeon/KABINI_me.bin");
90 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
91 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
92 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
93 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
94
95 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
96 MODULE_FIRMWARE("radeon/kabini_me.bin");
97 MODULE_FIRMWARE("radeon/kabini_ce.bin");
98 MODULE_FIRMWARE("radeon/kabini_mec.bin");
99 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
100 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
101
102 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
103 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
104 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
105 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
108
109 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
110 MODULE_FIRMWARE("radeon/mullins_me.bin");
111 MODULE_FIRMWARE("radeon/mullins_ce.bin");
112 MODULE_FIRMWARE("radeon/mullins_mec.bin");
113 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
114 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
115
116 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
117 static void cik_rlc_stop(struct radeon_device *rdev);
118 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
119 static void cik_program_aspm(struct radeon_device *rdev);
120 static void cik_init_pg(struct radeon_device *rdev);
121 static void cik_init_cg(struct radeon_device *rdev);
122 static void cik_fini_pg(struct radeon_device *rdev);
123 static void cik_fini_cg(struct radeon_device *rdev);
124 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
125 bool enable);
126
127 /* get temperature in millidegrees */
128 int ci_get_temp(struct radeon_device *rdev)
129 {
130 u32 temp;
131 int actual_temp = 0;
132
133 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
134 CTF_TEMP_SHIFT;
135
136 if (temp & 0x200)
137 actual_temp = 255;
138 else
139 actual_temp = temp & 0x1ff;
140
141 actual_temp = actual_temp * 1000;
142
143 return actual_temp;
144 }
145
146 /* get temperature in millidegrees */
147 int kv_get_temp(struct radeon_device *rdev)
148 {
149 u32 temp;
150 int actual_temp = 0;
151
152 temp = RREG32_SMC(0xC0300E0C);
153
154 if (temp)
155 actual_temp = (temp / 8) - 49;
156 else
157 actual_temp = 0;
158
159 actual_temp = actual_temp * 1000;
160
161 return actual_temp;
162 }
163
164 /*
165 * Indirect registers accessor
166 */
167 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
168 {
169 u32 r;
170
171 spin_lock(&rdev->pciep_idx_lock);
172 WREG32(PCIE_INDEX, reg);
173 (void)RREG32(PCIE_INDEX);
174 r = RREG32(PCIE_DATA);
175 spin_unlock(&rdev->pciep_idx_lock);
176 return r;
177 }
178
179 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
180 {
181 spin_lock(&rdev->pciep_idx_lock);
182 WREG32(PCIE_INDEX, reg);
183 (void)RREG32(PCIE_INDEX);
184 WREG32(PCIE_DATA, v);
185 (void)RREG32(PCIE_DATA);
186 spin_unlock(&rdev->pciep_idx_lock);
187 }
188
189 static const u32 spectre_rlc_save_restore_register_list[] =
190 {
191 (0x0e00 << 16) | (0xc12c >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0xc140 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0xc150 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0xc15c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0xc168 >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0xc170 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xc178 >> 2),
204 0x00000000,
205 (0x0e00 << 16) | (0xc204 >> 2),
206 0x00000000,
207 (0x0e00 << 16) | (0xc2b4 >> 2),
208 0x00000000,
209 (0x0e00 << 16) | (0xc2b8 >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0xc2bc >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc2c0 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0x8228 >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0x829c >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0x869c >> 2),
220 0x00000000,
221 (0x0600 << 16) | (0x98f4 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x98f8 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0x9900 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc260 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0x90e8 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0x3c000 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0x3c00c >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x8c1c >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x9700 >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0xcd20 >> 2),
240 0x00000000,
241 (0x4e00 << 16) | (0xcd20 >> 2),
242 0x00000000,
243 (0x5e00 << 16) | (0xcd20 >> 2),
244 0x00000000,
245 (0x6e00 << 16) | (0xcd20 >> 2),
246 0x00000000,
247 (0x7e00 << 16) | (0xcd20 >> 2),
248 0x00000000,
249 (0x8e00 << 16) | (0xcd20 >> 2),
250 0x00000000,
251 (0x9e00 << 16) | (0xcd20 >> 2),
252 0x00000000,
253 (0xae00 << 16) | (0xcd20 >> 2),
254 0x00000000,
255 (0xbe00 << 16) | (0xcd20 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0x89bc >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0x8900 >> 2),
260 0x00000000,
261 0x3,
262 (0x0e00 << 16) | (0xc130 >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc134 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc1fc >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc208 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc264 >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc268 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0xc26c >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0xc270 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0xc274 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0xc278 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc27c >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc280 >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc284 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc288 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc28c >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc290 >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc294 >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc298 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc29c >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc2a0 >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc2a4 >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc2a8 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc2ac >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc2b0 >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0x301d0 >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0x30238 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0x30250 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0x30254 >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0x30258 >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0x3025c >> 2),
321 0x00000000,
322 (0x4e00 << 16) | (0xc900 >> 2),
323 0x00000000,
324 (0x5e00 << 16) | (0xc900 >> 2),
325 0x00000000,
326 (0x6e00 << 16) | (0xc900 >> 2),
327 0x00000000,
328 (0x7e00 << 16) | (0xc900 >> 2),
329 0x00000000,
330 (0x8e00 << 16) | (0xc900 >> 2),
331 0x00000000,
332 (0x9e00 << 16) | (0xc900 >> 2),
333 0x00000000,
334 (0xae00 << 16) | (0xc900 >> 2),
335 0x00000000,
336 (0xbe00 << 16) | (0xc900 >> 2),
337 0x00000000,
338 (0x4e00 << 16) | (0xc904 >> 2),
339 0x00000000,
340 (0x5e00 << 16) | (0xc904 >> 2),
341 0x00000000,
342 (0x6e00 << 16) | (0xc904 >> 2),
343 0x00000000,
344 (0x7e00 << 16) | (0xc904 >> 2),
345 0x00000000,
346 (0x8e00 << 16) | (0xc904 >> 2),
347 0x00000000,
348 (0x9e00 << 16) | (0xc904 >> 2),
349 0x00000000,
350 (0xae00 << 16) | (0xc904 >> 2),
351 0x00000000,
352 (0xbe00 << 16) | (0xc904 >> 2),
353 0x00000000,
354 (0x4e00 << 16) | (0xc908 >> 2),
355 0x00000000,
356 (0x5e00 << 16) | (0xc908 >> 2),
357 0x00000000,
358 (0x6e00 << 16) | (0xc908 >> 2),
359 0x00000000,
360 (0x7e00 << 16) | (0xc908 >> 2),
361 0x00000000,
362 (0x8e00 << 16) | (0xc908 >> 2),
363 0x00000000,
364 (0x9e00 << 16) | (0xc908 >> 2),
365 0x00000000,
366 (0xae00 << 16) | (0xc908 >> 2),
367 0x00000000,
368 (0xbe00 << 16) | (0xc908 >> 2),
369 0x00000000,
370 (0x4e00 << 16) | (0xc90c >> 2),
371 0x00000000,
372 (0x5e00 << 16) | (0xc90c >> 2),
373 0x00000000,
374 (0x6e00 << 16) | (0xc90c >> 2),
375 0x00000000,
376 (0x7e00 << 16) | (0xc90c >> 2),
377 0x00000000,
378 (0x8e00 << 16) | (0xc90c >> 2),
379 0x00000000,
380 (0x9e00 << 16) | (0xc90c >> 2),
381 0x00000000,
382 (0xae00 << 16) | (0xc90c >> 2),
383 0x00000000,
384 (0xbe00 << 16) | (0xc90c >> 2),
385 0x00000000,
386 (0x4e00 << 16) | (0xc910 >> 2),
387 0x00000000,
388 (0x5e00 << 16) | (0xc910 >> 2),
389 0x00000000,
390 (0x6e00 << 16) | (0xc910 >> 2),
391 0x00000000,
392 (0x7e00 << 16) | (0xc910 >> 2),
393 0x00000000,
394 (0x8e00 << 16) | (0xc910 >> 2),
395 0x00000000,
396 (0x9e00 << 16) | (0xc910 >> 2),
397 0x00000000,
398 (0xae00 << 16) | (0xc910 >> 2),
399 0x00000000,
400 (0xbe00 << 16) | (0xc910 >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0xc99c >> 2),
403 0x00000000,
404 (0x0e00 << 16) | (0x9834 >> 2),
405 0x00000000,
406 (0x0000 << 16) | (0x30f00 >> 2),
407 0x00000000,
408 (0x0001 << 16) | (0x30f00 >> 2),
409 0x00000000,
410 (0x0000 << 16) | (0x30f04 >> 2),
411 0x00000000,
412 (0x0001 << 16) | (0x30f04 >> 2),
413 0x00000000,
414 (0x0000 << 16) | (0x30f08 >> 2),
415 0x00000000,
416 (0x0001 << 16) | (0x30f08 >> 2),
417 0x00000000,
418 (0x0000 << 16) | (0x30f0c >> 2),
419 0x00000000,
420 (0x0001 << 16) | (0x30f0c >> 2),
421 0x00000000,
422 (0x0600 << 16) | (0x9b7c >> 2),
423 0x00000000,
424 (0x0e00 << 16) | (0x8a14 >> 2),
425 0x00000000,
426 (0x0e00 << 16) | (0x8a18 >> 2),
427 0x00000000,
428 (0x0600 << 16) | (0x30a00 >> 2),
429 0x00000000,
430 (0x0e00 << 16) | (0x8bf0 >> 2),
431 0x00000000,
432 (0x0e00 << 16) | (0x8bcc >> 2),
433 0x00000000,
434 (0x0e00 << 16) | (0x8b24 >> 2),
435 0x00000000,
436 (0x0e00 << 16) | (0x30a04 >> 2),
437 0x00000000,
438 (0x0600 << 16) | (0x30a10 >> 2),
439 0x00000000,
440 (0x0600 << 16) | (0x30a14 >> 2),
441 0x00000000,
442 (0x0600 << 16) | (0x30a18 >> 2),
443 0x00000000,
444 (0x0600 << 16) | (0x30a2c >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0xc700 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0xc704 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0xc708 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0xc768 >> 2),
453 0x00000000,
454 (0x0400 << 16) | (0xc770 >> 2),
455 0x00000000,
456 (0x0400 << 16) | (0xc774 >> 2),
457 0x00000000,
458 (0x0400 << 16) | (0xc778 >> 2),
459 0x00000000,
460 (0x0400 << 16) | (0xc77c >> 2),
461 0x00000000,
462 (0x0400 << 16) | (0xc780 >> 2),
463 0x00000000,
464 (0x0400 << 16) | (0xc784 >> 2),
465 0x00000000,
466 (0x0400 << 16) | (0xc788 >> 2),
467 0x00000000,
468 (0x0400 << 16) | (0xc78c >> 2),
469 0x00000000,
470 (0x0400 << 16) | (0xc798 >> 2),
471 0x00000000,
472 (0x0400 << 16) | (0xc79c >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc7a0 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc7a4 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc7a8 >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc7ac >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc7b0 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc7b4 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9100 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0x3c010 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0x92a8 >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0x92ac >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0x92b4 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0x92b8 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0x92bc >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0x92c0 >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0x92c4 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x92c8 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x92cc >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x92d0 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x8c00 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8c04 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x8c20 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x8c38 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x8c3c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0xae00 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9604 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0xac08 >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0xac0c >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xac10 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xac14 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xac58 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0xac68 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0xac6c >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xac70 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0xac74 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0xac78 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac7c >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac80 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac84 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac88 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac8c >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x970c >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0x9714 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x9718 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x971c >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x31068 >> 2),
563 0x00000000,
564 (0x4e00 << 16) | (0x31068 >> 2),
565 0x00000000,
566 (0x5e00 << 16) | (0x31068 >> 2),
567 0x00000000,
568 (0x6e00 << 16) | (0x31068 >> 2),
569 0x00000000,
570 (0x7e00 << 16) | (0x31068 >> 2),
571 0x00000000,
572 (0x8e00 << 16) | (0x31068 >> 2),
573 0x00000000,
574 (0x9e00 << 16) | (0x31068 >> 2),
575 0x00000000,
576 (0xae00 << 16) | (0x31068 >> 2),
577 0x00000000,
578 (0xbe00 << 16) | (0x31068 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0xcd10 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0xcd14 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x88b0 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x88b4 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x88b8 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x88bc >> 2),
591 0x00000000,
592 (0x0400 << 16) | (0x89c0 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x88c4 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x88c8 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x88d0 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x88d4 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x88d8 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x8980 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x30938 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x3093c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0x30940 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0x89a0 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x30900 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x30904 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x89b4 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x3c210 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x3c214 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x3c218 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x8904 >> 2),
627 0x00000000,
628 0x5,
629 (0x0e00 << 16) | (0x8c28 >> 2),
630 (0x0e00 << 16) | (0x8c2c >> 2),
631 (0x0e00 << 16) | (0x8c30 >> 2),
632 (0x0e00 << 16) | (0x8c34 >> 2),
633 (0x0e00 << 16) | (0x9600 >> 2),
634 };
635
636 static const u32 kalindi_rlc_save_restore_register_list[] =
637 {
638 (0x0e00 << 16) | (0xc12c >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0xc140 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0xc150 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0xc15c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0xc168 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xc170 >> 2),
649 0x00000000,
650 (0x0e00 << 16) | (0xc204 >> 2),
651 0x00000000,
652 (0x0e00 << 16) | (0xc2b4 >> 2),
653 0x00000000,
654 (0x0e00 << 16) | (0xc2b8 >> 2),
655 0x00000000,
656 (0x0e00 << 16) | (0xc2bc >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xc2c0 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8228 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0x829c >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0x869c >> 2),
665 0x00000000,
666 (0x0600 << 16) | (0x98f4 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x98f8 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x9900 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc260 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0x90e8 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x3c000 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x3c00c >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x8c1c >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x9700 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0xcd20 >> 2),
685 0x00000000,
686 (0x4e00 << 16) | (0xcd20 >> 2),
687 0x00000000,
688 (0x5e00 << 16) | (0xcd20 >> 2),
689 0x00000000,
690 (0x6e00 << 16) | (0xcd20 >> 2),
691 0x00000000,
692 (0x7e00 << 16) | (0xcd20 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x89bc >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x8900 >> 2),
697 0x00000000,
698 0x3,
699 (0x0e00 << 16) | (0xc130 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0xc134 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0xc1fc >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0xc208 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0xc264 >> 2),
708 0x00000000,
709 (0x0e00 << 16) | (0xc268 >> 2),
710 0x00000000,
711 (0x0e00 << 16) | (0xc26c >> 2),
712 0x00000000,
713 (0x0e00 << 16) | (0xc270 >> 2),
714 0x00000000,
715 (0x0e00 << 16) | (0xc274 >> 2),
716 0x00000000,
717 (0x0e00 << 16) | (0xc28c >> 2),
718 0x00000000,
719 (0x0e00 << 16) | (0xc290 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc294 >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc298 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc2a0 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc2a4 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc2a8 >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc2ac >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0x301d0 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0x30238 >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0x30250 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0x30254 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0x30258 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0x3025c >> 2),
744 0x00000000,
745 (0x4e00 << 16) | (0xc900 >> 2),
746 0x00000000,
747 (0x5e00 << 16) | (0xc900 >> 2),
748 0x00000000,
749 (0x6e00 << 16) | (0xc900 >> 2),
750 0x00000000,
751 (0x7e00 << 16) | (0xc900 >> 2),
752 0x00000000,
753 (0x4e00 << 16) | (0xc904 >> 2),
754 0x00000000,
755 (0x5e00 << 16) | (0xc904 >> 2),
756 0x00000000,
757 (0x6e00 << 16) | (0xc904 >> 2),
758 0x00000000,
759 (0x7e00 << 16) | (0xc904 >> 2),
760 0x00000000,
761 (0x4e00 << 16) | (0xc908 >> 2),
762 0x00000000,
763 (0x5e00 << 16) | (0xc908 >> 2),
764 0x00000000,
765 (0x6e00 << 16) | (0xc908 >> 2),
766 0x00000000,
767 (0x7e00 << 16) | (0xc908 >> 2),
768 0x00000000,
769 (0x4e00 << 16) | (0xc90c >> 2),
770 0x00000000,
771 (0x5e00 << 16) | (0xc90c >> 2),
772 0x00000000,
773 (0x6e00 << 16) | (0xc90c >> 2),
774 0x00000000,
775 (0x7e00 << 16) | (0xc90c >> 2),
776 0x00000000,
777 (0x4e00 << 16) | (0xc910 >> 2),
778 0x00000000,
779 (0x5e00 << 16) | (0xc910 >> 2),
780 0x00000000,
781 (0x6e00 << 16) | (0xc910 >> 2),
782 0x00000000,
783 (0x7e00 << 16) | (0xc910 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc99c >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0x9834 >> 2),
788 0x00000000,
789 (0x0000 << 16) | (0x30f00 >> 2),
790 0x00000000,
791 (0x0000 << 16) | (0x30f04 >> 2),
792 0x00000000,
793 (0x0000 << 16) | (0x30f08 >> 2),
794 0x00000000,
795 (0x0000 << 16) | (0x30f0c >> 2),
796 0x00000000,
797 (0x0600 << 16) | (0x9b7c >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0x8a14 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x8a18 >> 2),
802 0x00000000,
803 (0x0600 << 16) | (0x30a00 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8bf0 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8bcc >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8b24 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x30a04 >> 2),
812 0x00000000,
813 (0x0600 << 16) | (0x30a10 >> 2),
814 0x00000000,
815 (0x0600 << 16) | (0x30a14 >> 2),
816 0x00000000,
817 (0x0600 << 16) | (0x30a18 >> 2),
818 0x00000000,
819 (0x0600 << 16) | (0x30a2c >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xc700 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xc704 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xc708 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xc768 >> 2),
828 0x00000000,
829 (0x0400 << 16) | (0xc770 >> 2),
830 0x00000000,
831 (0x0400 << 16) | (0xc774 >> 2),
832 0x00000000,
833 (0x0400 << 16) | (0xc798 >> 2),
834 0x00000000,
835 (0x0400 << 16) | (0xc79c >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0x9100 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0x3c010 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0x8c00 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0x8c04 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0x8c20 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0x8c38 >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x8c3c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0xae00 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9604 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0xac08 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0xac0c >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0xac10 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0xac14 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0xac58 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0xac68 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xac6c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xac70 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0xac74 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0xac78 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac7c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac80 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac84 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac88 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac8c >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x970c >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x9714 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x9718 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x971c >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x31068 >> 2),
894 0x00000000,
895 (0x4e00 << 16) | (0x31068 >> 2),
896 0x00000000,
897 (0x5e00 << 16) | (0x31068 >> 2),
898 0x00000000,
899 (0x6e00 << 16) | (0x31068 >> 2),
900 0x00000000,
901 (0x7e00 << 16) | (0x31068 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xcd10 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xcd14 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x88b0 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x88b4 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x88b8 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x88bc >> 2),
914 0x00000000,
915 (0x0400 << 16) | (0x89c0 >> 2),
916 0x00000000,
917 (0x0e00 << 16) | (0x88c4 >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x88c8 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x88d0 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x88d4 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x88d8 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8980 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x30938 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x3093c >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0x30940 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x89a0 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x30900 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x30904 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x89b4 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x3e1fc >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x3c210 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x3c214 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x3c218 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x8904 >> 2),
952 0x00000000,
953 0x5,
954 (0x0e00 << 16) | (0x8c28 >> 2),
955 (0x0e00 << 16) | (0x8c2c >> 2),
956 (0x0e00 << 16) | (0x8c30 >> 2),
957 (0x0e00 << 16) | (0x8c34 >> 2),
958 (0x0e00 << 16) | (0x9600 >> 2),
959 };
960
961 static const u32 bonaire_golden_spm_registers[] =
962 {
963 0x30800, 0xe0ffffff, 0xe0000000
964 };
965
966 static const u32 bonaire_golden_common_registers[] =
967 {
968 0xc770, 0xffffffff, 0x00000800,
969 0xc774, 0xffffffff, 0x00000800,
970 0xc798, 0xffffffff, 0x00007fbf,
971 0xc79c, 0xffffffff, 0x00007faf
972 };
973
974 static const u32 bonaire_golden_registers[] =
975 {
976 0x3354, 0x00000333, 0x00000333,
977 0x3350, 0x000c0fc0, 0x00040200,
978 0x9a10, 0x00010000, 0x00058208,
979 0x3c000, 0xffff1fff, 0x00140000,
980 0x3c200, 0xfdfc0fff, 0x00000100,
981 0x3c234, 0x40000000, 0x40000200,
982 0x9830, 0xffffffff, 0x00000000,
983 0x9834, 0xf00fffff, 0x00000400,
984 0x9838, 0x0002021c, 0x00020200,
985 0xc78, 0x00000080, 0x00000000,
986 0x5bb0, 0x000000f0, 0x00000070,
987 0x5bc0, 0xf0311fff, 0x80300000,
988 0x98f8, 0x73773777, 0x12010001,
989 0x350c, 0x00810000, 0x408af000,
990 0x7030, 0x31000111, 0x00000011,
991 0x2f48, 0x73773777, 0x12010001,
992 0x220c, 0x00007fb6, 0x0021a1b1,
993 0x2210, 0x00007fb6, 0x002021b1,
994 0x2180, 0x00007fb6, 0x00002191,
995 0x2218, 0x00007fb6, 0x002121b1,
996 0x221c, 0x00007fb6, 0x002021b1,
997 0x21dc, 0x00007fb6, 0x00002191,
998 0x21e0, 0x00007fb6, 0x00002191,
999 0x3628, 0x0000003f, 0x0000000a,
1000 0x362c, 0x0000003f, 0x0000000a,
1001 0x2ae4, 0x00073ffe, 0x000022a2,
1002 0x240c, 0x000007ff, 0x00000000,
1003 0x8a14, 0xf000003f, 0x00000007,
1004 0x8bf0, 0x00002001, 0x00000001,
1005 0x8b24, 0xffffffff, 0x00ffffff,
1006 0x30a04, 0x0000ff0f, 0x00000000,
1007 0x28a4c, 0x07ffffff, 0x06000000,
1008 0x4d8, 0x00000fff, 0x00000100,
1009 0x3e78, 0x00000001, 0x00000002,
1010 0x9100, 0x03000000, 0x0362c688,
1011 0x8c00, 0x000000ff, 0x00000001,
1012 0xe40, 0x00001fff, 0x00001fff,
1013 0x9060, 0x0000007f, 0x00000020,
1014 0x9508, 0x00010000, 0x00010000,
1015 0xac14, 0x000003ff, 0x000000f3,
1016 0xac0c, 0xffffffff, 0x00001032
1017 };
1018
1019 static const u32 bonaire_mgcg_cgcg_init[] =
1020 {
1021 0xc420, 0xffffffff, 0xfffffffc,
1022 0x30800, 0xffffffff, 0xe0000000,
1023 0x3c2a0, 0xffffffff, 0x00000100,
1024 0x3c208, 0xffffffff, 0x00000100,
1025 0x3c2c0, 0xffffffff, 0xc0000100,
1026 0x3c2c8, 0xffffffff, 0xc0000100,
1027 0x3c2c4, 0xffffffff, 0xc0000100,
1028 0x55e4, 0xffffffff, 0x00600100,
1029 0x3c280, 0xffffffff, 0x00000100,
1030 0x3c214, 0xffffffff, 0x06000100,
1031 0x3c220, 0xffffffff, 0x00000100,
1032 0x3c218, 0xffffffff, 0x06000100,
1033 0x3c204, 0xffffffff, 0x00000100,
1034 0x3c2e0, 0xffffffff, 0x00000100,
1035 0x3c224, 0xffffffff, 0x00000100,
1036 0x3c200, 0xffffffff, 0x00000100,
1037 0x3c230, 0xffffffff, 0x00000100,
1038 0x3c234, 0xffffffff, 0x00000100,
1039 0x3c250, 0xffffffff, 0x00000100,
1040 0x3c254, 0xffffffff, 0x00000100,
1041 0x3c258, 0xffffffff, 0x00000100,
1042 0x3c25c, 0xffffffff, 0x00000100,
1043 0x3c260, 0xffffffff, 0x00000100,
1044 0x3c27c, 0xffffffff, 0x00000100,
1045 0x3c278, 0xffffffff, 0x00000100,
1046 0x3c210, 0xffffffff, 0x06000100,
1047 0x3c290, 0xffffffff, 0x00000100,
1048 0x3c274, 0xffffffff, 0x00000100,
1049 0x3c2b4, 0xffffffff, 0x00000100,
1050 0x3c2b0, 0xffffffff, 0x00000100,
1051 0x3c270, 0xffffffff, 0x00000100,
1052 0x30800, 0xffffffff, 0xe0000000,
1053 0x3c020, 0xffffffff, 0x00010000,
1054 0x3c024, 0xffffffff, 0x00030002,
1055 0x3c028, 0xffffffff, 0x00040007,
1056 0x3c02c, 0xffffffff, 0x00060005,
1057 0x3c030, 0xffffffff, 0x00090008,
1058 0x3c034, 0xffffffff, 0x00010000,
1059 0x3c038, 0xffffffff, 0x00030002,
1060 0x3c03c, 0xffffffff, 0x00040007,
1061 0x3c040, 0xffffffff, 0x00060005,
1062 0x3c044, 0xffffffff, 0x00090008,
1063 0x3c048, 0xffffffff, 0x00010000,
1064 0x3c04c, 0xffffffff, 0x00030002,
1065 0x3c050, 0xffffffff, 0x00040007,
1066 0x3c054, 0xffffffff, 0x00060005,
1067 0x3c058, 0xffffffff, 0x00090008,
1068 0x3c05c, 0xffffffff, 0x00010000,
1069 0x3c060, 0xffffffff, 0x00030002,
1070 0x3c064, 0xffffffff, 0x00040007,
1071 0x3c068, 0xffffffff, 0x00060005,
1072 0x3c06c, 0xffffffff, 0x00090008,
1073 0x3c070, 0xffffffff, 0x00010000,
1074 0x3c074, 0xffffffff, 0x00030002,
1075 0x3c078, 0xffffffff, 0x00040007,
1076 0x3c07c, 0xffffffff, 0x00060005,
1077 0x3c080, 0xffffffff, 0x00090008,
1078 0x3c084, 0xffffffff, 0x00010000,
1079 0x3c088, 0xffffffff, 0x00030002,
1080 0x3c08c, 0xffffffff, 0x00040007,
1081 0x3c090, 0xffffffff, 0x00060005,
1082 0x3c094, 0xffffffff, 0x00090008,
1083 0x3c098, 0xffffffff, 0x00010000,
1084 0x3c09c, 0xffffffff, 0x00030002,
1085 0x3c0a0, 0xffffffff, 0x00040007,
1086 0x3c0a4, 0xffffffff, 0x00060005,
1087 0x3c0a8, 0xffffffff, 0x00090008,
1088 0x3c000, 0xffffffff, 0x96e00200,
1089 0x8708, 0xffffffff, 0x00900100,
1090 0xc424, 0xffffffff, 0x0020003f,
1091 0x38, 0xffffffff, 0x0140001c,
1092 0x3c, 0x000f0000, 0x000f0000,
1093 0x220, 0xffffffff, 0xC060000C,
1094 0x224, 0xc0000fff, 0x00000100,
1095 0xf90, 0xffffffff, 0x00000100,
1096 0xf98, 0x00000101, 0x00000000,
1097 0x20a8, 0xffffffff, 0x00000104,
1098 0x55e4, 0xff000fff, 0x00000100,
1099 0x30cc, 0xc0000fff, 0x00000104,
1100 0xc1e4, 0x00000001, 0x00000001,
1101 0xd00c, 0xff000ff0, 0x00000100,
1102 0xd80c, 0xff000ff0, 0x00000100
1103 };
1104
1105 static const u32 spectre_golden_spm_registers[] =
1106 {
1107 0x30800, 0xe0ffffff, 0xe0000000
1108 };
1109
1110 static const u32 spectre_golden_common_registers[] =
1111 {
1112 0xc770, 0xffffffff, 0x00000800,
1113 0xc774, 0xffffffff, 0x00000800,
1114 0xc798, 0xffffffff, 0x00007fbf,
1115 0xc79c, 0xffffffff, 0x00007faf
1116 };
1117
1118 static const u32 spectre_golden_registers[] =
1119 {
1120 0x3c000, 0xffff1fff, 0x96940200,
1121 0x3c00c, 0xffff0001, 0xff000000,
1122 0x3c200, 0xfffc0fff, 0x00000100,
1123 0x6ed8, 0x00010101, 0x00010000,
1124 0x9834, 0xf00fffff, 0x00000400,
1125 0x9838, 0xfffffffc, 0x00020200,
1126 0x5bb0, 0x000000f0, 0x00000070,
1127 0x5bc0, 0xf0311fff, 0x80300000,
1128 0x98f8, 0x73773777, 0x12010001,
1129 0x9b7c, 0x00ff0000, 0x00fc0000,
1130 0x2f48, 0x73773777, 0x12010001,
1131 0x8a14, 0xf000003f, 0x00000007,
1132 0x8b24, 0xffffffff, 0x00ffffff,
1133 0x28350, 0x3f3f3fff, 0x00000082,
1134 0x28354, 0x0000003f, 0x00000000,
1135 0x3e78, 0x00000001, 0x00000002,
1136 0x913c, 0xffff03df, 0x00000004,
1137 0xc768, 0x00000008, 0x00000008,
1138 0x8c00, 0x000008ff, 0x00000800,
1139 0x9508, 0x00010000, 0x00010000,
1140 0xac0c, 0xffffffff, 0x54763210,
1141 0x214f8, 0x01ff01ff, 0x00000002,
1142 0x21498, 0x007ff800, 0x00200000,
1143 0x2015c, 0xffffffff, 0x00000f40,
1144 0x30934, 0xffffffff, 0x00000001
1145 };
1146
1147 static const u32 spectre_mgcg_cgcg_init[] =
1148 {
1149 0xc420, 0xffffffff, 0xfffffffc,
1150 0x30800, 0xffffffff, 0xe0000000,
1151 0x3c2a0, 0xffffffff, 0x00000100,
1152 0x3c208, 0xffffffff, 0x00000100,
1153 0x3c2c0, 0xffffffff, 0x00000100,
1154 0x3c2c8, 0xffffffff, 0x00000100,
1155 0x3c2c4, 0xffffffff, 0x00000100,
1156 0x55e4, 0xffffffff, 0x00600100,
1157 0x3c280, 0xffffffff, 0x00000100,
1158 0x3c214, 0xffffffff, 0x06000100,
1159 0x3c220, 0xffffffff, 0x00000100,
1160 0x3c218, 0xffffffff, 0x06000100,
1161 0x3c204, 0xffffffff, 0x00000100,
1162 0x3c2e0, 0xffffffff, 0x00000100,
1163 0x3c224, 0xffffffff, 0x00000100,
1164 0x3c200, 0xffffffff, 0x00000100,
1165 0x3c230, 0xffffffff, 0x00000100,
1166 0x3c234, 0xffffffff, 0x00000100,
1167 0x3c250, 0xffffffff, 0x00000100,
1168 0x3c254, 0xffffffff, 0x00000100,
1169 0x3c258, 0xffffffff, 0x00000100,
1170 0x3c25c, 0xffffffff, 0x00000100,
1171 0x3c260, 0xffffffff, 0x00000100,
1172 0x3c27c, 0xffffffff, 0x00000100,
1173 0x3c278, 0xffffffff, 0x00000100,
1174 0x3c210, 0xffffffff, 0x06000100,
1175 0x3c290, 0xffffffff, 0x00000100,
1176 0x3c274, 0xffffffff, 0x00000100,
1177 0x3c2b4, 0xffffffff, 0x00000100,
1178 0x3c2b0, 0xffffffff, 0x00000100,
1179 0x3c270, 0xffffffff, 0x00000100,
1180 0x30800, 0xffffffff, 0xe0000000,
1181 0x3c020, 0xffffffff, 0x00010000,
1182 0x3c024, 0xffffffff, 0x00030002,
1183 0x3c028, 0xffffffff, 0x00040007,
1184 0x3c02c, 0xffffffff, 0x00060005,
1185 0x3c030, 0xffffffff, 0x00090008,
1186 0x3c034, 0xffffffff, 0x00010000,
1187 0x3c038, 0xffffffff, 0x00030002,
1188 0x3c03c, 0xffffffff, 0x00040007,
1189 0x3c040, 0xffffffff, 0x00060005,
1190 0x3c044, 0xffffffff, 0x00090008,
1191 0x3c048, 0xffffffff, 0x00010000,
1192 0x3c04c, 0xffffffff, 0x00030002,
1193 0x3c050, 0xffffffff, 0x00040007,
1194 0x3c054, 0xffffffff, 0x00060005,
1195 0x3c058, 0xffffffff, 0x00090008,
1196 0x3c05c, 0xffffffff, 0x00010000,
1197 0x3c060, 0xffffffff, 0x00030002,
1198 0x3c064, 0xffffffff, 0x00040007,
1199 0x3c068, 0xffffffff, 0x00060005,
1200 0x3c06c, 0xffffffff, 0x00090008,
1201 0x3c070, 0xffffffff, 0x00010000,
1202 0x3c074, 0xffffffff, 0x00030002,
1203 0x3c078, 0xffffffff, 0x00040007,
1204 0x3c07c, 0xffffffff, 0x00060005,
1205 0x3c080, 0xffffffff, 0x00090008,
1206 0x3c084, 0xffffffff, 0x00010000,
1207 0x3c088, 0xffffffff, 0x00030002,
1208 0x3c08c, 0xffffffff, 0x00040007,
1209 0x3c090, 0xffffffff, 0x00060005,
1210 0x3c094, 0xffffffff, 0x00090008,
1211 0x3c098, 0xffffffff, 0x00010000,
1212 0x3c09c, 0xffffffff, 0x00030002,
1213 0x3c0a0, 0xffffffff, 0x00040007,
1214 0x3c0a4, 0xffffffff, 0x00060005,
1215 0x3c0a8, 0xffffffff, 0x00090008,
1216 0x3c0ac, 0xffffffff, 0x00010000,
1217 0x3c0b0, 0xffffffff, 0x00030002,
1218 0x3c0b4, 0xffffffff, 0x00040007,
1219 0x3c0b8, 0xffffffff, 0x00060005,
1220 0x3c0bc, 0xffffffff, 0x00090008,
1221 0x3c000, 0xffffffff, 0x96e00200,
1222 0x8708, 0xffffffff, 0x00900100,
1223 0xc424, 0xffffffff, 0x0020003f,
1224 0x38, 0xffffffff, 0x0140001c,
1225 0x3c, 0x000f0000, 0x000f0000,
1226 0x220, 0xffffffff, 0xC060000C,
1227 0x224, 0xc0000fff, 0x00000100,
1228 0xf90, 0xffffffff, 0x00000100,
1229 0xf98, 0x00000101, 0x00000000,
1230 0x20a8, 0xffffffff, 0x00000104,
1231 0x55e4, 0xff000fff, 0x00000100,
1232 0x30cc, 0xc0000fff, 0x00000104,
1233 0xc1e4, 0x00000001, 0x00000001,
1234 0xd00c, 0xff000ff0, 0x00000100,
1235 0xd80c, 0xff000ff0, 0x00000100
1236 };
1237
1238 static const u32 kalindi_golden_spm_registers[] =
1239 {
1240 0x30800, 0xe0ffffff, 0xe0000000
1241 };
1242
1243 static const u32 kalindi_golden_common_registers[] =
1244 {
1245 0xc770, 0xffffffff, 0x00000800,
1246 0xc774, 0xffffffff, 0x00000800,
1247 0xc798, 0xffffffff, 0x00007fbf,
1248 0xc79c, 0xffffffff, 0x00007faf
1249 };
1250
1251 static const u32 kalindi_golden_registers[] =
1252 {
1253 0x3c000, 0xffffdfff, 0x6e944040,
1254 0x55e4, 0xff607fff, 0xfc000100,
1255 0x3c220, 0xff000fff, 0x00000100,
1256 0x3c224, 0xff000fff, 0x00000100,
1257 0x3c200, 0xfffc0fff, 0x00000100,
1258 0x6ed8, 0x00010101, 0x00010000,
1259 0x9830, 0xffffffff, 0x00000000,
1260 0x9834, 0xf00fffff, 0x00000400,
1261 0x5bb0, 0x000000f0, 0x00000070,
1262 0x5bc0, 0xf0311fff, 0x80300000,
1263 0x98f8, 0x73773777, 0x12010001,
1264 0x98fc, 0xffffffff, 0x00000010,
1265 0x9b7c, 0x00ff0000, 0x00fc0000,
1266 0x8030, 0x00001f0f, 0x0000100a,
1267 0x2f48, 0x73773777, 0x12010001,
1268 0x2408, 0x000fffff, 0x000c007f,
1269 0x8a14, 0xf000003f, 0x00000007,
1270 0x8b24, 0x3fff3fff, 0x00ffcfff,
1271 0x30a04, 0x0000ff0f, 0x00000000,
1272 0x28a4c, 0x07ffffff, 0x06000000,
1273 0x4d8, 0x00000fff, 0x00000100,
1274 0x3e78, 0x00000001, 0x00000002,
1275 0xc768, 0x00000008, 0x00000008,
1276 0x8c00, 0x000000ff, 0x00000003,
1277 0x214f8, 0x01ff01ff, 0x00000002,
1278 0x21498, 0x007ff800, 0x00200000,
1279 0x2015c, 0xffffffff, 0x00000f40,
1280 0x88c4, 0x001f3ae3, 0x00000082,
1281 0x88d4, 0x0000001f, 0x00000010,
1282 0x30934, 0xffffffff, 0x00000000
1283 };
1284
1285 static const u32 kalindi_mgcg_cgcg_init[] =
1286 {
1287 0xc420, 0xffffffff, 0xfffffffc,
1288 0x30800, 0xffffffff, 0xe0000000,
1289 0x3c2a0, 0xffffffff, 0x00000100,
1290 0x3c208, 0xffffffff, 0x00000100,
1291 0x3c2c0, 0xffffffff, 0x00000100,
1292 0x3c2c8, 0xffffffff, 0x00000100,
1293 0x3c2c4, 0xffffffff, 0x00000100,
1294 0x55e4, 0xffffffff, 0x00600100,
1295 0x3c280, 0xffffffff, 0x00000100,
1296 0x3c214, 0xffffffff, 0x06000100,
1297 0x3c220, 0xffffffff, 0x00000100,
1298 0x3c218, 0xffffffff, 0x06000100,
1299 0x3c204, 0xffffffff, 0x00000100,
1300 0x3c2e0, 0xffffffff, 0x00000100,
1301 0x3c224, 0xffffffff, 0x00000100,
1302 0x3c200, 0xffffffff, 0x00000100,
1303 0x3c230, 0xffffffff, 0x00000100,
1304 0x3c234, 0xffffffff, 0x00000100,
1305 0x3c250, 0xffffffff, 0x00000100,
1306 0x3c254, 0xffffffff, 0x00000100,
1307 0x3c258, 0xffffffff, 0x00000100,
1308 0x3c25c, 0xffffffff, 0x00000100,
1309 0x3c260, 0xffffffff, 0x00000100,
1310 0x3c27c, 0xffffffff, 0x00000100,
1311 0x3c278, 0xffffffff, 0x00000100,
1312 0x3c210, 0xffffffff, 0x06000100,
1313 0x3c290, 0xffffffff, 0x00000100,
1314 0x3c274, 0xffffffff, 0x00000100,
1315 0x3c2b4, 0xffffffff, 0x00000100,
1316 0x3c2b0, 0xffffffff, 0x00000100,
1317 0x3c270, 0xffffffff, 0x00000100,
1318 0x30800, 0xffffffff, 0xe0000000,
1319 0x3c020, 0xffffffff, 0x00010000,
1320 0x3c024, 0xffffffff, 0x00030002,
1321 0x3c028, 0xffffffff, 0x00040007,
1322 0x3c02c, 0xffffffff, 0x00060005,
1323 0x3c030, 0xffffffff, 0x00090008,
1324 0x3c034, 0xffffffff, 0x00010000,
1325 0x3c038, 0xffffffff, 0x00030002,
1326 0x3c03c, 0xffffffff, 0x00040007,
1327 0x3c040, 0xffffffff, 0x00060005,
1328 0x3c044, 0xffffffff, 0x00090008,
1329 0x3c000, 0xffffffff, 0x96e00200,
1330 0x8708, 0xffffffff, 0x00900100,
1331 0xc424, 0xffffffff, 0x0020003f,
1332 0x38, 0xffffffff, 0x0140001c,
1333 0x3c, 0x000f0000, 0x000f0000,
1334 0x220, 0xffffffff, 0xC060000C,
1335 0x224, 0xc0000fff, 0x00000100,
1336 0x20a8, 0xffffffff, 0x00000104,
1337 0x55e4, 0xff000fff, 0x00000100,
1338 0x30cc, 0xc0000fff, 0x00000104,
1339 0xc1e4, 0x00000001, 0x00000001,
1340 0xd00c, 0xff000ff0, 0x00000100,
1341 0xd80c, 0xff000ff0, 0x00000100
1342 };
1343
1344 static const u32 hawaii_golden_spm_registers[] =
1345 {
1346 0x30800, 0xe0ffffff, 0xe0000000
1347 };
1348
1349 static const u32 hawaii_golden_common_registers[] =
1350 {
1351 0x30800, 0xffffffff, 0xe0000000,
1352 0x28350, 0xffffffff, 0x3a00161a,
1353 0x28354, 0xffffffff, 0x0000002e,
1354 0x9a10, 0xffffffff, 0x00018208,
1355 0x98f8, 0xffffffff, 0x12011003
1356 };
1357
1358 static const u32 hawaii_golden_registers[] =
1359 {
1360 0x3354, 0x00000333, 0x00000333,
1361 0x9a10, 0x00010000, 0x00058208,
1362 0x9830, 0xffffffff, 0x00000000,
1363 0x9834, 0xf00fffff, 0x00000400,
1364 0x9838, 0x0002021c, 0x00020200,
1365 0xc78, 0x00000080, 0x00000000,
1366 0x5bb0, 0x000000f0, 0x00000070,
1367 0x5bc0, 0xf0311fff, 0x80300000,
1368 0x350c, 0x00810000, 0x408af000,
1369 0x7030, 0x31000111, 0x00000011,
1370 0x2f48, 0x73773777, 0x12010001,
1371 0x2120, 0x0000007f, 0x0000001b,
1372 0x21dc, 0x00007fb6, 0x00002191,
1373 0x3628, 0x0000003f, 0x0000000a,
1374 0x362c, 0x0000003f, 0x0000000a,
1375 0x2ae4, 0x00073ffe, 0x000022a2,
1376 0x240c, 0x000007ff, 0x00000000,
1377 0x8bf0, 0x00002001, 0x00000001,
1378 0x8b24, 0xffffffff, 0x00ffffff,
1379 0x30a04, 0x0000ff0f, 0x00000000,
1380 0x28a4c, 0x07ffffff, 0x06000000,
1381 0x3e78, 0x00000001, 0x00000002,
1382 0xc768, 0x00000008, 0x00000008,
1383 0xc770, 0x00000f00, 0x00000800,
1384 0xc774, 0x00000f00, 0x00000800,
1385 0xc798, 0x00ffffff, 0x00ff7fbf,
1386 0xc79c, 0x00ffffff, 0x00ff7faf,
1387 0x8c00, 0x000000ff, 0x00000800,
1388 0xe40, 0x00001fff, 0x00001fff,
1389 0x9060, 0x0000007f, 0x00000020,
1390 0x9508, 0x00010000, 0x00010000,
1391 0xae00, 0x00100000, 0x000ff07c,
1392 0xac14, 0x000003ff, 0x0000000f,
1393 0xac10, 0xffffffff, 0x7564fdec,
1394 0xac0c, 0xffffffff, 0x3120b9a8,
1395 0xac08, 0x20000000, 0x0f9c0000
1396 };
1397
1398 static const u32 hawaii_mgcg_cgcg_init[] =
1399 {
1400 0xc420, 0xffffffff, 0xfffffffd,
1401 0x30800, 0xffffffff, 0xe0000000,
1402 0x3c2a0, 0xffffffff, 0x00000100,
1403 0x3c208, 0xffffffff, 0x00000100,
1404 0x3c2c0, 0xffffffff, 0x00000100,
1405 0x3c2c8, 0xffffffff, 0x00000100,
1406 0x3c2c4, 0xffffffff, 0x00000100,
1407 0x55e4, 0xffffffff, 0x00200100,
1408 0x3c280, 0xffffffff, 0x00000100,
1409 0x3c214, 0xffffffff, 0x06000100,
1410 0x3c220, 0xffffffff, 0x00000100,
1411 0x3c218, 0xffffffff, 0x06000100,
1412 0x3c204, 0xffffffff, 0x00000100,
1413 0x3c2e0, 0xffffffff, 0x00000100,
1414 0x3c224, 0xffffffff, 0x00000100,
1415 0x3c200, 0xffffffff, 0x00000100,
1416 0x3c230, 0xffffffff, 0x00000100,
1417 0x3c234, 0xffffffff, 0x00000100,
1418 0x3c250, 0xffffffff, 0x00000100,
1419 0x3c254, 0xffffffff, 0x00000100,
1420 0x3c258, 0xffffffff, 0x00000100,
1421 0x3c25c, 0xffffffff, 0x00000100,
1422 0x3c260, 0xffffffff, 0x00000100,
1423 0x3c27c, 0xffffffff, 0x00000100,
1424 0x3c278, 0xffffffff, 0x00000100,
1425 0x3c210, 0xffffffff, 0x06000100,
1426 0x3c290, 0xffffffff, 0x00000100,
1427 0x3c274, 0xffffffff, 0x00000100,
1428 0x3c2b4, 0xffffffff, 0x00000100,
1429 0x3c2b0, 0xffffffff, 0x00000100,
1430 0x3c270, 0xffffffff, 0x00000100,
1431 0x30800, 0xffffffff, 0xe0000000,
1432 0x3c020, 0xffffffff, 0x00010000,
1433 0x3c024, 0xffffffff, 0x00030002,
1434 0x3c028, 0xffffffff, 0x00040007,
1435 0x3c02c, 0xffffffff, 0x00060005,
1436 0x3c030, 0xffffffff, 0x00090008,
1437 0x3c034, 0xffffffff, 0x00010000,
1438 0x3c038, 0xffffffff, 0x00030002,
1439 0x3c03c, 0xffffffff, 0x00040007,
1440 0x3c040, 0xffffffff, 0x00060005,
1441 0x3c044, 0xffffffff, 0x00090008,
1442 0x3c048, 0xffffffff, 0x00010000,
1443 0x3c04c, 0xffffffff, 0x00030002,
1444 0x3c050, 0xffffffff, 0x00040007,
1445 0x3c054, 0xffffffff, 0x00060005,
1446 0x3c058, 0xffffffff, 0x00090008,
1447 0x3c05c, 0xffffffff, 0x00010000,
1448 0x3c060, 0xffffffff, 0x00030002,
1449 0x3c064, 0xffffffff, 0x00040007,
1450 0x3c068, 0xffffffff, 0x00060005,
1451 0x3c06c, 0xffffffff, 0x00090008,
1452 0x3c070, 0xffffffff, 0x00010000,
1453 0x3c074, 0xffffffff, 0x00030002,
1454 0x3c078, 0xffffffff, 0x00040007,
1455 0x3c07c, 0xffffffff, 0x00060005,
1456 0x3c080, 0xffffffff, 0x00090008,
1457 0x3c084, 0xffffffff, 0x00010000,
1458 0x3c088, 0xffffffff, 0x00030002,
1459 0x3c08c, 0xffffffff, 0x00040007,
1460 0x3c090, 0xffffffff, 0x00060005,
1461 0x3c094, 0xffffffff, 0x00090008,
1462 0x3c098, 0xffffffff, 0x00010000,
1463 0x3c09c, 0xffffffff, 0x00030002,
1464 0x3c0a0, 0xffffffff, 0x00040007,
1465 0x3c0a4, 0xffffffff, 0x00060005,
1466 0x3c0a8, 0xffffffff, 0x00090008,
1467 0x3c0ac, 0xffffffff, 0x00010000,
1468 0x3c0b0, 0xffffffff, 0x00030002,
1469 0x3c0b4, 0xffffffff, 0x00040007,
1470 0x3c0b8, 0xffffffff, 0x00060005,
1471 0x3c0bc, 0xffffffff, 0x00090008,
1472 0x3c0c0, 0xffffffff, 0x00010000,
1473 0x3c0c4, 0xffffffff, 0x00030002,
1474 0x3c0c8, 0xffffffff, 0x00040007,
1475 0x3c0cc, 0xffffffff, 0x00060005,
1476 0x3c0d0, 0xffffffff, 0x00090008,
1477 0x3c0d4, 0xffffffff, 0x00010000,
1478 0x3c0d8, 0xffffffff, 0x00030002,
1479 0x3c0dc, 0xffffffff, 0x00040007,
1480 0x3c0e0, 0xffffffff, 0x00060005,
1481 0x3c0e4, 0xffffffff, 0x00090008,
1482 0x3c0e8, 0xffffffff, 0x00010000,
1483 0x3c0ec, 0xffffffff, 0x00030002,
1484 0x3c0f0, 0xffffffff, 0x00040007,
1485 0x3c0f4, 0xffffffff, 0x00060005,
1486 0x3c0f8, 0xffffffff, 0x00090008,
1487 0xc318, 0xffffffff, 0x00020200,
1488 0x3350, 0xffffffff, 0x00000200,
1489 0x15c0, 0xffffffff, 0x00000400,
1490 0x55e8, 0xffffffff, 0x00000000,
1491 0x2f50, 0xffffffff, 0x00000902,
1492 0x3c000, 0xffffffff, 0x96940200,
1493 0x8708, 0xffffffff, 0x00900100,
1494 0xc424, 0xffffffff, 0x0020003f,
1495 0x38, 0xffffffff, 0x0140001c,
1496 0x3c, 0x000f0000, 0x000f0000,
1497 0x220, 0xffffffff, 0xc060000c,
1498 0x224, 0xc0000fff, 0x00000100,
1499 0xf90, 0xffffffff, 0x00000100,
1500 0xf98, 0x00000101, 0x00000000,
1501 0x20a8, 0xffffffff, 0x00000104,
1502 0x55e4, 0xff000fff, 0x00000100,
1503 0x30cc, 0xc0000fff, 0x00000104,
1504 0xc1e4, 0x00000001, 0x00000001,
1505 0xd00c, 0xff000ff0, 0x00000100,
1506 0xd80c, 0xff000ff0, 0x00000100
1507 };
1508
1509 static const u32 godavari_golden_registers[] =
1510 {
1511 0x55e4, 0xff607fff, 0xfc000100,
1512 0x6ed8, 0x00010101, 0x00010000,
1513 0x9830, 0xffffffff, 0x00000000,
1514 0x98302, 0xf00fffff, 0x00000400,
1515 0x6130, 0xffffffff, 0x00010000,
1516 0x5bb0, 0x000000f0, 0x00000070,
1517 0x5bc0, 0xf0311fff, 0x80300000,
1518 0x98f8, 0x73773777, 0x12010001,
1519 0x98fc, 0xffffffff, 0x00000010,
1520 0x8030, 0x00001f0f, 0x0000100a,
1521 0x2f48, 0x73773777, 0x12010001,
1522 0x2408, 0x000fffff, 0x000c007f,
1523 0x8a14, 0xf000003f, 0x00000007,
1524 0x8b24, 0xffffffff, 0x00ff0fff,
1525 0x30a04, 0x0000ff0f, 0x00000000,
1526 0x28a4c, 0x07ffffff, 0x06000000,
1527 0x4d8, 0x00000fff, 0x00000100,
1528 0xd014, 0x00010000, 0x00810001,
1529 0xd814, 0x00010000, 0x00810001,
1530 0x3e78, 0x00000001, 0x00000002,
1531 0xc768, 0x00000008, 0x00000008,
1532 0xc770, 0x00000f00, 0x00000800,
1533 0xc774, 0x00000f00, 0x00000800,
1534 0xc798, 0x00ffffff, 0x00ff7fbf,
1535 0xc79c, 0x00ffffff, 0x00ff7faf,
1536 0x8c00, 0x000000ff, 0x00000001,
1537 0x214f8, 0x01ff01ff, 0x00000002,
1538 0x21498, 0x007ff800, 0x00200000,
1539 0x2015c, 0xffffffff, 0x00000f40,
1540 0x88c4, 0x001f3ae3, 0x00000082,
1541 0x88d4, 0x0000001f, 0x00000010,
1542 0x30934, 0xffffffff, 0x00000000
1543 };
1544
1545
1546 static void cik_init_golden_registers(struct radeon_device *rdev)
1547 {
1548 switch (rdev->family) {
1549 case CHIP_BONAIRE:
1550 radeon_program_register_sequence(rdev,
1551 bonaire_mgcg_cgcg_init,
1552 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1553 radeon_program_register_sequence(rdev,
1554 bonaire_golden_registers,
1555 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1556 radeon_program_register_sequence(rdev,
1557 bonaire_golden_common_registers,
1558 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1559 radeon_program_register_sequence(rdev,
1560 bonaire_golden_spm_registers,
1561 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1562 break;
1563 case CHIP_KABINI:
1564 radeon_program_register_sequence(rdev,
1565 kalindi_mgcg_cgcg_init,
1566 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1567 radeon_program_register_sequence(rdev,
1568 kalindi_golden_registers,
1569 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1570 radeon_program_register_sequence(rdev,
1571 kalindi_golden_common_registers,
1572 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1573 radeon_program_register_sequence(rdev,
1574 kalindi_golden_spm_registers,
1575 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1576 break;
1577 case CHIP_MULLINS:
1578 radeon_program_register_sequence(rdev,
1579 kalindi_mgcg_cgcg_init,
1580 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1581 radeon_program_register_sequence(rdev,
1582 godavari_golden_registers,
1583 (const u32)ARRAY_SIZE(godavari_golden_registers));
1584 radeon_program_register_sequence(rdev,
1585 kalindi_golden_common_registers,
1586 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1587 radeon_program_register_sequence(rdev,
1588 kalindi_golden_spm_registers,
1589 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1590 break;
1591 case CHIP_KAVERI:
1592 radeon_program_register_sequence(rdev,
1593 spectre_mgcg_cgcg_init,
1594 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1595 radeon_program_register_sequence(rdev,
1596 spectre_golden_registers,
1597 (const u32)ARRAY_SIZE(spectre_golden_registers));
1598 radeon_program_register_sequence(rdev,
1599 spectre_golden_common_registers,
1600 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1601 radeon_program_register_sequence(rdev,
1602 spectre_golden_spm_registers,
1603 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1604 break;
1605 case CHIP_HAWAII:
1606 radeon_program_register_sequence(rdev,
1607 hawaii_mgcg_cgcg_init,
1608 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1609 radeon_program_register_sequence(rdev,
1610 hawaii_golden_registers,
1611 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1612 radeon_program_register_sequence(rdev,
1613 hawaii_golden_common_registers,
1614 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1615 radeon_program_register_sequence(rdev,
1616 hawaii_golden_spm_registers,
1617 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1618 break;
1619 default:
1620 break;
1621 }
1622 }
1623
1624 /**
1625 * cik_get_xclk - get the xclk
1626 *
1627 * @rdev: radeon_device pointer
1628 *
1629 * Returns the reference clock used by the gfx engine
1630 * (CIK).
1631 */
1632 u32 cik_get_xclk(struct radeon_device *rdev)
1633 {
1634 u32 reference_clock = rdev->clock.spll.reference_freq;
1635
1636 if (rdev->flags & RADEON_IS_IGP) {
1637 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1638 return reference_clock / 2;
1639 } else {
1640 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1641 return reference_clock / 4;
1642 }
1643 return reference_clock;
1644 }
1645
1646 /**
1647 * cik_mm_rdoorbell - read a doorbell dword
1648 *
1649 * @rdev: radeon_device pointer
1650 * @index: doorbell index
1651 *
1652 * Returns the value in the doorbell aperture at the
1653 * requested doorbell index (CIK).
1654 */
1655 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1656 {
1657 if (index < rdev->doorbell.num_doorbells) {
1658 return readl(rdev->doorbell.ptr + index);
1659 } else {
1660 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1661 return 0;
1662 }
1663 }
1664
1665 /**
1666 * cik_mm_wdoorbell - write a doorbell dword
1667 *
1668 * @rdev: radeon_device pointer
1669 * @index: doorbell index
1670 * @v: value to write
1671 *
1672 * Writes @v to the doorbell aperture at the
1673 * requested doorbell index (CIK).
1674 */
1675 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1676 {
1677 if (index < rdev->doorbell.num_doorbells) {
1678 writel(v, rdev->doorbell.ptr + index);
1679 } else {
1680 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1681 }
1682 }
1683
1684 #define BONAIRE_IO_MC_REGS_SIZE 36
1685
1686 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1687 {
1688 {0x00000070, 0x04400000},
1689 {0x00000071, 0x80c01803},
1690 {0x00000072, 0x00004004},
1691 {0x00000073, 0x00000100},
1692 {0x00000074, 0x00ff0000},
1693 {0x00000075, 0x34000000},
1694 {0x00000076, 0x08000014},
1695 {0x00000077, 0x00cc08ec},
1696 {0x00000078, 0x00000400},
1697 {0x00000079, 0x00000000},
1698 {0x0000007a, 0x04090000},
1699 {0x0000007c, 0x00000000},
1700 {0x0000007e, 0x4408a8e8},
1701 {0x0000007f, 0x00000304},
1702 {0x00000080, 0x00000000},
1703 {0x00000082, 0x00000001},
1704 {0x00000083, 0x00000002},
1705 {0x00000084, 0xf3e4f400},
1706 {0x00000085, 0x052024e3},
1707 {0x00000087, 0x00000000},
1708 {0x00000088, 0x01000000},
1709 {0x0000008a, 0x1c0a0000},
1710 {0x0000008b, 0xff010000},
1711 {0x0000008d, 0xffffefff},
1712 {0x0000008e, 0xfff3efff},
1713 {0x0000008f, 0xfff3efbf},
1714 {0x00000092, 0xf7ffffff},
1715 {0x00000093, 0xffffff7f},
1716 {0x00000095, 0x00101101},
1717 {0x00000096, 0x00000fff},
1718 {0x00000097, 0x00116fff},
1719 {0x00000098, 0x60010000},
1720 {0x00000099, 0x10010000},
1721 {0x0000009a, 0x00006000},
1722 {0x0000009b, 0x00001000},
1723 {0x0000009f, 0x00b48000}
1724 };
1725
1726 #define HAWAII_IO_MC_REGS_SIZE 22
1727
1728 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1729 {
1730 {0x0000007d, 0x40000000},
1731 {0x0000007e, 0x40180304},
1732 {0x0000007f, 0x0000ff00},
1733 {0x00000081, 0x00000000},
1734 {0x00000083, 0x00000800},
1735 {0x00000086, 0x00000000},
1736 {0x00000087, 0x00000100},
1737 {0x00000088, 0x00020100},
1738 {0x00000089, 0x00000000},
1739 {0x0000008b, 0x00040000},
1740 {0x0000008c, 0x00000100},
1741 {0x0000008e, 0xff010000},
1742 {0x00000090, 0xffffefff},
1743 {0x00000091, 0xfff3efff},
1744 {0x00000092, 0xfff3efbf},
1745 {0x00000093, 0xf7ffffff},
1746 {0x00000094, 0xffffff7f},
1747 {0x00000095, 0x00000fff},
1748 {0x00000096, 0x00116fff},
1749 {0x00000097, 0x60010000},
1750 {0x00000098, 0x10010000},
1751 {0x0000009f, 0x00c79000}
1752 };
1753
1754
1755 /**
1756 * cik_srbm_select - select specific register instances
1757 *
1758 * @rdev: radeon_device pointer
1759 * @me: selected ME (micro engine)
1760 * @pipe: pipe
1761 * @queue: queue
1762 * @vmid: VMID
1763 *
1764 * Switches the currently active registers instances. Some
1765 * registers are instanced per VMID, others are instanced per
1766 * me/pipe/queue combination.
1767 */
1768 static void cik_srbm_select(struct radeon_device *rdev,
1769 u32 me, u32 pipe, u32 queue, u32 vmid)
1770 {
1771 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1772 MEID(me & 0x3) |
1773 VMID(vmid & 0xf) |
1774 QUEUEID(queue & 0x7));
1775 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1776 }
1777
1778 /* ucode loading */
1779 /**
1780 * ci_mc_load_microcode - load MC ucode into the hw
1781 *
1782 * @rdev: radeon_device pointer
1783 *
1784 * Load the GDDR MC ucode into the hw (CIK).
1785 * Returns 0 on success, error on failure.
1786 */
1787 int ci_mc_load_microcode(struct radeon_device *rdev)
1788 {
1789 const __be32 *fw_data = NULL;
1790 const __le32 *new_fw_data = NULL;
1791 u32 running, blackout = 0;
1792 u32 *io_mc_regs = NULL;
1793 const __le32 *new_io_mc_regs = NULL;
1794 int i, regs_size, ucode_size;
1795
1796 if (!rdev->mc_fw)
1797 return -EINVAL;
1798
1799 if (rdev->new_fw) {
1800 const struct mc_firmware_header_v1_0 *hdr =
1801 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1802
1803 radeon_ucode_print_mc_hdr(&hdr->header);
1804
1805 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1806 new_io_mc_regs = (const __le32 *)
1807 ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1808 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1809 new_fw_data = (const __le32 *)
1810 ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1811 } else {
1812 ucode_size = rdev->mc_fw->datasize / 4;
1813
1814 switch (rdev->family) {
1815 case CHIP_BONAIRE:
1816 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1817 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1818 break;
1819 case CHIP_HAWAII:
1820 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1821 regs_size = HAWAII_IO_MC_REGS_SIZE;
1822 break;
1823 default:
1824 return -EINVAL;
1825 }
1826 fw_data = (const __be32 *)rdev->mc_fw->data;
1827 }
1828
1829 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1830
1831 if (running == 0) {
1832 if (running) {
1833 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1834 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1835 }
1836
1837 /* reset the engine and set to writable */
1838 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1839 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1840
1841 /* load mc io regs */
1842 for (i = 0; i < regs_size; i++) {
1843 if (rdev->new_fw) {
1844 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1845 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1846 } else {
1847 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1848 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1849 }
1850 }
1851 /* load the MC ucode */
1852 for (i = 0; i < ucode_size; i++) {
1853 if (rdev->new_fw)
1854 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1855 else
1856 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1857 }
1858
1859 /* put the engine back into the active state */
1860 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1863
1864 /* wait for training to complete */
1865 for (i = 0; i < rdev->usec_timeout; i++) {
1866 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1867 break;
1868 udelay(1);
1869 }
1870 for (i = 0; i < rdev->usec_timeout; i++) {
1871 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1872 break;
1873 udelay(1);
1874 }
1875
1876 if (running)
1877 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1878 }
1879
1880 return 0;
1881 }
1882
1883 /**
1884 * cik_init_microcode - load ucode images from disk
1885 *
1886 * @rdev: radeon_device pointer
1887 *
1888 * Use the firmware interface to load the ucode images into
1889 * the driver (not loaded into hw).
1890 * Returns 0 on success, error on failure.
1891 */
1892 static int cik_init_microcode(struct radeon_device *rdev)
1893 {
1894 const char *chip_name;
1895 const char *new_chip_name;
1896 size_t pfp_req_size, me_req_size, ce_req_size,
1897 mec_req_size, rlc_req_size, mc_req_size = 0,
1898 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1899 char fw_name[30];
1900 int new_fw = 0;
1901 int err;
1902 int num_fw;
1903
1904 DRM_DEBUG("\n");
1905
1906 switch (rdev->family) {
1907 case CHIP_BONAIRE:
1908 chip_name = "BONAIRE";
1909 new_chip_name = "bonaire";
1910 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1911 me_req_size = CIK_ME_UCODE_SIZE * 4;
1912 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1913 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1914 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1915 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1916 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1917 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1918 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1919 num_fw = 8;
1920 break;
1921 case CHIP_HAWAII:
1922 chip_name = "HAWAII";
1923 new_chip_name = "hawaii";
1924 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1925 me_req_size = CIK_ME_UCODE_SIZE * 4;
1926 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1927 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1928 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1929 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1930 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1931 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1932 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1933 num_fw = 8;
1934 break;
1935 case CHIP_KAVERI:
1936 chip_name = "KAVERI";
1937 new_chip_name = "kaveri";
1938 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1939 me_req_size = CIK_ME_UCODE_SIZE * 4;
1940 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1941 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1942 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1943 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1944 num_fw = 7;
1945 break;
1946 case CHIP_KABINI:
1947 chip_name = "KABINI";
1948 new_chip_name = "kabini";
1949 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1950 me_req_size = CIK_ME_UCODE_SIZE * 4;
1951 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1952 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1953 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1954 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1955 num_fw = 6;
1956 break;
1957 case CHIP_MULLINS:
1958 chip_name = "MULLINS";
1959 new_chip_name = "mullins";
1960 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1961 me_req_size = CIK_ME_UCODE_SIZE * 4;
1962 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1963 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1964 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1965 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1966 num_fw = 6;
1967 break;
1968 default: BUG();
1969 }
1970
1971 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1972
1973 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", new_chip_name);
1974 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1975 if (err) {
1976 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
1977 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1978 if (err)
1979 goto out;
1980 if (rdev->pfp_fw->datasize != pfp_req_size) {
1981 printk(KERN_ERR
1982 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1983 rdev->pfp_fw->datasize, fw_name);
1984 err = -EINVAL;
1985 goto out;
1986 }
1987 } else {
1988 err = radeon_ucode_validate(rdev->pfp_fw);
1989 if (err) {
1990 printk(KERN_ERR
1991 "cik_fw: validation failed for firmware \"%s\"\n",
1992 fw_name);
1993 goto out;
1994 } else {
1995 new_fw++;
1996 }
1997 }
1998
1999 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", new_chip_name);
2000 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2001 if (err) {
2002 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
2003 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2004 if (err)
2005 goto out;
2006 if (rdev->me_fw->datasize != me_req_size) {
2007 printk(KERN_ERR
2008 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2009 rdev->me_fw->datasize, fw_name);
2010 err = -EINVAL;
2011 }
2012 } else {
2013 err = radeon_ucode_validate(rdev->me_fw);
2014 if (err) {
2015 printk(KERN_ERR
2016 "cik_fw: validation failed for firmware \"%s\"\n",
2017 fw_name);
2018 goto out;
2019 } else {
2020 new_fw++;
2021 }
2022 }
2023
2024 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", new_chip_name);
2025 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2026 if (err) {
2027 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name);
2028 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2029 if (err)
2030 goto out;
2031 if (rdev->ce_fw->datasize != ce_req_size) {
2032 printk(KERN_ERR
2033 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2034 rdev->ce_fw->datasize, fw_name);
2035 err = -EINVAL;
2036 }
2037 } else {
2038 err = radeon_ucode_validate(rdev->ce_fw);
2039 if (err) {
2040 printk(KERN_ERR
2041 "cik_fw: validation failed for firmware \"%s\"\n",
2042 fw_name);
2043 goto out;
2044 } else {
2045 new_fw++;
2046 }
2047 }
2048
2049 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", new_chip_name);
2050 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2051 if (err) {
2052 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", chip_name);
2053 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2054 if (err)
2055 goto out;
2056 if (rdev->mec_fw->datasize != mec_req_size) {
2057 printk(KERN_ERR
2058 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2059 rdev->mec_fw->datasize, fw_name);
2060 err = -EINVAL;
2061 }
2062 } else {
2063 err = radeon_ucode_validate(rdev->mec_fw);
2064 if (err) {
2065 printk(KERN_ERR
2066 "cik_fw: validation failed for firmware \"%s\"\n",
2067 fw_name);
2068 goto out;
2069 } else {
2070 new_fw++;
2071 }
2072 }
2073
2074 if (rdev->family == CHIP_KAVERI) {
2075 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec2", new_chip_name);
2076 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2077 if (err) {
2078 goto out;
2079 } else {
2080 err = radeon_ucode_validate(rdev->mec2_fw);
2081 if (err) {
2082 goto out;
2083 } else {
2084 new_fw++;
2085 }
2086 }
2087 }
2088
2089 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", new_chip_name);
2090 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2091 if (err) {
2092 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", chip_name);
2093 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2094 if (err)
2095 goto out;
2096 if (rdev->rlc_fw->datasize != rlc_req_size) {
2097 printk(KERN_ERR
2098 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2099 rdev->rlc_fw->datasize, fw_name);
2100 err = -EINVAL;
2101 }
2102 } else {
2103 err = radeon_ucode_validate(rdev->rlc_fw);
2104 if (err) {
2105 printk(KERN_ERR
2106 "cik_fw: validation failed for firmware \"%s\"\n",
2107 fw_name);
2108 goto out;
2109 } else {
2110 new_fw++;
2111 }
2112 }
2113
2114 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", new_chip_name);
2115 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2116 if (err) {
2117 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", chip_name);
2118 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2119 if (err)
2120 goto out;
2121 if (rdev->sdma_fw->datasize != sdma_req_size) {
2122 printk(KERN_ERR
2123 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2124 rdev->sdma_fw->datasize, fw_name);
2125 err = -EINVAL;
2126 }
2127 } else {
2128 err = radeon_ucode_validate(rdev->sdma_fw);
2129 if (err) {
2130 printk(KERN_ERR
2131 "cik_fw: validation failed for firmware \"%s\"\n",
2132 fw_name);
2133 goto out;
2134 } else {
2135 new_fw++;
2136 }
2137 }
2138
2139 /* No SMC, MC ucode on APUs */
2140 if (!(rdev->flags & RADEON_IS_IGP)) {
2141 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", new_chip_name);
2142 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2143 if (err) {
2144 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name);
2145 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2146 if (err) {
2147 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
2148 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2149 if (err)
2150 goto out;
2151 }
2152 if ((rdev->mc_fw->datasize != mc_req_size) &&
2153 (rdev->mc_fw->datasize != mc2_req_size)){
2154 printk(KERN_ERR
2155 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2156 rdev->mc_fw->datasize, fw_name);
2157 err = -EINVAL;
2158 }
2159 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize);
2160 } else {
2161 err = radeon_ucode_validate(rdev->mc_fw);
2162 if (err) {
2163 printk(KERN_ERR
2164 "cik_fw: validation failed for firmware \"%s\"\n",
2165 fw_name);
2166 goto out;
2167 } else {
2168 new_fw++;
2169 }
2170 }
2171
2172 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", new_chip_name);
2173 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2174 if (err) {
2175 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
2176 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2177 if (err) {
2178 printk(KERN_ERR
2179 "smc: error loading firmware \"%s\"\n",
2180 fw_name);
2181 release_firmware(rdev->smc_fw);
2182 rdev->smc_fw = NULL;
2183 err = 0;
2184 } else if (rdev->smc_fw->datasize != smc_req_size) {
2185 printk(KERN_ERR
2186 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2187 rdev->smc_fw->datasize, fw_name);
2188 err = -EINVAL;
2189 }
2190 } else {
2191 err = radeon_ucode_validate(rdev->smc_fw);
2192 if (err) {
2193 printk(KERN_ERR
2194 "cik_fw: validation failed for firmware \"%s\"\n",
2195 fw_name);
2196 goto out;
2197 } else {
2198 new_fw++;
2199 }
2200 }
2201 }
2202
2203 if (new_fw == 0) {
2204 rdev->new_fw = false;
2205 } else if (new_fw < num_fw) {
2206 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2207 err = -EINVAL;
2208 } else {
2209 rdev->new_fw = true;
2210 }
2211
2212 out:
2213 if (err) {
2214 if (err != -EINVAL)
2215 printk(KERN_ERR
2216 "cik_cp: Failed to load firmware \"%s\"\n",
2217 fw_name);
2218 release_firmware(rdev->pfp_fw);
2219 rdev->pfp_fw = NULL;
2220 release_firmware(rdev->me_fw);
2221 rdev->me_fw = NULL;
2222 release_firmware(rdev->ce_fw);
2223 rdev->ce_fw = NULL;
2224 release_firmware(rdev->mec_fw);
2225 rdev->mec_fw = NULL;
2226 release_firmware(rdev->mec2_fw);
2227 rdev->mec2_fw = NULL;
2228 release_firmware(rdev->rlc_fw);
2229 rdev->rlc_fw = NULL;
2230 release_firmware(rdev->sdma_fw);
2231 rdev->sdma_fw = NULL;
2232 release_firmware(rdev->mc_fw);
2233 rdev->mc_fw = NULL;
2234 release_firmware(rdev->smc_fw);
2235 rdev->smc_fw = NULL;
2236 }
2237 return err;
2238 }
2239
2240 /**
2241 * cik_fini_microcode - drop the firmwares image references
2242 *
2243 * @rdev: radeon_device pointer
2244 *
2245 * Drop the pfp, me, mec, mec2, rlc, sdma, mc, smc and ce firmware image references.
2246 * Called at driver shutdown.
2247 */
2248 static void cik_fini_microcode(struct radeon_device *rdev)
2249 {
2250 release_firmware(rdev->pfp_fw);
2251 rdev->pfp_fw = NULL;
2252 release_firmware(rdev->me_fw);
2253 rdev->me_fw = NULL;
2254 release_firmware(rdev->ce_fw);
2255 rdev->ce_fw = NULL;
2256 release_firmware(rdev->mec_fw);
2257 rdev->mec_fw = NULL;
2258 release_firmware(rdev->mec2_fw);
2259 rdev->mec2_fw = NULL;
2260 release_firmware(rdev->rlc_fw);
2261 rdev->rlc_fw = NULL;
2262 release_firmware(rdev->sdma_fw);
2263 rdev->sdma_fw = NULL;
2264 release_firmware(rdev->mc_fw);
2265 rdev->mc_fw = NULL;
2266 release_firmware(rdev->smc_fw);
2267 rdev->smc_fw = NULL;
2268 }
2269
2270 /*
2271 * Core functions
2272 */
2273 /**
2274 * cik_tiling_mode_table_init - init the hw tiling table
2275 *
2276 * @rdev: radeon_device pointer
2277 *
2278 * Starting with SI, the tiling setup is done globally in a
2279 * set of 32 tiling modes. Rather than selecting each set of
2280 * parameters per surface as on older asics, we just select
2281 * which index in the tiling table we want to use, and the
2282 * surface uses those parameters (CIK).
2283 */
2284 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2285 {
2286 const u32 num_tile_mode_states = 32;
2287 const u32 num_secondary_tile_mode_states = 16;
2288 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2289 u32 num_pipe_configs;
2290 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2291 rdev->config.cik.max_shader_engines;
2292
2293 switch (rdev->config.cik.mem_row_size_in_kb) {
2294 case 1:
2295 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2296 break;
2297 case 2:
2298 default:
2299 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2300 break;
2301 case 4:
2302 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2303 break;
2304 }
2305
2306 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2307 if (num_pipe_configs > 8)
2308 num_pipe_configs = 16;
2309
2310 if (num_pipe_configs == 16) {
2311 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2312 switch (reg_offset) {
2313 case 0:
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2318 break;
2319 case 1:
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2324 break;
2325 case 2:
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2330 break;
2331 case 3:
2332 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2333 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2334 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2335 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2336 break;
2337 case 4:
2338 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2339 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2340 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2341 TILE_SPLIT(split_equal_to_row_size));
2342 break;
2343 case 5:
2344 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2347 break;
2348 case 6:
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2350 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2351 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2352 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2353 break;
2354 case 7:
2355 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2356 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2357 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2358 TILE_SPLIT(split_equal_to_row_size));
2359 break;
2360 case 8:
2361 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2362 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2363 break;
2364 case 9:
2365 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2367 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2368 break;
2369 case 10:
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374 break;
2375 case 11:
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2377 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2378 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2379 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2380 break;
2381 case 12:
2382 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2384 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2385 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2386 break;
2387 case 13:
2388 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2389 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2390 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2391 break;
2392 case 14:
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 break;
2398 case 16:
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2400 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2401 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2402 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2403 break;
2404 case 17:
2405 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2408 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2409 break;
2410 case 27:
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2413 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2414 break;
2415 case 28:
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420 break;
2421 case 29:
2422 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2423 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2424 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2425 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2426 break;
2427 case 30:
2428 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2431 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2432 break;
2433 default:
2434 gb_tile_moden = 0;
2435 break;
2436 }
2437 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2438 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2439 }
2440 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2441 switch (reg_offset) {
2442 case 0:
2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2446 NUM_BANKS(ADDR_SURF_16_BANK));
2447 break;
2448 case 1:
2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2453 break;
2454 case 2:
2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 NUM_BANKS(ADDR_SURF_16_BANK));
2459 break;
2460 case 3:
2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_16_BANK));
2465 break;
2466 case 4:
2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 NUM_BANKS(ADDR_SURF_8_BANK));
2471 break;
2472 case 5:
2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2476 NUM_BANKS(ADDR_SURF_4_BANK));
2477 break;
2478 case 6:
2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2482 NUM_BANKS(ADDR_SURF_2_BANK));
2483 break;
2484 case 8:
2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2488 NUM_BANKS(ADDR_SURF_16_BANK));
2489 break;
2490 case 9:
2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2494 NUM_BANKS(ADDR_SURF_16_BANK));
2495 break;
2496 case 10:
2497 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_16_BANK));
2501 break;
2502 case 11:
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_8_BANK));
2507 break;
2508 case 12:
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 NUM_BANKS(ADDR_SURF_4_BANK));
2513 break;
2514 case 13:
2515 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2518 NUM_BANKS(ADDR_SURF_2_BANK));
2519 break;
2520 case 14:
2521 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2523 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2524 NUM_BANKS(ADDR_SURF_2_BANK));
2525 break;
2526 default:
2527 gb_tile_moden = 0;
2528 break;
2529 }
2530 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2531 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2532 }
2533 } else if (num_pipe_configs == 8) {
2534 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2535 switch (reg_offset) {
2536 case 0:
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2541 break;
2542 case 1:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2547 break;
2548 case 2:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2553 break;
2554 case 3:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2556 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2557 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2558 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2559 break;
2560 case 4:
2561 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2562 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2563 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2564 TILE_SPLIT(split_equal_to_row_size));
2565 break;
2566 case 5:
2567 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2570 break;
2571 case 6:
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2576 break;
2577 case 7:
2578 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2579 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2580 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2581 TILE_SPLIT(split_equal_to_row_size));
2582 break;
2583 case 8:
2584 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2585 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2586 break;
2587 case 9:
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2590 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2591 break;
2592 case 10:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597 break;
2598 case 11:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2602 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2603 break;
2604 case 12:
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2608 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2609 break;
2610 case 13:
2611 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2613 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2614 break;
2615 case 14:
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 break;
2621 case 16:
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2623 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2626 break;
2627 case 17:
2628 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2629 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2630 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2631 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2632 break;
2633 case 27:
2634 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2635 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2637 break;
2638 case 28:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 case 29:
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2646 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2648 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2649 break;
2650 case 30:
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655 break;
2656 default:
2657 gb_tile_moden = 0;
2658 break;
2659 }
2660 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2661 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2662 }
2663 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2664 switch (reg_offset) {
2665 case 0:
2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2669 NUM_BANKS(ADDR_SURF_16_BANK));
2670 break;
2671 case 1:
2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 NUM_BANKS(ADDR_SURF_16_BANK));
2676 break;
2677 case 2:
2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2681 NUM_BANKS(ADDR_SURF_16_BANK));
2682 break;
2683 case 3:
2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2687 NUM_BANKS(ADDR_SURF_16_BANK));
2688 break;
2689 case 4:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 NUM_BANKS(ADDR_SURF_8_BANK));
2694 break;
2695 case 5:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2699 NUM_BANKS(ADDR_SURF_4_BANK));
2700 break;
2701 case 6:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2705 NUM_BANKS(ADDR_SURF_2_BANK));
2706 break;
2707 case 8:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 9:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 10:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_16_BANK));
2724 break;
2725 case 11:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2729 NUM_BANKS(ADDR_SURF_16_BANK));
2730 break;
2731 case 12:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 NUM_BANKS(ADDR_SURF_8_BANK));
2736 break;
2737 case 13:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2741 NUM_BANKS(ADDR_SURF_4_BANK));
2742 break;
2743 case 14:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2747 NUM_BANKS(ADDR_SURF_2_BANK));
2748 break;
2749 default:
2750 gb_tile_moden = 0;
2751 break;
2752 }
2753 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2754 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2755 }
2756 } else if (num_pipe_configs == 4) {
2757 if (num_rbs == 4) {
2758 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2759 switch (reg_offset) {
2760 case 0:
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2765 break;
2766 case 1:
2767 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2771 break;
2772 case 2:
2773 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2777 break;
2778 case 3:
2779 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2780 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2781 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2782 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2783 break;
2784 case 4:
2785 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2786 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2787 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2788 TILE_SPLIT(split_equal_to_row_size));
2789 break;
2790 case 5:
2791 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2792 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2794 break;
2795 case 6:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2800 break;
2801 case 7:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2805 TILE_SPLIT(split_equal_to_row_size));
2806 break;
2807 case 8:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2809 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2810 break;
2811 case 9:
2812 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2813 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2814 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2815 break;
2816 case 10:
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821 break;
2822 case 11:
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2824 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2825 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2826 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2827 break;
2828 case 12:
2829 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2830 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2831 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2833 break;
2834 case 13:
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2836 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2837 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2838 break;
2839 case 14:
2840 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844 break;
2845 case 16:
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2847 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2849 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2850 break;
2851 case 17:
2852 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2853 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2854 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2855 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2856 break;
2857 case 27:
2858 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2859 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2861 break;
2862 case 28:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 case 29:
2869 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2870 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2871 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873 break;
2874 case 30:
2875 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2876 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2877 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2878 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2879 break;
2880 default:
2881 gb_tile_moden = 0;
2882 break;
2883 }
2884 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2885 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2886 }
2887 } else if (num_rbs < 4) {
2888 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2889 switch (reg_offset) {
2890 case 0:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2895 break;
2896 case 1:
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2901 break;
2902 case 2:
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2907 break;
2908 case 3:
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2910 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2911 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2913 break;
2914 case 4:
2915 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2916 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2917 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2918 TILE_SPLIT(split_equal_to_row_size));
2919 break;
2920 case 5:
2921 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2922 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2924 break;
2925 case 6:
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2927 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2928 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2929 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2930 break;
2931 case 7:
2932 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2933 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2934 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2935 TILE_SPLIT(split_equal_to_row_size));
2936 break;
2937 case 8:
2938 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2939 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2940 break;
2941 case 9:
2942 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2943 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2945 break;
2946 case 10:
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 break;
2952 case 11:
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2954 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2955 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2956 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2957 break;
2958 case 12:
2959 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2960 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2961 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2962 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2963 break;
2964 case 13:
2965 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2966 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2968 break;
2969 case 14:
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 break;
2975 case 16:
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2977 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2978 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2979 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2980 break;
2981 case 17:
2982 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2983 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2984 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2985 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2986 break;
2987 case 27:
2988 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2989 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2991 break;
2992 case 28:
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997 break;
2998 case 29:
2999 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3000 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3001 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3002 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3003 break;
3004 case 30:
3005 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3006 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3007 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3008 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3009 break;
3010 default:
3011 gb_tile_moden = 0;
3012 break;
3013 }
3014 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3015 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3016 }
3017 }
3018 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3019 switch (reg_offset) {
3020 case 0:
3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3024 NUM_BANKS(ADDR_SURF_16_BANK));
3025 break;
3026 case 1:
3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3031 break;
3032 case 2:
3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 NUM_BANKS(ADDR_SURF_16_BANK));
3037 break;
3038 case 3:
3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 NUM_BANKS(ADDR_SURF_16_BANK));
3043 break;
3044 case 4:
3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3048 NUM_BANKS(ADDR_SURF_16_BANK));
3049 break;
3050 case 5:
3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3054 NUM_BANKS(ADDR_SURF_8_BANK));
3055 break;
3056 case 6:
3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3060 NUM_BANKS(ADDR_SURF_4_BANK));
3061 break;
3062 case 8:
3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 NUM_BANKS(ADDR_SURF_16_BANK));
3067 break;
3068 case 9:
3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 NUM_BANKS(ADDR_SURF_16_BANK));
3073 break;
3074 case 10:
3075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3078 NUM_BANKS(ADDR_SURF_16_BANK));
3079 break;
3080 case 11:
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3084 NUM_BANKS(ADDR_SURF_16_BANK));
3085 break;
3086 case 12:
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3090 NUM_BANKS(ADDR_SURF_16_BANK));
3091 break;
3092 case 13:
3093 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3094 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3095 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3096 NUM_BANKS(ADDR_SURF_8_BANK));
3097 break;
3098 case 14:
3099 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3100 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3101 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3102 NUM_BANKS(ADDR_SURF_4_BANK));
3103 break;
3104 default:
3105 gb_tile_moden = 0;
3106 break;
3107 }
3108 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3109 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3110 }
3111 } else if (num_pipe_configs == 2) {
3112 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3113 switch (reg_offset) {
3114 case 0:
3115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 PIPE_CONFIG(ADDR_SURF_P2) |
3118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3119 break;
3120 case 1:
3121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 PIPE_CONFIG(ADDR_SURF_P2) |
3124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3125 break;
3126 case 2:
3127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 PIPE_CONFIG(ADDR_SURF_P2) |
3130 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3131 break;
3132 case 3:
3133 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3134 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3135 PIPE_CONFIG(ADDR_SURF_P2) |
3136 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3137 break;
3138 case 4:
3139 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3140 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3141 PIPE_CONFIG(ADDR_SURF_P2) |
3142 TILE_SPLIT(split_equal_to_row_size));
3143 break;
3144 case 5:
3145 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3146 PIPE_CONFIG(ADDR_SURF_P2) |
3147 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3148 break;
3149 case 6:
3150 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3151 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3152 PIPE_CONFIG(ADDR_SURF_P2) |
3153 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3154 break;
3155 case 7:
3156 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3157 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3158 PIPE_CONFIG(ADDR_SURF_P2) |
3159 TILE_SPLIT(split_equal_to_row_size));
3160 break;
3161 case 8:
3162 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3163 PIPE_CONFIG(ADDR_SURF_P2);
3164 break;
3165 case 9:
3166 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3167 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3168 PIPE_CONFIG(ADDR_SURF_P2));
3169 break;
3170 case 10:
3171 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 PIPE_CONFIG(ADDR_SURF_P2) |
3174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175 break;
3176 case 11:
3177 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3178 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3179 PIPE_CONFIG(ADDR_SURF_P2) |
3180 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3181 break;
3182 case 12:
3183 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3184 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3185 PIPE_CONFIG(ADDR_SURF_P2) |
3186 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3187 break;
3188 case 13:
3189 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3190 PIPE_CONFIG(ADDR_SURF_P2) |
3191 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3192 break;
3193 case 14:
3194 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3195 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 PIPE_CONFIG(ADDR_SURF_P2) |
3197 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198 break;
3199 case 16:
3200 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3201 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3202 PIPE_CONFIG(ADDR_SURF_P2) |
3203 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3204 break;
3205 case 17:
3206 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3207 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3208 PIPE_CONFIG(ADDR_SURF_P2) |
3209 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3210 break;
3211 case 27:
3212 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3213 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3214 PIPE_CONFIG(ADDR_SURF_P2));
3215 break;
3216 case 28:
3217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221 break;
3222 case 29:
3223 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3224 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3225 PIPE_CONFIG(ADDR_SURF_P2) |
3226 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3227 break;
3228 case 30:
3229 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3230 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3231 PIPE_CONFIG(ADDR_SURF_P2) |
3232 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3233 break;
3234 default:
3235 gb_tile_moden = 0;
3236 break;
3237 }
3238 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3239 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3240 }
3241 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3242 switch (reg_offset) {
3243 case 0:
3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 NUM_BANKS(ADDR_SURF_16_BANK));
3248 break;
3249 case 1:
3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 NUM_BANKS(ADDR_SURF_16_BANK));
3254 break;
3255 case 2:
3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 NUM_BANKS(ADDR_SURF_16_BANK));
3260 break;
3261 case 3:
3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 NUM_BANKS(ADDR_SURF_16_BANK));
3266 break;
3267 case 4:
3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3271 NUM_BANKS(ADDR_SURF_16_BANK));
3272 break;
3273 case 5:
3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 NUM_BANKS(ADDR_SURF_16_BANK));
3278 break;
3279 case 6:
3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3283 NUM_BANKS(ADDR_SURF_8_BANK));
3284 break;
3285 case 8:
3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 NUM_BANKS(ADDR_SURF_16_BANK));
3290 break;
3291 case 9:
3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 NUM_BANKS(ADDR_SURF_16_BANK));
3296 break;
3297 case 10:
3298 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 NUM_BANKS(ADDR_SURF_16_BANK));
3302 break;
3303 case 11:
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3308 break;
3309 case 12:
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3313 NUM_BANKS(ADDR_SURF_16_BANK));
3314 break;
3315 case 13:
3316 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3319 NUM_BANKS(ADDR_SURF_16_BANK));
3320 break;
3321 case 14:
3322 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3323 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3324 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3325 NUM_BANKS(ADDR_SURF_8_BANK));
3326 break;
3327 default:
3328 gb_tile_moden = 0;
3329 break;
3330 }
3331 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3332 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3333 }
3334 } else
3335 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3336 }
3337
3338 /**
3339 * cik_select_se_sh - select which SE, SH to address
3340 *
3341 * @rdev: radeon_device pointer
3342 * @se_num: shader engine to address
3343 * @sh_num: sh block to address
3344 *
3345 * Select which SE, SH combinations to address. Certain
3346 * registers are instanced per SE or SH. 0xffffffff means
3347 * broadcast to all SEs or SHs (CIK).
3348 */
3349 static void cik_select_se_sh(struct radeon_device *rdev,
3350 u32 se_num, u32 sh_num)
3351 {
3352 u32 data = INSTANCE_BROADCAST_WRITES;
3353
3354 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3355 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3356 else if (se_num == 0xffffffff)
3357 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3358 else if (sh_num == 0xffffffff)
3359 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3360 else
3361 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3362 WREG32(GRBM_GFX_INDEX, data);
3363 }
3364
3365 /**
3366 * cik_create_bitmask - create a bitmask
3367 *
3368 * @bit_width: length of the mask
3369 *
3370 * create a variable length bit mask (CIK).
3371 * Returns the bitmask.
3372 */
3373 static u32 cik_create_bitmask(u32 bit_width)
3374 {
3375 u32 i, mask = 0;
3376
3377 for (i = 0; i < bit_width; i++) {
3378 mask <<= 1;
3379 mask |= 1;
3380 }
3381 return mask;
3382 }
3383
3384 /**
3385 * cik_get_rb_disabled - computes the mask of disabled RBs
3386 *
3387 * @rdev: radeon_device pointer
3388 * @max_rb_num: max RBs (render backends) for the asic
3389 * @se_num: number of SEs (shader engines) for the asic
3390 * @sh_per_se: number of SH blocks per SE for the asic
3391 *
3392 * Calculates the bitmask of disabled RBs (CIK).
3393 * Returns the disabled RB bitmask.
3394 */
3395 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3396 u32 max_rb_num_per_se,
3397 u32 sh_per_se)
3398 {
3399 u32 data, mask;
3400
3401 data = RREG32(CC_RB_BACKEND_DISABLE);
3402 if (data & 1)
3403 data &= BACKEND_DISABLE_MASK;
3404 else
3405 data = 0;
3406 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3407
3408 data >>= BACKEND_DISABLE_SHIFT;
3409
3410 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3411
3412 return data & mask;
3413 }
3414
3415 /**
3416 * cik_setup_rb - setup the RBs on the asic
3417 *
3418 * @rdev: radeon_device pointer
3419 * @se_num: number of SEs (shader engines) for the asic
3420 * @sh_per_se: number of SH blocks per SE for the asic
3421 * @max_rb_num: max RBs (render backends) for the asic
3422 *
3423 * Configures per-SE/SH RB registers (CIK).
3424 */
3425 static void cik_setup_rb(struct radeon_device *rdev,
3426 u32 se_num, u32 sh_per_se,
3427 u32 max_rb_num_per_se)
3428 {
3429 int i, j;
3430 u32 data, mask;
3431 u32 disabled_rbs = 0;
3432 u32 enabled_rbs = 0;
3433
3434 for (i = 0; i < se_num; i++) {
3435 for (j = 0; j < sh_per_se; j++) {
3436 cik_select_se_sh(rdev, i, j);
3437 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3438 if (rdev->family == CHIP_HAWAII)
3439 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3440 else
3441 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3442 }
3443 }
3444 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3445
3446 mask = 1;
3447 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3448 if (!(disabled_rbs & mask))
3449 enabled_rbs |= mask;
3450 mask <<= 1;
3451 }
3452
3453 rdev->config.cik.backend_enable_mask = enabled_rbs;
3454
3455 for (i = 0; i < se_num; i++) {
3456 cik_select_se_sh(rdev, i, 0xffffffff);
3457 data = 0;
3458 for (j = 0; j < sh_per_se; j++) {
3459 switch (enabled_rbs & 3) {
3460 case 0:
3461 if (j == 0)
3462 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3463 else
3464 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3465 break;
3466 case 1:
3467 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3468 break;
3469 case 2:
3470 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3471 break;
3472 case 3:
3473 default:
3474 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3475 break;
3476 }
3477 enabled_rbs >>= 2;
3478 }
3479 WREG32(PA_SC_RASTER_CONFIG, data);
3480 }
3481 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3482 }
3483
3484 /**
3485 * cik_gpu_init - setup the 3D engine
3486 *
3487 * @rdev: radeon_device pointer
3488 *
3489 * Configures the 3D engine and tiling configuration
3490 * registers so that the 3D engine is usable.
3491 */
3492 static void cik_gpu_init(struct radeon_device *rdev)
3493 {
3494 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3495 u32 mc_shared_chmap, mc_arb_ramcfg;
3496 u32 hdp_host_path_cntl;
3497 u32 tmp;
3498 int i, j;
3499
3500 switch (rdev->family) {
3501 case CHIP_BONAIRE:
3502 rdev->config.cik.max_shader_engines = 2;
3503 rdev->config.cik.max_tile_pipes = 4;
3504 rdev->config.cik.max_cu_per_sh = 7;
3505 rdev->config.cik.max_sh_per_se = 1;
3506 rdev->config.cik.max_backends_per_se = 2;
3507 rdev->config.cik.max_texture_channel_caches = 4;
3508 rdev->config.cik.max_gprs = 256;
3509 rdev->config.cik.max_gs_threads = 32;
3510 rdev->config.cik.max_hw_contexts = 8;
3511
3512 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3513 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3514 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3515 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3516 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3517 break;
3518 case CHIP_HAWAII:
3519 rdev->config.cik.max_shader_engines = 4;
3520 rdev->config.cik.max_tile_pipes = 16;
3521 rdev->config.cik.max_cu_per_sh = 11;
3522 rdev->config.cik.max_sh_per_se = 1;
3523 rdev->config.cik.max_backends_per_se = 4;
3524 rdev->config.cik.max_texture_channel_caches = 16;
3525 rdev->config.cik.max_gprs = 256;
3526 rdev->config.cik.max_gs_threads = 32;
3527 rdev->config.cik.max_hw_contexts = 8;
3528
3529 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3530 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3531 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3532 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3533 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3534 break;
3535 case CHIP_KAVERI:
3536 rdev->config.cik.max_shader_engines = 1;
3537 rdev->config.cik.max_tile_pipes = 4;
3538 if ((rdev->pdev->device == 0x1304) ||
3539 (rdev->pdev->device == 0x1305) ||
3540 (rdev->pdev->device == 0x130C) ||
3541 (rdev->pdev->device == 0x130F) ||
3542 (rdev->pdev->device == 0x1310) ||
3543 (rdev->pdev->device == 0x1311) ||
3544 (rdev->pdev->device == 0x131C)) {
3545 rdev->config.cik.max_cu_per_sh = 8;
3546 rdev->config.cik.max_backends_per_se = 2;
3547 } else if ((rdev->pdev->device == 0x1309) ||
3548 (rdev->pdev->device == 0x130A) ||
3549 (rdev->pdev->device == 0x130D) ||
3550 (rdev->pdev->device == 0x1313) ||
3551 (rdev->pdev->device == 0x131D)) {
3552 rdev->config.cik.max_cu_per_sh = 6;
3553 rdev->config.cik.max_backends_per_se = 2;
3554 } else if ((rdev->pdev->device == 0x1306) ||
3555 (rdev->pdev->device == 0x1307) ||
3556 (rdev->pdev->device == 0x130B) ||
3557 (rdev->pdev->device == 0x130E) ||
3558 (rdev->pdev->device == 0x1315) ||
3559 (rdev->pdev->device == 0x1318) ||
3560 (rdev->pdev->device == 0x131B)) {
3561 rdev->config.cik.max_cu_per_sh = 4;
3562 rdev->config.cik.max_backends_per_se = 1;
3563 } else {
3564 rdev->config.cik.max_cu_per_sh = 3;
3565 rdev->config.cik.max_backends_per_se = 1;
3566 }
3567 rdev->config.cik.max_sh_per_se = 1;
3568 rdev->config.cik.max_texture_channel_caches = 4;
3569 rdev->config.cik.max_gprs = 256;
3570 rdev->config.cik.max_gs_threads = 16;
3571 rdev->config.cik.max_hw_contexts = 8;
3572
3573 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3574 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3575 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3576 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3577 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3578 break;
3579 case CHIP_KABINI:
3580 case CHIP_MULLINS:
3581 default:
3582 rdev->config.cik.max_shader_engines = 1;
3583 rdev->config.cik.max_tile_pipes = 2;
3584 rdev->config.cik.max_cu_per_sh = 2;
3585 rdev->config.cik.max_sh_per_se = 1;
3586 rdev->config.cik.max_backends_per_se = 1;
3587 rdev->config.cik.max_texture_channel_caches = 2;
3588 rdev->config.cik.max_gprs = 256;
3589 rdev->config.cik.max_gs_threads = 16;
3590 rdev->config.cik.max_hw_contexts = 8;
3591
3592 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3593 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3594 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3595 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3596 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3597 break;
3598 }
3599
3600 /* Initialize HDP */
3601 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3602 WREG32((0x2c14 + j), 0x00000000);
3603 WREG32((0x2c18 + j), 0x00000000);
3604 WREG32((0x2c1c + j), 0x00000000);
3605 WREG32((0x2c20 + j), 0x00000000);
3606 WREG32((0x2c24 + j), 0x00000000);
3607 }
3608
3609 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3610
3611 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3612
3613 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3614 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3615
3616 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3617 rdev->config.cik.mem_max_burst_length_bytes = 256;
3618 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3619 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3620 if (rdev->config.cik.mem_row_size_in_kb > 4)
3621 rdev->config.cik.mem_row_size_in_kb = 4;
3622 /* XXX use MC settings? */
3623 rdev->config.cik.shader_engine_tile_size = 32;
3624 rdev->config.cik.num_gpus = 1;
3625 rdev->config.cik.multi_gpu_tile_size = 64;
3626
3627 /* fix up row size */
3628 gb_addr_config &= ~ROW_SIZE_MASK;
3629 switch (rdev->config.cik.mem_row_size_in_kb) {
3630 case 1:
3631 default:
3632 gb_addr_config |= ROW_SIZE(0);
3633 break;
3634 case 2:
3635 gb_addr_config |= ROW_SIZE(1);
3636 break;
3637 case 4:
3638 gb_addr_config |= ROW_SIZE(2);
3639 break;
3640 }
3641
3642 /* setup tiling info dword. gb_addr_config is not adequate since it does
3643 * not have bank info, so create a custom tiling dword.
3644 * bits 3:0 num_pipes
3645 * bits 7:4 num_banks
3646 * bits 11:8 group_size
3647 * bits 15:12 row_size
3648 */
3649 rdev->config.cik.tile_config = 0;
3650 switch (rdev->config.cik.num_tile_pipes) {
3651 case 1:
3652 rdev->config.cik.tile_config |= (0 << 0);
3653 break;
3654 case 2:
3655 rdev->config.cik.tile_config |= (1 << 0);
3656 break;
3657 case 4:
3658 rdev->config.cik.tile_config |= (2 << 0);
3659 break;
3660 case 8:
3661 default:
3662 /* XXX what about 12? */
3663 rdev->config.cik.tile_config |= (3 << 0);
3664 break;
3665 }
3666 rdev->config.cik.tile_config |=
3667 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3668 rdev->config.cik.tile_config |=
3669 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3670 rdev->config.cik.tile_config |=
3671 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3672
3673 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3674 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3675 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3676 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3677 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3678 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3679 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3680 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3681
3682 cik_tiling_mode_table_init(rdev);
3683
3684 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3685 rdev->config.cik.max_sh_per_se,
3686 rdev->config.cik.max_backends_per_se);
3687
3688 rdev->config.cik.active_cus = 0;
3689 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3690 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3691 rdev->config.cik.active_cus +=
3692 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3693 }
3694 }
3695
3696 /* set HW defaults for 3D engine */
3697 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3698
3699 WREG32(SX_DEBUG_1, 0x20);
3700
3701 WREG32(TA_CNTL_AUX, 0x00010000);
3702
3703 tmp = RREG32(SPI_CONFIG_CNTL);
3704 tmp |= 0x03000000;
3705 WREG32(SPI_CONFIG_CNTL, tmp);
3706
3707 WREG32(SQ_CONFIG, 1);
3708
3709 WREG32(DB_DEBUG, 0);
3710
3711 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3712 tmp |= 0x00000400;
3713 WREG32(DB_DEBUG2, tmp);
3714
3715 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3716 tmp |= 0x00020200;
3717 WREG32(DB_DEBUG3, tmp);
3718
3719 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3720 tmp |= 0x00018208;
3721 WREG32(CB_HW_CONTROL, tmp);
3722
3723 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3724
3725 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3726 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3727 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3728 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3729
3730 WREG32(VGT_NUM_INSTANCES, 1);
3731
3732 WREG32(CP_PERFMON_CNTL, 0);
3733
3734 WREG32(SQ_CONFIG, 0);
3735
3736 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3737 FORCE_EOV_MAX_REZ_CNT(255)));
3738
3739 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3740 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3741
3742 WREG32(VGT_GS_VERTEX_REUSE, 16);
3743 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3744
3745 tmp = RREG32(HDP_MISC_CNTL);
3746 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3747 WREG32(HDP_MISC_CNTL, tmp);
3748
3749 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3750 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3751
3752 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3753 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3754
3755 udelay(50);
3756 }
3757
3758 /*
3759 * GPU scratch registers helpers function.
3760 */
3761 /**
3762 * cik_scratch_init - setup driver info for CP scratch regs
3763 *
3764 * @rdev: radeon_device pointer
3765 *
3766 * Set up the number and offset of the CP scratch registers.
3767 * NOTE: use of CP scratch registers is a legacy inferface and
3768 * is not used by default on newer asics (r6xx+). On newer asics,
3769 * memory buffers are used for fences rather than scratch regs.
3770 */
3771 static void cik_scratch_init(struct radeon_device *rdev)
3772 {
3773 int i;
3774
3775 rdev->scratch.num_reg = 7;
3776 rdev->scratch.reg_base = SCRATCH_REG0;
3777 for (i = 0; i < rdev->scratch.num_reg; i++) {
3778 rdev->scratch.free[i] = true;
3779 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3780 }
3781 }
3782
3783 /**
3784 * cik_ring_test - basic gfx ring test
3785 *
3786 * @rdev: radeon_device pointer
3787 * @ring: radeon_ring structure holding ring information
3788 *
3789 * Allocate a scratch register and write to it using the gfx ring (CIK).
3790 * Provides a basic gfx ring test to verify that the ring is working.
3791 * Used by cik_cp_gfx_resume();
3792 * Returns 0 on success, error on failure.
3793 */
3794 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3795 {
3796 uint32_t scratch;
3797 uint32_t tmp = 0;
3798 unsigned i;
3799 int r;
3800
3801 r = radeon_scratch_get(rdev, &scratch);
3802 if (r) {
3803 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3804 return r;
3805 }
3806 WREG32(scratch, 0xCAFEDEAD);
3807 r = radeon_ring_lock(rdev, ring, 3);
3808 if (r) {
3809 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3810 radeon_scratch_free(rdev, scratch);
3811 return r;
3812 }
3813 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3814 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3815 radeon_ring_write(ring, 0xDEADBEEF);
3816 radeon_ring_unlock_commit(rdev, ring, false);
3817
3818 for (i = 0; i < rdev->usec_timeout; i++) {
3819 tmp = RREG32(scratch);
3820 if (tmp == 0xDEADBEEF)
3821 break;
3822 DRM_UDELAY(1);
3823 }
3824 if (i < rdev->usec_timeout) {
3825 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3826 } else {
3827 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3828 ring->idx, scratch, tmp);
3829 r = -EINVAL;
3830 }
3831 radeon_scratch_free(rdev, scratch);
3832 return r;
3833 }
3834
3835 /**
3836 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3837 *
3838 * @rdev: radeon_device pointer
3839 * @ridx: radeon ring index
3840 *
3841 * Emits an hdp flush on the cp.
3842 */
3843 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3844 int ridx)
3845 {
3846 struct radeon_ring *ring = &rdev->ring[ridx];
3847 u32 ref_and_mask;
3848
3849 switch (ring->idx) {
3850 case CAYMAN_RING_TYPE_CP1_INDEX:
3851 case CAYMAN_RING_TYPE_CP2_INDEX:
3852 default:
3853 switch (ring->me) {
3854 case 0:
3855 ref_and_mask = CP2 << ring->pipe;
3856 break;
3857 case 1:
3858 ref_and_mask = CP6 << ring->pipe;
3859 break;
3860 default:
3861 return;
3862 }
3863 break;
3864 case RADEON_RING_TYPE_GFX_INDEX:
3865 ref_and_mask = CP0;
3866 break;
3867 }
3868
3869 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3870 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3871 WAIT_REG_MEM_FUNCTION(3) | /* == */
3872 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3873 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3874 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3875 radeon_ring_write(ring, ref_and_mask);
3876 radeon_ring_write(ring, ref_and_mask);
3877 radeon_ring_write(ring, 0x20); /* poll interval */
3878 }
3879
3880 /**
3881 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3882 *
3883 * @rdev: radeon_device pointer
3884 * @fence: radeon fence object
3885 *
3886 * Emits a fence sequnce number on the gfx ring and flushes
3887 * GPU caches.
3888 */
3889 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3890 struct radeon_fence *fence)
3891 {
3892 struct radeon_ring *ring = &rdev->ring[fence->ring];
3893 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3894
3895 /* EVENT_WRITE_EOP - flush caches, send int */
3896 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3897 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3898 EOP_TC_ACTION_EN |
3899 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3900 EVENT_INDEX(5)));
3901 radeon_ring_write(ring, addr & 0xfffffffc);
3902 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3903 radeon_ring_write(ring, fence->seq);
3904 radeon_ring_write(ring, 0);
3905 }
3906
3907 /**
3908 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3909 *
3910 * @rdev: radeon_device pointer
3911 * @fence: radeon fence object
3912 *
3913 * Emits a fence sequnce number on the compute ring and flushes
3914 * GPU caches.
3915 */
3916 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3917 struct radeon_fence *fence)
3918 {
3919 struct radeon_ring *ring = &rdev->ring[fence->ring];
3920 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3921
3922 /* RELEASE_MEM - flush caches, send int */
3923 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3924 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3925 EOP_TC_ACTION_EN |
3926 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3927 EVENT_INDEX(5)));
3928 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3929 radeon_ring_write(ring, addr & 0xfffffffc);
3930 radeon_ring_write(ring, upper_32_bits(addr));
3931 radeon_ring_write(ring, fence->seq);
3932 radeon_ring_write(ring, 0);
3933 }
3934
3935 /**
3936 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3937 *
3938 * @rdev: radeon_device pointer
3939 * @ring: radeon ring buffer object
3940 * @semaphore: radeon semaphore object
3941 * @emit_wait: Is this a sempahore wait?
3942 *
3943 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3944 * from running ahead of semaphore waits.
3945 */
3946 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3947 struct radeon_ring *ring,
3948 struct radeon_semaphore *semaphore,
3949 bool emit_wait)
3950 {
3951 uint64_t addr = semaphore->gpu_addr;
3952 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3953
3954 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3955 radeon_ring_write(ring, lower_32_bits(addr));
3956 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3957
3958 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3959 /* Prevent the PFP from running ahead of the semaphore wait */
3960 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3961 radeon_ring_write(ring, 0x0);
3962 }
3963
3964 return true;
3965 }
3966
3967 /**
3968 * cik_copy_cpdma - copy pages using the CP DMA engine
3969 *
3970 * @rdev: radeon_device pointer
3971 * @src_offset: src GPU address
3972 * @dst_offset: dst GPU address
3973 * @num_gpu_pages: number of GPU pages to xfer
3974 * @fence: radeon fence object
3975 *
3976 * Copy GPU paging using the CP DMA engine (CIK+).
3977 * Used by the radeon ttm implementation to move pages if
3978 * registered as the asic copy callback.
3979 */
3980 int cik_copy_cpdma(struct radeon_device *rdev,
3981 uint64_t src_offset, uint64_t dst_offset,
3982 unsigned num_gpu_pages,
3983 struct radeon_fence **fence)
3984 {
3985 struct radeon_semaphore *sem = NULL;
3986 int ring_index = rdev->asic->copy.blit_ring_index;
3987 struct radeon_ring *ring = &rdev->ring[ring_index];
3988 u32 size_in_bytes, cur_size_in_bytes, control;
3989 int i, num_loops;
3990 int r = 0;
3991
3992 r = radeon_semaphore_create(rdev, &sem);
3993 if (r) {
3994 DRM_ERROR("radeon: moving bo (%d).\n", r);
3995 return r;
3996 }
3997
3998 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3999 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4000 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4001 if (r) {
4002 DRM_ERROR("radeon: moving bo (%d).\n", r);
4003 radeon_semaphore_free(rdev, &sem, NULL);
4004 return r;
4005 }
4006
4007 radeon_semaphore_sync_to(sem, *fence);
4008 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
4009
4010 for (i = 0; i < num_loops; i++) {
4011 cur_size_in_bytes = size_in_bytes;
4012 if (cur_size_in_bytes > 0x1fffff)
4013 cur_size_in_bytes = 0x1fffff;
4014 size_in_bytes -= cur_size_in_bytes;
4015 control = 0;
4016 if (size_in_bytes == 0)
4017 control |= PACKET3_DMA_DATA_CP_SYNC;
4018 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4019 radeon_ring_write(ring, control);
4020 radeon_ring_write(ring, lower_32_bits(src_offset));
4021 radeon_ring_write(ring, upper_32_bits(src_offset));
4022 radeon_ring_write(ring, lower_32_bits(dst_offset));
4023 radeon_ring_write(ring, upper_32_bits(dst_offset));
4024 radeon_ring_write(ring, cur_size_in_bytes);
4025 src_offset += cur_size_in_bytes;
4026 dst_offset += cur_size_in_bytes;
4027 }
4028
4029 r = radeon_fence_emit(rdev, fence, ring->idx);
4030 if (r) {
4031 radeon_ring_unlock_undo(rdev, ring);
4032 radeon_semaphore_free(rdev, &sem, NULL);
4033 return r;
4034 }
4035
4036 radeon_ring_unlock_commit(rdev, ring, false);
4037 radeon_semaphore_free(rdev, &sem, *fence);
4038
4039 return r;
4040 }
4041
4042 /*
4043 * IB stuff
4044 */
4045 /**
4046 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4047 *
4048 * @rdev: radeon_device pointer
4049 * @ib: radeon indirect buffer object
4050 *
4051 * Emits an DE (drawing engine) or CE (constant engine) IB
4052 * on the gfx ring. IBs are usually generated by userspace
4053 * acceleration drivers and submitted to the kernel for
4054 * sheduling on the ring. This function schedules the IB
4055 * on the gfx ring for execution by the GPU.
4056 */
4057 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4058 {
4059 struct radeon_ring *ring = &rdev->ring[ib->ring];
4060 u32 header, control = INDIRECT_BUFFER_VALID;
4061
4062 if (ib->is_const_ib) {
4063 /* set switch buffer packet before const IB */
4064 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4065 radeon_ring_write(ring, 0);
4066
4067 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4068 } else {
4069 u32 next_rptr;
4070 if (ring->rptr_save_reg) {
4071 next_rptr = ring->wptr + 3 + 4;
4072 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4073 radeon_ring_write(ring, ((ring->rptr_save_reg -
4074 PACKET3_SET_UCONFIG_REG_START) >> 2));
4075 radeon_ring_write(ring, next_rptr);
4076 } else if (rdev->wb.enabled) {
4077 next_rptr = ring->wptr + 5 + 4;
4078 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4079 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4080 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4081 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4082 radeon_ring_write(ring, next_rptr);
4083 }
4084
4085 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4086 }
4087
4088 control |= ib->length_dw |
4089 (ib->vm ? (ib->vm->id << 24) : 0);
4090
4091 radeon_ring_write(ring, header);
4092 radeon_ring_write(ring,
4093 #ifdef __BIG_ENDIAN
4094 (2 << 0) |
4095 #endif
4096 (ib->gpu_addr & 0xFFFFFFFC));
4097 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4098 radeon_ring_write(ring, control);
4099 }
4100
4101 /**
4102 * cik_ib_test - basic gfx ring IB test
4103 *
4104 * @rdev: radeon_device pointer
4105 * @ring: radeon_ring structure holding ring information
4106 *
4107 * Allocate an IB and execute it on the gfx ring (CIK).
4108 * Provides a basic gfx ring test to verify that IBs are working.
4109 * Returns 0 on success, error on failure.
4110 */
4111 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4112 {
4113 struct radeon_ib ib;
4114 uint32_t scratch;
4115 uint32_t tmp = 0;
4116 unsigned i;
4117 int r;
4118
4119 r = radeon_scratch_get(rdev, &scratch);
4120 if (r) {
4121 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4122 return r;
4123 }
4124 WREG32(scratch, 0xCAFEDEAD);
4125 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4126 if (r) {
4127 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4128 radeon_scratch_free(rdev, scratch);
4129 return r;
4130 }
4131 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4132 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4133 ib.ptr[2] = 0xDEADBEEF;
4134 ib.length_dw = 3;
4135 r = radeon_ib_schedule(rdev, &ib, NULL, false);
4136 if (r) {
4137 radeon_scratch_free(rdev, scratch);
4138 radeon_ib_free(rdev, &ib);
4139 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4140 return r;
4141 }
4142 r = radeon_fence_wait(ib.fence, false);
4143 if (r) {
4144 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4145 radeon_scratch_free(rdev, scratch);
4146 radeon_ib_free(rdev, &ib);
4147 return r;
4148 }
4149 for (i = 0; i < rdev->usec_timeout; i++) {
4150 tmp = RREG32(scratch);
4151 if (tmp == 0xDEADBEEF)
4152 break;
4153 DRM_UDELAY(1);
4154 }
4155 if (i < rdev->usec_timeout) {
4156 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4157 } else {
4158 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4159 scratch, tmp);
4160 r = -EINVAL;
4161 }
4162 radeon_scratch_free(rdev, scratch);
4163 radeon_ib_free(rdev, &ib);
4164 return r;
4165 }
4166
4167 /*
4168 * CP.
4169 * On CIK, gfx and compute now have independant command processors.
4170 *
4171 * GFX
4172 * Gfx consists of a single ring and can process both gfx jobs and
4173 * compute jobs. The gfx CP consists of three microengines (ME):
4174 * PFP - Pre-Fetch Parser
4175 * ME - Micro Engine
4176 * CE - Constant Engine
4177 * The PFP and ME make up what is considered the Drawing Engine (DE).
4178 * The CE is an asynchronous engine used for updating buffer desciptors
4179 * used by the DE so that they can be loaded into cache in parallel
4180 * while the DE is processing state update packets.
4181 *
4182 * Compute
4183 * The compute CP consists of two microengines (ME):
4184 * MEC1 - Compute MicroEngine 1
4185 * MEC2 - Compute MicroEngine 2
4186 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4187 * The queues are exposed to userspace and are programmed directly
4188 * by the compute runtime.
4189 */
4190 /**
4191 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4192 *
4193 * @rdev: radeon_device pointer
4194 * @enable: enable or disable the MEs
4195 *
4196 * Halts or unhalts the gfx MEs.
4197 */
4198 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4199 {
4200 if (enable)
4201 WREG32(CP_ME_CNTL, 0);
4202 else {
4203 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4204 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4205 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4206 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4207 }
4208 udelay(50);
4209 }
4210
4211 /**
4212 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4213 *
4214 * @rdev: radeon_device pointer
4215 *
4216 * Loads the gfx PFP, ME, and CE ucode.
4217 * Returns 0 for success, -EINVAL if the ucode is not available.
4218 */
4219 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4220 {
4221 int i;
4222
4223 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4224 return -EINVAL;
4225
4226 cik_cp_gfx_enable(rdev, false);
4227
4228 if (rdev->new_fw) {
4229 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4230 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4231 const struct gfx_firmware_header_v1_0 *ce_hdr =
4232 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4233 const struct gfx_firmware_header_v1_0 *me_hdr =
4234 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4235 const __le32 *fw_data;
4236 u32 fw_size;
4237
4238 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4239 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4240 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4241
4242 /* PFP */
4243 fw_data = (const __le32 *)
4244 ((const char *)rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4245 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4246 WREG32(CP_PFP_UCODE_ADDR, 0);
4247 for (i = 0; i < fw_size; i++)
4248 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4249 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
4250
4251 /* CE */
4252 fw_data = (const __le32 *)
4253 ((const char *)rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4254 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4255 WREG32(CP_CE_UCODE_ADDR, 0);
4256 for (i = 0; i < fw_size; i++)
4257 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4258 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
4259
4260 /* ME */
4261 fw_data = (const __be32 *)
4262 ((const char *)rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4263 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4264 WREG32(CP_ME_RAM_WADDR, 0);
4265 for (i = 0; i < fw_size; i++)
4266 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4267 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4268 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
4269 } else {
4270 const __be32 *fw_data;
4271
4272 /* PFP */
4273 fw_data = (const __be32 *)rdev->pfp_fw->data;
4274 WREG32(CP_PFP_UCODE_ADDR, 0);
4275 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4276 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4277 WREG32(CP_PFP_UCODE_ADDR, 0);
4278
4279 /* CE */
4280 fw_data = (const __be32 *)rdev->ce_fw->data;
4281 WREG32(CP_CE_UCODE_ADDR, 0);
4282 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4283 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4284 WREG32(CP_CE_UCODE_ADDR, 0);
4285
4286 /* ME */
4287 fw_data = (const __be32 *)rdev->me_fw->data;
4288 WREG32(CP_ME_RAM_WADDR, 0);
4289 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4290 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4291 WREG32(CP_ME_RAM_WADDR, 0);
4292 }
4293
4294 return 0;
4295 }
4296
4297 /**
4298 * cik_cp_gfx_start - start the gfx ring
4299 *
4300 * @rdev: radeon_device pointer
4301 *
4302 * Enables the ring and loads the clear state context and other
4303 * packets required to init the ring.
4304 * Returns 0 for success, error for failure.
4305 */
4306 static int cik_cp_gfx_start(struct radeon_device *rdev)
4307 {
4308 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4309 int r, i;
4310
4311 /* init the CP */
4312 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4313 WREG32(CP_ENDIAN_SWAP, 0);
4314 WREG32(CP_DEVICE_ID, 1);
4315
4316 cik_cp_gfx_enable(rdev, true);
4317
4318 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4319 if (r) {
4320 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4321 return r;
4322 }
4323
4324 /* init the CE partitions. CE only used for gfx on CIK */
4325 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4326 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4327 radeon_ring_write(ring, 0x8000);
4328 radeon_ring_write(ring, 0x8000);
4329
4330 /* setup clear context state */
4331 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4332 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4333
4334 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4335 radeon_ring_write(ring, 0x80000000);
4336 radeon_ring_write(ring, 0x80000000);
4337
4338 for (i = 0; i < cik_default_size; i++)
4339 radeon_ring_write(ring, cik_default_state[i]);
4340
4341 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4342 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4343
4344 /* set clear context state */
4345 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4346 radeon_ring_write(ring, 0);
4347
4348 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4349 radeon_ring_write(ring, 0x00000316);
4350 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4351 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4352
4353 radeon_ring_unlock_commit(rdev, ring, false);
4354
4355 return 0;
4356 }
4357
4358 /**
4359 * cik_cp_gfx_fini - stop the gfx ring
4360 *
4361 * @rdev: radeon_device pointer
4362 *
4363 * Stop the gfx ring and tear down the driver ring
4364 * info.
4365 */
4366 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4367 {
4368 cik_cp_gfx_enable(rdev, false);
4369 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4370 }
4371
4372 /**
4373 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4374 *
4375 * @rdev: radeon_device pointer
4376 *
4377 * Program the location and size of the gfx ring buffer
4378 * and test it to make sure it's working.
4379 * Returns 0 for success, error for failure.
4380 */
4381 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4382 {
4383 struct radeon_ring *ring;
4384 u32 tmp;
4385 u32 rb_bufsz;
4386 u64 rb_addr;
4387 int r;
4388
4389 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4390 if (rdev->family != CHIP_HAWAII)
4391 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4392
4393 /* Set the write pointer delay */
4394 WREG32(CP_RB_WPTR_DELAY, 0);
4395
4396 /* set the RB to use vmid 0 */
4397 WREG32(CP_RB_VMID, 0);
4398
4399 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4400
4401 /* ring 0 - compute and gfx */
4402 /* Set ring buffer size */
4403 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4404 rb_bufsz = order_base_2(ring->ring_size / 8);
4405 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4406 #ifdef __BIG_ENDIAN
4407 tmp |= BUF_SWAP_32BIT;
4408 #endif
4409 WREG32(CP_RB0_CNTL, tmp);
4410
4411 /* Initialize the ring buffer's read and write pointers */
4412 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4413 ring->wptr = 0;
4414 WREG32(CP_RB0_WPTR, ring->wptr);
4415
4416 /* set the wb address wether it's enabled or not */
4417 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4418 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4419
4420 /* scratch register shadowing is no longer supported */
4421 WREG32(SCRATCH_UMSK, 0);
4422
4423 if (!rdev->wb.enabled)
4424 tmp |= RB_NO_UPDATE;
4425
4426 mdelay(1);
4427 WREG32(CP_RB0_CNTL, tmp);
4428
4429 rb_addr = ring->gpu_addr >> 8;
4430 WREG32(CP_RB0_BASE, rb_addr);
4431 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4432
4433 /* start the ring */
4434 cik_cp_gfx_start(rdev);
4435 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4436 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4437 if (r) {
4438 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4439 return r;
4440 }
4441
4442 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4443 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4444
4445 return 0;
4446 }
4447
4448 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4449 struct radeon_ring *ring)
4450 {
4451 u32 rptr;
4452
4453 if (rdev->wb.enabled)
4454 rptr = rdev->wb.wb[ring->rptr_offs/4];
4455 else
4456 rptr = RREG32(CP_RB0_RPTR);
4457
4458 return rptr;
4459 }
4460
4461 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4462 struct radeon_ring *ring)
4463 {
4464 u32 wptr;
4465
4466 wptr = RREG32(CP_RB0_WPTR);
4467
4468 return wptr;
4469 }
4470
4471 void cik_gfx_set_wptr(struct radeon_device *rdev,
4472 struct radeon_ring *ring)
4473 {
4474 WREG32(CP_RB0_WPTR, ring->wptr);
4475 (void)RREG32(CP_RB0_WPTR);
4476 }
4477
4478 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4479 struct radeon_ring *ring)
4480 {
4481 u32 rptr;
4482
4483 if (rdev->wb.enabled) {
4484 rptr = rdev->wb.wb[ring->rptr_offs/4];
4485 } else {
4486 spin_lock(&rdev->srbm_mutex);
4487 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4488 rptr = RREG32(CP_HQD_PQ_RPTR);
4489 cik_srbm_select(rdev, 0, 0, 0, 0);
4490 spin_unlock(&rdev->srbm_mutex);
4491 }
4492
4493 return rptr;
4494 }
4495
4496 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4497 struct radeon_ring *ring)
4498 {
4499 u32 wptr;
4500
4501 if (rdev->wb.enabled) {
4502 /* XXX check if swapping is necessary on BE */
4503 wptr = rdev->wb.wb[ring->wptr_offs/4];
4504 } else {
4505 spin_lock(&rdev->srbm_mutex);
4506 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4507 wptr = RREG32(CP_HQD_PQ_WPTR);
4508 cik_srbm_select(rdev, 0, 0, 0, 0);
4509 spin_unlock(&rdev->srbm_mutex);
4510 }
4511
4512 return wptr;
4513 }
4514
4515 void cik_compute_set_wptr(struct radeon_device *rdev,
4516 struct radeon_ring *ring)
4517 {
4518 /* XXX check if swapping is necessary on BE */
4519 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4520 WDOORBELL32(ring->doorbell_index, ring->wptr);
4521 }
4522
4523 /**
4524 * cik_cp_compute_enable - enable/disable the compute CP MEs
4525 *
4526 * @rdev: radeon_device pointer
4527 * @enable: enable or disable the MEs
4528 *
4529 * Halts or unhalts the compute MEs.
4530 */
4531 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4532 {
4533 if (enable)
4534 WREG32(CP_MEC_CNTL, 0);
4535 else {
4536 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4537 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4538 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4539 }
4540 udelay(50);
4541 }
4542
4543 /**
4544 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4545 *
4546 * @rdev: radeon_device pointer
4547 *
4548 * Loads the compute MEC1&2 ucode.
4549 * Returns 0 for success, -EINVAL if the ucode is not available.
4550 */
4551 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4552 {
4553 int i;
4554
4555 if (!rdev->mec_fw)
4556 return -EINVAL;
4557
4558 cik_cp_compute_enable(rdev, false);
4559
4560 if (rdev->new_fw) {
4561 const struct gfx_firmware_header_v1_0 *mec_hdr =
4562 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4563 const __le32 *fw_data;
4564 u32 fw_size;
4565
4566 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4567
4568 /* MEC1 */
4569 fw_data = (const __le32 *)
4570 ((const char *)rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4571 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4572 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4573 for (i = 0; i < fw_size; i++)
4574 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4575 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4576
4577 /* MEC2 */
4578 if (rdev->family == CHIP_KAVERI) {
4579 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4580 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4581
4582 fw_data = (const __le32 *)
4583 ((const char *)rdev->mec2_fw->data +
4584 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4585 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4586 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4587 for (i = 0; i < fw_size; i++)
4588 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4589 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4590 }
4591 } else {
4592 const __be32 *fw_data;
4593
4594 /* MEC1 */
4595 fw_data = (const __be32 *)rdev->mec_fw->data;
4596 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4597 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4598 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4599 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4600
4601 if (rdev->family == CHIP_KAVERI) {
4602 /* MEC2 */
4603 fw_data = (const __be32 *)rdev->mec_fw->data;
4604 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4605 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4606 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4607 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4608 }
4609 }
4610
4611 return 0;
4612 }
4613
4614 /**
4615 * cik_cp_compute_start - start the compute queues
4616 *
4617 * @rdev: radeon_device pointer
4618 *
4619 * Enable the compute queues.
4620 * Returns 0 for success, error for failure.
4621 */
4622 static int cik_cp_compute_start(struct radeon_device *rdev)
4623 {
4624 cik_cp_compute_enable(rdev, true);
4625
4626 return 0;
4627 }
4628
4629 /**
4630 * cik_cp_compute_fini - stop the compute queues
4631 *
4632 * @rdev: radeon_device pointer
4633 *
4634 * Stop the compute queues and tear down the driver queue
4635 * info.
4636 */
4637 static void cik_cp_compute_fini(struct radeon_device *rdev)
4638 {
4639 int i, idx, r;
4640
4641 cik_cp_compute_enable(rdev, false);
4642
4643 for (i = 0; i < 2; i++) {
4644 if (i == 0)
4645 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4646 else
4647 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4648
4649 if (rdev->ring[idx].mqd_obj) {
4650 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4651 if (unlikely(r != 0))
4652 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4653
4654 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4655 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4656
4657 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4658 rdev->ring[idx].mqd_obj = NULL;
4659 }
4660 }
4661 }
4662
4663 static void cik_mec_fini(struct radeon_device *rdev)
4664 {
4665 int r;
4666
4667 if (rdev->mec.hpd_eop_obj) {
4668 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4669 if (unlikely(r != 0))
4670 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4671 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4672 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4673
4674 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4675 rdev->mec.hpd_eop_obj = NULL;
4676 }
4677 }
4678
4679 #define MEC_HPD_SIZE 2048
4680
4681 static int cik_mec_init(struct radeon_device *rdev)
4682 {
4683 int r;
4684 u32 *hpd;
4685
4686 /*
4687 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4688 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4689 */
4690 if (rdev->family == CHIP_KAVERI)
4691 rdev->mec.num_mec = 2;
4692 else
4693 rdev->mec.num_mec = 1;
4694 rdev->mec.num_pipe = 4;
4695 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4696
4697 if (rdev->mec.hpd_eop_obj == NULL) {
4698 r = radeon_bo_create(rdev,
4699 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4700 PAGE_SIZE, true,
4701 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4702 &rdev->mec.hpd_eop_obj);
4703 if (r) {
4704 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4705 return r;
4706 }
4707 }
4708
4709 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4710 if (unlikely(r != 0)) {
4711 cik_mec_fini(rdev);
4712 return r;
4713 }
4714 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4715 &rdev->mec.hpd_eop_gpu_addr);
4716 if (r) {
4717 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4718 cik_mec_fini(rdev);
4719 return r;
4720 }
4721 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4722 if (r) {
4723 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4724 cik_mec_fini(rdev);
4725 return r;
4726 }
4727
4728 /* clear memory. Not sure if this is required or not */
4729 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4730
4731 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4732 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4733
4734 return 0;
4735 }
4736
4737 struct hqd_registers
4738 {
4739 u32 cp_mqd_base_addr;
4740 u32 cp_mqd_base_addr_hi;
4741 u32 cp_hqd_active;
4742 u32 cp_hqd_vmid;
4743 u32 cp_hqd_persistent_state;
4744 u32 cp_hqd_pipe_priority;
4745 u32 cp_hqd_queue_priority;
4746 u32 cp_hqd_quantum;
4747 u32 cp_hqd_pq_base;
4748 u32 cp_hqd_pq_base_hi;
4749 u32 cp_hqd_pq_rptr;
4750 u32 cp_hqd_pq_rptr_report_addr;
4751 u32 cp_hqd_pq_rptr_report_addr_hi;
4752 u32 cp_hqd_pq_wptr_poll_addr;
4753 u32 cp_hqd_pq_wptr_poll_addr_hi;
4754 u32 cp_hqd_pq_doorbell_control;
4755 u32 cp_hqd_pq_wptr;
4756 u32 cp_hqd_pq_control;
4757 u32 cp_hqd_ib_base_addr;
4758 u32 cp_hqd_ib_base_addr_hi;
4759 u32 cp_hqd_ib_rptr;
4760 u32 cp_hqd_ib_control;
4761 u32 cp_hqd_iq_timer;
4762 u32 cp_hqd_iq_rptr;
4763 u32 cp_hqd_dequeue_request;
4764 u32 cp_hqd_dma_offload;
4765 u32 cp_hqd_sema_cmd;
4766 u32 cp_hqd_msg_type;
4767 u32 cp_hqd_atomic0_preop_lo;
4768 u32 cp_hqd_atomic0_preop_hi;
4769 u32 cp_hqd_atomic1_preop_lo;
4770 u32 cp_hqd_atomic1_preop_hi;
4771 u32 cp_hqd_hq_scheduler0;
4772 u32 cp_hqd_hq_scheduler1;
4773 u32 cp_mqd_control;
4774 };
4775
4776 struct bonaire_mqd
4777 {
4778 u32 header;
4779 u32 dispatch_initiator;
4780 u32 dimensions[3];
4781 u32 start_idx[3];
4782 u32 num_threads[3];
4783 u32 pipeline_stat_enable;
4784 u32 perf_counter_enable;
4785 u32 pgm[2];
4786 u32 tba[2];
4787 u32 tma[2];
4788 u32 pgm_rsrc[2];
4789 u32 vmid;
4790 u32 resource_limits;
4791 u32 static_thread_mgmt01[2];
4792 u32 tmp_ring_size;
4793 u32 static_thread_mgmt23[2];
4794 u32 restart[3];
4795 u32 thread_trace_enable;
4796 u32 reserved1;
4797 u32 user_data[16];
4798 u32 vgtcs_invoke_count[2];
4799 struct hqd_registers queue_state;
4800 u32 dequeue_cntr;
4801 u32 interrupt_queue[64];
4802 };
4803
4804 /**
4805 * cik_cp_compute_resume - setup the compute queue registers
4806 *
4807 * @rdev: radeon_device pointer
4808 *
4809 * Program the compute queues and test them to make sure they
4810 * are working.
4811 * Returns 0 for success, error for failure.
4812 */
4813 static int cik_cp_compute_resume(struct radeon_device *rdev)
4814 {
4815 int r, i, j, idx;
4816 u32 tmp;
4817 bool use_doorbell = true;
4818 u64 hqd_gpu_addr;
4819 u64 mqd_gpu_addr;
4820 u64 eop_gpu_addr;
4821 u64 wb_gpu_addr;
4822 u32 *buf;
4823 struct bonaire_mqd *mqd;
4824
4825 r = cik_cp_compute_start(rdev);
4826 if (r)
4827 return r;
4828
4829 /* fix up chicken bits */
4830 tmp = RREG32(CP_CPF_DEBUG);
4831 tmp |= (1 << 23);
4832 WREG32(CP_CPF_DEBUG, tmp);
4833
4834 /* init the pipes */
4835 spin_lock(&rdev->srbm_mutex);
4836 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4837 int me = (i < 4) ? 1 : 2;
4838 int pipe = (i < 4) ? i : (i - 4);
4839
4840 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4841
4842 cik_srbm_select(rdev, me, pipe, 0, 0);
4843
4844 /* write the EOP addr */
4845 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4846 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4847
4848 /* set the VMID assigned */
4849 WREG32(CP_HPD_EOP_VMID, 0);
4850
4851 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4852 tmp = RREG32(CP_HPD_EOP_CONTROL);
4853 tmp &= ~EOP_SIZE_MASK;
4854 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4855 WREG32(CP_HPD_EOP_CONTROL, tmp);
4856 }
4857 cik_srbm_select(rdev, 0, 0, 0, 0);
4858 spin_unlock(&rdev->srbm_mutex);
4859
4860 /* init the queues. Just two for now. */
4861 for (i = 0; i < 2; i++) {
4862 if (i == 0)
4863 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4864 else
4865 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4866
4867 if (rdev->ring[idx].mqd_obj == NULL) {
4868 r = radeon_bo_create(rdev,
4869 sizeof(struct bonaire_mqd),
4870 PAGE_SIZE, true,
4871 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4872 &rdev->ring[idx].mqd_obj);
4873 if (r) {
4874 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4875 return r;
4876 }
4877 }
4878
4879 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4880 if (unlikely(r != 0)) {
4881 cik_cp_compute_fini(rdev);
4882 return r;
4883 }
4884 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4885 &mqd_gpu_addr);
4886 if (r) {
4887 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4888 cik_cp_compute_fini(rdev);
4889 return r;
4890 }
4891 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4892 if (r) {
4893 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4894 cik_cp_compute_fini(rdev);
4895 return r;
4896 }
4897
4898 /* init the mqd struct */
4899 memset(buf, 0, sizeof(struct bonaire_mqd));
4900
4901 mqd = (struct bonaire_mqd *)buf;
4902 mqd->header = 0xC0310800;
4903 mqd->static_thread_mgmt01[0] = 0xffffffff;
4904 mqd->static_thread_mgmt01[1] = 0xffffffff;
4905 mqd->static_thread_mgmt23[0] = 0xffffffff;
4906 mqd->static_thread_mgmt23[1] = 0xffffffff;
4907
4908 spin_lock(&rdev->srbm_mutex);
4909 cik_srbm_select(rdev, rdev->ring[idx].me,
4910 rdev->ring[idx].pipe,
4911 rdev->ring[idx].queue, 0);
4912
4913 /* disable wptr polling */
4914 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4915 tmp &= ~WPTR_POLL_EN;
4916 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4917
4918 /* enable doorbell? */
4919 mqd->queue_state.cp_hqd_pq_doorbell_control =
4920 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4921 if (use_doorbell)
4922 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4923 else
4924 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4925 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4926 mqd->queue_state.cp_hqd_pq_doorbell_control);
4927
4928 /* disable the queue if it's active */
4929 mqd->queue_state.cp_hqd_dequeue_request = 0;
4930 mqd->queue_state.cp_hqd_pq_rptr = 0;
4931 mqd->queue_state.cp_hqd_pq_wptr= 0;
4932 if (RREG32(CP_HQD_ACTIVE) & 1) {
4933 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4934 for (j = 0; j < rdev->usec_timeout; j++) {
4935 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4936 break;
4937 udelay(1);
4938 }
4939 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4940 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4941 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4942 }
4943
4944 /* set the pointer to the MQD */
4945 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4946 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4947 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4948 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4949 /* set MQD vmid to 0 */
4950 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4951 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4952 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4953
4954 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4955 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4956 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4957 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4958 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4959 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4960
4961 /* set up the HQD, this is similar to CP_RB0_CNTL */
4962 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4963 mqd->queue_state.cp_hqd_pq_control &=
4964 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4965
4966 mqd->queue_state.cp_hqd_pq_control |=
4967 order_base_2(rdev->ring[idx].ring_size / 8);
4968 mqd->queue_state.cp_hqd_pq_control |=
4969 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4970 #ifdef __BIG_ENDIAN
4971 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4972 #endif
4973 mqd->queue_state.cp_hqd_pq_control &=
4974 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4975 mqd->queue_state.cp_hqd_pq_control |=
4976 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4977 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4978
4979 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4980 if (i == 0)
4981 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4982 else
4983 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4984 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4985 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4986 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4987 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4988 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4989
4990 /* set the wb address wether it's enabled or not */
4991 if (i == 0)
4992 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4993 else
4994 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4995 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4996 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4997 upper_32_bits(wb_gpu_addr) & 0xffff;
4998 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4999 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5000 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5001 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5002
5003 /* enable the doorbell if requested */
5004 if (use_doorbell) {
5005 mqd->queue_state.cp_hqd_pq_doorbell_control =
5006 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5007 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5008 mqd->queue_state.cp_hqd_pq_doorbell_control |=
5009 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5010 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5011 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5012 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5013
5014 } else {
5015 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5016 }
5017 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5018 mqd->queue_state.cp_hqd_pq_doorbell_control);
5019
5020 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5021 rdev->ring[idx].wptr = 0;
5022 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5023 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5024 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5025
5026 /* set the vmid for the queue */
5027 mqd->queue_state.cp_hqd_vmid = 0;
5028 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5029
5030 /* activate the queue */
5031 mqd->queue_state.cp_hqd_active = 1;
5032 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5033
5034 cik_srbm_select(rdev, 0, 0, 0, 0);
5035 spin_unlock(&rdev->srbm_mutex);
5036
5037 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5038 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5039
5040 rdev->ring[idx].ready = true;
5041 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5042 if (r)
5043 rdev->ring[idx].ready = false;
5044 }
5045
5046 return 0;
5047 }
5048
5049 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5050 {
5051 cik_cp_gfx_enable(rdev, enable);
5052 cik_cp_compute_enable(rdev, enable);
5053 }
5054
5055 static int cik_cp_load_microcode(struct radeon_device *rdev)
5056 {
5057 int r;
5058
5059 r = cik_cp_gfx_load_microcode(rdev);
5060 if (r)
5061 return r;
5062 r = cik_cp_compute_load_microcode(rdev);
5063 if (r)
5064 return r;
5065
5066 return 0;
5067 }
5068
5069 static void cik_cp_fini(struct radeon_device *rdev)
5070 {
5071 cik_cp_gfx_fini(rdev);
5072 cik_cp_compute_fini(rdev);
5073 }
5074
5075 static int cik_cp_resume(struct radeon_device *rdev)
5076 {
5077 int r;
5078
5079 cik_enable_gui_idle_interrupt(rdev, false);
5080
5081 r = cik_cp_load_microcode(rdev);
5082 if (r)
5083 return r;
5084
5085 r = cik_cp_gfx_resume(rdev);
5086 if (r)
5087 return r;
5088 r = cik_cp_compute_resume(rdev);
5089 if (r)
5090 return r;
5091
5092 cik_enable_gui_idle_interrupt(rdev, true);
5093
5094 return 0;
5095 }
5096
5097 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5098 {
5099 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5100 RREG32(GRBM_STATUS));
5101 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5102 RREG32(GRBM_STATUS2));
5103 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5104 RREG32(GRBM_STATUS_SE0));
5105 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5106 RREG32(GRBM_STATUS_SE1));
5107 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5108 RREG32(GRBM_STATUS_SE2));
5109 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5110 RREG32(GRBM_STATUS_SE3));
5111 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5112 RREG32(SRBM_STATUS));
5113 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5114 RREG32(SRBM_STATUS2));
5115 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5116 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5117 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5118 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5119 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5120 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5121 RREG32(CP_STALLED_STAT1));
5122 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5123 RREG32(CP_STALLED_STAT2));
5124 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5125 RREG32(CP_STALLED_STAT3));
5126 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5127 RREG32(CP_CPF_BUSY_STAT));
5128 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5129 RREG32(CP_CPF_STALLED_STAT1));
5130 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5131 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5132 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5133 RREG32(CP_CPC_STALLED_STAT1));
5134 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5135 }
5136
5137 /**
5138 * cik_gpu_check_soft_reset - check which blocks are busy
5139 *
5140 * @rdev: radeon_device pointer
5141 *
5142 * Check which blocks are busy and return the relevant reset
5143 * mask to be used by cik_gpu_soft_reset().
5144 * Returns a mask of the blocks to be reset.
5145 */
5146 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5147 {
5148 u32 reset_mask = 0;
5149 u32 tmp;
5150
5151 /* GRBM_STATUS */
5152 tmp = RREG32(GRBM_STATUS);
5153 if (tmp & (PA_BUSY | SC_BUSY |
5154 BCI_BUSY | SX_BUSY |
5155 TA_BUSY | VGT_BUSY |
5156 DB_BUSY | CB_BUSY |
5157 GDS_BUSY | SPI_BUSY |
5158 IA_BUSY | IA_BUSY_NO_DMA))
5159 reset_mask |= RADEON_RESET_GFX;
5160
5161 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5162 reset_mask |= RADEON_RESET_CP;
5163
5164 /* GRBM_STATUS2 */
5165 tmp = RREG32(GRBM_STATUS2);
5166 if (tmp & RLC_BUSY)
5167 reset_mask |= RADEON_RESET_RLC;
5168
5169 /* SDMA0_STATUS_REG */
5170 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5171 if (!(tmp & SDMA_IDLE))
5172 reset_mask |= RADEON_RESET_DMA;
5173
5174 /* SDMA1_STATUS_REG */
5175 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5176 if (!(tmp & SDMA_IDLE))
5177 reset_mask |= RADEON_RESET_DMA1;
5178
5179 /* SRBM_STATUS2 */
5180 tmp = RREG32(SRBM_STATUS2);
5181 if (tmp & SDMA_BUSY)
5182 reset_mask |= RADEON_RESET_DMA;
5183
5184 if (tmp & SDMA1_BUSY)
5185 reset_mask |= RADEON_RESET_DMA1;
5186
5187 /* SRBM_STATUS */
5188 tmp = RREG32(SRBM_STATUS);
5189
5190 if (tmp & IH_BUSY)
5191 reset_mask |= RADEON_RESET_IH;
5192
5193 if (tmp & SEM_BUSY)
5194 reset_mask |= RADEON_RESET_SEM;
5195
5196 if (tmp & GRBM_RQ_PENDING)
5197 reset_mask |= RADEON_RESET_GRBM;
5198
5199 if (tmp & VMC_BUSY)
5200 reset_mask |= RADEON_RESET_VMC;
5201
5202 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5203 MCC_BUSY | MCD_BUSY))
5204 reset_mask |= RADEON_RESET_MC;
5205
5206 if (evergreen_is_display_hung(rdev))
5207 reset_mask |= RADEON_RESET_DISPLAY;
5208
5209 /* Skip MC reset as it's mostly likely not hung, just busy */
5210 if (reset_mask & RADEON_RESET_MC) {
5211 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5212 reset_mask &= ~RADEON_RESET_MC;
5213 }
5214
5215 return reset_mask;
5216 }
5217
5218 /**
5219 * cik_gpu_soft_reset - soft reset GPU
5220 *
5221 * @rdev: radeon_device pointer
5222 * @reset_mask: mask of which blocks to reset
5223 *
5224 * Soft reset the blocks specified in @reset_mask.
5225 */
5226 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5227 {
5228 struct evergreen_mc_save save;
5229 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5230 u32 tmp;
5231
5232 if (reset_mask == 0)
5233 return;
5234
5235 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5236
5237 cik_print_gpu_status_regs(rdev);
5238 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5239 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5240 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5241 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5242
5243 /* disable CG/PG */
5244 cik_fini_pg(rdev);
5245 cik_fini_cg(rdev);
5246
5247 /* stop the rlc */
5248 cik_rlc_stop(rdev);
5249
5250 /* Disable GFX parsing/prefetching */
5251 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5252
5253 /* Disable MEC parsing/prefetching */
5254 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5255
5256 if (reset_mask & RADEON_RESET_DMA) {
5257 /* sdma0 */
5258 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5259 tmp |= SDMA_HALT;
5260 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5261 }
5262 if (reset_mask & RADEON_RESET_DMA1) {
5263 /* sdma1 */
5264 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5265 tmp |= SDMA_HALT;
5266 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5267 }
5268
5269 evergreen_mc_stop(rdev, &save);
5270 if (evergreen_mc_wait_for_idle(rdev)) {
5271 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5272 }
5273
5274 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5275 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5276
5277 if (reset_mask & RADEON_RESET_CP) {
5278 grbm_soft_reset |= SOFT_RESET_CP;
5279
5280 srbm_soft_reset |= SOFT_RESET_GRBM;
5281 }
5282
5283 if (reset_mask & RADEON_RESET_DMA)
5284 srbm_soft_reset |= SOFT_RESET_SDMA;
5285
5286 if (reset_mask & RADEON_RESET_DMA1)
5287 srbm_soft_reset |= SOFT_RESET_SDMA1;
5288
5289 if (reset_mask & RADEON_RESET_DISPLAY)
5290 srbm_soft_reset |= SOFT_RESET_DC;
5291
5292 if (reset_mask & RADEON_RESET_RLC)
5293 grbm_soft_reset |= SOFT_RESET_RLC;
5294
5295 if (reset_mask & RADEON_RESET_SEM)
5296 srbm_soft_reset |= SOFT_RESET_SEM;
5297
5298 if (reset_mask & RADEON_RESET_IH)
5299 srbm_soft_reset |= SOFT_RESET_IH;
5300
5301 if (reset_mask & RADEON_RESET_GRBM)
5302 srbm_soft_reset |= SOFT_RESET_GRBM;
5303
5304 if (reset_mask & RADEON_RESET_VMC)
5305 srbm_soft_reset |= SOFT_RESET_VMC;
5306
5307 if (!(rdev->flags & RADEON_IS_IGP)) {
5308 if (reset_mask & RADEON_RESET_MC)
5309 srbm_soft_reset |= SOFT_RESET_MC;
5310 }
5311
5312 if (grbm_soft_reset) {
5313 tmp = RREG32(GRBM_SOFT_RESET);
5314 tmp |= grbm_soft_reset;
5315 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5316 WREG32(GRBM_SOFT_RESET, tmp);
5317 tmp = RREG32(GRBM_SOFT_RESET);
5318
5319 udelay(50);
5320
5321 tmp &= ~grbm_soft_reset;
5322 WREG32(GRBM_SOFT_RESET, tmp);
5323 tmp = RREG32(GRBM_SOFT_RESET);
5324 }
5325
5326 if (srbm_soft_reset) {
5327 tmp = RREG32(SRBM_SOFT_RESET);
5328 tmp |= srbm_soft_reset;
5329 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5330 WREG32(SRBM_SOFT_RESET, tmp);
5331 tmp = RREG32(SRBM_SOFT_RESET);
5332
5333 udelay(50);
5334
5335 tmp &= ~srbm_soft_reset;
5336 WREG32(SRBM_SOFT_RESET, tmp);
5337 tmp = RREG32(SRBM_SOFT_RESET);
5338 }
5339
5340 /* Wait a little for things to settle down */
5341 udelay(50);
5342
5343 evergreen_mc_resume(rdev, &save);
5344 udelay(50);
5345
5346 cik_print_gpu_status_regs(rdev);
5347 }
5348
5349 struct kv_reset_save_regs {
5350 u32 gmcon_reng_execute;
5351 u32 gmcon_misc;
5352 u32 gmcon_misc3;
5353 };
5354
5355 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5356 struct kv_reset_save_regs *save)
5357 {
5358 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5359 save->gmcon_misc = RREG32(GMCON_MISC);
5360 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5361
5362 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5363 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5364 STCTRL_STUTTER_EN));
5365 }
5366
5367 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5368 struct kv_reset_save_regs *save)
5369 {
5370 int i;
5371
5372 WREG32(GMCON_PGFSM_WRITE, 0);
5373 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5374
5375 for (i = 0; i < 5; i++)
5376 WREG32(GMCON_PGFSM_WRITE, 0);
5377
5378 WREG32(GMCON_PGFSM_WRITE, 0);
5379 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5380
5381 for (i = 0; i < 5; i++)
5382 WREG32(GMCON_PGFSM_WRITE, 0);
5383
5384 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5385 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5386
5387 for (i = 0; i < 5; i++)
5388 WREG32(GMCON_PGFSM_WRITE, 0);
5389
5390 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5391 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5392
5393 for (i = 0; i < 5; i++)
5394 WREG32(GMCON_PGFSM_WRITE, 0);
5395
5396 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5397 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5398
5399 for (i = 0; i < 5; i++)
5400 WREG32(GMCON_PGFSM_WRITE, 0);
5401
5402 WREG32(GMCON_PGFSM_WRITE, 0);
5403 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5404
5405 for (i = 0; i < 5; i++)
5406 WREG32(GMCON_PGFSM_WRITE, 0);
5407
5408 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5409 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5410
5411 for (i = 0; i < 5; i++)
5412 WREG32(GMCON_PGFSM_WRITE, 0);
5413
5414 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5415 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5416
5417 for (i = 0; i < 5; i++)
5418 WREG32(GMCON_PGFSM_WRITE, 0);
5419
5420 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5421 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5422
5423 for (i = 0; i < 5; i++)
5424 WREG32(GMCON_PGFSM_WRITE, 0);
5425
5426 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5427 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5428
5429 for (i = 0; i < 5; i++)
5430 WREG32(GMCON_PGFSM_WRITE, 0);
5431
5432 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5433 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5434
5435 WREG32(GMCON_MISC3, save->gmcon_misc3);
5436 WREG32(GMCON_MISC, save->gmcon_misc);
5437 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5438 }
5439
5440 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5441 {
5442 struct evergreen_mc_save save;
5443 struct kv_reset_save_regs kv_save = { 0 };
5444 u32 tmp, i;
5445
5446 dev_info(rdev->dev, "GPU pci config reset\n");
5447
5448 /* disable dpm? */
5449
5450 /* disable cg/pg */
5451 cik_fini_pg(rdev);
5452 cik_fini_cg(rdev);
5453
5454 /* Disable GFX parsing/prefetching */
5455 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5456
5457 /* Disable MEC parsing/prefetching */
5458 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5459
5460 /* sdma0 */
5461 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5462 tmp |= SDMA_HALT;
5463 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5464 /* sdma1 */
5465 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5466 tmp |= SDMA_HALT;
5467 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5468 /* XXX other engines? */
5469
5470 /* halt the rlc, disable cp internal ints */
5471 cik_rlc_stop(rdev);
5472
5473 udelay(50);
5474
5475 /* disable mem access */
5476 evergreen_mc_stop(rdev, &save);
5477 if (evergreen_mc_wait_for_idle(rdev)) {
5478 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5479 }
5480
5481 if (rdev->flags & RADEON_IS_IGP)
5482 kv_save_regs_for_reset(rdev, &kv_save);
5483
5484 /* disable BM */
5485 pci_disable_busmaster(rdev->pdev->dev.bsddev);
5486 /* reset */
5487 radeon_pci_config_reset(rdev);
5488
5489 udelay(100);
5490
5491 /* wait for asic to come out of reset */
5492 for (i = 0; i < rdev->usec_timeout; i++) {
5493 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5494 break;
5495 udelay(1);
5496 }
5497
5498 /* does asic init need to be run first??? */
5499 if (rdev->flags & RADEON_IS_IGP)
5500 kv_restore_regs_for_reset(rdev, &kv_save);
5501 }
5502
5503 /**
5504 * cik_asic_reset - soft reset GPU
5505 *
5506 * @rdev: radeon_device pointer
5507 *
5508 * Look up which blocks are hung and attempt
5509 * to reset them.
5510 * Returns 0 for success.
5511 */
5512 int cik_asic_reset(struct radeon_device *rdev)
5513 {
5514 u32 reset_mask;
5515
5516 reset_mask = cik_gpu_check_soft_reset(rdev);
5517
5518 if (reset_mask)
5519 r600_set_bios_scratch_engine_hung(rdev, true);
5520
5521 /* try soft reset */
5522 cik_gpu_soft_reset(rdev, reset_mask);
5523
5524 reset_mask = cik_gpu_check_soft_reset(rdev);
5525
5526 /* try pci config reset */
5527 if (reset_mask && radeon_hard_reset)
5528 cik_gpu_pci_config_reset(rdev);
5529
5530 reset_mask = cik_gpu_check_soft_reset(rdev);
5531
5532 if (!reset_mask)
5533 r600_set_bios_scratch_engine_hung(rdev, false);
5534
5535 return 0;
5536 }
5537
5538 /**
5539 * cik_gfx_is_lockup - check if the 3D engine is locked up
5540 *
5541 * @rdev: radeon_device pointer
5542 * @ring: radeon_ring structure holding ring information
5543 *
5544 * Check if the 3D engine is locked up (CIK).
5545 * Returns true if the engine is locked, false if not.
5546 */
5547 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5548 {
5549 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5550
5551 if (!(reset_mask & (RADEON_RESET_GFX |
5552 RADEON_RESET_COMPUTE |
5553 RADEON_RESET_CP))) {
5554 radeon_ring_lockup_update(rdev, ring);
5555 return false;
5556 }
5557 return radeon_ring_test_lockup(rdev, ring);
5558 }
5559
5560 /* MC */
5561 /**
5562 * cik_mc_program - program the GPU memory controller
5563 *
5564 * @rdev: radeon_device pointer
5565 *
5566 * Set the location of vram, gart, and AGP in the GPU's
5567 * physical address space (CIK).
5568 */
5569 static void cik_mc_program(struct radeon_device *rdev)
5570 {
5571 struct evergreen_mc_save save;
5572 u32 tmp;
5573 int i, j;
5574
5575 /* Initialize HDP */
5576 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5577 WREG32((0x2c14 + j), 0x00000000);
5578 WREG32((0x2c18 + j), 0x00000000);
5579 WREG32((0x2c1c + j), 0x00000000);
5580 WREG32((0x2c20 + j), 0x00000000);
5581 WREG32((0x2c24 + j), 0x00000000);
5582 }
5583 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5584
5585 evergreen_mc_stop(rdev, &save);
5586 if (radeon_mc_wait_for_idle(rdev)) {
5587 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5588 }
5589 /* Lockout access through VGA aperture*/
5590 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5591 /* Update configuration */
5592 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5593 rdev->mc.vram_start >> 12);
5594 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5595 rdev->mc.vram_end >> 12);
5596 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5597 rdev->vram_scratch.gpu_addr >> 12);
5598 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5599 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5600 WREG32(MC_VM_FB_LOCATION, tmp);
5601 /* XXX double check these! */
5602 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5603 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5604 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5605 WREG32(MC_VM_AGP_BASE, 0);
5606 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5607 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5608 if (radeon_mc_wait_for_idle(rdev)) {
5609 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5610 }
5611 evergreen_mc_resume(rdev, &save);
5612 /* we need to own VRAM, so turn off the VGA renderer here
5613 * to stop it overwriting our objects */
5614 rv515_vga_render_disable(rdev);
5615 }
5616
5617 /**
5618 * cik_mc_init - initialize the memory controller driver params
5619 *
5620 * @rdev: radeon_device pointer
5621 *
5622 * Look up the amount of vram, vram width, and decide how to place
5623 * vram and gart within the GPU's physical address space (CIK).
5624 * Returns 0 for success.
5625 */
5626 static int cik_mc_init(struct radeon_device *rdev)
5627 {
5628 u32 tmp;
5629 int chansize, numchan;
5630
5631 /* Get VRAM informations */
5632 rdev->mc.vram_is_ddr = true;
5633 tmp = RREG32(MC_ARB_RAMCFG);
5634 if (tmp & CHANSIZE_MASK) {
5635 chansize = 64;
5636 } else {
5637 chansize = 32;
5638 }
5639 tmp = RREG32(MC_SHARED_CHMAP);
5640 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5641 case 0:
5642 default:
5643 numchan = 1;
5644 break;
5645 case 1:
5646 numchan = 2;
5647 break;
5648 case 2:
5649 numchan = 4;
5650 break;
5651 case 3:
5652 numchan = 8;
5653 break;
5654 case 4:
5655 numchan = 3;
5656 break;
5657 case 5:
5658 numchan = 6;
5659 break;
5660 case 6:
5661 numchan = 10;
5662 break;
5663 case 7:
5664 numchan = 12;
5665 break;
5666 case 8:
5667 numchan = 16;
5668 break;
5669 }
5670 rdev->mc.vram_width = numchan * chansize;
5671 /* Could aper size report 0 ? */
5672 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5673 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5674 /* size in MB on si */
5675 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5676 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5677 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5678 si_vram_gtt_location(rdev, &rdev->mc);
5679 radeon_update_bandwidth_info(rdev);
5680
5681 return 0;
5682 }
5683
5684 /*
5685 * GART
5686 * VMID 0 is the physical GPU addresses as used by the kernel.
5687 * VMIDs 1-15 are used for userspace clients and are handled
5688 * by the radeon vm/hsa code.
5689 */
5690 /**
5691 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5692 *
5693 * @rdev: radeon_device pointer
5694 *
5695 * Flush the TLB for the VMID 0 page table (CIK).
5696 */
5697 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5698 {
5699 /* flush hdp cache */
5700 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5701
5702 /* bits 0-15 are the VM contexts0-15 */
5703 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5704 }
5705
5706 /**
5707 * cik_pcie_gart_enable - gart enable
5708 *
5709 * @rdev: radeon_device pointer
5710 *
5711 * This sets up the TLBs, programs the page tables for VMID0,
5712 * sets up the hw for VMIDs 1-15 which are allocated on
5713 * demand, and sets up the global locations for the LDS, GDS,
5714 * and GPUVM for FSA64 clients (CIK).
5715 * Returns 0 for success, errors for failure.
5716 */
5717 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5718 {
5719 int r, i;
5720
5721 if (rdev->gart.robj == NULL) {
5722 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5723 return -EINVAL;
5724 }
5725 r = radeon_gart_table_vram_pin(rdev);
5726 if (r)
5727 return r;
5728 /* Setup TLB control */
5729 WREG32(MC_VM_MX_L1_TLB_CNTL,
5730 (0xA << 7) |
5731 ENABLE_L1_TLB |
5732 ENABLE_L1_FRAGMENT_PROCESSING |
5733 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5734 ENABLE_ADVANCED_DRIVER_MODEL |
5735 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5736 /* Setup L2 cache */
5737 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5738 ENABLE_L2_FRAGMENT_PROCESSING |
5739 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5740 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5741 EFFECTIVE_L2_QUEUE_SIZE(7) |
5742 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5743 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5744 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5745 BANK_SELECT(4) |
5746 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5747 /* setup context0 */
5748 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5749 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5750 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5751 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5752 (u32)(rdev->dummy_page.addr >> 12));
5753 WREG32(VM_CONTEXT0_CNTL2, 0);
5754 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5755 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5756
5757 WREG32(0x15D4, 0);
5758 WREG32(0x15D8, 0);
5759 WREG32(0x15DC, 0);
5760
5761 /* restore context1-15 */
5762 /* set vm size, must be a multiple of 4 */
5763 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5764 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5765 for (i = 1; i < 16; i++) {
5766 if (i < 8)
5767 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5768 rdev->vm_manager.saved_table_addr[i]);
5769 else
5770 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5771 rdev->vm_manager.saved_table_addr[i]);
5772 }
5773
5774 /* enable context1-15 */
5775 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5776 (u32)(rdev->dummy_page.addr >> 12));
5777 WREG32(VM_CONTEXT1_CNTL2, 4);
5778 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5779 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5780 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5781 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5782 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5783 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5784 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5785 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5786 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5787 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5788 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5789 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5790 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5791 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5792
5793 if (rdev->family == CHIP_KAVERI) {
5794 u32 tmp = RREG32(CHUB_CONTROL);
5795 tmp &= ~BYPASS_VM;
5796 WREG32(CHUB_CONTROL, tmp);
5797 }
5798
5799 /* XXX SH_MEM regs */
5800 /* where to put LDS, scratch, GPUVM in FSA64 space */
5801 spin_lock(&rdev->srbm_mutex);
5802 for (i = 0; i < 16; i++) {
5803 cik_srbm_select(rdev, 0, 0, 0, i);
5804 /* CP and shaders */
5805 WREG32(SH_MEM_CONFIG, 0);
5806 WREG32(SH_MEM_APE1_BASE, 1);
5807 WREG32(SH_MEM_APE1_LIMIT, 0);
5808 WREG32(SH_MEM_BASES, 0);
5809 /* SDMA GFX */
5810 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5811 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5812 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5813 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5814 /* XXX SDMA RLC - todo */
5815 }
5816 cik_srbm_select(rdev, 0, 0, 0, 0);
5817 spin_unlock(&rdev->srbm_mutex);
5818
5819 cik_pcie_gart_tlb_flush(rdev);
5820 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5821 (unsigned)(rdev->mc.gtt_size >> 20),
5822 (unsigned long long)rdev->gart.table_addr);
5823 rdev->gart.ready = true;
5824 return 0;
5825 }
5826
5827 /**
5828 * cik_pcie_gart_disable - gart disable
5829 *
5830 * @rdev: radeon_device pointer
5831 *
5832 * This disables all VM page table (CIK).
5833 */
5834 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5835 {
5836 unsigned i;
5837
5838 for (i = 1; i < 16; ++i) {
5839 uint32_t reg;
5840 if (i < 8)
5841 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5842 else
5843 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5844 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5845 }
5846
5847 /* Disable all tables */
5848 WREG32(VM_CONTEXT0_CNTL, 0);
5849 WREG32(VM_CONTEXT1_CNTL, 0);
5850 /* Setup TLB control */
5851 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5852 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5853 /* Setup L2 cache */
5854 WREG32(VM_L2_CNTL,
5855 ENABLE_L2_FRAGMENT_PROCESSING |
5856 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5857 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5858 EFFECTIVE_L2_QUEUE_SIZE(7) |
5859 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5860 WREG32(VM_L2_CNTL2, 0);
5861 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5862 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5863 radeon_gart_table_vram_unpin(rdev);
5864 }
5865
5866 /**
5867 * cik_pcie_gart_fini - vm fini callback
5868 *
5869 * @rdev: radeon_device pointer
5870 *
5871 * Tears down the driver GART/VM setup (CIK).
5872 */
5873 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5874 {
5875 cik_pcie_gart_disable(rdev);
5876 radeon_gart_table_vram_free(rdev);
5877 radeon_gart_fini(rdev);
5878 }
5879
5880 /* vm parser */
5881 /**
5882 * cik_ib_parse - vm ib_parse callback
5883 *
5884 * @rdev: radeon_device pointer
5885 * @ib: indirect buffer pointer
5886 *
5887 * CIK uses hw IB checking so this is a nop (CIK).
5888 */
5889 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5890 {
5891 return 0;
5892 }
5893
5894 /*
5895 * vm
5896 * VMID 0 is the physical GPU addresses as used by the kernel.
5897 * VMIDs 1-15 are used for userspace clients and are handled
5898 * by the radeon vm/hsa code.
5899 */
5900 /**
5901 * cik_vm_init - cik vm init callback
5902 *
5903 * @rdev: radeon_device pointer
5904 *
5905 * Inits cik specific vm parameters (number of VMs, base of vram for
5906 * VMIDs 1-15) (CIK).
5907 * Returns 0 for success.
5908 */
5909 int cik_vm_init(struct radeon_device *rdev)
5910 {
5911 /* number of VMs */
5912 rdev->vm_manager.nvm = 16;
5913 /* base offset of vram pages */
5914 if (rdev->flags & RADEON_IS_IGP) {
5915 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5916 tmp <<= 22;
5917 rdev->vm_manager.vram_base_offset = tmp;
5918 } else
5919 rdev->vm_manager.vram_base_offset = 0;
5920
5921 return 0;
5922 }
5923
5924 /**
5925 * cik_vm_fini - cik vm fini callback
5926 *
5927 * @rdev: radeon_device pointer
5928 *
5929 * Tear down any asic specific VM setup (CIK).
5930 */
5931 void cik_vm_fini(struct radeon_device *rdev)
5932 {
5933 }
5934
5935 /**
5936 * cik_vm_decode_fault - print human readable fault info
5937 *
5938 * @rdev: radeon_device pointer
5939 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5940 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5941 *
5942 * Print human readable fault information (CIK).
5943 */
5944 static void cik_vm_decode_fault(struct radeon_device *rdev,
5945 u32 status, u32 addr, u32 mc_client)
5946 {
5947 u32 mc_id;
5948 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5949 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5950 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5951 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5952
5953 if (rdev->family == CHIP_HAWAII)
5954 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5955 else
5956 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5957
5958 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5959 protections, vmid, addr,
5960 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5961 block, mc_client, mc_id);
5962 }
5963
5964 /**
5965 * cik_vm_flush - cik vm flush using the CP
5966 *
5967 * @rdev: radeon_device pointer
5968 *
5969 * Update the page table base and flush the VM TLB
5970 * using the CP (CIK).
5971 */
5972 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5973 {
5974 struct radeon_ring *ring = &rdev->ring[ridx];
5975 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5976
5977 if (vm == NULL)
5978 return;
5979
5980 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5981 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5982 WRITE_DATA_DST_SEL(0)));
5983 if (vm->id < 8) {
5984 radeon_ring_write(ring,
5985 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5986 } else {
5987 radeon_ring_write(ring,
5988 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5989 }
5990 radeon_ring_write(ring, 0);
5991 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5992
5993 /* update SH_MEM_* regs */
5994 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5995 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5996 WRITE_DATA_DST_SEL(0)));
5997 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5998 radeon_ring_write(ring, 0);
5999 radeon_ring_write(ring, VMID(vm->id));
6000
6001 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
6002 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6003 WRITE_DATA_DST_SEL(0)));
6004 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6005 radeon_ring_write(ring, 0);
6006
6007 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6008 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6009 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6010 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
6011
6012 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6013 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6014 WRITE_DATA_DST_SEL(0)));
6015 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6016 radeon_ring_write(ring, 0);
6017 radeon_ring_write(ring, VMID(0));
6018
6019 /* HDP flush */
6020 cik_hdp_flush_cp_ring_emit(rdev, ridx);
6021
6022 /* bits 0-15 are the VM contexts0-15 */
6023 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6024 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6025 WRITE_DATA_DST_SEL(0)));
6026 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6027 radeon_ring_write(ring, 0);
6028 radeon_ring_write(ring, 1 << vm->id);
6029
6030 /* compute doesn't have PFP */
6031 if (usepfp) {
6032 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6033 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6034 radeon_ring_write(ring, 0x0);
6035 }
6036 }
6037
6038 /*
6039 * RLC
6040 * The RLC is a multi-purpose microengine that handles a
6041 * variety of functions, the most important of which is
6042 * the interrupt controller.
6043 */
6044 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6045 bool enable)
6046 {
6047 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6048
6049 if (enable)
6050 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6051 else
6052 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6053 WREG32(CP_INT_CNTL_RING0, tmp);
6054 }
6055
6056 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6057 {
6058 u32 tmp;
6059
6060 tmp = RREG32(RLC_LB_CNTL);
6061 if (enable)
6062 tmp |= LOAD_BALANCE_ENABLE;
6063 else
6064 tmp &= ~LOAD_BALANCE_ENABLE;
6065 WREG32(RLC_LB_CNTL, tmp);
6066 }
6067
6068 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6069 {
6070 u32 i, j, k;
6071 u32 mask;
6072
6073 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6074 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6075 cik_select_se_sh(rdev, i, j);
6076 for (k = 0; k < rdev->usec_timeout; k++) {
6077 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6078 break;
6079 udelay(1);
6080 }
6081 }
6082 }
6083 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6084
6085 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6086 for (k = 0; k < rdev->usec_timeout; k++) {
6087 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6088 break;
6089 udelay(1);
6090 }
6091 }
6092
6093 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6094 {
6095 u32 tmp;
6096
6097 tmp = RREG32(RLC_CNTL);
6098 if (tmp != rlc)
6099 WREG32(RLC_CNTL, rlc);
6100 }
6101
6102 static u32 cik_halt_rlc(struct radeon_device *rdev)
6103 {
6104 u32 data, orig;
6105
6106 orig = data = RREG32(RLC_CNTL);
6107
6108 if (data & RLC_ENABLE) {
6109 u32 i;
6110
6111 data &= ~RLC_ENABLE;
6112 WREG32(RLC_CNTL, data);
6113
6114 for (i = 0; i < rdev->usec_timeout; i++) {
6115 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6116 break;
6117 udelay(1);
6118 }
6119
6120 cik_wait_for_rlc_serdes(rdev);
6121 }
6122
6123 return orig;
6124 }
6125
6126 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6127 {
6128 u32 tmp, i, mask;
6129
6130 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6131 WREG32(RLC_GPR_REG2, tmp);
6132
6133 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6134 for (i = 0; i < rdev->usec_timeout; i++) {
6135 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6136 break;
6137 udelay(1);
6138 }
6139
6140 for (i = 0; i < rdev->usec_timeout; i++) {
6141 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6142 break;
6143 udelay(1);
6144 }
6145 }
6146
6147 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6148 {
6149 u32 tmp;
6150
6151 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6152 WREG32(RLC_GPR_REG2, tmp);
6153 }
6154
6155 /**
6156 * cik_rlc_stop - stop the RLC ME
6157 *
6158 * @rdev: radeon_device pointer
6159 *
6160 * Halt the RLC ME (MicroEngine) (CIK).
6161 */
6162 static void cik_rlc_stop(struct radeon_device *rdev)
6163 {
6164 WREG32(RLC_CNTL, 0);
6165
6166 cik_enable_gui_idle_interrupt(rdev, false);
6167
6168 cik_wait_for_rlc_serdes(rdev);
6169 }
6170
6171 /**
6172 * cik_rlc_start - start the RLC ME
6173 *
6174 * @rdev: radeon_device pointer
6175 *
6176 * Unhalt the RLC ME (MicroEngine) (CIK).
6177 */
6178 static void cik_rlc_start(struct radeon_device *rdev)
6179 {
6180 WREG32(RLC_CNTL, RLC_ENABLE);
6181
6182 cik_enable_gui_idle_interrupt(rdev, true);
6183
6184 udelay(50);
6185 }
6186
6187 /**
6188 * cik_rlc_resume - setup the RLC hw
6189 *
6190 * @rdev: radeon_device pointer
6191 *
6192 * Initialize the RLC registers, load the ucode,
6193 * and start the RLC (CIK).
6194 * Returns 0 for success, -EINVAL if the ucode is not available.
6195 */
6196 static int cik_rlc_resume(struct radeon_device *rdev)
6197 {
6198 u32 i, size, tmp;
6199
6200 if (!rdev->rlc_fw)
6201 return -EINVAL;
6202
6203 cik_rlc_stop(rdev);
6204
6205 /* disable CG */
6206 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6207 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6208
6209 si_rlc_reset(rdev);
6210
6211 cik_init_pg(rdev);
6212
6213 cik_init_cg(rdev);
6214
6215 WREG32(RLC_LB_CNTR_INIT, 0);
6216 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6217
6218 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6219 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6220 WREG32(RLC_LB_PARAMS, 0x00600408);
6221 WREG32(RLC_LB_CNTL, 0x80000004);
6222
6223 WREG32(RLC_MC_CNTL, 0);
6224 WREG32(RLC_UCODE_CNTL, 0);
6225
6226 if (rdev->new_fw) {
6227 const struct rlc_firmware_header_v1_0 *hdr =
6228 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6229 const __le32 *fw_data = (const __le32 *)
6230 ((const char *)rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6231
6232 radeon_ucode_print_rlc_hdr(&hdr->header);
6233
6234 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6235 WREG32(RLC_GPM_UCODE_ADDR, 0);
6236 for (i = 0; i < size; i++)
6237 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6238 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6239 } else {
6240 const __be32 *fw_data;
6241
6242 switch (rdev->family) {
6243 case CHIP_BONAIRE:
6244 case CHIP_HAWAII:
6245 default:
6246 size = BONAIRE_RLC_UCODE_SIZE;
6247 break;
6248 case CHIP_KAVERI:
6249 size = KV_RLC_UCODE_SIZE;
6250 break;
6251 case CHIP_KABINI:
6252 size = KB_RLC_UCODE_SIZE;
6253 break;
6254 case CHIP_MULLINS:
6255 size = ML_RLC_UCODE_SIZE;
6256 break;
6257 }
6258
6259 fw_data = (const __be32 *)rdev->rlc_fw->data;
6260 WREG32(RLC_GPM_UCODE_ADDR, 0);
6261 for (i = 0; i < size; i++)
6262 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6263 WREG32(RLC_GPM_UCODE_ADDR, 0);
6264 }
6265
6266 /* XXX - find out what chips support lbpw */
6267 cik_enable_lbpw(rdev, false);
6268
6269 if (rdev->family == CHIP_BONAIRE)
6270 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6271
6272 cik_rlc_start(rdev);
6273
6274 return 0;
6275 }
6276
6277 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6278 {
6279 u32 data, orig, tmp, tmp2;
6280
6281 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6282
6283 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6284 cik_enable_gui_idle_interrupt(rdev, true);
6285
6286 tmp = cik_halt_rlc(rdev);
6287
6288 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6289 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6290 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6291 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6292 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6293
6294 cik_update_rlc(rdev, tmp);
6295
6296 data |= CGCG_EN | CGLS_EN;
6297 } else {
6298 cik_enable_gui_idle_interrupt(rdev, false);
6299
6300 RREG32(CB_CGTT_SCLK_CTRL);
6301 RREG32(CB_CGTT_SCLK_CTRL);
6302 RREG32(CB_CGTT_SCLK_CTRL);
6303 RREG32(CB_CGTT_SCLK_CTRL);
6304
6305 data &= ~(CGCG_EN | CGLS_EN);
6306 }
6307
6308 if (orig != data)
6309 WREG32(RLC_CGCG_CGLS_CTRL, data);
6310
6311 }
6312
6313 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6314 {
6315 u32 data, orig, tmp = 0;
6316
6317 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6318 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6319 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6320 orig = data = RREG32(CP_MEM_SLP_CNTL);
6321 data |= CP_MEM_LS_EN;
6322 if (orig != data)
6323 WREG32(CP_MEM_SLP_CNTL, data);
6324 }
6325 }
6326
6327 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6328 data &= 0xfffffffd;
6329 if (orig != data)
6330 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6331
6332 tmp = cik_halt_rlc(rdev);
6333
6334 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6335 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6336 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6337 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6338 WREG32(RLC_SERDES_WR_CTRL, data);
6339
6340 cik_update_rlc(rdev, tmp);
6341
6342 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6343 orig = data = RREG32(CGTS_SM_CTRL_REG);
6344 data &= ~SM_MODE_MASK;
6345 data |= SM_MODE(0x2);
6346 data |= SM_MODE_ENABLE;
6347 data &= ~CGTS_OVERRIDE;
6348 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6349 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6350 data &= ~CGTS_LS_OVERRIDE;
6351 data &= ~ON_MONITOR_ADD_MASK;
6352 data |= ON_MONITOR_ADD_EN;
6353 data |= ON_MONITOR_ADD(0x96);
6354 if (orig != data)
6355 WREG32(CGTS_SM_CTRL_REG, data);
6356 }
6357 } else {
6358 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6359 data |= 0x00000002;
6360 if (orig != data)
6361 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6362
6363 data = RREG32(RLC_MEM_SLP_CNTL);
6364 if (data & RLC_MEM_LS_EN) {
6365 data &= ~RLC_MEM_LS_EN;
6366 WREG32(RLC_MEM_SLP_CNTL, data);
6367 }
6368
6369 data = RREG32(CP_MEM_SLP_CNTL);
6370 if (data & CP_MEM_LS_EN) {
6371 data &= ~CP_MEM_LS_EN;
6372 WREG32(CP_MEM_SLP_CNTL, data);
6373 }
6374
6375 orig = data = RREG32(CGTS_SM_CTRL_REG);
6376 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6377 if (orig != data)
6378 WREG32(CGTS_SM_CTRL_REG, data);
6379
6380 tmp = cik_halt_rlc(rdev);
6381
6382 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6383 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6384 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6385 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6386 WREG32(RLC_SERDES_WR_CTRL, data);
6387
6388 cik_update_rlc(rdev, tmp);
6389 }
6390 }
6391
6392 static const u32 mc_cg_registers[] =
6393 {
6394 MC_HUB_MISC_HUB_CG,
6395 MC_HUB_MISC_SIP_CG,
6396 MC_HUB_MISC_VM_CG,
6397 MC_XPB_CLK_GAT,
6398 ATC_MISC_CG,
6399 MC_CITF_MISC_WR_CG,
6400 MC_CITF_MISC_RD_CG,
6401 MC_CITF_MISC_VM_CG,
6402 VM_L2_CG,
6403 };
6404
6405 static void cik_enable_mc_ls(struct radeon_device *rdev,
6406 bool enable)
6407 {
6408 int i;
6409 u32 orig, data;
6410
6411 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6412 orig = data = RREG32(mc_cg_registers[i]);
6413 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6414 data |= MC_LS_ENABLE;
6415 else
6416 data &= ~MC_LS_ENABLE;
6417 if (data != orig)
6418 WREG32(mc_cg_registers[i], data);
6419 }
6420 }
6421
6422 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6423 bool enable)
6424 {
6425 int i;
6426 u32 orig, data;
6427
6428 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6429 orig = data = RREG32(mc_cg_registers[i]);
6430 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6431 data |= MC_CG_ENABLE;
6432 else
6433 data &= ~MC_CG_ENABLE;
6434 if (data != orig)
6435 WREG32(mc_cg_registers[i], data);
6436 }
6437 }
6438
6439 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6440 bool enable)
6441 {
6442 u32 orig, data;
6443
6444 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6445 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6446 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6447 } else {
6448 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6449 data |= 0xff000000;
6450 if (data != orig)
6451 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6452
6453 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6454 data |= 0xff000000;
6455 if (data != orig)
6456 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6457 }
6458 }
6459
6460 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6461 bool enable)
6462 {
6463 u32 orig, data;
6464
6465 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6466 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6467 data |= 0x100;
6468 if (orig != data)
6469 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6470
6471 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6472 data |= 0x100;
6473 if (orig != data)
6474 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6475 } else {
6476 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6477 data &= ~0x100;
6478 if (orig != data)
6479 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6480
6481 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6482 data &= ~0x100;
6483 if (orig != data)
6484 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6485 }
6486 }
6487
6488 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6489 bool enable)
6490 {
6491 u32 orig, data;
6492
6493 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6494 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6495 data = 0xfff;
6496 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6497
6498 orig = data = RREG32(UVD_CGC_CTRL);
6499 data |= DCM;
6500 if (orig != data)
6501 WREG32(UVD_CGC_CTRL, data);
6502 } else {
6503 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6504 data &= ~0xfff;
6505 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6506
6507 orig = data = RREG32(UVD_CGC_CTRL);
6508 data &= ~DCM;
6509 if (orig != data)
6510 WREG32(UVD_CGC_CTRL, data);
6511 }
6512 }
6513
6514 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6515 bool enable)
6516 {
6517 u32 orig, data;
6518
6519 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6520
6521 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6522 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6523 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6524 else
6525 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6526 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6527
6528 if (orig != data)
6529 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6530 }
6531
6532 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6533 bool enable)
6534 {
6535 u32 orig, data;
6536
6537 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6538
6539 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6540 data &= ~CLOCK_GATING_DIS;
6541 else
6542 data |= CLOCK_GATING_DIS;
6543
6544 if (orig != data)
6545 WREG32(HDP_HOST_PATH_CNTL, data);
6546 }
6547
6548 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6549 bool enable)
6550 {
6551 u32 orig, data;
6552
6553 orig = data = RREG32(HDP_MEM_POWER_LS);
6554
6555 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6556 data |= HDP_LS_ENABLE;
6557 else
6558 data &= ~HDP_LS_ENABLE;
6559
6560 if (orig != data)
6561 WREG32(HDP_MEM_POWER_LS, data);
6562 }
6563
6564 void cik_update_cg(struct radeon_device *rdev,
6565 u32 block, bool enable)
6566 {
6567
6568 if (block & RADEON_CG_BLOCK_GFX) {
6569 cik_enable_gui_idle_interrupt(rdev, false);
6570 /* order matters! */
6571 if (enable) {
6572 cik_enable_mgcg(rdev, true);
6573 cik_enable_cgcg(rdev, true);
6574 } else {
6575 cik_enable_cgcg(rdev, false);
6576 cik_enable_mgcg(rdev, false);
6577 }
6578 cik_enable_gui_idle_interrupt(rdev, true);
6579 }
6580
6581 if (block & RADEON_CG_BLOCK_MC) {
6582 if (!(rdev->flags & RADEON_IS_IGP)) {
6583 cik_enable_mc_mgcg(rdev, enable);
6584 cik_enable_mc_ls(rdev, enable);
6585 }
6586 }
6587
6588 if (block & RADEON_CG_BLOCK_SDMA) {
6589 cik_enable_sdma_mgcg(rdev, enable);
6590 cik_enable_sdma_mgls(rdev, enable);
6591 }
6592
6593 if (block & RADEON_CG_BLOCK_BIF) {
6594 cik_enable_bif_mgls(rdev, enable);
6595 }
6596
6597 if (block & RADEON_CG_BLOCK_UVD) {
6598 if (rdev->has_uvd)
6599 cik_enable_uvd_mgcg(rdev, enable);
6600 }
6601
6602 if (block & RADEON_CG_BLOCK_HDP) {
6603 cik_enable_hdp_mgcg(rdev, enable);
6604 cik_enable_hdp_ls(rdev, enable);
6605 }
6606
6607 if (block & RADEON_CG_BLOCK_VCE) {
6608 vce_v2_0_enable_mgcg(rdev, enable);
6609 }
6610 }
6611
6612 static void cik_init_cg(struct radeon_device *rdev)
6613 {
6614
6615 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6616
6617 if (rdev->has_uvd)
6618 si_init_uvd_internal_cg(rdev);
6619
6620 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6621 RADEON_CG_BLOCK_SDMA |
6622 RADEON_CG_BLOCK_BIF |
6623 RADEON_CG_BLOCK_UVD |
6624 RADEON_CG_BLOCK_HDP), true);
6625 }
6626
6627 static void cik_fini_cg(struct radeon_device *rdev)
6628 {
6629 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6630 RADEON_CG_BLOCK_SDMA |
6631 RADEON_CG_BLOCK_BIF |
6632 RADEON_CG_BLOCK_UVD |
6633 RADEON_CG_BLOCK_HDP), false);
6634
6635 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6636 }
6637
6638 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6639 bool enable)
6640 {
6641 u32 data, orig;
6642
6643 orig = data = RREG32(RLC_PG_CNTL);
6644 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6645 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6646 else
6647 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6648 if (orig != data)
6649 WREG32(RLC_PG_CNTL, data);
6650 }
6651
6652 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6653 bool enable)
6654 {
6655 u32 data, orig;
6656
6657 orig = data = RREG32(RLC_PG_CNTL);
6658 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6659 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6660 else
6661 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6662 if (orig != data)
6663 WREG32(RLC_PG_CNTL, data);
6664 }
6665
6666 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6667 {
6668 u32 data, orig;
6669
6670 orig = data = RREG32(RLC_PG_CNTL);
6671 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6672 data &= ~DISABLE_CP_PG;
6673 else
6674 data |= DISABLE_CP_PG;
6675 if (orig != data)
6676 WREG32(RLC_PG_CNTL, data);
6677 }
6678
6679 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6680 {
6681 u32 data, orig;
6682
6683 orig = data = RREG32(RLC_PG_CNTL);
6684 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6685 data &= ~DISABLE_GDS_PG;
6686 else
6687 data |= DISABLE_GDS_PG;
6688 if (orig != data)
6689 WREG32(RLC_PG_CNTL, data);
6690 }
6691
6692 #define CP_ME_TABLE_SIZE 96
6693 #define CP_ME_TABLE_OFFSET 2048
6694 #define CP_MEC_TABLE_OFFSET 4096
6695
6696 void cik_init_cp_pg_table(struct radeon_device *rdev)
6697 {
6698 volatile u32 *dst_ptr;
6699 int me, i, max_me = 4;
6700 u32 bo_offset = 0;
6701 u32 table_offset, table_size;
6702
6703 if (rdev->family == CHIP_KAVERI)
6704 max_me = 5;
6705
6706 if (rdev->rlc.cp_table_ptr == NULL)
6707 return;
6708
6709 /* write the cp table buffer */
6710 dst_ptr = rdev->rlc.cp_table_ptr;
6711 for (me = 0; me < max_me; me++) {
6712 if (rdev->new_fw) {
6713 const __le32 *fw_data;
6714 const struct gfx_firmware_header_v1_0 *hdr;
6715
6716 if (me == 0) {
6717 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6718 fw_data = (const __le32 *)
6719 ((const char *)rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6720 table_offset = le32_to_cpu(hdr->jt_offset);
6721 table_size = le32_to_cpu(hdr->jt_size);
6722 } else if (me == 1) {
6723 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6724 fw_data = (const __le32 *)
6725 ((const char *)rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6726 table_offset = le32_to_cpu(hdr->jt_offset);
6727 table_size = le32_to_cpu(hdr->jt_size);
6728 } else if (me == 2) {
6729 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6730 fw_data = (const __le32 *)
6731 ((const char *)rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6732 table_offset = le32_to_cpu(hdr->jt_offset);
6733 table_size = le32_to_cpu(hdr->jt_size);
6734 } else if (me == 3) {
6735 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6736 fw_data = (const __le32 *)
6737 ((const char *)rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6738 table_offset = le32_to_cpu(hdr->jt_offset);
6739 table_size = le32_to_cpu(hdr->jt_size);
6740 } else {
6741 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6742 fw_data = (const __le32 *)
6743 ((const char *)rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6744 table_offset = le32_to_cpu(hdr->jt_offset);
6745 table_size = le32_to_cpu(hdr->jt_size);
6746 }
6747
6748 for (i = 0; i < table_size; i ++) {
6749 dst_ptr[bo_offset + i] =
6750 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6751 }
6752 bo_offset += table_size;
6753 } else {
6754 const __be32 *fw_data;
6755 table_size = CP_ME_TABLE_SIZE;
6756
6757 if (me == 0) {
6758 fw_data = (const __be32 *)rdev->ce_fw->data;
6759 table_offset = CP_ME_TABLE_OFFSET;
6760 } else if (me == 1) {
6761 fw_data = (const __be32 *)rdev->pfp_fw->data;
6762 table_offset = CP_ME_TABLE_OFFSET;
6763 } else if (me == 2) {
6764 fw_data = (const __be32 *)rdev->me_fw->data;
6765 table_offset = CP_ME_TABLE_OFFSET;
6766 } else {
6767 fw_data = (const __be32 *)rdev->mec_fw->data;
6768 table_offset = CP_MEC_TABLE_OFFSET;
6769 }
6770
6771 for (i = 0; i < table_size; i ++) {
6772 dst_ptr[bo_offset + i] =
6773 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6774 }
6775 bo_offset += table_size;
6776 }
6777 }
6778 }
6779
6780 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6781 bool enable)
6782 {
6783 u32 data, orig;
6784
6785 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6786 orig = data = RREG32(RLC_PG_CNTL);
6787 data |= GFX_PG_ENABLE;
6788 if (orig != data)
6789 WREG32(RLC_PG_CNTL, data);
6790
6791 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6792 data |= AUTO_PG_EN;
6793 if (orig != data)
6794 WREG32(RLC_AUTO_PG_CTRL, data);
6795 } else {
6796 orig = data = RREG32(RLC_PG_CNTL);
6797 data &= ~GFX_PG_ENABLE;
6798 if (orig != data)
6799 WREG32(RLC_PG_CNTL, data);
6800
6801 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6802 data &= ~AUTO_PG_EN;
6803 if (orig != data)
6804 WREG32(RLC_AUTO_PG_CTRL, data);
6805
6806 data = RREG32(DB_RENDER_CONTROL);
6807 }
6808 }
6809
6810 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6811 {
6812 u32 mask = 0, tmp, tmp1;
6813 int i;
6814
6815 cik_select_se_sh(rdev, se, sh);
6816 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6817 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6818 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6819
6820 tmp &= 0xffff0000;
6821
6822 tmp |= tmp1;
6823 tmp >>= 16;
6824
6825 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6826 mask <<= 1;
6827 mask |= 1;
6828 }
6829
6830 return (~tmp) & mask;
6831 }
6832
6833 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6834 {
6835 u32 i, j, k, active_cu_number = 0;
6836 u32 mask, counter, cu_bitmap;
6837 u32 tmp = 0;
6838
6839 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6840 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6841 mask = 1;
6842 cu_bitmap = 0;
6843 counter = 0;
6844 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6845 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6846 if (counter < 2)
6847 cu_bitmap |= mask;
6848 counter ++;
6849 }
6850 mask <<= 1;
6851 }
6852
6853 active_cu_number += counter;
6854 tmp |= (cu_bitmap << (i * 16 + j * 8));
6855 }
6856 }
6857
6858 WREG32(RLC_PG_AO_CU_MASK, tmp);
6859
6860 tmp = RREG32(RLC_MAX_PG_CU);
6861 tmp &= ~MAX_PU_CU_MASK;
6862 tmp |= MAX_PU_CU(active_cu_number);
6863 WREG32(RLC_MAX_PG_CU, tmp);
6864 }
6865
6866 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6867 bool enable)
6868 {
6869 u32 data, orig;
6870
6871 orig = data = RREG32(RLC_PG_CNTL);
6872 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6873 data |= STATIC_PER_CU_PG_ENABLE;
6874 else
6875 data &= ~STATIC_PER_CU_PG_ENABLE;
6876 if (orig != data)
6877 WREG32(RLC_PG_CNTL, data);
6878 }
6879
6880 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6881 bool enable)
6882 {
6883 u32 data, orig;
6884
6885 orig = data = RREG32(RLC_PG_CNTL);
6886 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6887 data |= DYN_PER_CU_PG_ENABLE;
6888 else
6889 data &= ~DYN_PER_CU_PG_ENABLE;
6890 if (orig != data)
6891 WREG32(RLC_PG_CNTL, data);
6892 }
6893
6894 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6895 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6896
6897 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6898 {
6899 u32 data, orig;
6900 u32 i;
6901
6902 if (rdev->rlc.cs_data) {
6903 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6904 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6905 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6906 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6907 } else {
6908 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6909 for (i = 0; i < 3; i++)
6910 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6911 }
6912 if (rdev->rlc.reg_list) {
6913 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6914 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6915 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6916 }
6917
6918 orig = data = RREG32(RLC_PG_CNTL);
6919 data |= GFX_PG_SRC;
6920 if (orig != data)
6921 WREG32(RLC_PG_CNTL, data);
6922
6923 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6924 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6925
6926 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6927 data &= ~IDLE_POLL_COUNT_MASK;
6928 data |= IDLE_POLL_COUNT(0x60);
6929 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6930
6931 data = 0x10101010;
6932 WREG32(RLC_PG_DELAY, data);
6933
6934 data = RREG32(RLC_PG_DELAY_2);
6935 data &= ~0xff;
6936 data |= 0x3;
6937 WREG32(RLC_PG_DELAY_2, data);
6938
6939 data = RREG32(RLC_AUTO_PG_CTRL);
6940 data &= ~GRBM_REG_SGIT_MASK;
6941 data |= GRBM_REG_SGIT(0x700);
6942 WREG32(RLC_AUTO_PG_CTRL, data);
6943
6944 }
6945
6946 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6947 {
6948 cik_enable_gfx_cgpg(rdev, enable);
6949 cik_enable_gfx_static_mgpg(rdev, enable);
6950 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6951 }
6952
6953 u32 cik_get_csb_size(struct radeon_device *rdev)
6954 {
6955 u32 count = 0;
6956 const struct cs_section_def *sect = NULL;
6957 const struct cs_extent_def *ext = NULL;
6958
6959 if (rdev->rlc.cs_data == NULL)
6960 return 0;
6961
6962 /* begin clear state */
6963 count += 2;
6964 /* context control state */
6965 count += 3;
6966
6967 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6968 for (ext = sect->section; ext->extent != NULL; ++ext) {
6969 if (sect->id == SECT_CONTEXT)
6970 count += 2 + ext->reg_count;
6971 else
6972 return 0;
6973 }
6974 }
6975 /* pa_sc_raster_config/pa_sc_raster_config1 */
6976 count += 4;
6977 /* end clear state */
6978 count += 2;
6979 /* clear state */
6980 count += 2;
6981
6982 return count;
6983 }
6984
6985 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6986 {
6987 u32 count = 0, i;
6988 const struct cs_section_def *sect = NULL;
6989 const struct cs_extent_def *ext = NULL;
6990
6991 if (rdev->rlc.cs_data == NULL)
6992 return;
6993 if (buffer == NULL)
6994 return;
6995
6996 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6997 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6998
6999 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7000 buffer[count++] = cpu_to_le32(0x80000000);
7001 buffer[count++] = cpu_to_le32(0x80000000);
7002
7003 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7004 for (ext = sect->section; ext->extent != NULL; ++ext) {
7005 if (sect->id == SECT_CONTEXT) {
7006 buffer[count++] =
7007 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7008 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
7009 for (i = 0; i < ext->reg_count; i++)
7010 buffer[count++] = cpu_to_le32(ext->extent[i]);
7011 } else {
7012 return;
7013 }
7014 }
7015 }
7016
7017 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7018 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7019 switch (rdev->family) {
7020 case CHIP_BONAIRE:
7021 buffer[count++] = cpu_to_le32(0x16000012);
7022 buffer[count++] = cpu_to_le32(0x00000000);
7023 break;
7024 case CHIP_KAVERI:
7025 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7026 buffer[count++] = cpu_to_le32(0x00000000);
7027 break;
7028 case CHIP_KABINI:
7029 case CHIP_MULLINS:
7030 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7031 buffer[count++] = cpu_to_le32(0x00000000);
7032 break;
7033 case CHIP_HAWAII:
7034 buffer[count++] = cpu_to_le32(0x3a00161a);
7035 buffer[count++] = cpu_to_le32(0x0000002e);
7036 break;
7037 default:
7038 buffer[count++] = cpu_to_le32(0x00000000);
7039 buffer[count++] = cpu_to_le32(0x00000000);
7040 break;
7041 }
7042
7043 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7044 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7045
7046 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7047 buffer[count++] = cpu_to_le32(0);
7048 }
7049
7050 static void cik_init_pg(struct radeon_device *rdev)
7051 {
7052 if (rdev->pg_flags) {
7053 cik_enable_sck_slowdown_on_pu(rdev, true);
7054 cik_enable_sck_slowdown_on_pd(rdev, true);
7055 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7056 cik_init_gfx_cgpg(rdev);
7057 cik_enable_cp_pg(rdev, true);
7058 cik_enable_gds_pg(rdev, true);
7059 }
7060 cik_init_ao_cu_mask(rdev);
7061 cik_update_gfx_pg(rdev, true);
7062 }
7063 }
7064
7065 static void cik_fini_pg(struct radeon_device *rdev)
7066 {
7067 if (rdev->pg_flags) {
7068 cik_update_gfx_pg(rdev, false);
7069 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7070 cik_enable_cp_pg(rdev, false);
7071 cik_enable_gds_pg(rdev, false);
7072 }
7073 }
7074 }
7075
7076 /*
7077 * Interrupts
7078 * Starting with r6xx, interrupts are handled via a ring buffer.
7079 * Ring buffers are areas of GPU accessible memory that the GPU
7080 * writes interrupt vectors into and the host reads vectors out of.
7081 * There is a rptr (read pointer) that determines where the
7082 * host is currently reading, and a wptr (write pointer)
7083 * which determines where the GPU has written. When the
7084 * pointers are equal, the ring is idle. When the GPU
7085 * writes vectors to the ring buffer, it increments the
7086 * wptr. When there is an interrupt, the host then starts
7087 * fetching commands and processing them until the pointers are
7088 * equal again at which point it updates the rptr.
7089 */
7090
7091 /**
7092 * cik_enable_interrupts - Enable the interrupt ring buffer
7093 *
7094 * @rdev: radeon_device pointer
7095 *
7096 * Enable the interrupt ring buffer (CIK).
7097 */
7098 static void cik_enable_interrupts(struct radeon_device *rdev)
7099 {
7100 u32 ih_cntl = RREG32(IH_CNTL);
7101 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7102
7103 ih_cntl |= ENABLE_INTR;
7104 ih_rb_cntl |= IH_RB_ENABLE;
7105 WREG32(IH_CNTL, ih_cntl);
7106 WREG32(IH_RB_CNTL, ih_rb_cntl);
7107 rdev->ih.enabled = true;
7108 }
7109
7110 /**
7111 * cik_disable_interrupts - Disable the interrupt ring buffer
7112 *
7113 * @rdev: radeon_device pointer
7114 *
7115 * Disable the interrupt ring buffer (CIK).
7116 */
7117 static void cik_disable_interrupts(struct radeon_device *rdev)
7118 {
7119 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7120 u32 ih_cntl = RREG32(IH_CNTL);
7121
7122 ih_rb_cntl &= ~IH_RB_ENABLE;
7123 ih_cntl &= ~ENABLE_INTR;
7124 WREG32(IH_RB_CNTL, ih_rb_cntl);
7125 WREG32(IH_CNTL, ih_cntl);
7126 /* set rptr, wptr to 0 */
7127 WREG32(IH_RB_RPTR, 0);
7128 WREG32(IH_RB_WPTR, 0);
7129 rdev->ih.enabled = false;
7130 rdev->ih.rptr = 0;
7131 }
7132
7133 /**
7134 * cik_disable_interrupt_state - Disable all interrupt sources
7135 *
7136 * @rdev: radeon_device pointer
7137 *
7138 * Clear all interrupt enable bits used by the driver (CIK).
7139 */
7140 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7141 {
7142 u32 tmp;
7143
7144 /* gfx ring */
7145 tmp = RREG32(CP_INT_CNTL_RING0) &
7146 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7147 WREG32(CP_INT_CNTL_RING0, tmp);
7148 /* sdma */
7149 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7150 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7151 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7152 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7153 /* compute queues */
7154 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7155 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7156 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7157 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7158 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7159 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7160 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7161 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7162 /* grbm */
7163 WREG32(GRBM_INT_CNTL, 0);
7164 /* vline/vblank, etc. */
7165 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7166 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7167 if (rdev->num_crtc >= 4) {
7168 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7169 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7170 }
7171 if (rdev->num_crtc >= 6) {
7172 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7173 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7174 }
7175 /* pflip */
7176 if (rdev->num_crtc >= 2) {
7177 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7178 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7179 }
7180 if (rdev->num_crtc >= 4) {
7181 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7182 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7183 }
7184 if (rdev->num_crtc >= 6) {
7185 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7186 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7187 }
7188
7189 /* dac hotplug */
7190 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7191
7192 /* digital hotplug */
7193 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7194 WREG32(DC_HPD1_INT_CONTROL, tmp);
7195 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7196 WREG32(DC_HPD2_INT_CONTROL, tmp);
7197 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7198 WREG32(DC_HPD3_INT_CONTROL, tmp);
7199 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7200 WREG32(DC_HPD4_INT_CONTROL, tmp);
7201 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7202 WREG32(DC_HPD5_INT_CONTROL, tmp);
7203 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7204 WREG32(DC_HPD6_INT_CONTROL, tmp);
7205
7206 }
7207
7208 /**
7209 * cik_irq_init - init and enable the interrupt ring
7210 *
7211 * @rdev: radeon_device pointer
7212 *
7213 * Allocate a ring buffer for the interrupt controller,
7214 * enable the RLC, disable interrupts, enable the IH
7215 * ring buffer and enable it (CIK).
7216 * Called at device load and reume.
7217 * Returns 0 for success, errors for failure.
7218 */
7219 static int cik_irq_init(struct radeon_device *rdev)
7220 {
7221 int ret = 0;
7222 int rb_bufsz;
7223 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7224
7225 /* allocate ring */
7226 ret = r600_ih_ring_alloc(rdev);
7227 if (ret)
7228 return ret;
7229
7230 /* disable irqs */
7231 cik_disable_interrupts(rdev);
7232
7233 /* init rlc */
7234 ret = cik_rlc_resume(rdev);
7235 if (ret) {
7236 r600_ih_ring_fini(rdev);
7237 return ret;
7238 }
7239
7240 /* setup interrupt control */
7241 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7242 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7243 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7244 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7245 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7246 */
7247 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7248 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7249 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7250 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7251
7252 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7253 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7254
7255 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7256 IH_WPTR_OVERFLOW_CLEAR |
7257 (rb_bufsz << 1));
7258
7259 if (rdev->wb.enabled)
7260 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7261
7262 /* set the writeback address whether it's enabled or not */
7263 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7264 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7265
7266 WREG32(IH_RB_CNTL, ih_rb_cntl);
7267
7268 /* set rptr, wptr to 0 */
7269 WREG32(IH_RB_RPTR, 0);
7270 WREG32(IH_RB_WPTR, 0);
7271
7272 /* Default settings for IH_CNTL (disabled at first) */
7273 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7274 /* RPTR_REARM only works if msi's are enabled */
7275 if (rdev->msi_enabled)
7276 ih_cntl |= RPTR_REARM;
7277 WREG32(IH_CNTL, ih_cntl);
7278
7279 /* force the active interrupt state to all disabled */
7280 cik_disable_interrupt_state(rdev);
7281
7282 pci_enable_busmaster(rdev->pdev->dev.bsddev);
7283
7284 /* enable irqs */
7285 cik_enable_interrupts(rdev);
7286
7287 return ret;
7288 }
7289
7290 /**
7291 * cik_irq_set - enable/disable interrupt sources
7292 *
7293 * @rdev: radeon_device pointer
7294 *
7295 * Enable interrupt sources on the GPU (vblanks, hpd,
7296 * etc.) (CIK).
7297 * Returns 0 for success, errors for failure.
7298 */
7299 int cik_irq_set(struct radeon_device *rdev)
7300 {
7301 u32 cp_int_cntl;
7302 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7303 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7304 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7305 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7306 u32 grbm_int_cntl = 0;
7307 u32 dma_cntl, dma_cntl1;
7308 u32 thermal_int;
7309
7310 if (!rdev->irq.installed) {
7311 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7312 return -EINVAL;
7313 }
7314 /* don't enable anything if the ih is disabled */
7315 if (!rdev->ih.enabled) {
7316 cik_disable_interrupts(rdev);
7317 /* force the active interrupt state to all disabled */
7318 cik_disable_interrupt_state(rdev);
7319 return 0;
7320 }
7321
7322 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7323 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7324 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7325
7326 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7327 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7328 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7329 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7330 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7331 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7332
7333 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7334 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7335
7336 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7337 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7338 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7339 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7340 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7341 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7342 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7343 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7344
7345 if (rdev->flags & RADEON_IS_IGP)
7346 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7347 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7348 else
7349 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7350 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7351
7352 /* enable CP interrupts on all rings */
7353 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7354 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7355 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7356 }
7357 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7358 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7359 DRM_DEBUG("si_irq_set: sw int cp1\n");
7360 if (ring->me == 1) {
7361 switch (ring->pipe) {
7362 case 0:
7363 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7364 break;
7365 case 1:
7366 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7367 break;
7368 case 2:
7369 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7370 break;
7371 case 3:
7372 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7373 break;
7374 default:
7375 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7376 break;
7377 }
7378 } else if (ring->me == 2) {
7379 switch (ring->pipe) {
7380 case 0:
7381 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7382 break;
7383 case 1:
7384 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7385 break;
7386 case 2:
7387 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7388 break;
7389 case 3:
7390 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7391 break;
7392 default:
7393 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7394 break;
7395 }
7396 } else {
7397 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7398 }
7399 }
7400 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7401 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7402 DRM_DEBUG("si_irq_set: sw int cp2\n");
7403 if (ring->me == 1) {
7404 switch (ring->pipe) {
7405 case 0:
7406 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7407 break;
7408 case 1:
7409 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7410 break;
7411 case 2:
7412 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7413 break;
7414 case 3:
7415 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7416 break;
7417 default:
7418 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7419 break;
7420 }
7421 } else if (ring->me == 2) {
7422 switch (ring->pipe) {
7423 case 0:
7424 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7425 break;
7426 case 1:
7427 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7428 break;
7429 case 2:
7430 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7431 break;
7432 case 3:
7433 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7434 break;
7435 default:
7436 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7437 break;
7438 }
7439 } else {
7440 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7441 }
7442 }
7443
7444 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7445 DRM_DEBUG("cik_irq_set: sw int dma\n");
7446 dma_cntl |= TRAP_ENABLE;
7447 }
7448
7449 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7450 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7451 dma_cntl1 |= TRAP_ENABLE;
7452 }
7453
7454 if (rdev->irq.crtc_vblank_int[0] ||
7455 atomic_read(&rdev->irq.pflip[0])) {
7456 DRM_DEBUG("cik_irq_set: vblank 0\n");
7457 crtc1 |= VBLANK_INTERRUPT_MASK;
7458 }
7459 if (rdev->irq.crtc_vblank_int[1] ||
7460 atomic_read(&rdev->irq.pflip[1])) {
7461 DRM_DEBUG("cik_irq_set: vblank 1\n");
7462 crtc2 |= VBLANK_INTERRUPT_MASK;
7463 }
7464 if (rdev->irq.crtc_vblank_int[2] ||
7465 atomic_read(&rdev->irq.pflip[2])) {
7466 DRM_DEBUG("cik_irq_set: vblank 2\n");
7467 crtc3 |= VBLANK_INTERRUPT_MASK;
7468 }
7469 if (rdev->irq.crtc_vblank_int[3] ||
7470 atomic_read(&rdev->irq.pflip[3])) {
7471 DRM_DEBUG("cik_irq_set: vblank 3\n");
7472 crtc4 |= VBLANK_INTERRUPT_MASK;
7473 }
7474 if (rdev->irq.crtc_vblank_int[4] ||
7475 atomic_read(&rdev->irq.pflip[4])) {
7476 DRM_DEBUG("cik_irq_set: vblank 4\n");
7477 crtc5 |= VBLANK_INTERRUPT_MASK;
7478 }
7479 if (rdev->irq.crtc_vblank_int[5] ||
7480 atomic_read(&rdev->irq.pflip[5])) {
7481 DRM_DEBUG("cik_irq_set: vblank 5\n");
7482 crtc6 |= VBLANK_INTERRUPT_MASK;
7483 }
7484 if (rdev->irq.hpd[0]) {
7485 DRM_DEBUG("cik_irq_set: hpd 1\n");
7486 hpd1 |= DC_HPDx_INT_EN;
7487 }
7488 if (rdev->irq.hpd[1]) {
7489 DRM_DEBUG("cik_irq_set: hpd 2\n");
7490 hpd2 |= DC_HPDx_INT_EN;
7491 }
7492 if (rdev->irq.hpd[2]) {
7493 DRM_DEBUG("cik_irq_set: hpd 3\n");
7494 hpd3 |= DC_HPDx_INT_EN;
7495 }
7496 if (rdev->irq.hpd[3]) {
7497 DRM_DEBUG("cik_irq_set: hpd 4\n");
7498 hpd4 |= DC_HPDx_INT_EN;
7499 }
7500 if (rdev->irq.hpd[4]) {
7501 DRM_DEBUG("cik_irq_set: hpd 5\n");
7502 hpd5 |= DC_HPDx_INT_EN;
7503 }
7504 if (rdev->irq.hpd[5]) {
7505 DRM_DEBUG("cik_irq_set: hpd 6\n");
7506 hpd6 |= DC_HPDx_INT_EN;
7507 }
7508
7509 if (rdev->irq.dpm_thermal) {
7510 DRM_DEBUG("dpm thermal\n");
7511 if (rdev->flags & RADEON_IS_IGP)
7512 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7513 else
7514 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7515 }
7516
7517 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7518
7519 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7520 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7521
7522 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7523 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7524 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7525 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7526 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7527 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7528 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7529 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7530
7531 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7532
7533 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7534 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7535 if (rdev->num_crtc >= 4) {
7536 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7537 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7538 }
7539 if (rdev->num_crtc >= 6) {
7540 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7541 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7542 }
7543
7544 if (rdev->num_crtc >= 2) {
7545 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7546 GRPH_PFLIP_INT_MASK);
7547 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7548 GRPH_PFLIP_INT_MASK);
7549 }
7550 if (rdev->num_crtc >= 4) {
7551 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7552 GRPH_PFLIP_INT_MASK);
7553 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7554 GRPH_PFLIP_INT_MASK);
7555 }
7556 if (rdev->num_crtc >= 6) {
7557 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7558 GRPH_PFLIP_INT_MASK);
7559 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7560 GRPH_PFLIP_INT_MASK);
7561 }
7562
7563 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7564 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7565 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7566 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7567 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7568 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7569
7570 if (rdev->flags & RADEON_IS_IGP)
7571 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7572 else
7573 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7574
7575 return 0;
7576 }
7577
7578 /**
7579 * cik_irq_ack - ack interrupt sources
7580 *
7581 * @rdev: radeon_device pointer
7582 *
7583 * Ack interrupt sources on the GPU (vblanks, hpd,
7584 * etc.) (CIK). Certain interrupts sources are sw
7585 * generated and do not require an explicit ack.
7586 */
7587 static inline void cik_irq_ack(struct radeon_device *rdev)
7588 {
7589 u32 tmp;
7590
7591 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7592 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7593 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7594 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7595 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7596 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7597 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7598
7599 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7600 EVERGREEN_CRTC0_REGISTER_OFFSET);
7601 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7602 EVERGREEN_CRTC1_REGISTER_OFFSET);
7603 if (rdev->num_crtc >= 4) {
7604 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7605 EVERGREEN_CRTC2_REGISTER_OFFSET);
7606 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7607 EVERGREEN_CRTC3_REGISTER_OFFSET);
7608 }
7609 if (rdev->num_crtc >= 6) {
7610 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7611 EVERGREEN_CRTC4_REGISTER_OFFSET);
7612 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7613 EVERGREEN_CRTC5_REGISTER_OFFSET);
7614 }
7615
7616 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7617 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7618 GRPH_PFLIP_INT_CLEAR);
7619 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7620 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7621 GRPH_PFLIP_INT_CLEAR);
7622 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7623 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7624 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7625 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7626 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7627 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7628 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7629 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7630
7631 if (rdev->num_crtc >= 4) {
7632 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7634 GRPH_PFLIP_INT_CLEAR);
7635 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7637 GRPH_PFLIP_INT_CLEAR);
7638 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7639 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7640 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7641 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7642 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7643 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7644 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7645 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7646 }
7647
7648 if (rdev->num_crtc >= 6) {
7649 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7650 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7651 GRPH_PFLIP_INT_CLEAR);
7652 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7653 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7654 GRPH_PFLIP_INT_CLEAR);
7655 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7656 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7657 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7658 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7659 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7660 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7661 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7662 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7663 }
7664
7665 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7666 tmp = RREG32(DC_HPD1_INT_CONTROL);
7667 tmp |= DC_HPDx_INT_ACK;
7668 WREG32(DC_HPD1_INT_CONTROL, tmp);
7669 }
7670 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7671 tmp = RREG32(DC_HPD2_INT_CONTROL);
7672 tmp |= DC_HPDx_INT_ACK;
7673 WREG32(DC_HPD2_INT_CONTROL, tmp);
7674 }
7675 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7676 tmp = RREG32(DC_HPD3_INT_CONTROL);
7677 tmp |= DC_HPDx_INT_ACK;
7678 WREG32(DC_HPD3_INT_CONTROL, tmp);
7679 }
7680 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7681 tmp = RREG32(DC_HPD4_INT_CONTROL);
7682 tmp |= DC_HPDx_INT_ACK;
7683 WREG32(DC_HPD4_INT_CONTROL, tmp);
7684 }
7685 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7686 tmp = RREG32(DC_HPD5_INT_CONTROL);
7687 tmp |= DC_HPDx_INT_ACK;
7688 WREG32(DC_HPD5_INT_CONTROL, tmp);
7689 }
7690 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7691 tmp = RREG32(DC_HPD5_INT_CONTROL);
7692 tmp |= DC_HPDx_INT_ACK;
7693 WREG32(DC_HPD6_INT_CONTROL, tmp);
7694 }
7695 }
7696
7697 /**
7698 * cik_irq_disable - disable interrupts
7699 *
7700 * @rdev: radeon_device pointer
7701 *
7702 * Disable interrupts on the hw (CIK).
7703 */
7704 static void cik_irq_disable(struct radeon_device *rdev)
7705 {
7706 cik_disable_interrupts(rdev);
7707 /* Wait and acknowledge irq */
7708 mdelay(1);
7709 cik_irq_ack(rdev);
7710 cik_disable_interrupt_state(rdev);
7711 }
7712
7713 /**
7714 * cik_irq_disable - disable interrupts for suspend
7715 *
7716 * @rdev: radeon_device pointer
7717 *
7718 * Disable interrupts and stop the RLC (CIK).
7719 * Used for suspend.
7720 */
7721 static void cik_irq_suspend(struct radeon_device *rdev)
7722 {
7723 cik_irq_disable(rdev);
7724 cik_rlc_stop(rdev);
7725 }
7726
7727 /**
7728 * cik_irq_fini - tear down interrupt support
7729 *
7730 * @rdev: radeon_device pointer
7731 *
7732 * Disable interrupts on the hw and free the IH ring
7733 * buffer (CIK).
7734 * Used for driver unload.
7735 */
7736 static void cik_irq_fini(struct radeon_device *rdev)
7737 {
7738 cik_irq_suspend(rdev);
7739 r600_ih_ring_fini(rdev);
7740 }
7741
7742 /**
7743 * cik_get_ih_wptr - get the IH ring buffer wptr
7744 *
7745 * @rdev: radeon_device pointer
7746 *
7747 * Get the IH ring buffer wptr from either the register
7748 * or the writeback memory buffer (CIK). Also check for
7749 * ring buffer overflow and deal with it.
7750 * Used by cik_irq_process().
7751 * Returns the value of the wptr.
7752 */
7753 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7754 {
7755 u32 wptr, tmp;
7756
7757 if (rdev->wb.enabled)
7758 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7759 else
7760 wptr = RREG32(IH_RB_WPTR);
7761
7762 if (wptr & RB_OVERFLOW) {
7763 wptr &= ~RB_OVERFLOW;
7764 /* When a ring buffer overflow happen start parsing interrupt
7765 * from the last not overwritten vector (wptr + 16). Hopefully
7766 * this should allow us to catchup.
7767 */
7768 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7769 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7770 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7771 tmp = RREG32(IH_RB_CNTL);
7772 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7773 WREG32(IH_RB_CNTL, tmp);
7774 }
7775 return (wptr & rdev->ih.ptr_mask);
7776 }
7777
7778 /* CIK IV Ring
7779 * Each IV ring entry is 128 bits:
7780 * [7:0] - interrupt source id
7781 * [31:8] - reserved
7782 * [59:32] - interrupt source data
7783 * [63:60] - reserved
7784 * [71:64] - RINGID
7785 * CP:
7786 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7787 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7788 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7789 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7790 * PIPE_ID - ME0 0=3D
7791 * - ME1&2 compute dispatcher (4 pipes each)
7792 * SDMA:
7793 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7794 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7795 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7796 * [79:72] - VMID
7797 * [95:80] - PASID
7798 * [127:96] - reserved
7799 */
7800 /**
7801 * cik_irq_process - interrupt handler
7802 *
7803 * @rdev: radeon_device pointer
7804 *
7805 * Interrupt hander (CIK). Walk the IH ring,
7806 * ack interrupts and schedule work to handle
7807 * interrupt events.
7808 * Returns irq process return code.
7809 */
7810 irqreturn_t cik_irq_process(struct radeon_device *rdev)
7811 {
7812 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7813 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7814 u32 wptr;
7815 u32 rptr;
7816 u32 src_id, src_data, ring_id;
7817 u8 me_id, pipe_id, queue_id;
7818 u32 ring_index;
7819 bool queue_hotplug = false;
7820 bool queue_reset = false;
7821 u32 addr, status, mc_client;
7822 bool queue_thermal = false;
7823
7824 if (!rdev->ih.enabled || rdev->shutdown)
7825 return IRQ_NONE;
7826
7827 wptr = cik_get_ih_wptr(rdev);
7828
7829 restart_ih:
7830 /* is somebody else already processing irqs? */
7831 if (atomic_xchg(&rdev->ih.lock, 1))
7832 return IRQ_NONE;
7833
7834 rptr = rdev->ih.rptr;
7835 DRM_DEBUG_VBLANK("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7836
7837 /* Order reading of wptr vs. reading of IH ring data */
7838 rmb();
7839
7840 /* display interrupts */
7841 cik_irq_ack(rdev);
7842
7843 while (rptr != wptr) {
7844 /* wptr/rptr are in bytes! */
7845 ring_index = rptr / 4;
7846 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7847 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7848 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7849
7850 switch (src_id) {
7851 case 1: /* D1 vblank/vline */
7852 switch (src_data) {
7853 case 0: /* D1 vblank */
7854 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7855 if (rdev->irq.crtc_vblank_int[0]) {
7856 drm_handle_vblank(rdev->ddev, 0);
7857 rdev->pm.vblank_sync = true;
7858 wake_up(&rdev->irq.vblank_queue);
7859 }
7860 if (atomic_read(&rdev->irq.pflip[0]))
7861 radeon_crtc_handle_vblank(rdev, 0);
7862 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7863 DRM_DEBUG_VBLANK("IH: D1 vblank\n");
7864 }
7865 break;
7866 case 1: /* D1 vline */
7867 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7868 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7869 DRM_DEBUG_VBLANK("IH: D1 vline\n");
7870 }
7871 break;
7872 default:
7873 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7874 break;
7875 }
7876 break;
7877 case 2: /* D2 vblank/vline */
7878 switch (src_data) {
7879 case 0: /* D2 vblank */
7880 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7881 if (rdev->irq.crtc_vblank_int[1]) {
7882 drm_handle_vblank(rdev->ddev, 1);
7883 rdev->pm.vblank_sync = true;
7884 wake_up(&rdev->irq.vblank_queue);
7885 }
7886 if (atomic_read(&rdev->irq.pflip[1]))
7887 radeon_crtc_handle_vblank(rdev, 1);
7888 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7889 DRM_DEBUG_VBLANK("IH: D2 vblank\n");
7890 }
7891 break;
7892 case 1: /* D2 vline */
7893 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7894 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7895 DRM_DEBUG_VBLANK("IH: D2 vline\n");
7896 }
7897 break;
7898 default:
7899 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7900 break;
7901 }
7902 break;
7903 case 3: /* D3 vblank/vline */
7904 switch (src_data) {
7905 case 0: /* D3 vblank */
7906 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7907 if (rdev->irq.crtc_vblank_int[2]) {
7908 drm_handle_vblank(rdev->ddev, 2);
7909 rdev->pm.vblank_sync = true;
7910 wake_up(&rdev->irq.vblank_queue);
7911 }
7912 if (atomic_read(&rdev->irq.pflip[2]))
7913 radeon_crtc_handle_vblank(rdev, 2);
7914 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7915 DRM_DEBUG_VBLANK("IH: D3 vblank\n");
7916 }
7917 break;
7918 case 1: /* D3 vline */
7919 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7920 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7921 DRM_DEBUG_VBLANK("IH: D3 vline\n");
7922 }
7923 break;
7924 default:
7925 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7926 break;
7927 }
7928 break;
7929 case 4: /* D4 vblank/vline */
7930 switch (src_data) {
7931 case 0: /* D4 vblank */
7932 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7933 if (rdev->irq.crtc_vblank_int[3]) {
7934 drm_handle_vblank(rdev->ddev, 3);
7935 rdev->pm.vblank_sync = true;
7936 wake_up(&rdev->irq.vblank_queue);
7937 }
7938 if (atomic_read(&rdev->irq.pflip[3]))
7939 radeon_crtc_handle_vblank(rdev, 3);
7940 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7941 DRM_DEBUG_VBLANK("IH: D4 vblank\n");
7942 }
7943 break;
7944 case 1: /* D4 vline */
7945 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7946 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7947 DRM_DEBUG_VBLANK("IH: D4 vline\n");
7948 }
7949 break;
7950 default:
7951 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7952 break;
7953 }
7954 break;
7955 case 5: /* D5 vblank/vline */
7956 switch (src_data) {
7957 case 0: /* D5 vblank */
7958 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7959 if (rdev->irq.crtc_vblank_int[4]) {
7960 drm_handle_vblank(rdev->ddev, 4);
7961 rdev->pm.vblank_sync = true;
7962 wake_up(&rdev->irq.vblank_queue);
7963 }
7964 if (atomic_read(&rdev->irq.pflip[4]))
7965 radeon_crtc_handle_vblank(rdev, 4);
7966 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7967 DRM_DEBUG_VBLANK("IH: D5 vblank\n");
7968 }
7969 break;
7970 case 1: /* D5 vline */
7971 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7972 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7973 DRM_DEBUG_VBLANK("IH: D5 vline\n");
7974 }
7975 break;
7976 default:
7977 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7978 break;
7979 }
7980 break;
7981 case 6: /* D6 vblank/vline */
7982 switch (src_data) {
7983 case 0: /* D6 vblank */
7984 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7985 if (rdev->irq.crtc_vblank_int[5]) {
7986 drm_handle_vblank(rdev->ddev, 5);
7987 rdev->pm.vblank_sync = true;
7988 wake_up(&rdev->irq.vblank_queue);
7989 }
7990 if (atomic_read(&rdev->irq.pflip[5]))
7991 radeon_crtc_handle_vblank(rdev, 5);
7992 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7993 DRM_DEBUG_VBLANK("IH: D6 vblank\n");
7994 }
7995 break;
7996 case 1: /* D6 vline */
7997 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7998 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7999 DRM_DEBUG_VBLANK("IH: D6 vline\n");
8000 }
8001 break;
8002 default:
8003 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8004 break;
8005 }
8006 break;
8007 case 8: /* D1 page flip */
8008 case 10: /* D2 page flip */
8009 case 12: /* D3 page flip */
8010 case 14: /* D4 page flip */
8011 case 16: /* D5 page flip */
8012 case 18: /* D6 page flip */
8013 DRM_DEBUG_VBLANK("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
8014 if (radeon_use_pflipirq > 0)
8015 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8016 break;
8017 case 42: /* HPD hotplug */
8018 switch (src_data) {
8019 case 0:
8020 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8021 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8022 queue_hotplug = true;
8023 DRM_DEBUG("IH: HPD1\n");
8024 }
8025 break;
8026 case 1:
8027 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8028 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8029 queue_hotplug = true;
8030 DRM_DEBUG("IH: HPD2\n");
8031 }
8032 break;
8033 case 2:
8034 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8035 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8036 queue_hotplug = true;
8037 DRM_DEBUG("IH: HPD3\n");
8038 }
8039 break;
8040 case 3:
8041 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8042 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8043 queue_hotplug = true;
8044 DRM_DEBUG("IH: HPD4\n");
8045 }
8046 break;
8047 case 4:
8048 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8049 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8050 queue_hotplug = true;
8051 DRM_DEBUG("IH: HPD5\n");
8052 }
8053 break;
8054 case 5:
8055 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8056 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8057 queue_hotplug = true;
8058 DRM_DEBUG("IH: HPD6\n");
8059 }
8060 break;
8061 default:
8062 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8063 break;
8064 }
8065 break;
8066 case 124: /* UVD */
8067 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8068 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8069 break;
8070 case 146:
8071 case 147:
8072 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8073 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8074 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8075 /* reset addr and status */
8076 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8077 if (addr == 0x0 && status == 0x0)
8078 break;
8079 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8080 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8081 addr);
8082 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8083 status);
8084 cik_vm_decode_fault(rdev, status, addr, mc_client);
8085 break;
8086 case 167: /* VCE */
8087 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8088 switch (src_data) {
8089 case 0:
8090 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8091 break;
8092 case 1:
8093 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8094 break;
8095 default:
8096 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8097 break;
8098 }
8099 break;
8100 case 176: /* GFX RB CP_INT */
8101 case 177: /* GFX IB CP_INT */
8102 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8103 break;
8104 case 181: /* CP EOP event */
8105 DRM_DEBUG("IH: CP EOP\n");
8106 /* XXX check the bitfield order! */
8107 me_id = (ring_id & 0x60) >> 5;
8108 pipe_id = (ring_id & 0x18) >> 3;
8109 queue_id = (ring_id & 0x7) >> 0;
8110 switch (me_id) {
8111 case 0:
8112 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8113 break;
8114 case 1:
8115 case 2:
8116 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8117 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8118 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8119 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8120 break;
8121 }
8122 break;
8123 case 184: /* CP Privileged reg access */
8124 DRM_ERROR("Illegal register access in command stream\n");
8125 /* XXX check the bitfield order! */
8126 me_id = (ring_id & 0x60) >> 5;
8127 pipe_id = (ring_id & 0x18) >> 3;
8128 queue_id = (ring_id & 0x7) >> 0;
8129 switch (me_id) {
8130 case 0:
8131 /* This results in a full GPU reset, but all we need to do is soft
8132 * reset the CP for gfx
8133 */
8134 queue_reset = true;
8135 break;
8136 case 1:
8137 /* XXX compute */
8138 queue_reset = true;
8139 break;
8140 case 2:
8141 /* XXX compute */
8142 queue_reset = true;
8143 break;
8144 }
8145 break;
8146 case 185: /* CP Privileged inst */
8147 DRM_ERROR("Illegal instruction in command stream\n");
8148 /* XXX check the bitfield order! */
8149 me_id = (ring_id & 0x60) >> 5;
8150 pipe_id = (ring_id & 0x18) >> 3;
8151 queue_id = (ring_id & 0x7) >> 0;
8152 switch (me_id) {
8153 case 0:
8154 /* This results in a full GPU reset, but all we need to do is soft
8155 * reset the CP for gfx
8156 */
8157 queue_reset = true;
8158 break;
8159 case 1:
8160 /* XXX compute */
8161 queue_reset = true;
8162 break;
8163 case 2:
8164 /* XXX compute */
8165 queue_reset = true;
8166 break;
8167 }
8168 break;
8169 case 224: /* SDMA trap event */
8170 /* XXX check the bitfield order! */
8171 me_id = (ring_id & 0x3) >> 0;
8172 queue_id = (ring_id & 0xc) >> 2;
8173 DRM_DEBUG("IH: SDMA trap\n");
8174 switch (me_id) {
8175 case 0:
8176 switch (queue_id) {
8177 case 0:
8178 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8179 break;
8180 case 1:
8181 /* XXX compute */
8182 break;
8183 case 2:
8184 /* XXX compute */
8185 break;
8186 }
8187 break;
8188 case 1:
8189 switch (queue_id) {
8190 case 0:
8191 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8192 break;
8193 case 1:
8194 /* XXX compute */
8195 break;
8196 case 2:
8197 /* XXX compute */
8198 break;
8199 }
8200 break;
8201 }
8202 break;
8203 case 230: /* thermal low to high */
8204 DRM_DEBUG("IH: thermal low to high\n");
8205 rdev->pm.dpm.thermal.high_to_low = false;
8206 queue_thermal = true;
8207 break;
8208 case 231: /* thermal high to low */
8209 DRM_DEBUG("IH: thermal high to low\n");
8210 rdev->pm.dpm.thermal.high_to_low = true;
8211 queue_thermal = true;
8212 break;
8213 case 233: /* GUI IDLE */
8214 DRM_DEBUG("IH: GUI idle\n");
8215 break;
8216 case 241: /* SDMA Privileged inst */
8217 case 247: /* SDMA Privileged inst */
8218 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8219 /* XXX check the bitfield order! */
8220 me_id = (ring_id & 0x3) >> 0;
8221 queue_id = (ring_id & 0xc) >> 2;
8222 switch (me_id) {
8223 case 0:
8224 switch (queue_id) {
8225 case 0:
8226 queue_reset = true;
8227 break;
8228 case 1:
8229 /* XXX compute */
8230 queue_reset = true;
8231 break;
8232 case 2:
8233 /* XXX compute */
8234 queue_reset = true;
8235 break;
8236 }
8237 break;
8238 case 1:
8239 switch (queue_id) {
8240 case 0:
8241 queue_reset = true;
8242 break;
8243 case 1:
8244 /* XXX compute */
8245 queue_reset = true;
8246 break;
8247 case 2:
8248 /* XXX compute */
8249 queue_reset = true;
8250 break;
8251 }
8252 break;
8253 }
8254 break;
8255 default:
8256 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8257 break;
8258 }
8259
8260 /* wptr/rptr are in bytes! */
8261 rptr += 16;
8262 rptr &= rdev->ih.ptr_mask;
8263 WREG32(IH_RB_RPTR, rptr);
8264 }
8265 if (queue_hotplug)
8266 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
8267 if (queue_reset) {
8268 rdev->needs_reset = true;
8269 wake_up_all(&rdev->fence_queue);
8270 }
8271 if (queue_thermal)
8272 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work);
8273 rdev->ih.rptr = rptr;
8274 atomic_set(&rdev->ih.lock, 0);
8275
8276 /* make sure wptr hasn't changed while processing */
8277 wptr = cik_get_ih_wptr(rdev);
8278 if (wptr != rptr)
8279 goto restart_ih;
8280
8281 return IRQ_HANDLED;
8282 }
8283
8284 /*
8285 * startup/shutdown callbacks
8286 */
8287 /**
8288 * cik_startup - program the asic to a functional state
8289 *
8290 * @rdev: radeon_device pointer
8291 *
8292 * Programs the asic to a functional state (CIK).
8293 * Called by cik_init() and cik_resume().
8294 * Returns 0 for success, error for failure.
8295 */
8296 static int cik_startup(struct radeon_device *rdev)
8297 {
8298 struct radeon_ring *ring;
8299 u32 nop;
8300 int r;
8301
8302 /* enable pcie gen2/3 link */
8303 cik_pcie_gen3_enable(rdev);
8304 /* enable aspm */
8305 cik_program_aspm(rdev);
8306
8307 /* scratch needs to be initialized before MC */
8308 r = r600_vram_scratch_init(rdev);
8309 if (r)
8310 return r;
8311
8312 cik_mc_program(rdev);
8313
8314 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8315 r = ci_mc_load_microcode(rdev);
8316 if (r) {
8317 DRM_ERROR("Failed to load MC firmware!\n");
8318 return r;
8319 }
8320 }
8321
8322 r = cik_pcie_gart_enable(rdev);
8323 if (r)
8324 return r;
8325 cik_gpu_init(rdev);
8326
8327 /* allocate rlc buffers */
8328 if (rdev->flags & RADEON_IS_IGP) {
8329 if (rdev->family == CHIP_KAVERI) {
8330 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8331 rdev->rlc.reg_list_size =
8332 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8333 } else {
8334 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8335 rdev->rlc.reg_list_size =
8336 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8337 }
8338 }
8339 rdev->rlc.cs_data = ci_cs_data;
8340 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8341 r = sumo_rlc_init(rdev);
8342 if (r) {
8343 DRM_ERROR("Failed to init rlc BOs!\n");
8344 return r;
8345 }
8346
8347 /* allocate wb buffer */
8348 r = radeon_wb_init(rdev);
8349 if (r)
8350 return r;
8351
8352 /* allocate mec buffers */
8353 r = cik_mec_init(rdev);
8354 if (r) {
8355 DRM_ERROR("Failed to init MEC BOs!\n");
8356 return r;
8357 }
8358
8359 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8360 if (r) {
8361 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8362 return r;
8363 }
8364
8365 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8366 if (r) {
8367 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8368 return r;
8369 }
8370
8371 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8372 if (r) {
8373 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8374 return r;
8375 }
8376
8377 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8378 if (r) {
8379 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8380 return r;
8381 }
8382
8383 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8384 if (r) {
8385 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8386 return r;
8387 }
8388
8389 r = radeon_uvd_resume(rdev);
8390 if (!r) {
8391 r = uvd_v4_2_resume(rdev);
8392 if (!r) {
8393 r = radeon_fence_driver_start_ring(rdev,
8394 R600_RING_TYPE_UVD_INDEX);
8395 if (r)
8396 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8397 }
8398 }
8399 if (r)
8400 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8401
8402 r = radeon_vce_resume(rdev);
8403 if (!r) {
8404 r = vce_v2_0_resume(rdev);
8405 if (!r)
8406 r = radeon_fence_driver_start_ring(rdev,
8407 TN_RING_TYPE_VCE1_INDEX);
8408 if (!r)
8409 r = radeon_fence_driver_start_ring(rdev,
8410 TN_RING_TYPE_VCE2_INDEX);
8411 }
8412 if (r) {
8413 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8414 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8415 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8416 }
8417
8418 /* Enable IRQ */
8419 if (!rdev->irq.installed) {
8420 r = radeon_irq_kms_init(rdev);
8421 if (r)
8422 return r;
8423 }
8424
8425 r = cik_irq_init(rdev);
8426 if (r) {
8427 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8428 radeon_irq_kms_fini(rdev);
8429 return r;
8430 }
8431 cik_irq_set(rdev);
8432
8433 if (rdev->family == CHIP_HAWAII) {
8434 if (rdev->new_fw)
8435 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8436 else
8437 nop = RADEON_CP_PACKET2;
8438 } else {
8439 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8440 }
8441
8442 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8443 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8444 nop);
8445 if (r)
8446 return r;
8447
8448 /* set up the compute queues */
8449 /* type-2 packets are deprecated on MEC, use type-3 instead */
8450 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8451 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8452 nop);
8453 if (r)
8454 return r;
8455 ring->me = 1; /* first MEC */
8456 ring->pipe = 0; /* first pipe */
8457 ring->queue = 0; /* first queue */
8458 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8459
8460 /* type-2 packets are deprecated on MEC, use type-3 instead */
8461 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8462 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8463 nop);
8464 if (r)
8465 return r;
8466 /* dGPU only have 1 MEC */
8467 ring->me = 1; /* first MEC */
8468 ring->pipe = 0; /* first pipe */
8469 ring->queue = 1; /* second queue */
8470 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8471
8472 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8473 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8474 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8475 if (r)
8476 return r;
8477
8478 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8479 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8480 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8481 if (r)
8482 return r;
8483
8484 r = cik_cp_resume(rdev);
8485 if (r)
8486 return r;
8487
8488 r = cik_sdma_resume(rdev);
8489 if (r)
8490 return r;
8491
8492 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8493 if (ring->ring_size) {
8494 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8495 RADEON_CP_PACKET2);
8496 if (!r)
8497 r = uvd_v1_0_init(rdev);
8498 if (r)
8499 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8500 }
8501
8502 r = -ENOENT;
8503
8504 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8505 if (ring->ring_size)
8506 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8507 VCE_CMD_NO_OP);
8508
8509 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8510 if (ring->ring_size)
8511 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8512 VCE_CMD_NO_OP);
8513
8514 if (!r)
8515 r = vce_v1_0_init(rdev);
8516 else if (r != -ENOENT)
8517 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8518
8519 r = radeon_ib_pool_init(rdev);
8520 if (r) {
8521 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8522 return r;
8523 }
8524
8525 r = radeon_vm_manager_init(rdev);
8526 if (r) {
8527 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8528 return r;
8529 }
8530
8531 r = dce6_audio_init(rdev);
8532 if (r)
8533 return r;
8534
8535 return 0;
8536 }
8537
8538 /**
8539 * cik_resume - resume the asic to a functional state
8540 *
8541 * @rdev: radeon_device pointer
8542 *
8543 * Programs the asic to a functional state (CIK).
8544 * Called at resume.
8545 * Returns 0 for success, error for failure.
8546 */
8547 int cik_resume(struct radeon_device *rdev)
8548 {
8549 int r;
8550
8551 /* post card */
8552 atom_asic_init(rdev->mode_info.atom_context);
8553
8554 /* init golden registers */
8555 cik_init_golden_registers(rdev);
8556
8557 if (rdev->pm.pm_method == PM_METHOD_DPM)
8558 radeon_pm_resume(rdev);
8559
8560 rdev->accel_working = true;
8561 r = cik_startup(rdev);
8562 if (r) {
8563 DRM_ERROR("cik startup failed on resume\n");
8564 rdev->accel_working = false;
8565 return r;
8566 }
8567
8568 return r;
8569
8570 }
8571
8572 /**
8573 * cik_suspend - suspend the asic
8574 *
8575 * @rdev: radeon_device pointer
8576 *
8577 * Bring the chip into a state suitable for suspend (CIK).
8578 * Called at suspend.
8579 * Returns 0 for success.
8580 */
8581 int cik_suspend(struct radeon_device *rdev)
8582 {
8583 radeon_pm_suspend(rdev);
8584 dce6_audio_fini(rdev);
8585 radeon_vm_manager_fini(rdev);
8586 cik_cp_enable(rdev, false);
8587 cik_sdma_enable(rdev, false);
8588 uvd_v1_0_fini(rdev);
8589 radeon_uvd_suspend(rdev);
8590 radeon_vce_suspend(rdev);
8591 cik_fini_pg(rdev);
8592 cik_fini_cg(rdev);
8593 cik_irq_suspend(rdev);
8594 radeon_wb_disable(rdev);
8595 cik_pcie_gart_disable(rdev);
8596 return 0;
8597 }
8598
8599 /* Plan is to move initialization in that function and use
8600 * helper function so that radeon_device_init pretty much
8601 * do nothing more than calling asic specific function. This
8602 * should also allow to remove a bunch of callback function
8603 * like vram_info.
8604 */
8605 /**
8606 * cik_init - asic specific driver and hw init
8607 *
8608 * @rdev: radeon_device pointer
8609 *
8610 * Setup asic specific driver variables and program the hw
8611 * to a functional state (CIK).
8612 * Called at driver startup.
8613 * Returns 0 for success, errors for failure.
8614 */
8615 int cik_init(struct radeon_device *rdev)
8616 {
8617 struct radeon_ring *ring;
8618 int r;
8619
8620 /* Read BIOS */
8621 if (!radeon_get_bios(rdev)) {
8622 if (ASIC_IS_AVIVO(rdev))
8623 return -EINVAL;
8624 }
8625 /* Must be an ATOMBIOS */
8626 if (!rdev->is_atom_bios) {
8627 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8628 return -EINVAL;
8629 }
8630 r = radeon_atombios_init(rdev);
8631 if (r)
8632 return r;
8633
8634 /* Post card if necessary */
8635 if (!radeon_card_posted(rdev)) {
8636 if (!rdev->bios) {
8637 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8638 return -EINVAL;
8639 }
8640 DRM_INFO("GPU not posted. posting now...\n");
8641 atom_asic_init(rdev->mode_info.atom_context);
8642 }
8643 /* init golden registers */
8644 cik_init_golden_registers(rdev);
8645 /* Initialize scratch registers */
8646 cik_scratch_init(rdev);
8647 /* Initialize surface registers */
8648 radeon_surface_init(rdev);
8649 /* Initialize clocks */
8650 radeon_get_clock_info(rdev->ddev);
8651
8652 /* Fence driver */
8653 r = radeon_fence_driver_init(rdev);
8654 if (r)
8655 return r;
8656
8657 /* initialize memory controller */
8658 r = cik_mc_init(rdev);
8659 if (r)
8660 return r;
8661 /* Memory manager */
8662 r = radeon_bo_init(rdev);
8663 if (r)
8664 return r;
8665
8666 if (rdev->flags & RADEON_IS_IGP) {
8667 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8668 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8669 r = cik_init_microcode(rdev);
8670 if (r) {
8671 DRM_ERROR("Failed to load firmware!\n");
8672 return r;
8673 }
8674 }
8675 } else {
8676 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8677 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8678 !rdev->mc_fw) {
8679 r = cik_init_microcode(rdev);
8680 if (r) {
8681 DRM_ERROR("Failed to load firmware!\n");
8682 return r;
8683 }
8684 }
8685 }
8686
8687 /* Initialize power management */
8688 radeon_pm_init(rdev);
8689
8690 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8691 ring->ring_obj = NULL;
8692 r600_ring_init(rdev, ring, 1024 * 1024);
8693
8694 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8695 ring->ring_obj = NULL;
8696 r600_ring_init(rdev, ring, 1024 * 1024);
8697 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8698 if (r)
8699 return r;
8700
8701 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8702 ring->ring_obj = NULL;
8703 r600_ring_init(rdev, ring, 1024 * 1024);
8704 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8705 if (r)
8706 return r;
8707
8708 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8709 ring->ring_obj = NULL;
8710 r600_ring_init(rdev, ring, 256 * 1024);
8711
8712 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8713 ring->ring_obj = NULL;
8714 r600_ring_init(rdev, ring, 256 * 1024);
8715
8716 r = radeon_uvd_init(rdev);
8717 if (!r) {
8718 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8719 ring->ring_obj = NULL;
8720 r600_ring_init(rdev, ring, 4096);
8721 }
8722
8723 r = radeon_vce_init(rdev);
8724 if (!r) {
8725 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8726 ring->ring_obj = NULL;
8727 r600_ring_init(rdev, ring, 4096);
8728
8729 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8730 ring->ring_obj = NULL;
8731 r600_ring_init(rdev, ring, 4096);
8732 }
8733
8734 rdev->ih.ring_obj = NULL;
8735 r600_ih_ring_init(rdev, 64 * 1024);
8736
8737 r = r600_pcie_gart_init(rdev);
8738 if (r)
8739 return r;
8740
8741 rdev->accel_working = true;
8742 r = cik_startup(rdev);
8743 if (r) {
8744 dev_err(rdev->dev, "disabling GPU acceleration\n");
8745 cik_cp_fini(rdev);
8746 cik_sdma_fini(rdev);
8747 cik_irq_fini(rdev);
8748 sumo_rlc_fini(rdev);
8749 cik_mec_fini(rdev);
8750 radeon_wb_fini(rdev);
8751 radeon_ib_pool_fini(rdev);
8752 radeon_vm_manager_fini(rdev);
8753 radeon_irq_kms_fini(rdev);
8754 cik_pcie_gart_fini(rdev);
8755 rdev->accel_working = false;
8756 }
8757
8758 /* Don't start up if the MC ucode is missing.
8759 * The default clocks and voltages before the MC ucode
8760 * is loaded are not suffient for advanced operations.
8761 */
8762 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8763 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8764 return -EINVAL;
8765 }
8766
8767 return 0;
8768 }
8769
8770 /**
8771 * cik_fini - asic specific driver and hw fini
8772 *
8773 * @rdev: radeon_device pointer
8774 *
8775 * Tear down the asic specific driver variables and program the hw
8776 * to an idle state (CIK).
8777 * Called at driver unload.
8778 */
8779 void cik_fini(struct radeon_device *rdev)
8780 {
8781 radeon_pm_fini(rdev);
8782 cik_cp_fini(rdev);
8783 cik_sdma_fini(rdev);
8784 cik_fini_pg(rdev);
8785 cik_fini_cg(rdev);
8786 cik_irq_fini(rdev);
8787 sumo_rlc_fini(rdev);
8788 cik_mec_fini(rdev);
8789 radeon_wb_fini(rdev);
8790 radeon_vm_manager_fini(rdev);
8791 radeon_ib_pool_fini(rdev);
8792 radeon_irq_kms_fini(rdev);
8793 uvd_v1_0_fini(rdev);
8794 radeon_uvd_fini(rdev);
8795 radeon_vce_fini(rdev);
8796 cik_pcie_gart_fini(rdev);
8797 r600_vram_scratch_fini(rdev);
8798 radeon_gem_fini(rdev);
8799 radeon_fence_driver_fini(rdev);
8800 radeon_bo_fini(rdev);
8801 radeon_atombios_fini(rdev);
8802 cik_fini_microcode(rdev);
8803 kfree(rdev->bios);
8804 rdev->bios = NULL;
8805 }
8806
8807 void dce8_program_fmt(struct drm_encoder *encoder)
8808 {
8809 struct drm_device *dev = encoder->dev;
8810 struct radeon_device *rdev = dev->dev_private;
8811 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8812 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8813 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8814 int bpc = 0;
8815 u32 tmp = 0;
8816 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8817
8818 if (connector) {
8819 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8820 bpc = radeon_get_monitor_bpc(connector);
8821 dither = radeon_connector->dither;
8822 }
8823
8824 /* LVDS/eDP FMT is set up by atom */
8825 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8826 return;
8827
8828 /* not needed for analog */
8829 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8830 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8831 return;
8832
8833 if (bpc == 0)
8834 return;
8835
8836 switch (bpc) {
8837 case 6:
8838 if (dither == RADEON_FMT_DITHER_ENABLE)
8839 /* XXX sort out optimal dither settings */
8840 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8841 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8842 else
8843 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8844 break;
8845 case 8:
8846 if (dither == RADEON_FMT_DITHER_ENABLE)
8847 /* XXX sort out optimal dither settings */
8848 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8849 FMT_RGB_RANDOM_ENABLE |
8850 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8851 else
8852 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8853 break;
8854 case 10:
8855 if (dither == RADEON_FMT_DITHER_ENABLE)
8856 /* XXX sort out optimal dither settings */
8857 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8858 FMT_RGB_RANDOM_ENABLE |
8859 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8860 else
8861 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8862 break;
8863 default:
8864 /* not needed */
8865 break;
8866 }
8867
8868 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8869 }
8870
8871 /* display watermark setup */
8872 /**
8873 * dce8_line_buffer_adjust - Set up the line buffer
8874 *
8875 * @rdev: radeon_device pointer
8876 * @radeon_crtc: the selected display controller
8877 * @mode: the current display mode on the selected display
8878 * controller
8879 *
8880 * Setup up the line buffer allocation for
8881 * the selected display controller (CIK).
8882 * Returns the line buffer size in pixels.
8883 */
8884 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8885 struct radeon_crtc *radeon_crtc,
8886 struct drm_display_mode *mode)
8887 {
8888 u32 tmp, buffer_alloc, i;
8889 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8890 /*
8891 * Line Buffer Setup
8892 * There are 6 line buffers, one for each display controllers.
8893 * There are 3 partitions per LB. Select the number of partitions
8894 * to enable based on the display width. For display widths larger
8895 * than 4096, you need use to use 2 display controllers and combine
8896 * them using the stereo blender.
8897 */
8898 if (radeon_crtc->base.enabled && mode) {
8899 if (mode->crtc_hdisplay < 1920) {
8900 tmp = 1;
8901 buffer_alloc = 2;
8902 } else if (mode->crtc_hdisplay < 2560) {
8903 tmp = 2;
8904 buffer_alloc = 2;
8905 } else if (mode->crtc_hdisplay < 4096) {
8906 tmp = 0;
8907 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8908 } else {
8909 DRM_DEBUG_KMS("Mode too big for LB!\n");
8910 tmp = 0;
8911 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8912 }
8913 } else {
8914 tmp = 1;
8915 buffer_alloc = 0;
8916 }
8917
8918 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8919 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8920
8921 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8922 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8923 for (i = 0; i < rdev->usec_timeout; i++) {
8924 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8925 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8926 break;
8927 udelay(1);
8928 }
8929
8930 if (radeon_crtc->base.enabled && mode) {
8931 switch (tmp) {
8932 case 0:
8933 default:
8934 return 4096 * 2;
8935 case 1:
8936 return 1920 * 2;
8937 case 2:
8938 return 2560 * 2;
8939 }
8940 }
8941
8942 /* controller not enabled, so no lb used */
8943 return 0;
8944 }
8945
8946 /**
8947 * cik_get_number_of_dram_channels - get the number of dram channels
8948 *
8949 * @rdev: radeon_device pointer
8950 *
8951 * Look up the number of video ram channels (CIK).
8952 * Used for display watermark bandwidth calculations
8953 * Returns the number of dram channels
8954 */
8955 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8956 {
8957 u32 tmp = RREG32(MC_SHARED_CHMAP);
8958
8959 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8960 case 0:
8961 default:
8962 return 1;
8963 case 1:
8964 return 2;
8965 case 2:
8966 return 4;
8967 case 3:
8968 return 8;
8969 case 4:
8970 return 3;
8971 case 5:
8972 return 6;
8973 case 6:
8974 return 10;
8975 case 7:
8976 return 12;
8977 case 8:
8978 return 16;
8979 }
8980 }
8981
8982 struct dce8_wm_params {
8983 u32 dram_channels; /* number of dram channels */
8984 u32 yclk; /* bandwidth per dram data pin in kHz */
8985 u32 sclk; /* engine clock in kHz */
8986 u32 disp_clk; /* display clock in kHz */
8987 u32 src_width; /* viewport width */
8988 u32 active_time; /* active display time in ns */
8989 u32 blank_time; /* blank time in ns */
8990 bool interlaced; /* mode is interlaced */
8991 fixed20_12 vsc; /* vertical scale ratio */
8992 u32 num_heads; /* number of active crtcs */
8993 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8994 u32 lb_size; /* line buffer allocated to pipe */
8995 u32 vtaps; /* vertical scaler taps */
8996 };
8997
8998 /**
8999 * dce8_dram_bandwidth - get the dram bandwidth
9000 *
9001 * @wm: watermark calculation data
9002 *
9003 * Calculate the raw dram bandwidth (CIK).
9004 * Used for display watermark bandwidth calculations
9005 * Returns the dram bandwidth in MBytes/s
9006 */
9007 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9008 {
9009 /* Calculate raw DRAM Bandwidth */
9010 fixed20_12 dram_efficiency; /* 0.7 */
9011 fixed20_12 yclk, dram_channels, bandwidth;
9012 fixed20_12 a;
9013
9014 a.full = dfixed_const(1000);
9015 yclk.full = dfixed_const(wm->yclk);
9016 yclk.full = dfixed_div(yclk, a);
9017 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9018 a.full = dfixed_const(10);
9019 dram_efficiency.full = dfixed_const(7);
9020 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9021 bandwidth.full = dfixed_mul(dram_channels, yclk);
9022 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9023
9024 return dfixed_trunc(bandwidth);
9025 }
9026
9027 /**
9028 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9029 *
9030 * @wm: watermark calculation data
9031 *
9032 * Calculate the dram bandwidth used for display (CIK).
9033 * Used for display watermark bandwidth calculations
9034 * Returns the dram bandwidth for display in MBytes/s
9035 */
9036 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9037 {
9038 /* Calculate DRAM Bandwidth and the part allocated to display. */
9039 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9040 fixed20_12 yclk, dram_channels, bandwidth;
9041 fixed20_12 a;
9042
9043 a.full = dfixed_const(1000);
9044 yclk.full = dfixed_const(wm->yclk);
9045 yclk.full = dfixed_div(yclk, a);
9046 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9047 a.full = dfixed_const(10);
9048 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9049 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9050 bandwidth.full = dfixed_mul(dram_channels, yclk);
9051 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9052
9053 return dfixed_trunc(bandwidth);
9054 }
9055
9056 /**
9057 * dce8_data_return_bandwidth - get the data return bandwidth
9058 *
9059 * @wm: watermark calculation data
9060 *
9061 * Calculate the data return bandwidth used for display (CIK).
9062 * Used for display watermark bandwidth calculations
9063 * Returns the data return bandwidth in MBytes/s
9064 */
9065 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9066 {
9067 /* Calculate the display Data return Bandwidth */
9068 fixed20_12 return_efficiency; /* 0.8 */
9069 fixed20_12 sclk, bandwidth;
9070 fixed20_12 a;
9071
9072 a.full = dfixed_const(1000);
9073 sclk.full = dfixed_const(wm->sclk);
9074 sclk.full = dfixed_div(sclk, a);
9075 a.full = dfixed_const(10);
9076 return_efficiency.full = dfixed_const(8);
9077 return_efficiency.full = dfixed_div(return_efficiency, a);
9078 a.full = dfixed_const(32);
9079 bandwidth.full = dfixed_mul(a, sclk);
9080 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9081
9082 return dfixed_trunc(bandwidth);
9083 }
9084
9085 /**
9086 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9087 *
9088 * @wm: watermark calculation data
9089 *
9090 * Calculate the dmif bandwidth used for display (CIK).
9091 * Used for display watermark bandwidth calculations
9092 * Returns the dmif bandwidth in MBytes/s
9093 */
9094 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9095 {
9096 /* Calculate the DMIF Request Bandwidth */
9097 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9098 fixed20_12 disp_clk, bandwidth;
9099 fixed20_12 a, b;
9100
9101 a.full = dfixed_const(1000);
9102 disp_clk.full = dfixed_const(wm->disp_clk);
9103 disp_clk.full = dfixed_div(disp_clk, a);
9104 a.full = dfixed_const(32);
9105 b.full = dfixed_mul(a, disp_clk);
9106
9107 a.full = dfixed_const(10);
9108 disp_clk_request_efficiency.full = dfixed_const(8);
9109 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9110
9111 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9112
9113 return dfixed_trunc(bandwidth);
9114 }
9115
9116 /**
9117 * dce8_available_bandwidth - get the min available bandwidth
9118 *
9119 * @wm: watermark calculation data
9120 *
9121 * Calculate the min available bandwidth used for display (CIK).
9122 * Used for display watermark bandwidth calculations
9123 * Returns the min available bandwidth in MBytes/s
9124 */
9125 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9126 {
9127 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9128 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9129 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9130 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9131
9132 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9133 }
9134
9135 /**
9136 * dce8_average_bandwidth - get the average available bandwidth
9137 *
9138 * @wm: watermark calculation data
9139 *
9140 * Calculate the average available bandwidth used for display (CIK).
9141 * Used for display watermark bandwidth calculations
9142 * Returns the average available bandwidth in MBytes/s
9143 */
9144 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9145 {
9146 /* Calculate the display mode Average Bandwidth
9147 * DisplayMode should contain the source and destination dimensions,
9148 * timing, etc.
9149 */
9150 fixed20_12 bpp;
9151 fixed20_12 line_time;
9152 fixed20_12 src_width;
9153 fixed20_12 bandwidth;
9154 fixed20_12 a;
9155
9156 a.full = dfixed_const(1000);
9157 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9158 line_time.full = dfixed_div(line_time, a);
9159 bpp.full = dfixed_const(wm->bytes_per_pixel);
9160 src_width.full = dfixed_const(wm->src_width);
9161 bandwidth.full = dfixed_mul(src_width, bpp);
9162 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9163 bandwidth.full = dfixed_div(bandwidth, line_time);
9164
9165 return dfixed_trunc(bandwidth);
9166 }
9167
9168 /**
9169 * dce8_latency_watermark - get the latency watermark
9170 *
9171 * @wm: watermark calculation data
9172 *
9173 * Calculate the latency watermark (CIK).
9174 * Used for display watermark bandwidth calculations
9175 * Returns the latency watermark in ns
9176 */
9177 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9178 {
9179 /* First calculate the latency in ns */
9180 u32 mc_latency = 2000; /* 2000 ns. */
9181 u32 available_bandwidth = dce8_available_bandwidth(wm);
9182 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9183 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9184 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9185 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9186 (wm->num_heads * cursor_line_pair_return_time);
9187 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9188 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9189 u32 tmp, dmif_size = 12288;
9190 fixed20_12 a, b, c;
9191
9192 if (wm->num_heads == 0)
9193 return 0;
9194
9195 a.full = dfixed_const(2);
9196 b.full = dfixed_const(1);
9197 if ((wm->vsc.full > a.full) ||
9198 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9199 (wm->vtaps >= 5) ||
9200 ((wm->vsc.full >= a.full) && wm->interlaced))
9201 max_src_lines_per_dst_line = 4;
9202 else
9203 max_src_lines_per_dst_line = 2;
9204
9205 a.full = dfixed_const(available_bandwidth);
9206 b.full = dfixed_const(wm->num_heads);
9207 a.full = dfixed_div(a, b);
9208
9209 b.full = dfixed_const(mc_latency + 512);
9210 c.full = dfixed_const(wm->disp_clk);
9211 b.full = dfixed_div(b, c);
9212
9213 c.full = dfixed_const(dmif_size);
9214 b.full = dfixed_div(c, b);
9215
9216 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9217
9218 b.full = dfixed_const(1000);
9219 c.full = dfixed_const(wm->disp_clk);
9220 b.full = dfixed_div(c, b);
9221 c.full = dfixed_const(wm->bytes_per_pixel);
9222 b.full = dfixed_mul(b, c);
9223
9224 lb_fill_bw = min(tmp, dfixed_trunc(b));
9225
9226 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9227 b.full = dfixed_const(1000);
9228 c.full = dfixed_const(lb_fill_bw);
9229 b.full = dfixed_div(c, b);
9230 a.full = dfixed_div(a, b);
9231 line_fill_time = dfixed_trunc(a);
9232
9233 if (line_fill_time < wm->active_time)
9234 return latency;
9235 else
9236 return latency + (line_fill_time - wm->active_time);
9237
9238 }
9239
9240 /**
9241 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9242 * average and available dram bandwidth
9243 *
9244 * @wm: watermark calculation data
9245 *
9246 * Check if the display average bandwidth fits in the display
9247 * dram bandwidth (CIK).
9248 * Used for display watermark bandwidth calculations
9249 * Returns true if the display fits, false if not.
9250 */
9251 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9252 {
9253 if (dce8_average_bandwidth(wm) <=
9254 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9255 return true;
9256 else
9257 return false;
9258 }
9259
9260 /**
9261 * dce8_average_bandwidth_vs_available_bandwidth - check
9262 * average and available bandwidth
9263 *
9264 * @wm: watermark calculation data
9265 *
9266 * Check if the display average bandwidth fits in the display
9267 * available bandwidth (CIK).
9268 * Used for display watermark bandwidth calculations
9269 * Returns true if the display fits, false if not.
9270 */
9271 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9272 {
9273 if (dce8_average_bandwidth(wm) <=
9274 (dce8_available_bandwidth(wm) / wm->num_heads))
9275 return true;
9276 else
9277 return false;
9278 }
9279
9280 /**
9281 * dce8_check_latency_hiding - check latency hiding
9282 *
9283 * @wm: watermark calculation data
9284 *
9285 * Check latency hiding (CIK).
9286 * Used for display watermark bandwidth calculations
9287 * Returns true if the display fits, false if not.
9288 */
9289 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9290 {
9291 u32 lb_partitions = wm->lb_size / wm->src_width;
9292 u32 line_time = wm->active_time + wm->blank_time;
9293 u32 latency_tolerant_lines;
9294 u32 latency_hiding;
9295 fixed20_12 a;
9296
9297 a.full = dfixed_const(1);
9298 if (wm->vsc.full > a.full)
9299 latency_tolerant_lines = 1;
9300 else {
9301 if (lb_partitions <= (wm->vtaps + 1))
9302 latency_tolerant_lines = 1;
9303 else
9304 latency_tolerant_lines = 2;
9305 }
9306
9307 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9308
9309 if (dce8_latency_watermark(wm) <= latency_hiding)
9310 return true;
9311 else
9312 return false;
9313 }
9314
9315 /**
9316 * dce8_program_watermarks - program display watermarks
9317 *
9318 * @rdev: radeon_device pointer
9319 * @radeon_crtc: the selected display controller
9320 * @lb_size: line buffer size
9321 * @num_heads: number of display controllers in use
9322 *
9323 * Calculate and program the display watermarks for the
9324 * selected display controller (CIK).
9325 */
9326 static void dce8_program_watermarks(struct radeon_device *rdev,
9327 struct radeon_crtc *radeon_crtc,
9328 u32 lb_size, u32 num_heads)
9329 {
9330 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9331 struct dce8_wm_params wm_low, wm_high;
9332 u32 pixel_period;
9333 u32 line_time = 0;
9334 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9335 u32 tmp, wm_mask;
9336
9337 if (radeon_crtc->base.enabled && num_heads && mode) {
9338 pixel_period = 1000000 / (u32)mode->clock;
9339 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9340
9341 /* watermark for high clocks */
9342 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9343 rdev->pm.dpm_enabled) {
9344 wm_high.yclk =
9345 radeon_dpm_get_mclk(rdev, false) * 10;
9346 wm_high.sclk =
9347 radeon_dpm_get_sclk(rdev, false) * 10;
9348 } else {
9349 wm_high.yclk = rdev->pm.current_mclk * 10;
9350 wm_high.sclk = rdev->pm.current_sclk * 10;
9351 }
9352
9353 wm_high.disp_clk = mode->clock;
9354 wm_high.src_width = mode->crtc_hdisplay;
9355 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9356 wm_high.blank_time = line_time - wm_high.active_time;
9357 wm_high.interlaced = false;
9358 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9359 wm_high.interlaced = true;
9360 wm_high.vsc = radeon_crtc->vsc;
9361 wm_high.vtaps = 1;
9362 if (radeon_crtc->rmx_type != RMX_OFF)
9363 wm_high.vtaps = 2;
9364 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9365 wm_high.lb_size = lb_size;
9366 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9367 wm_high.num_heads = num_heads;
9368
9369 /* set for high clocks */
9370 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9371
9372 /* possibly force display priority to high */
9373 /* should really do this at mode validation time... */
9374 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9375 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9376 !dce8_check_latency_hiding(&wm_high) ||
9377 (rdev->disp_priority == 2)) {
9378 DRM_DEBUG_KMS("force priority to high\n");
9379 }
9380
9381 /* watermark for low clocks */
9382 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9383 rdev->pm.dpm_enabled) {
9384 wm_low.yclk =
9385 radeon_dpm_get_mclk(rdev, true) * 10;
9386 wm_low.sclk =
9387 radeon_dpm_get_sclk(rdev, true) * 10;
9388 } else {
9389 wm_low.yclk = rdev->pm.current_mclk * 10;
9390 wm_low.sclk = rdev->pm.current_sclk * 10;
9391 }
9392
9393 wm_low.disp_clk = mode->clock;
9394 wm_low.src_width = mode->crtc_hdisplay;
9395 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9396 wm_low.blank_time = line_time - wm_low.active_time;
9397 wm_low.interlaced = false;
9398 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9399 wm_low.interlaced = true;
9400 wm_low.vsc = radeon_crtc->vsc;
9401 wm_low.vtaps = 1;
9402 if (radeon_crtc->rmx_type != RMX_OFF)
9403 wm_low.vtaps = 2;
9404 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9405 wm_low.lb_size = lb_size;
9406 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9407 wm_low.num_heads = num_heads;
9408
9409 /* set for low clocks */
9410 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9411
9412 /* possibly force display priority to high */
9413 /* should really do this at mode validation time... */
9414 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9415 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9416 !dce8_check_latency_hiding(&wm_low) ||
9417 (rdev->disp_priority == 2)) {
9418 DRM_DEBUG_KMS("force priority to high\n");
9419 }
9420 }
9421
9422 /* select wm A */
9423 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9424 tmp = wm_mask;
9425 tmp &= ~LATENCY_WATERMARK_MASK(3);
9426 tmp |= LATENCY_WATERMARK_MASK(1);
9427 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9428 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9429 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9430 LATENCY_HIGH_WATERMARK(line_time)));
9431 /* select wm B */
9432 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9433 tmp &= ~LATENCY_WATERMARK_MASK(3);
9434 tmp |= LATENCY_WATERMARK_MASK(2);
9435 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9436 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9437 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9438 LATENCY_HIGH_WATERMARK(line_time)));
9439 /* restore original selection */
9440 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9441
9442 /* save values for DPM */
9443 radeon_crtc->line_time = line_time;
9444 radeon_crtc->wm_high = latency_watermark_a;
9445 radeon_crtc->wm_low = latency_watermark_b;
9446 }
9447
9448 /**
9449 * dce8_bandwidth_update - program display watermarks
9450 *
9451 * @rdev: radeon_device pointer
9452 *
9453 * Calculate and program the display watermarks and line
9454 * buffer allocation (CIK).
9455 */
9456 void dce8_bandwidth_update(struct radeon_device *rdev)
9457 {
9458 struct drm_display_mode *mode = NULL;
9459 u32 num_heads = 0, lb_size;
9460 int i;
9461
9462 if (!rdev->mode_info.mode_config_initialized)
9463 return;
9464
9465 radeon_update_display_priority(rdev);
9466
9467 for (i = 0; i < rdev->num_crtc; i++) {
9468 if (rdev->mode_info.crtcs[i]->base.enabled)
9469 num_heads++;
9470 }
9471 for (i = 0; i < rdev->num_crtc; i++) {
9472 mode = &rdev->mode_info.crtcs[i]->base.mode;
9473 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9474 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9475 }
9476 }
9477
9478 /**
9479 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9480 *
9481 * @rdev: radeon_device pointer
9482 *
9483 * Fetches a GPU clock counter snapshot (SI).
9484 * Returns the 64 bit clock counter snapshot.
9485 */
9486 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9487 {
9488 uint64_t clock;
9489
9490 spin_lock(&rdev->gpu_clock_mutex);
9491 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9492 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9493 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9494 spin_unlock(&rdev->gpu_clock_mutex);
9495 return clock;
9496 }
9497
9498 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9499 u32 cntl_reg, u32 status_reg)
9500 {
9501 int r, i;
9502 struct atom_clock_dividers dividers;
9503 uint32_t tmp;
9504
9505 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9506 clock, false, &dividers);
9507 if (r)
9508 return r;
9509
9510 tmp = RREG32_SMC(cntl_reg);
9511 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9512 tmp |= dividers.post_divider;
9513 WREG32_SMC(cntl_reg, tmp);
9514
9515 for (i = 0; i < 100; i++) {
9516 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9517 break;
9518 mdelay(10);
9519 }
9520 if (i == 100)
9521 return -ETIMEDOUT;
9522
9523 return 0;
9524 }
9525
9526 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9527 {
9528 int r = 0;
9529
9530 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9531 if (r)
9532 return r;
9533
9534 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9535 return r;
9536 }
9537
9538 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9539 {
9540 int r, i;
9541 struct atom_clock_dividers dividers;
9542 u32 tmp;
9543
9544 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9545 ecclk, false, &dividers);
9546 if (r)
9547 return r;
9548
9549 for (i = 0; i < 100; i++) {
9550 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9551 break;
9552 mdelay(10);
9553 }
9554 if (i == 100)
9555 return -ETIMEDOUT;
9556
9557 tmp = RREG32_SMC(CG_ECLK_CNTL);
9558 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9559 tmp |= dividers.post_divider;
9560 WREG32_SMC(CG_ECLK_CNTL, tmp);
9561
9562 for (i = 0; i < 100; i++) {
9563 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9564 break;
9565 mdelay(10);
9566 }
9567 if (i == 100)
9568 return -ETIMEDOUT;
9569
9570 return 0;
9571 }
9572
9573 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9574 {
9575 struct pci_dev *root = rdev->pdev->bus->self;
9576 int bridge_pos, gpu_pos;
9577 u32 speed_cntl, mask, current_data_rate;
9578 int ret, i;
9579 u16 tmp16;
9580
9581 if (radeon_pcie_gen2 == 0)
9582 return;
9583
9584 if (rdev->flags & RADEON_IS_IGP)
9585 return;
9586
9587 if (!(rdev->flags & RADEON_IS_PCIE))
9588 return;
9589
9590 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9591 if (ret != 0)
9592 return;
9593
9594 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9595 return;
9596
9597 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9598 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9599 LC_CURRENT_DATA_RATE_SHIFT;
9600 if (mask & DRM_PCIE_SPEED_80) {
9601 if (current_data_rate == 2) {
9602 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9603 return;
9604 }
9605 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9606 } else if (mask & DRM_PCIE_SPEED_50) {
9607 if (current_data_rate == 1) {
9608 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9609 return;
9610 }
9611 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9612 }
9613
9614 bridge_pos = pci_get_pciecap_ptr(root->dev.bsddev);
9615 if (!bridge_pos)
9616 return;
9617
9618 gpu_pos = pci_get_pciecap_ptr(rdev->pdev->dev.bsddev);
9619 if (!gpu_pos)
9620 return;
9621
9622 if (mask & DRM_PCIE_SPEED_80) {
9623 /* re-try equalization if gen3 is not already enabled */
9624 if (current_data_rate != 2) {
9625 u16 bridge_cfg, gpu_cfg;
9626 u16 bridge_cfg2, gpu_cfg2;
9627 u32 max_lw, current_lw, tmp;
9628
9629 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9630 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9631
9632 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9633 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9634
9635 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9636 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9637
9638 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9639 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9640 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9641
9642 if (current_lw < max_lw) {
9643 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9644 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9645 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9646 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9647 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9648 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9649 }
9650 }
9651
9652 for (i = 0; i < 10; i++) {
9653 /* check status */
9654 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9655 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9656 break;
9657
9658 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9659 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9660
9661 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9662 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9663
9664 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9665 tmp |= LC_SET_QUIESCE;
9666 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9667
9668 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9669 tmp |= LC_REDO_EQ;
9670 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9671
9672 mdelay(100);
9673
9674 /* linkctl */
9675 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9676 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9677 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9678 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9679
9680 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9681 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9682 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9683 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9684
9685 /* linkctl2 */
9686 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9687 tmp16 &= ~((1 << 4) | (7 << 9));
9688 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9689 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9690
9691 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9692 tmp16 &= ~((1 << 4) | (7 << 9));
9693 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9694 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9695
9696 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9697 tmp &= ~LC_SET_QUIESCE;
9698 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9699 }
9700 }
9701 }
9702
9703 /* set the link speed */
9704 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9705 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9706 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9707
9708 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9709 tmp16 &= ~0xf;
9710 if (mask & DRM_PCIE_SPEED_80)
9711 tmp16 |= 3; /* gen3 */
9712 else if (mask & DRM_PCIE_SPEED_50)
9713 tmp16 |= 2; /* gen2 */
9714 else
9715 tmp16 |= 1; /* gen1 */
9716 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9717
9718 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9719 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9720 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9721
9722 for (i = 0; i < rdev->usec_timeout; i++) {
9723 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9724 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9725 break;
9726 udelay(1);
9727 }
9728 }
9729
9730 static void cik_program_aspm(struct radeon_device *rdev)
9731 {
9732 u32 data, orig;
9733 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9734 bool disable_clkreq = false;
9735
9736 if (radeon_aspm == 0)
9737 return;
9738
9739 /* XXX double check IGPs */
9740 if (rdev->flags & RADEON_IS_IGP)
9741 return;
9742
9743 if (!(rdev->flags & RADEON_IS_PCIE))
9744 return;
9745
9746 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9747 data &= ~LC_XMIT_N_FTS_MASK;
9748 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9749 if (orig != data)
9750 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9751
9752 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9753 data |= LC_GO_TO_RECOVERY;
9754 if (orig != data)
9755 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9756
9757 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9758 data |= P_IGNORE_EDB_ERR;
9759 if (orig != data)
9760 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9761
9762 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9763 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9764 data |= LC_PMI_TO_L1_DIS;
9765 if (!disable_l0s)
9766 data |= LC_L0S_INACTIVITY(7);
9767
9768 if (!disable_l1) {
9769 data |= LC_L1_INACTIVITY(7);
9770 data &= ~LC_PMI_TO_L1_DIS;
9771 if (orig != data)
9772 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9773
9774 if (!disable_plloff_in_l1) {
9775 bool clk_req_support;
9776
9777 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9778 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9779 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9780 if (orig != data)
9781 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9782
9783 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9784 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9785 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9786 if (orig != data)
9787 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9788
9789 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9790 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9791 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9792 if (orig != data)
9793 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9794
9795 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9796 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9797 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9798 if (orig != data)
9799 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9800
9801 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9802 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9803 data |= LC_DYN_LANES_PWR_STATE(3);
9804 if (orig != data)
9805 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9806
9807 if (!disable_clkreq) {
9808 #ifdef zMN_TODO
9809 struct pci_dev *root = rdev->pdev->bus->self;
9810 u32 lnkcap;
9811
9812 clk_req_support = false;
9813 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9814 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9815 clk_req_support = true;
9816 #else
9817 clk_req_support = false;
9818 #endif
9819 } else {
9820 clk_req_support = false;
9821 }
9822
9823 if (clk_req_support) {
9824 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9825 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9826 if (orig != data)
9827 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9828
9829 orig = data = RREG32_SMC(THM_CLK_CNTL);
9830 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9831 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9832 if (orig != data)
9833 WREG32_SMC(THM_CLK_CNTL, data);
9834
9835 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9836 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9837 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9838 if (orig != data)
9839 WREG32_SMC(MISC_CLK_CTRL, data);
9840
9841 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9842 data &= ~BCLK_AS_XCLK;
9843 if (orig != data)
9844 WREG32_SMC(CG_CLKPIN_CNTL, data);
9845
9846 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9847 data &= ~FORCE_BIF_REFCLK_EN;
9848 if (orig != data)
9849 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9850
9851 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9852 data &= ~MPLL_CLKOUT_SEL_MASK;
9853 data |= MPLL_CLKOUT_SEL(4);
9854 if (orig != data)
9855 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9856 }
9857 }
9858 } else {
9859 if (orig != data)
9860 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9861 }
9862
9863 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9864 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9865 if (orig != data)
9866 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9867
9868 if (!disable_l0s) {
9869 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9870 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9871 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9872 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9873 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9874 data &= ~LC_L0S_INACTIVITY_MASK;
9875 if (orig != data)
9876 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9877 }
9878 }
9879 }
9880 }
DragonFly BSD