?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'pci-v3.13-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/helgaa...
[linux-2.6.git] / drivers / gpu / drm / radeon / evergreen.c
blob56f6bec34af538fa899c381ebeb9b79d37c856fc
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
37
38 static const u32 crtc_offsets[6] =
39 {
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46 };
47
48 #include "clearstate_evergreen.h"
49
50 static const u32 sumo_rlc_save_restore_register_list[] =
51 {
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133 };
134
135 static void evergreen_gpu_init(struct radeon_device *rdev);
136 void evergreen_fini(struct radeon_device *rdev);
137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
138 void evergreen_program_aspm(struct radeon_device *rdev);
139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
143 void cik_init_cp_pg_table(struct radeon_device *rdev);
144
145 extern u32 si_get_csb_size(struct radeon_device *rdev);
146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
149
150 static const u32 evergreen_golden_registers[] =
151 {
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5cc, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194 };
195
196 static const u32 evergreen_golden_registers2[] =
197 {
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212 };
213
214 static const u32 cypress_mgcg_init[] =
215 {
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365 };
366
367 static const u32 redwood_mgcg_init[] =
368 {
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437 };
438
439 static const u32 cedar_golden_registers[] =
440 {
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5cc, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480 };
481
482 static const u32 cedar_mgcg_init[] =
483 {
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534 };
535
536 static const u32 juniper_mgcg_init[] =
537 {
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632 };
633
634 static const u32 supersumo_golden_registers[] =
635 {
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5cc, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707 };
708
709 static const u32 sumo_golden_registers[] =
710 {
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716 };
717
718 static const u32 wrestler_golden_registers[] =
719 {
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5cc, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769 };
770
771 static const u32 barts_golden_registers[] =
772 {
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818 };
819
820 static const u32 turks_golden_registers[] =
821 {
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869 };
870
871 static const u32 caicos_golden_registers[] =
872 {
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920 };
921
922 static void evergreen_init_golden_registers(struct radeon_device *rdev)
923 {
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006 }
1007
1008 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011 {
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037 }
1038
1039 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041 {
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061 }
1062
1063 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064 {
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080 done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084 }
1085
1086 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087 {
1088 /* start off with something large */
1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1090 int r;
1091
1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
1140 if (fb_div < 307200)
1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173 }
1174
1175 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176 {
1177 int readrq;
1178 u16 v;
1179
1180 readrq = pcie_get_readrq(rdev->pdev);
1181 v = ffs(readrq) - 8;
1182 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183 * to avoid hangs or perfomance issues
1184 */
1185 if ((v == 0) || (v == 6) || (v == 7))
1186 pcie_set_readrq(rdev->pdev, 512);
1187 }
1188
1189 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1190 {
1191 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1192 return true;
1193 else
1194 return false;
1195 }
1196
1197 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1198 {
1199 u32 pos1, pos2;
1200
1201 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1203
1204 if (pos1 != pos2)
1205 return true;
1206 else
1207 return false;
1208 }
1209
1210 /**
1211 * dce4_wait_for_vblank - vblank wait asic callback.
1212 *
1213 * @rdev: radeon_device pointer
1214 * @crtc: crtc to wait for vblank on
1215 *
1216 * Wait for vblank on the requested crtc (evergreen+).
1217 */
1218 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1219 {
1220 unsigned i = 0;
1221
1222 if (crtc >= rdev->num_crtc)
1223 return;
1224
1225 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1226 return;
1227
1228 /* depending on when we hit vblank, we may be close to active; if so,
1229 * wait for another frame.
1230 */
1231 while (dce4_is_in_vblank(rdev, crtc)) {
1232 if (i++ % 100 == 0) {
1233 if (!dce4_is_counter_moving(rdev, crtc))
1234 break;
1235 }
1236 }
1237
1238 while (!dce4_is_in_vblank(rdev, crtc)) {
1239 if (i++ % 100 == 0) {
1240 if (!dce4_is_counter_moving(rdev, crtc))
1241 break;
1242 }
1243 }
1244 }
1245
1246 /**
1247 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1248 *
1249 * @rdev: radeon_device pointer
1250 * @crtc: crtc to prepare for pageflip on
1251 *
1252 * Pre-pageflip callback (evergreen+).
1253 * Enables the pageflip irq (vblank irq).
1254 */
1255 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1256 {
1257 /* enable the pflip int */
1258 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1259 }
1260
1261 /**
1262 * evergreen_post_page_flip - pos-pageflip callback.
1263 *
1264 * @rdev: radeon_device pointer
1265 * @crtc: crtc to cleanup pageflip on
1266 *
1267 * Post-pageflip callback (evergreen+).
1268 * Disables the pageflip irq (vblank irq).
1269 */
1270 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1271 {
1272 /* disable the pflip int */
1273 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1274 }
1275
1276 /**
1277 * evergreen_page_flip - pageflip callback.
1278 *
1279 * @rdev: radeon_device pointer
1280 * @crtc_id: crtc to cleanup pageflip on
1281 * @crtc_base: new address of the crtc (GPU MC address)
1282 *
1283 * Does the actual pageflip (evergreen+).
1284 * During vblank we take the crtc lock and wait for the update_pending
1285 * bit to go high, when it does, we release the lock, and allow the
1286 * double buffered update to take place.
1287 * Returns the current update pending status.
1288 */
1289 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1290 {
1291 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1292 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1293 int i;
1294
1295 /* Lock the graphics update lock */
1296 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1297 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1298
1299 /* update the scanout addresses */
1300 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1301 upper_32_bits(crtc_base));
1302 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1303 (u32)crtc_base);
1304
1305 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1306 upper_32_bits(crtc_base));
1307 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1308 (u32)crtc_base);
1309
1310 /* Wait for update_pending to go high. */
1311 for (i = 0; i < rdev->usec_timeout; i++) {
1312 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1313 break;
1314 udelay(1);
1315 }
1316 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1317
1318 /* Unlock the lock, so double-buffering can take place inside vblank */
1319 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1320 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1321
1322 /* Return current update_pending status: */
1323 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1324 }
1325
1326 /* get temperature in millidegrees */
1327 int evergreen_get_temp(struct radeon_device *rdev)
1328 {
1329 u32 temp, toffset;
1330 int actual_temp = 0;
1331
1332 if (rdev->family == CHIP_JUNIPER) {
1333 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1334 TOFFSET_SHIFT;
1335 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1336 TS0_ADC_DOUT_SHIFT;
1337
1338 if (toffset & 0x100)
1339 actual_temp = temp / 2 - (0x200 - toffset);
1340 else
1341 actual_temp = temp / 2 + toffset;
1342
1343 actual_temp = actual_temp * 1000;
1344
1345 } else {
1346 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1347 ASIC_T_SHIFT;
1348
1349 if (temp & 0x400)
1350 actual_temp = -256;
1351 else if (temp & 0x200)
1352 actual_temp = 255;
1353 else if (temp & 0x100) {
1354 actual_temp = temp & 0x1ff;
1355 actual_temp |= ~0x1ff;
1356 } else
1357 actual_temp = temp & 0xff;
1358
1359 actual_temp = (actual_temp * 1000) / 2;
1360 }
1361
1362 return actual_temp;
1363 }
1364
1365 int sumo_get_temp(struct radeon_device *rdev)
1366 {
1367 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1368 int actual_temp = temp - 49;
1369
1370 return actual_temp * 1000;
1371 }
1372
1373 /**
1374 * sumo_pm_init_profile - Initialize power profiles callback.
1375 *
1376 * @rdev: radeon_device pointer
1377 *
1378 * Initialize the power states used in profile mode
1379 * (sumo, trinity, SI).
1380 * Used for profile mode only.
1381 */
1382 void sumo_pm_init_profile(struct radeon_device *rdev)
1383 {
1384 int idx;
1385
1386 /* default */
1387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1391
1392 /* low,mid sh/mh */
1393 if (rdev->flags & RADEON_IS_MOBILITY)
1394 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1395 else
1396 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1397
1398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1401 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1402
1403 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1407
1408 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1411 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1412
1413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1417
1418 /* high sh/mh */
1419 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1420 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1424 rdev->pm.power_state[idx].num_clock_modes - 1;
1425
1426 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1430 rdev->pm.power_state[idx].num_clock_modes - 1;
1431 }
1432
1433 /**
1434 * btc_pm_init_profile - Initialize power profiles callback.
1435 *
1436 * @rdev: radeon_device pointer
1437 *
1438 * Initialize the power states used in profile mode
1439 * (BTC, cayman).
1440 * Used for profile mode only.
1441 */
1442 void btc_pm_init_profile(struct radeon_device *rdev)
1443 {
1444 int idx;
1445
1446 /* default */
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1451 /* starting with BTC, there is one state that is used for both
1452 * MH and SH. Difference is that we always use the high clock index for
1453 * mclk.
1454 */
1455 if (rdev->flags & RADEON_IS_MOBILITY)
1456 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1457 else
1458 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1459 /* low sh */
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1464 /* mid sh */
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1469 /* high sh */
1470 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1473 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1474 /* low mh */
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1478 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1479 /* mid mh */
1480 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1484 /* high mh */
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1489 }
1490
1491 /**
1492 * evergreen_pm_misc - set additional pm hw parameters callback.
1493 *
1494 * @rdev: radeon_device pointer
1495 *
1496 * Set non-clock parameters associated with a power state
1497 * (voltage, etc.) (evergreen+).
1498 */
1499 void evergreen_pm_misc(struct radeon_device *rdev)
1500 {
1501 int req_ps_idx = rdev->pm.requested_power_state_index;
1502 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1503 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1504 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1505
1506 if (voltage->type == VOLTAGE_SW) {
1507 /* 0xff0x are flags rather then an actual voltage */
1508 if ((voltage->voltage & 0xff00) == 0xff00)
1509 return;
1510 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1511 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1512 rdev->pm.current_vddc = voltage->voltage;
1513 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1514 }
1515
1516 /* starting with BTC, there is one state that is used for both
1517 * MH and SH. Difference is that we always use the high clock index for
1518 * mclk and vddci.
1519 */
1520 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1521 (rdev->family >= CHIP_BARTS) &&
1522 rdev->pm.active_crtc_count &&
1523 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1524 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1525 voltage = &rdev->pm.power_state[req_ps_idx].
1526 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1527
1528 /* 0xff0x are flags rather then an actual voltage */
1529 if ((voltage->vddci & 0xff00) == 0xff00)
1530 return;
1531 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1532 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1533 rdev->pm.current_vddci = voltage->vddci;
1534 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1535 }
1536 }
1537 }
1538
1539 /**
1540 * evergreen_pm_prepare - pre-power state change callback.
1541 *
1542 * @rdev: radeon_device pointer
1543 *
1544 * Prepare for a power state change (evergreen+).
1545 */
1546 void evergreen_pm_prepare(struct radeon_device *rdev)
1547 {
1548 struct drm_device *ddev = rdev->ddev;
1549 struct drm_crtc *crtc;
1550 struct radeon_crtc *radeon_crtc;
1551 u32 tmp;
1552
1553 /* disable any active CRTCs */
1554 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1555 radeon_crtc = to_radeon_crtc(crtc);
1556 if (radeon_crtc->enabled) {
1557 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1558 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1559 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1560 }
1561 }
1562 }
1563
1564 /**
1565 * evergreen_pm_finish - post-power state change callback.
1566 *
1567 * @rdev: radeon_device pointer
1568 *
1569 * Clean up after a power state change (evergreen+).
1570 */
1571 void evergreen_pm_finish(struct radeon_device *rdev)
1572 {
1573 struct drm_device *ddev = rdev->ddev;
1574 struct drm_crtc *crtc;
1575 struct radeon_crtc *radeon_crtc;
1576 u32 tmp;
1577
1578 /* enable any active CRTCs */
1579 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1580 radeon_crtc = to_radeon_crtc(crtc);
1581 if (radeon_crtc->enabled) {
1582 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1583 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1584 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1585 }
1586 }
1587 }
1588
1589 /**
1590 * evergreen_hpd_sense - hpd sense callback.
1591 *
1592 * @rdev: radeon_device pointer
1593 * @hpd: hpd (hotplug detect) pin
1594 *
1595 * Checks if a digital monitor is connected (evergreen+).
1596 * Returns true if connected, false if not connected.
1597 */
1598 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1599 {
1600 bool connected = false;
1601
1602 switch (hpd) {
1603 case RADEON_HPD_1:
1604 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1605 connected = true;
1606 break;
1607 case RADEON_HPD_2:
1608 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1609 connected = true;
1610 break;
1611 case RADEON_HPD_3:
1612 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1613 connected = true;
1614 break;
1615 case RADEON_HPD_4:
1616 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1617 connected = true;
1618 break;
1619 case RADEON_HPD_5:
1620 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1621 connected = true;
1622 break;
1623 case RADEON_HPD_6:
1624 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1625 connected = true;
1626 break;
1627 default:
1628 break;
1629 }
1630
1631 return connected;
1632 }
1633
1634 /**
1635 * evergreen_hpd_set_polarity - hpd set polarity callback.
1636 *
1637 * @rdev: radeon_device pointer
1638 * @hpd: hpd (hotplug detect) pin
1639 *
1640 * Set the polarity of the hpd pin (evergreen+).
1641 */
1642 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1643 enum radeon_hpd_id hpd)
1644 {
1645 u32 tmp;
1646 bool connected = evergreen_hpd_sense(rdev, hpd);
1647
1648 switch (hpd) {
1649 case RADEON_HPD_1:
1650 tmp = RREG32(DC_HPD1_INT_CONTROL);
1651 if (connected)
1652 tmp &= ~DC_HPDx_INT_POLARITY;
1653 else
1654 tmp |= DC_HPDx_INT_POLARITY;
1655 WREG32(DC_HPD1_INT_CONTROL, tmp);
1656 break;
1657 case RADEON_HPD_2:
1658 tmp = RREG32(DC_HPD2_INT_CONTROL);
1659 if (connected)
1660 tmp &= ~DC_HPDx_INT_POLARITY;
1661 else
1662 tmp |= DC_HPDx_INT_POLARITY;
1663 WREG32(DC_HPD2_INT_CONTROL, tmp);
1664 break;
1665 case RADEON_HPD_3:
1666 tmp = RREG32(DC_HPD3_INT_CONTROL);
1667 if (connected)
1668 tmp &= ~DC_HPDx_INT_POLARITY;
1669 else
1670 tmp |= DC_HPDx_INT_POLARITY;
1671 WREG32(DC_HPD3_INT_CONTROL, tmp);
1672 break;
1673 case RADEON_HPD_4:
1674 tmp = RREG32(DC_HPD4_INT_CONTROL);
1675 if (connected)
1676 tmp &= ~DC_HPDx_INT_POLARITY;
1677 else
1678 tmp |= DC_HPDx_INT_POLARITY;
1679 WREG32(DC_HPD4_INT_CONTROL, tmp);
1680 break;
1681 case RADEON_HPD_5:
1682 tmp = RREG32(DC_HPD5_INT_CONTROL);
1683 if (connected)
1684 tmp &= ~DC_HPDx_INT_POLARITY;
1685 else
1686 tmp |= DC_HPDx_INT_POLARITY;
1687 WREG32(DC_HPD5_INT_CONTROL, tmp);
1688 break;
1689 case RADEON_HPD_6:
1690 tmp = RREG32(DC_HPD6_INT_CONTROL);
1691 if (connected)
1692 tmp &= ~DC_HPDx_INT_POLARITY;
1693 else
1694 tmp |= DC_HPDx_INT_POLARITY;
1695 WREG32(DC_HPD6_INT_CONTROL, tmp);
1696 break;
1697 default:
1698 break;
1699 }
1700 }
1701
1702 /**
1703 * evergreen_hpd_init - hpd setup callback.
1704 *
1705 * @rdev: radeon_device pointer
1706 *
1707 * Setup the hpd pins used by the card (evergreen+).
1708 * Enable the pin, set the polarity, and enable the hpd interrupts.
1709 */
1710 void evergreen_hpd_init(struct radeon_device *rdev)
1711 {
1712 struct drm_device *dev = rdev->ddev;
1713 struct drm_connector *connector;
1714 unsigned enabled = 0;
1715 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1716 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1717
1718 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1719 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1720
1721 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1722 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1723 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1724 * aux dp channel on imac and help (but not completely fix)
1725 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1726 * also avoid interrupt storms during dpms.
1727 */
1728 continue;
1729 }
1730 switch (radeon_connector->hpd.hpd) {
1731 case RADEON_HPD_1:
1732 WREG32(DC_HPD1_CONTROL, tmp);
1733 break;
1734 case RADEON_HPD_2:
1735 WREG32(DC_HPD2_CONTROL, tmp);
1736 break;
1737 case RADEON_HPD_3:
1738 WREG32(DC_HPD3_CONTROL, tmp);
1739 break;
1740 case RADEON_HPD_4:
1741 WREG32(DC_HPD4_CONTROL, tmp);
1742 break;
1743 case RADEON_HPD_5:
1744 WREG32(DC_HPD5_CONTROL, tmp);
1745 break;
1746 case RADEON_HPD_6:
1747 WREG32(DC_HPD6_CONTROL, tmp);
1748 break;
1749 default:
1750 break;
1751 }
1752 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1753 enabled |= 1 << radeon_connector->hpd.hpd;
1754 }
1755 radeon_irq_kms_enable_hpd(rdev, enabled);
1756 }
1757
1758 /**
1759 * evergreen_hpd_fini - hpd tear down callback.
1760 *
1761 * @rdev: radeon_device pointer
1762 *
1763 * Tear down the hpd pins used by the card (evergreen+).
1764 * Disable the hpd interrupts.
1765 */
1766 void evergreen_hpd_fini(struct radeon_device *rdev)
1767 {
1768 struct drm_device *dev = rdev->ddev;
1769 struct drm_connector *connector;
1770 unsigned disabled = 0;
1771
1772 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1774 switch (radeon_connector->hpd.hpd) {
1775 case RADEON_HPD_1:
1776 WREG32(DC_HPD1_CONTROL, 0);
1777 break;
1778 case RADEON_HPD_2:
1779 WREG32(DC_HPD2_CONTROL, 0);
1780 break;
1781 case RADEON_HPD_3:
1782 WREG32(DC_HPD3_CONTROL, 0);
1783 break;
1784 case RADEON_HPD_4:
1785 WREG32(DC_HPD4_CONTROL, 0);
1786 break;
1787 case RADEON_HPD_5:
1788 WREG32(DC_HPD5_CONTROL, 0);
1789 break;
1790 case RADEON_HPD_6:
1791 WREG32(DC_HPD6_CONTROL, 0);
1792 break;
1793 default:
1794 break;
1795 }
1796 disabled |= 1 << radeon_connector->hpd.hpd;
1797 }
1798 radeon_irq_kms_disable_hpd(rdev, disabled);
1799 }
1800
1801 /* watermark setup */
1802
1803 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1804 struct radeon_crtc *radeon_crtc,
1805 struct drm_display_mode *mode,
1806 struct drm_display_mode *other_mode)
1807 {
1808 u32 tmp, buffer_alloc, i;
1809 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1810 /*
1811 * Line Buffer Setup
1812 * There are 3 line buffers, each one shared by 2 display controllers.
1813 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1814 * the display controllers. The paritioning is done via one of four
1815 * preset allocations specified in bits 2:0:
1816 * first display controller
1817 * 0 - first half of lb (3840 * 2)
1818 * 1 - first 3/4 of lb (5760 * 2)
1819 * 2 - whole lb (7680 * 2), other crtc must be disabled
1820 * 3 - first 1/4 of lb (1920 * 2)
1821 * second display controller
1822 * 4 - second half of lb (3840 * 2)
1823 * 5 - second 3/4 of lb (5760 * 2)
1824 * 6 - whole lb (7680 * 2), other crtc must be disabled
1825 * 7 - last 1/4 of lb (1920 * 2)
1826 */
1827 /* this can get tricky if we have two large displays on a paired group
1828 * of crtcs. Ideally for multiple large displays we'd assign them to
1829 * non-linked crtcs for maximum line buffer allocation.
1830 */
1831 if (radeon_crtc->base.enabled && mode) {
1832 if (other_mode) {
1833 tmp = 0; /* 1/2 */
1834 buffer_alloc = 1;
1835 } else {
1836 tmp = 2; /* whole */
1837 buffer_alloc = 2;
1838 }
1839 } else {
1840 tmp = 0;
1841 buffer_alloc = 0;
1842 }
1843
1844 /* second controller of the pair uses second half of the lb */
1845 if (radeon_crtc->crtc_id % 2)
1846 tmp += 4;
1847 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1848
1849 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1850 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1851 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1852 for (i = 0; i < rdev->usec_timeout; i++) {
1853 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1854 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1855 break;
1856 udelay(1);
1857 }
1858 }
1859
1860 if (radeon_crtc->base.enabled && mode) {
1861 switch (tmp) {
1862 case 0:
1863 case 4:
1864 default:
1865 if (ASIC_IS_DCE5(rdev))
1866 return 4096 * 2;
1867 else
1868 return 3840 * 2;
1869 case 1:
1870 case 5:
1871 if (ASIC_IS_DCE5(rdev))
1872 return 6144 * 2;
1873 else
1874 return 5760 * 2;
1875 case 2:
1876 case 6:
1877 if (ASIC_IS_DCE5(rdev))
1878 return 8192 * 2;
1879 else
1880 return 7680 * 2;
1881 case 3:
1882 case 7:
1883 if (ASIC_IS_DCE5(rdev))
1884 return 2048 * 2;
1885 else
1886 return 1920 * 2;
1887 }
1888 }
1889
1890 /* controller not enabled, so no lb used */
1891 return 0;
1892 }
1893
1894 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1895 {
1896 u32 tmp = RREG32(MC_SHARED_CHMAP);
1897
1898 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1899 case 0:
1900 default:
1901 return 1;
1902 case 1:
1903 return 2;
1904 case 2:
1905 return 4;
1906 case 3:
1907 return 8;
1908 }
1909 }
1910
1911 struct evergreen_wm_params {
1912 u32 dram_channels; /* number of dram channels */
1913 u32 yclk; /* bandwidth per dram data pin in kHz */
1914 u32 sclk; /* engine clock in kHz */
1915 u32 disp_clk; /* display clock in kHz */
1916 u32 src_width; /* viewport width */
1917 u32 active_time; /* active display time in ns */
1918 u32 blank_time; /* blank time in ns */
1919 bool interlaced; /* mode is interlaced */
1920 fixed20_12 vsc; /* vertical scale ratio */
1921 u32 num_heads; /* number of active crtcs */
1922 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1923 u32 lb_size; /* line buffer allocated to pipe */
1924 u32 vtaps; /* vertical scaler taps */
1925 };
1926
1927 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1928 {
1929 /* Calculate DRAM Bandwidth and the part allocated to display. */
1930 fixed20_12 dram_efficiency; /* 0.7 */
1931 fixed20_12 yclk, dram_channels, bandwidth;
1932 fixed20_12 a;
1933
1934 a.full = dfixed_const(1000);
1935 yclk.full = dfixed_const(wm->yclk);
1936 yclk.full = dfixed_div(yclk, a);
1937 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1938 a.full = dfixed_const(10);
1939 dram_efficiency.full = dfixed_const(7);
1940 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1941 bandwidth.full = dfixed_mul(dram_channels, yclk);
1942 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1943
1944 return dfixed_trunc(bandwidth);
1945 }
1946
1947 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1948 {
1949 /* Calculate DRAM Bandwidth and the part allocated to display. */
1950 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1951 fixed20_12 yclk, dram_channels, bandwidth;
1952 fixed20_12 a;
1953
1954 a.full = dfixed_const(1000);
1955 yclk.full = dfixed_const(wm->yclk);
1956 yclk.full = dfixed_div(yclk, a);
1957 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1958 a.full = dfixed_const(10);
1959 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1960 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1961 bandwidth.full = dfixed_mul(dram_channels, yclk);
1962 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1963
1964 return dfixed_trunc(bandwidth);
1965 }
1966
1967 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1968 {
1969 /* Calculate the display Data return Bandwidth */
1970 fixed20_12 return_efficiency; /* 0.8 */
1971 fixed20_12 sclk, bandwidth;
1972 fixed20_12 a;
1973
1974 a.full = dfixed_const(1000);
1975 sclk.full = dfixed_const(wm->sclk);
1976 sclk.full = dfixed_div(sclk, a);
1977 a.full = dfixed_const(10);
1978 return_efficiency.full = dfixed_const(8);
1979 return_efficiency.full = dfixed_div(return_efficiency, a);
1980 a.full = dfixed_const(32);
1981 bandwidth.full = dfixed_mul(a, sclk);
1982 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1983
1984 return dfixed_trunc(bandwidth);
1985 }
1986
1987 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1988 {
1989 /* Calculate the DMIF Request Bandwidth */
1990 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1991 fixed20_12 disp_clk, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 disp_clk.full = dfixed_const(wm->disp_clk);
1996 disp_clk.full = dfixed_div(disp_clk, a);
1997 a.full = dfixed_const(10);
1998 disp_clk_request_efficiency.full = dfixed_const(8);
1999 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2000 a.full = dfixed_const(32);
2001 bandwidth.full = dfixed_mul(a, disp_clk);
2002 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2003
2004 return dfixed_trunc(bandwidth);
2005 }
2006
2007 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2008 {
2009 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2010 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2011 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2012 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2013
2014 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2015 }
2016
2017 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2018 {
2019 /* Calculate the display mode Average Bandwidth
2020 * DisplayMode should contain the source and destination dimensions,
2021 * timing, etc.
2022 */
2023 fixed20_12 bpp;
2024 fixed20_12 line_time;
2025 fixed20_12 src_width;
2026 fixed20_12 bandwidth;
2027 fixed20_12 a;
2028
2029 a.full = dfixed_const(1000);
2030 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2031 line_time.full = dfixed_div(line_time, a);
2032 bpp.full = dfixed_const(wm->bytes_per_pixel);
2033 src_width.full = dfixed_const(wm->src_width);
2034 bandwidth.full = dfixed_mul(src_width, bpp);
2035 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2036 bandwidth.full = dfixed_div(bandwidth, line_time);
2037
2038 return dfixed_trunc(bandwidth);
2039 }
2040
2041 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2042 {
2043 /* First calcualte the latency in ns */
2044 u32 mc_latency = 2000; /* 2000 ns. */
2045 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2046 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2047 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2048 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2049 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2050 (wm->num_heads * cursor_line_pair_return_time);
2051 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2052 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2053 fixed20_12 a, b, c;
2054
2055 if (wm->num_heads == 0)
2056 return 0;
2057
2058 a.full = dfixed_const(2);
2059 b.full = dfixed_const(1);
2060 if ((wm->vsc.full > a.full) ||
2061 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2062 (wm->vtaps >= 5) ||
2063 ((wm->vsc.full >= a.full) && wm->interlaced))
2064 max_src_lines_per_dst_line = 4;
2065 else
2066 max_src_lines_per_dst_line = 2;
2067
2068 a.full = dfixed_const(available_bandwidth);
2069 b.full = dfixed_const(wm->num_heads);
2070 a.full = dfixed_div(a, b);
2071
2072 b.full = dfixed_const(1000);
2073 c.full = dfixed_const(wm->disp_clk);
2074 b.full = dfixed_div(c, b);
2075 c.full = dfixed_const(wm->bytes_per_pixel);
2076 b.full = dfixed_mul(b, c);
2077
2078 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2079
2080 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2081 b.full = dfixed_const(1000);
2082 c.full = dfixed_const(lb_fill_bw);
2083 b.full = dfixed_div(c, b);
2084 a.full = dfixed_div(a, b);
2085 line_fill_time = dfixed_trunc(a);
2086
2087 if (line_fill_time < wm->active_time)
2088 return latency;
2089 else
2090 return latency + (line_fill_time - wm->active_time);
2091
2092 }
2093
2094 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2095 {
2096 if (evergreen_average_bandwidth(wm) <=
2097 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2098 return true;
2099 else
2100 return false;
2101 };
2102
2103 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2104 {
2105 if (evergreen_average_bandwidth(wm) <=
2106 (evergreen_available_bandwidth(wm) / wm->num_heads))
2107 return true;
2108 else
2109 return false;
2110 };
2111
2112 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2113 {
2114 u32 lb_partitions = wm->lb_size / wm->src_width;
2115 u32 line_time = wm->active_time + wm->blank_time;
2116 u32 latency_tolerant_lines;
2117 u32 latency_hiding;
2118 fixed20_12 a;
2119
2120 a.full = dfixed_const(1);
2121 if (wm->vsc.full > a.full)
2122 latency_tolerant_lines = 1;
2123 else {
2124 if (lb_partitions <= (wm->vtaps + 1))
2125 latency_tolerant_lines = 1;
2126 else
2127 latency_tolerant_lines = 2;
2128 }
2129
2130 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2131
2132 if (evergreen_latency_watermark(wm) <= latency_hiding)
2133 return true;
2134 else
2135 return false;
2136 }
2137
2138 static void evergreen_program_watermarks(struct radeon_device *rdev,
2139 struct radeon_crtc *radeon_crtc,
2140 u32 lb_size, u32 num_heads)
2141 {
2142 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2143 struct evergreen_wm_params wm_low, wm_high;
2144 u32 dram_channels;
2145 u32 pixel_period;
2146 u32 line_time = 0;
2147 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2148 u32 priority_a_mark = 0, priority_b_mark = 0;
2149 u32 priority_a_cnt = PRIORITY_OFF;
2150 u32 priority_b_cnt = PRIORITY_OFF;
2151 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2152 u32 tmp, arb_control3;
2153 fixed20_12 a, b, c;
2154
2155 if (radeon_crtc->base.enabled && num_heads && mode) {
2156 pixel_period = 1000000 / (u32)mode->clock;
2157 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2158 priority_a_cnt = 0;
2159 priority_b_cnt = 0;
2160 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2161
2162 /* watermark for high clocks */
2163 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2164 wm_high.yclk =
2165 radeon_dpm_get_mclk(rdev, false) * 10;
2166 wm_high.sclk =
2167 radeon_dpm_get_sclk(rdev, false) * 10;
2168 } else {
2169 wm_high.yclk = rdev->pm.current_mclk * 10;
2170 wm_high.sclk = rdev->pm.current_sclk * 10;
2171 }
2172
2173 wm_high.disp_clk = mode->clock;
2174 wm_high.src_width = mode->crtc_hdisplay;
2175 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2176 wm_high.blank_time = line_time - wm_high.active_time;
2177 wm_high.interlaced = false;
2178 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2179 wm_high.interlaced = true;
2180 wm_high.vsc = radeon_crtc->vsc;
2181 wm_high.vtaps = 1;
2182 if (radeon_crtc->rmx_type != RMX_OFF)
2183 wm_high.vtaps = 2;
2184 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2185 wm_high.lb_size = lb_size;
2186 wm_high.dram_channels = dram_channels;
2187 wm_high.num_heads = num_heads;
2188
2189 /* watermark for low clocks */
2190 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2191 wm_low.yclk =
2192 radeon_dpm_get_mclk(rdev, true) * 10;
2193 wm_low.sclk =
2194 radeon_dpm_get_sclk(rdev, true) * 10;
2195 } else {
2196 wm_low.yclk = rdev->pm.current_mclk * 10;
2197 wm_low.sclk = rdev->pm.current_sclk * 10;
2198 }
2199
2200 wm_low.disp_clk = mode->clock;
2201 wm_low.src_width = mode->crtc_hdisplay;
2202 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2203 wm_low.blank_time = line_time - wm_low.active_time;
2204 wm_low.interlaced = false;
2205 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2206 wm_low.interlaced = true;
2207 wm_low.vsc = radeon_crtc->vsc;
2208 wm_low.vtaps = 1;
2209 if (radeon_crtc->rmx_type != RMX_OFF)
2210 wm_low.vtaps = 2;
2211 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2212 wm_low.lb_size = lb_size;
2213 wm_low.dram_channels = dram_channels;
2214 wm_low.num_heads = num_heads;
2215
2216 /* set for high clocks */
2217 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2218 /* set for low clocks */
2219 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2220
2221 /* possibly force display priority to high */
2222 /* should really do this at mode validation time... */
2223 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2224 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2225 !evergreen_check_latency_hiding(&wm_high) ||
2226 (rdev->disp_priority == 2)) {
2227 DRM_DEBUG_KMS("force priority a to high\n");
2228 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2229 }
2230 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2231 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2232 !evergreen_check_latency_hiding(&wm_low) ||
2233 (rdev->disp_priority == 2)) {
2234 DRM_DEBUG_KMS("force priority b to high\n");
2235 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2236 }
2237
2238 a.full = dfixed_const(1000);
2239 b.full = dfixed_const(mode->clock);
2240 b.full = dfixed_div(b, a);
2241 c.full = dfixed_const(latency_watermark_a);
2242 c.full = dfixed_mul(c, b);
2243 c.full = dfixed_mul(c, radeon_crtc->hsc);
2244 c.full = dfixed_div(c, a);
2245 a.full = dfixed_const(16);
2246 c.full = dfixed_div(c, a);
2247 priority_a_mark = dfixed_trunc(c);
2248 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2249
2250 a.full = dfixed_const(1000);
2251 b.full = dfixed_const(mode->clock);
2252 b.full = dfixed_div(b, a);
2253 c.full = dfixed_const(latency_watermark_b);
2254 c.full = dfixed_mul(c, b);
2255 c.full = dfixed_mul(c, radeon_crtc->hsc);
2256 c.full = dfixed_div(c, a);
2257 a.full = dfixed_const(16);
2258 c.full = dfixed_div(c, a);
2259 priority_b_mark = dfixed_trunc(c);
2260 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2261 }
2262
2263 /* select wm A */
2264 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2265 tmp = arb_control3;
2266 tmp &= ~LATENCY_WATERMARK_MASK(3);
2267 tmp |= LATENCY_WATERMARK_MASK(1);
2268 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2269 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2270 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2271 LATENCY_HIGH_WATERMARK(line_time)));
2272 /* select wm B */
2273 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2274 tmp &= ~LATENCY_WATERMARK_MASK(3);
2275 tmp |= LATENCY_WATERMARK_MASK(2);
2276 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2277 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2278 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2279 LATENCY_HIGH_WATERMARK(line_time)));
2280 /* restore original selection */
2281 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2282
2283 /* write the priority marks */
2284 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2285 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2286
2287 /* save values for DPM */
2288 radeon_crtc->line_time = line_time;
2289 radeon_crtc->wm_high = latency_watermark_a;
2290 radeon_crtc->wm_low = latency_watermark_b;
2291 }
2292
2293 /**
2294 * evergreen_bandwidth_update - update display watermarks callback.
2295 *
2296 * @rdev: radeon_device pointer
2297 *
2298 * Update the display watermarks based on the requested mode(s)
2299 * (evergreen+).
2300 */
2301 void evergreen_bandwidth_update(struct radeon_device *rdev)
2302 {
2303 struct drm_display_mode *mode0 = NULL;
2304 struct drm_display_mode *mode1 = NULL;
2305 u32 num_heads = 0, lb_size;
2306 int i;
2307
2308 radeon_update_display_priority(rdev);
2309
2310 for (i = 0; i < rdev->num_crtc; i++) {
2311 if (rdev->mode_info.crtcs[i]->base.enabled)
2312 num_heads++;
2313 }
2314 for (i = 0; i < rdev->num_crtc; i += 2) {
2315 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2316 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2317 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2318 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2319 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2320 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2321 }
2322 }
2323
2324 /**
2325 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2326 *
2327 * @rdev: radeon_device pointer
2328 *
2329 * Wait for the MC (memory controller) to be idle.
2330 * (evergreen+).
2331 * Returns 0 if the MC is idle, -1 if not.
2332 */
2333 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2334 {
2335 unsigned i;
2336 u32 tmp;
2337
2338 for (i = 0; i < rdev->usec_timeout; i++) {
2339 /* read MC_STATUS */
2340 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2341 if (!tmp)
2342 return 0;
2343 udelay(1);
2344 }
2345 return -1;
2346 }
2347
2348 /*
2349 * GART
2350 */
2351 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2352 {
2353 unsigned i;
2354 u32 tmp;
2355
2356 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2357
2358 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2359 for (i = 0; i < rdev->usec_timeout; i++) {
2360 /* read MC_STATUS */
2361 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2362 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2363 if (tmp == 2) {
2364 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2365 return;
2366 }
2367 if (tmp) {
2368 return;
2369 }
2370 udelay(1);
2371 }
2372 }
2373
2374 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2375 {
2376 u32 tmp;
2377 int r;
2378
2379 if (rdev->gart.robj == NULL) {
2380 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2381 return -EINVAL;
2382 }
2383 r = radeon_gart_table_vram_pin(rdev);
2384 if (r)
2385 return r;
2386 radeon_gart_restore(rdev);
2387 /* Setup L2 cache */
2388 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2389 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2390 EFFECTIVE_L2_QUEUE_SIZE(7));
2391 WREG32(VM_L2_CNTL2, 0);
2392 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2393 /* Setup TLB control */
2394 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2395 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2396 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2397 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2398 if (rdev->flags & RADEON_IS_IGP) {
2399 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2400 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2401 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2402 } else {
2403 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2404 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2405 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2406 if ((rdev->family == CHIP_JUNIPER) ||
2407 (rdev->family == CHIP_CYPRESS) ||
2408 (rdev->family == CHIP_HEMLOCK) ||
2409 (rdev->family == CHIP_BARTS))
2410 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2411 }
2412 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2413 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2414 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2415 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2416 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2417 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2418 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2419 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2420 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2421 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2422 (u32)(rdev->dummy_page.addr >> 12));
2423 WREG32(VM_CONTEXT1_CNTL, 0);
2424
2425 evergreen_pcie_gart_tlb_flush(rdev);
2426 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2427 (unsigned)(rdev->mc.gtt_size >> 20),
2428 (unsigned long long)rdev->gart.table_addr);
2429 rdev->gart.ready = true;
2430 return 0;
2431 }
2432
2433 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2434 {
2435 u32 tmp;
2436
2437 /* Disable all tables */
2438 WREG32(VM_CONTEXT0_CNTL, 0);
2439 WREG32(VM_CONTEXT1_CNTL, 0);
2440
2441 /* Setup L2 cache */
2442 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2443 EFFECTIVE_L2_QUEUE_SIZE(7));
2444 WREG32(VM_L2_CNTL2, 0);
2445 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2446 /* Setup TLB control */
2447 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2448 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2449 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2450 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2451 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2452 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2453 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2454 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2455 radeon_gart_table_vram_unpin(rdev);
2456 }
2457
2458 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2459 {
2460 evergreen_pcie_gart_disable(rdev);
2461 radeon_gart_table_vram_free(rdev);
2462 radeon_gart_fini(rdev);
2463 }
2464
2465
2466 static void evergreen_agp_enable(struct radeon_device *rdev)
2467 {
2468 u32 tmp;
2469
2470 /* Setup L2 cache */
2471 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2472 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2473 EFFECTIVE_L2_QUEUE_SIZE(7));
2474 WREG32(VM_L2_CNTL2, 0);
2475 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2476 /* Setup TLB control */
2477 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2478 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2479 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2480 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2481 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2482 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2483 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2484 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2485 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2486 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2487 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2488 WREG32(VM_CONTEXT0_CNTL, 0);
2489 WREG32(VM_CONTEXT1_CNTL, 0);
2490 }
2491
2492 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2493 {
2494 u32 crtc_enabled, tmp, frame_count, blackout;
2495 int i, j;
2496
2497 if (!ASIC_IS_NODCE(rdev)) {
2498 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2499 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2500
2501 /* disable VGA render */
2502 WREG32(VGA_RENDER_CONTROL, 0);
2503 }
2504 /* blank the display controllers */
2505 for (i = 0; i < rdev->num_crtc; i++) {
2506 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2507 if (crtc_enabled) {
2508 save->crtc_enabled[i] = true;
2509 if (ASIC_IS_DCE6(rdev)) {
2510 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2511 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2512 radeon_wait_for_vblank(rdev, i);
2513 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2514 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2515 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2516 }
2517 } else {
2518 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2519 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2520 radeon_wait_for_vblank(rdev, i);
2521 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2522 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2523 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2524 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2525 }
2526 }
2527 /* wait for the next frame */
2528 frame_count = radeon_get_vblank_counter(rdev, i);
2529 for (j = 0; j < rdev->usec_timeout; j++) {
2530 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2531 break;
2532 udelay(1);
2533 }
2534
2535 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2536 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2537 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2538 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2539 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2540 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2541 save->crtc_enabled[i] = false;
2542 /* ***** */
2543 } else {
2544 save->crtc_enabled[i] = false;
2545 }
2546 }
2547
2548 radeon_mc_wait_for_idle(rdev);
2549
2550 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2551 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2552 /* Block CPU access */
2553 WREG32(BIF_FB_EN, 0);
2554 /* blackout the MC */
2555 blackout &= ~BLACKOUT_MODE_MASK;
2556 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2557 }
2558 /* wait for the MC to settle */
2559 udelay(100);
2560
2561 /* lock double buffered regs */
2562 for (i = 0; i < rdev->num_crtc; i++) {
2563 if (save->crtc_enabled[i]) {
2564 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2565 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2566 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2567 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2568 }
2569 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2570 if (!(tmp & 1)) {
2571 tmp |= 1;
2572 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2573 }
2574 }
2575 }
2576 }
2577
2578 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2579 {
2580 u32 tmp, frame_count;
2581 int i, j;
2582
2583 /* update crtc base addresses */
2584 for (i = 0; i < rdev->num_crtc; i++) {
2585 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2586 upper_32_bits(rdev->mc.vram_start));
2587 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2588 upper_32_bits(rdev->mc.vram_start));
2589 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2590 (u32)rdev->mc.vram_start);
2591 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2592 (u32)rdev->mc.vram_start);
2593 }
2594
2595 if (!ASIC_IS_NODCE(rdev)) {
2596 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2597 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2598 }
2599
2600 /* unlock regs and wait for update */
2601 for (i = 0; i < rdev->num_crtc; i++) {
2602 if (save->crtc_enabled[i]) {
2603 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2604 if ((tmp & 0x3) != 0) {
2605 tmp &= ~0x3;
2606 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2607 }
2608 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2609 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2610 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2611 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2612 }
2613 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2614 if (tmp & 1) {
2615 tmp &= ~1;
2616 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2617 }
2618 for (j = 0; j < rdev->usec_timeout; j++) {
2619 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2620 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2621 break;
2622 udelay(1);
2623 }
2624 }
2625 }
2626
2627 /* unblackout the MC */
2628 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2629 tmp &= ~BLACKOUT_MODE_MASK;
2630 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2631 /* allow CPU access */
2632 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2633
2634 for (i = 0; i < rdev->num_crtc; i++) {
2635 if (save->crtc_enabled[i]) {
2636 if (ASIC_IS_DCE6(rdev)) {
2637 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2638 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2639 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2640 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2641 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2642 } else {
2643 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2644 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2645 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2646 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2647 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2648 }
2649 /* wait for the next frame */
2650 frame_count = radeon_get_vblank_counter(rdev, i);
2651 for (j = 0; j < rdev->usec_timeout; j++) {
2652 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2653 break;
2654 udelay(1);
2655 }
2656 }
2657 }
2658 if (!ASIC_IS_NODCE(rdev)) {
2659 /* Unlock vga access */
2660 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2661 mdelay(1);
2662 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2663 }
2664 }
2665
2666 void evergreen_mc_program(struct radeon_device *rdev)
2667 {
2668 struct evergreen_mc_save save;
2669 u32 tmp;
2670 int i, j;
2671
2672 /* Initialize HDP */
2673 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2674 WREG32((0x2c14 + j), 0x00000000);
2675 WREG32((0x2c18 + j), 0x00000000);
2676 WREG32((0x2c1c + j), 0x00000000);
2677 WREG32((0x2c20 + j), 0x00000000);
2678 WREG32((0x2c24 + j), 0x00000000);
2679 }
2680 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2681
2682 evergreen_mc_stop(rdev, &save);
2683 if (evergreen_mc_wait_for_idle(rdev)) {
2684 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2685 }
2686 /* Lockout access through VGA aperture*/
2687 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2688 /* Update configuration */
2689 if (rdev->flags & RADEON_IS_AGP) {
2690 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2691 /* VRAM before AGP */
2692 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2693 rdev->mc.vram_start >> 12);
2694 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2695 rdev->mc.gtt_end >> 12);
2696 } else {
2697 /* VRAM after AGP */
2698 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2699 rdev->mc.gtt_start >> 12);
2700 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2701 rdev->mc.vram_end >> 12);
2702 }
2703 } else {
2704 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2705 rdev->mc.vram_start >> 12);
2706 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2707 rdev->mc.vram_end >> 12);
2708 }
2709 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2710 /* llano/ontario only */
2711 if ((rdev->family == CHIP_PALM) ||
2712 (rdev->family == CHIP_SUMO) ||
2713 (rdev->family == CHIP_SUMO2)) {
2714 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2715 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2716 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2717 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2718 }
2719 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2720 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2721 WREG32(MC_VM_FB_LOCATION, tmp);
2722 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2723 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2724 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2725 if (rdev->flags & RADEON_IS_AGP) {
2726 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2727 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2728 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2729 } else {
2730 WREG32(MC_VM_AGP_BASE, 0);
2731 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2732 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2733 }
2734 if (evergreen_mc_wait_for_idle(rdev)) {
2735 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2736 }
2737 evergreen_mc_resume(rdev, &save);
2738 /* we need to own VRAM, so turn off the VGA renderer here
2739 * to stop it overwriting our objects */
2740 rv515_vga_render_disable(rdev);
2741 }
2742
2743 /*
2744 * CP.
2745 */
2746 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2747 {
2748 struct radeon_ring *ring = &rdev->ring[ib->ring];
2749 u32 next_rptr;
2750
2751 /* set to DX10/11 mode */
2752 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2753 radeon_ring_write(ring, 1);
2754
2755 if (ring->rptr_save_reg) {
2756 next_rptr = ring->wptr + 3 + 4;
2757 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2758 radeon_ring_write(ring, ((ring->rptr_save_reg -
2759 PACKET3_SET_CONFIG_REG_START) >> 2));
2760 radeon_ring_write(ring, next_rptr);
2761 } else if (rdev->wb.enabled) {
2762 next_rptr = ring->wptr + 5 + 4;
2763 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2764 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2765 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2766 radeon_ring_write(ring, next_rptr);
2767 radeon_ring_write(ring, 0);
2768 }
2769
2770 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2771 radeon_ring_write(ring,
2772 #ifdef __BIG_ENDIAN
2773 (2 << 0) |
2774 #endif
2775 (ib->gpu_addr & 0xFFFFFFFC));
2776 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2777 radeon_ring_write(ring, ib->length_dw);
2778 }
2779
2780
2781 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2782 {
2783 const __be32 *fw_data;
2784 int i;
2785
2786 if (!rdev->me_fw || !rdev->pfp_fw)
2787 return -EINVAL;
2788
2789 r700_cp_stop(rdev);
2790 WREG32(CP_RB_CNTL,
2791 #ifdef __BIG_ENDIAN
2792 BUF_SWAP_32BIT |
2793 #endif
2794 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2795
2796 fw_data = (const __be32 *)rdev->pfp_fw->data;
2797 WREG32(CP_PFP_UCODE_ADDR, 0);
2798 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2799 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2800 WREG32(CP_PFP_UCODE_ADDR, 0);
2801
2802 fw_data = (const __be32 *)rdev->me_fw->data;
2803 WREG32(CP_ME_RAM_WADDR, 0);
2804 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2805 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2806
2807 WREG32(CP_PFP_UCODE_ADDR, 0);
2808 WREG32(CP_ME_RAM_WADDR, 0);
2809 WREG32(CP_ME_RAM_RADDR, 0);
2810 return 0;
2811 }
2812
2813 static int evergreen_cp_start(struct radeon_device *rdev)
2814 {
2815 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2816 int r, i;
2817 uint32_t cp_me;
2818
2819 r = radeon_ring_lock(rdev, ring, 7);
2820 if (r) {
2821 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2822 return r;
2823 }
2824 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2825 radeon_ring_write(ring, 0x1);
2826 radeon_ring_write(ring, 0x0);
2827 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2828 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2829 radeon_ring_write(ring, 0);
2830 radeon_ring_write(ring, 0);
2831 radeon_ring_unlock_commit(rdev, ring);
2832
2833 cp_me = 0xff;
2834 WREG32(CP_ME_CNTL, cp_me);
2835
2836 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2837 if (r) {
2838 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2839 return r;
2840 }
2841
2842 /* setup clear context state */
2843 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2844 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2845
2846 for (i = 0; i < evergreen_default_size; i++)
2847 radeon_ring_write(ring, evergreen_default_state[i]);
2848
2849 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2850 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2851
2852 /* set clear context state */
2853 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2854 radeon_ring_write(ring, 0);
2855
2856 /* SQ_VTX_BASE_VTX_LOC */
2857 radeon_ring_write(ring, 0xc0026f00);
2858 radeon_ring_write(ring, 0x00000000);
2859 radeon_ring_write(ring, 0x00000000);
2860 radeon_ring_write(ring, 0x00000000);
2861
2862 /* Clear consts */
2863 radeon_ring_write(ring, 0xc0036f00);
2864 radeon_ring_write(ring, 0x00000bc4);
2865 radeon_ring_write(ring, 0xffffffff);
2866 radeon_ring_write(ring, 0xffffffff);
2867 radeon_ring_write(ring, 0xffffffff);
2868
2869 radeon_ring_write(ring, 0xc0026900);
2870 radeon_ring_write(ring, 0x00000316);
2871 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2872 radeon_ring_write(ring, 0x00000010); /* */
2873
2874 radeon_ring_unlock_commit(rdev, ring);
2875
2876 return 0;
2877 }
2878
2879 static int evergreen_cp_resume(struct radeon_device *rdev)
2880 {
2881 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2882 u32 tmp;
2883 u32 rb_bufsz;
2884 int r;
2885
2886 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2887 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2888 SOFT_RESET_PA |
2889 SOFT_RESET_SH |
2890 SOFT_RESET_VGT |
2891 SOFT_RESET_SPI |
2892 SOFT_RESET_SX));
2893 RREG32(GRBM_SOFT_RESET);
2894 mdelay(15);
2895 WREG32(GRBM_SOFT_RESET, 0);
2896 RREG32(GRBM_SOFT_RESET);
2897
2898 /* Set ring buffer size */
2899 rb_bufsz = order_base_2(ring->ring_size / 8);
2900 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2901 #ifdef __BIG_ENDIAN
2902 tmp |= BUF_SWAP_32BIT;
2903 #endif
2904 WREG32(CP_RB_CNTL, tmp);
2905 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2906 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2907
2908 /* Set the write pointer delay */
2909 WREG32(CP_RB_WPTR_DELAY, 0);
2910
2911 /* Initialize the ring buffer's read and write pointers */
2912 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2913 WREG32(CP_RB_RPTR_WR, 0);
2914 ring->wptr = 0;
2915 WREG32(CP_RB_WPTR, ring->wptr);
2916
2917 /* set the wb address whether it's enabled or not */
2918 WREG32(CP_RB_RPTR_ADDR,
2919 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2920 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2921 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2922
2923 if (rdev->wb.enabled)
2924 WREG32(SCRATCH_UMSK, 0xff);
2925 else {
2926 tmp |= RB_NO_UPDATE;
2927 WREG32(SCRATCH_UMSK, 0);
2928 }
2929
2930 mdelay(1);
2931 WREG32(CP_RB_CNTL, tmp);
2932
2933 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2934 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2935
2936 ring->rptr = RREG32(CP_RB_RPTR);
2937
2938 evergreen_cp_start(rdev);
2939 ring->ready = true;
2940 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2941 if (r) {
2942 ring->ready = false;
2943 return r;
2944 }
2945 return 0;
2946 }
2947
2948 /*
2949 * Core functions
2950 */
2951 static void evergreen_gpu_init(struct radeon_device *rdev)
2952 {
2953 u32 gb_addr_config;
2954 u32 mc_shared_chmap, mc_arb_ramcfg;
2955 u32 sx_debug_1;
2956 u32 smx_dc_ctl0;
2957 u32 sq_config;
2958 u32 sq_lds_resource_mgmt;
2959 u32 sq_gpr_resource_mgmt_1;
2960 u32 sq_gpr_resource_mgmt_2;
2961 u32 sq_gpr_resource_mgmt_3;
2962 u32 sq_thread_resource_mgmt;
2963 u32 sq_thread_resource_mgmt_2;
2964 u32 sq_stack_resource_mgmt_1;
2965 u32 sq_stack_resource_mgmt_2;
2966 u32 sq_stack_resource_mgmt_3;
2967 u32 vgt_cache_invalidation;
2968 u32 hdp_host_path_cntl, tmp;
2969 u32 disabled_rb_mask;
2970 int i, j, num_shader_engines, ps_thread_count;
2971
2972 switch (rdev->family) {
2973 case CHIP_CYPRESS:
2974 case CHIP_HEMLOCK:
2975 rdev->config.evergreen.num_ses = 2;
2976 rdev->config.evergreen.max_pipes = 4;
2977 rdev->config.evergreen.max_tile_pipes = 8;
2978 rdev->config.evergreen.max_simds = 10;
2979 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2980 rdev->config.evergreen.max_gprs = 256;
2981 rdev->config.evergreen.max_threads = 248;
2982 rdev->config.evergreen.max_gs_threads = 32;
2983 rdev->config.evergreen.max_stack_entries = 512;
2984 rdev->config.evergreen.sx_num_of_sets = 4;
2985 rdev->config.evergreen.sx_max_export_size = 256;
2986 rdev->config.evergreen.sx_max_export_pos_size = 64;
2987 rdev->config.evergreen.sx_max_export_smx_size = 192;
2988 rdev->config.evergreen.max_hw_contexts = 8;
2989 rdev->config.evergreen.sq_num_cf_insts = 2;
2990
2991 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2992 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2993 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2994 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2995 break;
2996 case CHIP_JUNIPER:
2997 rdev->config.evergreen.num_ses = 1;
2998 rdev->config.evergreen.max_pipes = 4;
2999 rdev->config.evergreen.max_tile_pipes = 4;
3000 rdev->config.evergreen.max_simds = 10;
3001 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3002 rdev->config.evergreen.max_gprs = 256;
3003 rdev->config.evergreen.max_threads = 248;
3004 rdev->config.evergreen.max_gs_threads = 32;
3005 rdev->config.evergreen.max_stack_entries = 512;
3006 rdev->config.evergreen.sx_num_of_sets = 4;
3007 rdev->config.evergreen.sx_max_export_size = 256;
3008 rdev->config.evergreen.sx_max_export_pos_size = 64;
3009 rdev->config.evergreen.sx_max_export_smx_size = 192;
3010 rdev->config.evergreen.max_hw_contexts = 8;
3011 rdev->config.evergreen.sq_num_cf_insts = 2;
3012
3013 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3014 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3015 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3016 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3017 break;
3018 case CHIP_REDWOOD:
3019 rdev->config.evergreen.num_ses = 1;
3020 rdev->config.evergreen.max_pipes = 4;
3021 rdev->config.evergreen.max_tile_pipes = 4;
3022 rdev->config.evergreen.max_simds = 5;
3023 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3024 rdev->config.evergreen.max_gprs = 256;
3025 rdev->config.evergreen.max_threads = 248;
3026 rdev->config.evergreen.max_gs_threads = 32;
3027 rdev->config.evergreen.max_stack_entries = 256;
3028 rdev->config.evergreen.sx_num_of_sets = 4;
3029 rdev->config.evergreen.sx_max_export_size = 256;
3030 rdev->config.evergreen.sx_max_export_pos_size = 64;
3031 rdev->config.evergreen.sx_max_export_smx_size = 192;
3032 rdev->config.evergreen.max_hw_contexts = 8;
3033 rdev->config.evergreen.sq_num_cf_insts = 2;
3034
3035 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3036 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3037 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3038 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3039 break;
3040 case CHIP_CEDAR:
3041 default:
3042 rdev->config.evergreen.num_ses = 1;
3043 rdev->config.evergreen.max_pipes = 2;
3044 rdev->config.evergreen.max_tile_pipes = 2;
3045 rdev->config.evergreen.max_simds = 2;
3046 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3047 rdev->config.evergreen.max_gprs = 256;
3048 rdev->config.evergreen.max_threads = 192;
3049 rdev->config.evergreen.max_gs_threads = 16;
3050 rdev->config.evergreen.max_stack_entries = 256;
3051 rdev->config.evergreen.sx_num_of_sets = 4;
3052 rdev->config.evergreen.sx_max_export_size = 128;
3053 rdev->config.evergreen.sx_max_export_pos_size = 32;
3054 rdev->config.evergreen.sx_max_export_smx_size = 96;
3055 rdev->config.evergreen.max_hw_contexts = 4;
3056 rdev->config.evergreen.sq_num_cf_insts = 1;
3057
3058 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3059 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3060 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3061 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3062 break;
3063 case CHIP_PALM:
3064 rdev->config.evergreen.num_ses = 1;
3065 rdev->config.evergreen.max_pipes = 2;
3066 rdev->config.evergreen.max_tile_pipes = 2;
3067 rdev->config.evergreen.max_simds = 2;
3068 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3069 rdev->config.evergreen.max_gprs = 256;
3070 rdev->config.evergreen.max_threads = 192;
3071 rdev->config.evergreen.max_gs_threads = 16;
3072 rdev->config.evergreen.max_stack_entries = 256;
3073 rdev->config.evergreen.sx_num_of_sets = 4;
3074 rdev->config.evergreen.sx_max_export_size = 128;
3075 rdev->config.evergreen.sx_max_export_pos_size = 32;
3076 rdev->config.evergreen.sx_max_export_smx_size = 96;
3077 rdev->config.evergreen.max_hw_contexts = 4;
3078 rdev->config.evergreen.sq_num_cf_insts = 1;
3079
3080 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3081 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3082 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3083 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3084 break;
3085 case CHIP_SUMO:
3086 rdev->config.evergreen.num_ses = 1;
3087 rdev->config.evergreen.max_pipes = 4;
3088 rdev->config.evergreen.max_tile_pipes = 4;
3089 if (rdev->pdev->device == 0x9648)
3090 rdev->config.evergreen.max_simds = 3;
3091 else if ((rdev->pdev->device == 0x9647) ||
3092 (rdev->pdev->device == 0x964a))
3093 rdev->config.evergreen.max_simds = 4;
3094 else
3095 rdev->config.evergreen.max_simds = 5;
3096 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3097 rdev->config.evergreen.max_gprs = 256;
3098 rdev->config.evergreen.max_threads = 248;
3099 rdev->config.evergreen.max_gs_threads = 32;
3100 rdev->config.evergreen.max_stack_entries = 256;
3101 rdev->config.evergreen.sx_num_of_sets = 4;
3102 rdev->config.evergreen.sx_max_export_size = 256;
3103 rdev->config.evergreen.sx_max_export_pos_size = 64;
3104 rdev->config.evergreen.sx_max_export_smx_size = 192;
3105 rdev->config.evergreen.max_hw_contexts = 8;
3106 rdev->config.evergreen.sq_num_cf_insts = 2;
3107
3108 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3109 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3110 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3111 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3112 break;
3113 case CHIP_SUMO2:
3114 rdev->config.evergreen.num_ses = 1;
3115 rdev->config.evergreen.max_pipes = 4;
3116 rdev->config.evergreen.max_tile_pipes = 4;
3117 rdev->config.evergreen.max_simds = 2;
3118 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3119 rdev->config.evergreen.max_gprs = 256;
3120 rdev->config.evergreen.max_threads = 248;
3121 rdev->config.evergreen.max_gs_threads = 32;
3122 rdev->config.evergreen.max_stack_entries = 512;
3123 rdev->config.evergreen.sx_num_of_sets = 4;
3124 rdev->config.evergreen.sx_max_export_size = 256;
3125 rdev->config.evergreen.sx_max_export_pos_size = 64;
3126 rdev->config.evergreen.sx_max_export_smx_size = 192;
3127 rdev->config.evergreen.max_hw_contexts = 4;
3128 rdev->config.evergreen.sq_num_cf_insts = 2;
3129
3130 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3131 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3132 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3133 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3134 break;
3135 case CHIP_BARTS:
3136 rdev->config.evergreen.num_ses = 2;
3137 rdev->config.evergreen.max_pipes = 4;
3138 rdev->config.evergreen.max_tile_pipes = 8;
3139 rdev->config.evergreen.max_simds = 7;
3140 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3141 rdev->config.evergreen.max_gprs = 256;
3142 rdev->config.evergreen.max_threads = 248;
3143 rdev->config.evergreen.max_gs_threads = 32;
3144 rdev->config.evergreen.max_stack_entries = 512;
3145 rdev->config.evergreen.sx_num_of_sets = 4;
3146 rdev->config.evergreen.sx_max_export_size = 256;
3147 rdev->config.evergreen.sx_max_export_pos_size = 64;
3148 rdev->config.evergreen.sx_max_export_smx_size = 192;
3149 rdev->config.evergreen.max_hw_contexts = 8;
3150 rdev->config.evergreen.sq_num_cf_insts = 2;
3151
3152 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3153 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3154 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3155 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3156 break;
3157 case CHIP_TURKS:
3158 rdev->config.evergreen.num_ses = 1;
3159 rdev->config.evergreen.max_pipes = 4;
3160 rdev->config.evergreen.max_tile_pipes = 4;
3161 rdev->config.evergreen.max_simds = 6;
3162 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3163 rdev->config.evergreen.max_gprs = 256;
3164 rdev->config.evergreen.max_threads = 248;
3165 rdev->config.evergreen.max_gs_threads = 32;
3166 rdev->config.evergreen.max_stack_entries = 256;
3167 rdev->config.evergreen.sx_num_of_sets = 4;
3168 rdev->config.evergreen.sx_max_export_size = 256;
3169 rdev->config.evergreen.sx_max_export_pos_size = 64;
3170 rdev->config.evergreen.sx_max_export_smx_size = 192;
3171 rdev->config.evergreen.max_hw_contexts = 8;
3172 rdev->config.evergreen.sq_num_cf_insts = 2;
3173
3174 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3175 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3177 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3178 break;
3179 case CHIP_CAICOS:
3180 rdev->config.evergreen.num_ses = 1;
3181 rdev->config.evergreen.max_pipes = 2;
3182 rdev->config.evergreen.max_tile_pipes = 2;
3183 rdev->config.evergreen.max_simds = 2;
3184 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3185 rdev->config.evergreen.max_gprs = 256;
3186 rdev->config.evergreen.max_threads = 192;
3187 rdev->config.evergreen.max_gs_threads = 16;
3188 rdev->config.evergreen.max_stack_entries = 256;
3189 rdev->config.evergreen.sx_num_of_sets = 4;
3190 rdev->config.evergreen.sx_max_export_size = 128;
3191 rdev->config.evergreen.sx_max_export_pos_size = 32;
3192 rdev->config.evergreen.sx_max_export_smx_size = 96;
3193 rdev->config.evergreen.max_hw_contexts = 4;
3194 rdev->config.evergreen.sq_num_cf_insts = 1;
3195
3196 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3197 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3198 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3199 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3200 break;
3201 }
3202
3203 /* Initialize HDP */
3204 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3205 WREG32((0x2c14 + j), 0x00000000);
3206 WREG32((0x2c18 + j), 0x00000000);
3207 WREG32((0x2c1c + j), 0x00000000);
3208 WREG32((0x2c20 + j), 0x00000000);
3209 WREG32((0x2c24 + j), 0x00000000);
3210 }
3211
3212 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3213
3214 evergreen_fix_pci_max_read_req_size(rdev);
3215
3216 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3217 if ((rdev->family == CHIP_PALM) ||
3218 (rdev->family == CHIP_SUMO) ||
3219 (rdev->family == CHIP_SUMO2))
3220 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3221 else
3222 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3223
3224 /* setup tiling info dword. gb_addr_config is not adequate since it does
3225 * not have bank info, so create a custom tiling dword.
3226 * bits 3:0 num_pipes
3227 * bits 7:4 num_banks
3228 * bits 11:8 group_size
3229 * bits 15:12 row_size
3230 */
3231 rdev->config.evergreen.tile_config = 0;
3232 switch (rdev->config.evergreen.max_tile_pipes) {
3233 case 1:
3234 default:
3235 rdev->config.evergreen.tile_config |= (0 << 0);
3236 break;
3237 case 2:
3238 rdev->config.evergreen.tile_config |= (1 << 0);
3239 break;
3240 case 4:
3241 rdev->config.evergreen.tile_config |= (2 << 0);
3242 break;
3243 case 8:
3244 rdev->config.evergreen.tile_config |= (3 << 0);
3245 break;
3246 }
3247 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3248 if (rdev->flags & RADEON_IS_IGP)
3249 rdev->config.evergreen.tile_config |= 1 << 4;
3250 else {
3251 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3252 case 0: /* four banks */
3253 rdev->config.evergreen.tile_config |= 0 << 4;
3254 break;
3255 case 1: /* eight banks */
3256 rdev->config.evergreen.tile_config |= 1 << 4;
3257 break;
3258 case 2: /* sixteen banks */
3259 default:
3260 rdev->config.evergreen.tile_config |= 2 << 4;
3261 break;
3262 }
3263 }
3264 rdev->config.evergreen.tile_config |= 0 << 8;
3265 rdev->config.evergreen.tile_config |=
3266 ((gb_addr_config & 0x30000000) >> 28) << 12;
3267
3268 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3269
3270 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3271 u32 efuse_straps_4;
3272 u32 efuse_straps_3;
3273
3274 efuse_straps_4 = RREG32_RCU(0x204);
3275 efuse_straps_3 = RREG32_RCU(0x203);
3276 tmp = (((efuse_straps_4 & 0xf) << 4) |
3277 ((efuse_straps_3 & 0xf0000000) >> 28));
3278 } else {
3279 tmp = 0;
3280 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3281 u32 rb_disable_bitmap;
3282
3283 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3284 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3285 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3286 tmp <<= 4;
3287 tmp |= rb_disable_bitmap;
3288 }
3289 }
3290 /* enabled rb are just the one not disabled :) */
3291 disabled_rb_mask = tmp;
3292 tmp = 0;
3293 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3294 tmp |= (1 << i);
3295 /* if all the backends are disabled, fix it up here */
3296 if ((disabled_rb_mask & tmp) == tmp) {
3297 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3298 disabled_rb_mask &= ~(1 << i);
3299 }
3300
3301 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3302 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3303
3304 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3305 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3306 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3307 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3308 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3309 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3310 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3311
3312 if ((rdev->config.evergreen.max_backends == 1) &&
3313 (rdev->flags & RADEON_IS_IGP)) {
3314 if ((disabled_rb_mask & 3) == 1) {
3315 /* RB0 disabled, RB1 enabled */
3316 tmp = 0x11111111;
3317 } else {
3318 /* RB1 disabled, RB0 enabled */
3319 tmp = 0x00000000;
3320 }
3321 } else {
3322 tmp = gb_addr_config & NUM_PIPES_MASK;
3323 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3324 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3325 }
3326 WREG32(GB_BACKEND_MAP, tmp);
3327
3328 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3329 WREG32(CGTS_TCC_DISABLE, 0);
3330 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3331 WREG32(CGTS_USER_TCC_DISABLE, 0);
3332
3333 /* set HW defaults for 3D engine */
3334 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3335 ROQ_IB2_START(0x2b)));
3336
3337 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3338
3339 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3340 SYNC_GRADIENT |
3341 SYNC_WALKER |
3342 SYNC_ALIGNER));
3343
3344 sx_debug_1 = RREG32(SX_DEBUG_1);
3345 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3346 WREG32(SX_DEBUG_1, sx_debug_1);
3347
3348
3349 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3350 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3351 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3352 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3353
3354 if (rdev->family <= CHIP_SUMO2)
3355 WREG32(SMX_SAR_CTL0, 0x00010000);
3356
3357 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3358 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3359 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3360
3361 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3362 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3363 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3364
3365 WREG32(VGT_NUM_INSTANCES, 1);
3366 WREG32(SPI_CONFIG_CNTL, 0);
3367 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3368 WREG32(CP_PERFMON_CNTL, 0);
3369
3370 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3371 FETCH_FIFO_HIWATER(0x4) |
3372 DONE_FIFO_HIWATER(0xe0) |
3373 ALU_UPDATE_FIFO_HIWATER(0x8)));
3374
3375 sq_config = RREG32(SQ_CONFIG);
3376 sq_config &= ~(PS_PRIO(3) |
3377 VS_PRIO(3) |
3378 GS_PRIO(3) |
3379 ES_PRIO(3));
3380 sq_config |= (VC_ENABLE |
3381 EXPORT_SRC_C |
3382 PS_PRIO(0) |
3383 VS_PRIO(1) |
3384 GS_PRIO(2) |
3385 ES_PRIO(3));
3386
3387 switch (rdev->family) {
3388 case CHIP_CEDAR:
3389 case CHIP_PALM:
3390 case CHIP_SUMO:
3391 case CHIP_SUMO2:
3392 case CHIP_CAICOS:
3393 /* no vertex cache */
3394 sq_config &= ~VC_ENABLE;
3395 break;
3396 default:
3397 break;
3398 }
3399
3400 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3401
3402 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3403 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3404 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3405 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3406 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3407 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3408 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3409
3410 switch (rdev->family) {
3411 case CHIP_CEDAR:
3412 case CHIP_PALM:
3413 case CHIP_SUMO:
3414 case CHIP_SUMO2:
3415 ps_thread_count = 96;
3416 break;
3417 default:
3418 ps_thread_count = 128;
3419 break;
3420 }
3421
3422 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3423 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3424 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3425 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3426 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3427 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3428
3429 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3430 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3431 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3432 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3433 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3434 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3435
3436 WREG32(SQ_CONFIG, sq_config);
3437 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3438 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3439 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3440 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3441 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3442 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3443 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3444 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3445 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3446 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3447
3448 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3449 FORCE_EOV_MAX_REZ_CNT(255)));
3450
3451 switch (rdev->family) {
3452 case CHIP_CEDAR:
3453 case CHIP_PALM:
3454 case CHIP_SUMO:
3455 case CHIP_SUMO2:
3456 case CHIP_CAICOS:
3457 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3458 break;
3459 default:
3460 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3461 break;
3462 }
3463 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3464 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3465
3466 WREG32(VGT_GS_VERTEX_REUSE, 16);
3467 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3468 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3469
3470 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3471 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3472
3473 WREG32(CB_PERF_CTR0_SEL_0, 0);
3474 WREG32(CB_PERF_CTR0_SEL_1, 0);
3475 WREG32(CB_PERF_CTR1_SEL_0, 0);
3476 WREG32(CB_PERF_CTR1_SEL_1, 0);
3477 WREG32(CB_PERF_CTR2_SEL_0, 0);
3478 WREG32(CB_PERF_CTR2_SEL_1, 0);
3479 WREG32(CB_PERF_CTR3_SEL_0, 0);
3480 WREG32(CB_PERF_CTR3_SEL_1, 0);
3481
3482 /* clear render buffer base addresses */
3483 WREG32(CB_COLOR0_BASE, 0);
3484 WREG32(CB_COLOR1_BASE, 0);
3485 WREG32(CB_COLOR2_BASE, 0);
3486 WREG32(CB_COLOR3_BASE, 0);
3487 WREG32(CB_COLOR4_BASE, 0);
3488 WREG32(CB_COLOR5_BASE, 0);
3489 WREG32(CB_COLOR6_BASE, 0);
3490 WREG32(CB_COLOR7_BASE, 0);
3491 WREG32(CB_COLOR8_BASE, 0);
3492 WREG32(CB_COLOR9_BASE, 0);
3493 WREG32(CB_COLOR10_BASE, 0);
3494 WREG32(CB_COLOR11_BASE, 0);
3495
3496 /* set the shader const cache sizes to 0 */
3497 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3498 WREG32(i, 0);
3499 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3500 WREG32(i, 0);
3501
3502 tmp = RREG32(HDP_MISC_CNTL);
3503 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3504 WREG32(HDP_MISC_CNTL, tmp);
3505
3506 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3507 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3508
3509 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3510
3511 udelay(50);
3512
3513 }
3514
3515 int evergreen_mc_init(struct radeon_device *rdev)
3516 {
3517 u32 tmp;
3518 int chansize, numchan;
3519
3520 /* Get VRAM informations */
3521 rdev->mc.vram_is_ddr = true;
3522 if ((rdev->family == CHIP_PALM) ||
3523 (rdev->family == CHIP_SUMO) ||
3524 (rdev->family == CHIP_SUMO2))
3525 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3526 else
3527 tmp = RREG32(MC_ARB_RAMCFG);
3528 if (tmp & CHANSIZE_OVERRIDE) {
3529 chansize = 16;
3530 } else if (tmp & CHANSIZE_MASK) {
3531 chansize = 64;
3532 } else {
3533 chansize = 32;
3534 }
3535 tmp = RREG32(MC_SHARED_CHMAP);
3536 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3537 case 0:
3538 default:
3539 numchan = 1;
3540 break;
3541 case 1:
3542 numchan = 2;
3543 break;
3544 case 2:
3545 numchan = 4;
3546 break;
3547 case 3:
3548 numchan = 8;
3549 break;
3550 }
3551 rdev->mc.vram_width = numchan * chansize;
3552 /* Could aper size report 0 ? */
3553 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3554 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3555 /* Setup GPU memory space */
3556 if ((rdev->family == CHIP_PALM) ||
3557 (rdev->family == CHIP_SUMO) ||
3558 (rdev->family == CHIP_SUMO2)) {
3559 /* size in bytes on fusion */
3560 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3561 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3562 } else {
3563 /* size in MB on evergreen/cayman/tn */
3564 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3565 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3566 }
3567 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3568 r700_vram_gtt_location(rdev, &rdev->mc);
3569 radeon_update_bandwidth_info(rdev);
3570
3571 return 0;
3572 }
3573
3574 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3575 {
3576 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3577 RREG32(GRBM_STATUS));
3578 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3579 RREG32(GRBM_STATUS_SE0));
3580 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3581 RREG32(GRBM_STATUS_SE1));
3582 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3583 RREG32(SRBM_STATUS));
3584 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3585 RREG32(SRBM_STATUS2));
3586 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3587 RREG32(CP_STALLED_STAT1));
3588 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3589 RREG32(CP_STALLED_STAT2));
3590 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3591 RREG32(CP_BUSY_STAT));
3592 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3593 RREG32(CP_STAT));
3594 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3595 RREG32(DMA_STATUS_REG));
3596 if (rdev->family >= CHIP_CAYMAN) {
3597 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3598 RREG32(DMA_STATUS_REG + 0x800));
3599 }
3600 }
3601
3602 bool evergreen_is_display_hung(struct radeon_device *rdev)
3603 {
3604 u32 crtc_hung = 0;
3605 u32 crtc_status[6];
3606 u32 i, j, tmp;
3607
3608 for (i = 0; i < rdev->num_crtc; i++) {
3609 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3610 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3611 crtc_hung |= (1 << i);
3612 }
3613 }
3614
3615 for (j = 0; j < 10; j++) {
3616 for (i = 0; i < rdev->num_crtc; i++) {
3617 if (crtc_hung & (1 << i)) {
3618 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3619 if (tmp != crtc_status[i])
3620 crtc_hung &= ~(1 << i);
3621 }
3622 }
3623 if (crtc_hung == 0)
3624 return false;
3625 udelay(100);
3626 }
3627
3628 return true;
3629 }
3630
3631 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3632 {
3633 u32 reset_mask = 0;
3634 u32 tmp;
3635
3636 /* GRBM_STATUS */
3637 tmp = RREG32(GRBM_STATUS);
3638 if (tmp & (PA_BUSY | SC_BUSY |
3639 SH_BUSY | SX_BUSY |
3640 TA_BUSY | VGT_BUSY |
3641 DB_BUSY | CB_BUSY |
3642 SPI_BUSY | VGT_BUSY_NO_DMA))
3643 reset_mask |= RADEON_RESET_GFX;
3644
3645 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3646 CP_BUSY | CP_COHERENCY_BUSY))
3647 reset_mask |= RADEON_RESET_CP;
3648
3649 if (tmp & GRBM_EE_BUSY)
3650 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3651
3652 /* DMA_STATUS_REG */
3653 tmp = RREG32(DMA_STATUS_REG);
3654 if (!(tmp & DMA_IDLE))
3655 reset_mask |= RADEON_RESET_DMA;
3656
3657 /* SRBM_STATUS2 */
3658 tmp = RREG32(SRBM_STATUS2);
3659 if (tmp & DMA_BUSY)
3660 reset_mask |= RADEON_RESET_DMA;
3661
3662 /* SRBM_STATUS */
3663 tmp = RREG32(SRBM_STATUS);
3664 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3665 reset_mask |= RADEON_RESET_RLC;
3666
3667 if (tmp & IH_BUSY)
3668 reset_mask |= RADEON_RESET_IH;
3669
3670 if (tmp & SEM_BUSY)
3671 reset_mask |= RADEON_RESET_SEM;
3672
3673 if (tmp & GRBM_RQ_PENDING)
3674 reset_mask |= RADEON_RESET_GRBM;
3675
3676 if (tmp & VMC_BUSY)
3677 reset_mask |= RADEON_RESET_VMC;
3678
3679 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3680 MCC_BUSY | MCD_BUSY))
3681 reset_mask |= RADEON_RESET_MC;
3682
3683 if (evergreen_is_display_hung(rdev))
3684 reset_mask |= RADEON_RESET_DISPLAY;
3685
3686 /* VM_L2_STATUS */
3687 tmp = RREG32(VM_L2_STATUS);
3688 if (tmp & L2_BUSY)
3689 reset_mask |= RADEON_RESET_VMC;
3690
3691 /* Skip MC reset as it's mostly likely not hung, just busy */
3692 if (reset_mask & RADEON_RESET_MC) {
3693 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3694 reset_mask &= ~RADEON_RESET_MC;
3695 }
3696
3697 return reset_mask;
3698 }
3699
3700 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3701 {
3702 struct evergreen_mc_save save;
3703 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3704 u32 tmp;
3705
3706 if (reset_mask == 0)
3707 return;
3708
3709 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3710
3711 evergreen_print_gpu_status_regs(rdev);
3712
3713 /* Disable CP parsing/prefetching */
3714 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3715
3716 if (reset_mask & RADEON_RESET_DMA) {
3717 /* Disable DMA */
3718 tmp = RREG32(DMA_RB_CNTL);
3719 tmp &= ~DMA_RB_ENABLE;
3720 WREG32(DMA_RB_CNTL, tmp);
3721 }
3722
3723 udelay(50);
3724
3725 evergreen_mc_stop(rdev, &save);
3726 if (evergreen_mc_wait_for_idle(rdev)) {
3727 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3728 }
3729
3730 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3731 grbm_soft_reset |= SOFT_RESET_DB |
3732 SOFT_RESET_CB |
3733 SOFT_RESET_PA |
3734 SOFT_RESET_SC |
3735 SOFT_RESET_SPI |
3736 SOFT_RESET_SX |
3737 SOFT_RESET_SH |
3738 SOFT_RESET_TC |
3739 SOFT_RESET_TA |
3740 SOFT_RESET_VC |
3741 SOFT_RESET_VGT;
3742 }
3743
3744 if (reset_mask & RADEON_RESET_CP) {
3745 grbm_soft_reset |= SOFT_RESET_CP |
3746 SOFT_RESET_VGT;
3747
3748 srbm_soft_reset |= SOFT_RESET_GRBM;
3749 }
3750
3751 if (reset_mask & RADEON_RESET_DMA)
3752 srbm_soft_reset |= SOFT_RESET_DMA;
3753
3754 if (reset_mask & RADEON_RESET_DISPLAY)
3755 srbm_soft_reset |= SOFT_RESET_DC;
3756
3757 if (reset_mask & RADEON_RESET_RLC)
3758 srbm_soft_reset |= SOFT_RESET_RLC;
3759
3760 if (reset_mask & RADEON_RESET_SEM)
3761 srbm_soft_reset |= SOFT_RESET_SEM;
3762
3763 if (reset_mask & RADEON_RESET_IH)
3764 srbm_soft_reset |= SOFT_RESET_IH;
3765
3766 if (reset_mask & RADEON_RESET_GRBM)
3767 srbm_soft_reset |= SOFT_RESET_GRBM;
3768
3769 if (reset_mask & RADEON_RESET_VMC)
3770 srbm_soft_reset |= SOFT_RESET_VMC;
3771
3772 if (!(rdev->flags & RADEON_IS_IGP)) {
3773 if (reset_mask & RADEON_RESET_MC)
3774 srbm_soft_reset |= SOFT_RESET_MC;
3775 }
3776
3777 if (grbm_soft_reset) {
3778 tmp = RREG32(GRBM_SOFT_RESET);
3779 tmp |= grbm_soft_reset;
3780 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3781 WREG32(GRBM_SOFT_RESET, tmp);
3782 tmp = RREG32(GRBM_SOFT_RESET);
3783
3784 udelay(50);
3785
3786 tmp &= ~grbm_soft_reset;
3787 WREG32(GRBM_SOFT_RESET, tmp);
3788 tmp = RREG32(GRBM_SOFT_RESET);
3789 }
3790
3791 if (srbm_soft_reset) {
3792 tmp = RREG32(SRBM_SOFT_RESET);
3793 tmp |= srbm_soft_reset;
3794 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3795 WREG32(SRBM_SOFT_RESET, tmp);
3796 tmp = RREG32(SRBM_SOFT_RESET);
3797
3798 udelay(50);
3799
3800 tmp &= ~srbm_soft_reset;
3801 WREG32(SRBM_SOFT_RESET, tmp);
3802 tmp = RREG32(SRBM_SOFT_RESET);
3803 }
3804
3805 /* Wait a little for things to settle down */
3806 udelay(50);
3807
3808 evergreen_mc_resume(rdev, &save);
3809 udelay(50);
3810
3811 evergreen_print_gpu_status_regs(rdev);
3812 }
3813
3814 int evergreen_asic_reset(struct radeon_device *rdev)
3815 {
3816 u32 reset_mask;
3817
3818 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3819
3820 if (reset_mask)
3821 r600_set_bios_scratch_engine_hung(rdev, true);
3822
3823 evergreen_gpu_soft_reset(rdev, reset_mask);
3824
3825 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3826
3827 if (!reset_mask)
3828 r600_set_bios_scratch_engine_hung(rdev, false);
3829
3830 return 0;
3831 }
3832
3833 /**
3834 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3835 *
3836 * @rdev: radeon_device pointer
3837 * @ring: radeon_ring structure holding ring information
3838 *
3839 * Check if the GFX engine is locked up.
3840 * Returns true if the engine appears to be locked up, false if not.
3841 */
3842 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3843 {
3844 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3845
3846 if (!(reset_mask & (RADEON_RESET_GFX |
3847 RADEON_RESET_COMPUTE |
3848 RADEON_RESET_CP))) {
3849 radeon_ring_lockup_update(ring);
3850 return false;
3851 }
3852 /* force CP activities */
3853 radeon_ring_force_activity(rdev, ring);
3854 return radeon_ring_test_lockup(rdev, ring);
3855 }
3856
3857 /*
3858 * RLC
3859 */
3860 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3861 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3862
3863 void sumo_rlc_fini(struct radeon_device *rdev)
3864 {
3865 int r;
3866
3867 /* save restore block */
3868 if (rdev->rlc.save_restore_obj) {
3869 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3870 if (unlikely(r != 0))
3871 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3872 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3873 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3874
3875 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3876 rdev->rlc.save_restore_obj = NULL;
3877 }
3878
3879 /* clear state block */
3880 if (rdev->rlc.clear_state_obj) {
3881 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3882 if (unlikely(r != 0))
3883 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3884 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3885 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3886
3887 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3888 rdev->rlc.clear_state_obj = NULL;
3889 }
3890
3891 /* clear state block */
3892 if (rdev->rlc.cp_table_obj) {
3893 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3894 if (unlikely(r != 0))
3895 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3896 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3897 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3898
3899 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3900 rdev->rlc.cp_table_obj = NULL;
3901 }
3902 }
3903
3904 #define CP_ME_TABLE_SIZE 96
3905
3906 int sumo_rlc_init(struct radeon_device *rdev)
3907 {
3908 const u32 *src_ptr;
3909 volatile u32 *dst_ptr;
3910 u32 dws, data, i, j, k, reg_num;
3911 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
3912 u64 reg_list_mc_addr;
3913 const struct cs_section_def *cs_data;
3914 int r;
3915
3916 src_ptr = rdev->rlc.reg_list;
3917 dws = rdev->rlc.reg_list_size;
3918 if (rdev->family >= CHIP_BONAIRE) {
3919 dws += (5 * 16) + 48 + 48 + 64;
3920 }
3921 cs_data = rdev->rlc.cs_data;
3922
3923 if (src_ptr) {
3924 /* save restore block */
3925 if (rdev->rlc.save_restore_obj == NULL) {
3926 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3927 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3928 if (r) {
3929 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3930 return r;
3931 }
3932 }
3933
3934 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3935 if (unlikely(r != 0)) {
3936 sumo_rlc_fini(rdev);
3937 return r;
3938 }
3939 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3940 &rdev->rlc.save_restore_gpu_addr);
3941 if (r) {
3942 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3943 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3944 sumo_rlc_fini(rdev);
3945 return r;
3946 }
3947
3948 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3949 if (r) {
3950 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3951 sumo_rlc_fini(rdev);
3952 return r;
3953 }
3954 /* write the sr buffer */
3955 dst_ptr = rdev->rlc.sr_ptr;
3956 if (rdev->family >= CHIP_TAHITI) {
3957 /* SI */
3958 for (i = 0; i < rdev->rlc.reg_list_size; i++)
3959 dst_ptr[i] = src_ptr[i];
3960 } else {
3961 /* ON/LN/TN */
3962 /* format:
3963 * dw0: (reg2 << 16) | reg1
3964 * dw1: reg1 save space
3965 * dw2: reg2 save space
3966 */
3967 for (i = 0; i < dws; i++) {
3968 data = src_ptr[i] >> 2;
3969 i++;
3970 if (i < dws)
3971 data |= (src_ptr[i] >> 2) << 16;
3972 j = (((i - 1) * 3) / 2);
3973 dst_ptr[j] = data;
3974 }
3975 j = ((i * 3) / 2);
3976 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3977 }
3978 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3979 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3980 }
3981
3982 if (cs_data) {
3983 /* clear state block */
3984 if (rdev->family >= CHIP_BONAIRE) {
3985 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
3986 } else if (rdev->family >= CHIP_TAHITI) {
3987 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
3988 dws = rdev->rlc.clear_state_size + (256 / 4);
3989 } else {
3990 reg_list_num = 0;
3991 dws = 0;
3992 for (i = 0; cs_data[i].section != NULL; i++) {
3993 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3994 reg_list_num++;
3995 dws += cs_data[i].section[j].reg_count;
3996 }
3997 }
3998 reg_list_blk_index = (3 * reg_list_num + 2);
3999 dws += reg_list_blk_index;
4000 rdev->rlc.clear_state_size = dws;
4001 }
4002
4003 if (rdev->rlc.clear_state_obj == NULL) {
4004 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4005 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4006 if (r) {
4007 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4008 sumo_rlc_fini(rdev);
4009 return r;
4010 }
4011 }
4012 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4013 if (unlikely(r != 0)) {
4014 sumo_rlc_fini(rdev);
4015 return r;
4016 }
4017 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4018 &rdev->rlc.clear_state_gpu_addr);
4019 if (r) {
4020 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4021 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4022 sumo_rlc_fini(rdev);
4023 return r;
4024 }
4025
4026 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4027 if (r) {
4028 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4029 sumo_rlc_fini(rdev);
4030 return r;
4031 }
4032 /* set up the cs buffer */
4033 dst_ptr = rdev->rlc.cs_ptr;
4034 if (rdev->family >= CHIP_BONAIRE) {
4035 cik_get_csb_buffer(rdev, dst_ptr);
4036 } else if (rdev->family >= CHIP_TAHITI) {
4037 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4038 dst_ptr[0] = upper_32_bits(reg_list_mc_addr);
4039 dst_ptr[1] = lower_32_bits(reg_list_mc_addr);
4040 dst_ptr[2] = rdev->rlc.clear_state_size;
4041 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4042 } else {
4043 reg_list_hdr_blk_index = 0;
4044 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4045 data = upper_32_bits(reg_list_mc_addr);
4046 dst_ptr[reg_list_hdr_blk_index] = data;
4047 reg_list_hdr_blk_index++;
4048 for (i = 0; cs_data[i].section != NULL; i++) {
4049 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4050 reg_num = cs_data[i].section[j].reg_count;
4051 data = reg_list_mc_addr & 0xffffffff;
4052 dst_ptr[reg_list_hdr_blk_index] = data;
4053 reg_list_hdr_blk_index++;
4054
4055 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4056 dst_ptr[reg_list_hdr_blk_index] = data;
4057 reg_list_hdr_blk_index++;
4058
4059 data = 0x08000000 | (reg_num * 4);
4060 dst_ptr[reg_list_hdr_blk_index] = data;
4061 reg_list_hdr_blk_index++;
4062
4063 for (k = 0; k < reg_num; k++) {
4064 data = cs_data[i].section[j].extent[k];
4065 dst_ptr[reg_list_blk_index + k] = data;
4066 }
4067 reg_list_mc_addr += reg_num * 4;
4068 reg_list_blk_index += reg_num;
4069 }
4070 }
4071 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4072 }
4073 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4074 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4075 }
4076
4077 if (rdev->rlc.cp_table_size) {
4078 if (rdev->rlc.cp_table_obj == NULL) {
4079 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4080 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4081 if (r) {
4082 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4083 sumo_rlc_fini(rdev);
4084 return r;
4085 }
4086 }
4087
4088 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4089 if (unlikely(r != 0)) {
4090 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4091 sumo_rlc_fini(rdev);
4092 return r;
4093 }
4094 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4095 &rdev->rlc.cp_table_gpu_addr);
4096 if (r) {
4097 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4098 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4099 sumo_rlc_fini(rdev);
4100 return r;
4101 }
4102 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4103 if (r) {
4104 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4105 sumo_rlc_fini(rdev);
4106 return r;
4107 }
4108
4109 cik_init_cp_pg_table(rdev);
4110
4111 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4112 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4113
4114 }
4115
4116 return 0;
4117 }
4118
4119 static void evergreen_rlc_start(struct radeon_device *rdev)
4120 {
4121 u32 mask = RLC_ENABLE;
4122
4123 if (rdev->flags & RADEON_IS_IGP) {
4124 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4125 }
4126
4127 WREG32(RLC_CNTL, mask);
4128 }
4129
4130 int evergreen_rlc_resume(struct radeon_device *rdev)
4131 {
4132 u32 i;
4133 const __be32 *fw_data;
4134
4135 if (!rdev->rlc_fw)
4136 return -EINVAL;
4137
4138 r600_rlc_stop(rdev);
4139
4140 WREG32(RLC_HB_CNTL, 0);
4141
4142 if (rdev->flags & RADEON_IS_IGP) {
4143 if (rdev->family == CHIP_ARUBA) {
4144 u32 always_on_bitmap =
4145 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4146 /* find out the number of active simds */
4147 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4148 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4149 tmp = hweight32(~tmp);
4150 if (tmp == rdev->config.cayman.max_simds_per_se) {
4151 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4152 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4153 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4154 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4155 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4156 }
4157 } else {
4158 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4159 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4160 }
4161 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4162 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4163 } else {
4164 WREG32(RLC_HB_BASE, 0);
4165 WREG32(RLC_HB_RPTR, 0);
4166 WREG32(RLC_HB_WPTR, 0);
4167 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4168 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4169 }
4170 WREG32(RLC_MC_CNTL, 0);
4171 WREG32(RLC_UCODE_CNTL, 0);
4172
4173 fw_data = (const __be32 *)rdev->rlc_fw->data;
4174 if (rdev->family >= CHIP_ARUBA) {
4175 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4176 WREG32(RLC_UCODE_ADDR, i);
4177 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4178 }
4179 } else if (rdev->family >= CHIP_CAYMAN) {
4180 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4181 WREG32(RLC_UCODE_ADDR, i);
4182 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4183 }
4184 } else {
4185 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4186 WREG32(RLC_UCODE_ADDR, i);
4187 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4188 }
4189 }
4190 WREG32(RLC_UCODE_ADDR, 0);
4191
4192 evergreen_rlc_start(rdev);
4193
4194 return 0;
4195 }
4196
4197 /* Interrupts */
4198
4199 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4200 {
4201 if (crtc >= rdev->num_crtc)
4202 return 0;
4203 else
4204 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4205 }
4206
4207 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4208 {
4209 u32 tmp;
4210
4211 if (rdev->family >= CHIP_CAYMAN) {
4212 cayman_cp_int_cntl_setup(rdev, 0,
4213 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4214 cayman_cp_int_cntl_setup(rdev, 1, 0);
4215 cayman_cp_int_cntl_setup(rdev, 2, 0);
4216 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4217 WREG32(CAYMAN_DMA1_CNTL, tmp);
4218 } else
4219 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4220 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4221 WREG32(DMA_CNTL, tmp);
4222 WREG32(GRBM_INT_CNTL, 0);
4223 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4224 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4225 if (rdev->num_crtc >= 4) {
4226 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4227 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4228 }
4229 if (rdev->num_crtc >= 6) {
4230 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4231 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4232 }
4233
4234 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4235 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4236 if (rdev->num_crtc >= 4) {
4237 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4238 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4239 }
4240 if (rdev->num_crtc >= 6) {
4241 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4242 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4243 }
4244
4245 /* only one DAC on DCE6 */
4246 if (!ASIC_IS_DCE6(rdev))
4247 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4248 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4249
4250 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4251 WREG32(DC_HPD1_INT_CONTROL, tmp);
4252 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4253 WREG32(DC_HPD2_INT_CONTROL, tmp);
4254 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4255 WREG32(DC_HPD3_INT_CONTROL, tmp);
4256 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4257 WREG32(DC_HPD4_INT_CONTROL, tmp);
4258 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4259 WREG32(DC_HPD5_INT_CONTROL, tmp);
4260 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4261 WREG32(DC_HPD6_INT_CONTROL, tmp);
4262
4263 }
4264
4265 int evergreen_irq_set(struct radeon_device *rdev)
4266 {
4267 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4268 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4269 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4270 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4271 u32 grbm_int_cntl = 0;
4272 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4273 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4274 u32 dma_cntl, dma_cntl1 = 0;
4275 u32 thermal_int = 0;
4276
4277 if (!rdev->irq.installed) {
4278 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4279 return -EINVAL;
4280 }
4281 /* don't enable anything if the ih is disabled */
4282 if (!rdev->ih.enabled) {
4283 r600_disable_interrupts(rdev);
4284 /* force the active interrupt state to all disabled */
4285 evergreen_disable_interrupt_state(rdev);
4286 return 0;
4287 }
4288
4289 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4290 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4291 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4292 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4293 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4294 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4295 if (rdev->family == CHIP_ARUBA)
4296 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4297 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4298 else
4299 thermal_int = RREG32(CG_THERMAL_INT) &
4300 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4301
4302 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4303 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4304 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4305 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4306 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4307 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4308
4309 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4310
4311 if (rdev->family >= CHIP_CAYMAN) {
4312 /* enable CP interrupts on all rings */
4313 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4314 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4315 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4316 }
4317 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4318 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4319 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4320 }
4321 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4322 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4323 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4324 }
4325 } else {
4326 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4327 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4328 cp_int_cntl |= RB_INT_ENABLE;
4329 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4330 }
4331 }
4332
4333 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4334 DRM_DEBUG("r600_irq_set: sw int dma\n");
4335 dma_cntl |= TRAP_ENABLE;
4336 }
4337
4338 if (rdev->family >= CHIP_CAYMAN) {
4339 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4340 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4341 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4342 dma_cntl1 |= TRAP_ENABLE;
4343 }
4344 }
4345
4346 if (rdev->irq.dpm_thermal) {
4347 DRM_DEBUG("dpm thermal\n");
4348 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4349 }
4350
4351 if (rdev->irq.crtc_vblank_int[0] ||
4352 atomic_read(&rdev->irq.pflip[0])) {
4353 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4354 crtc1 |= VBLANK_INT_MASK;
4355 }
4356 if (rdev->irq.crtc_vblank_int[1] ||
4357 atomic_read(&rdev->irq.pflip[1])) {
4358 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4359 crtc2 |= VBLANK_INT_MASK;
4360 }
4361 if (rdev->irq.crtc_vblank_int[2] ||
4362 atomic_read(&rdev->irq.pflip[2])) {
4363 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4364 crtc3 |= VBLANK_INT_MASK;
4365 }
4366 if (rdev->irq.crtc_vblank_int[3] ||
4367 atomic_read(&rdev->irq.pflip[3])) {
4368 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4369 crtc4 |= VBLANK_INT_MASK;
4370 }
4371 if (rdev->irq.crtc_vblank_int[4] ||
4372 atomic_read(&rdev->irq.pflip[4])) {
4373 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4374 crtc5 |= VBLANK_INT_MASK;
4375 }
4376 if (rdev->irq.crtc_vblank_int[5] ||
4377 atomic_read(&rdev->irq.pflip[5])) {
4378 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4379 crtc6 |= VBLANK_INT_MASK;
4380 }
4381 if (rdev->irq.hpd[0]) {
4382 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4383 hpd1 |= DC_HPDx_INT_EN;
4384 }
4385 if (rdev->irq.hpd[1]) {
4386 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4387 hpd2 |= DC_HPDx_INT_EN;
4388 }
4389 if (rdev->irq.hpd[2]) {
4390 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4391 hpd3 |= DC_HPDx_INT_EN;
4392 }
4393 if (rdev->irq.hpd[3]) {
4394 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4395 hpd4 |= DC_HPDx_INT_EN;
4396 }
4397 if (rdev->irq.hpd[4]) {
4398 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4399 hpd5 |= DC_HPDx_INT_EN;
4400 }
4401 if (rdev->irq.hpd[5]) {
4402 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4403 hpd6 |= DC_HPDx_INT_EN;
4404 }
4405 if (rdev->irq.afmt[0]) {
4406 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4407 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4408 }
4409 if (rdev->irq.afmt[1]) {
4410 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4411 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4412 }
4413 if (rdev->irq.afmt[2]) {
4414 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4415 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4416 }
4417 if (rdev->irq.afmt[3]) {
4418 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4419 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4420 }
4421 if (rdev->irq.afmt[4]) {
4422 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4423 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4424 }
4425 if (rdev->irq.afmt[5]) {
4426 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4427 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4428 }
4429
4430 if (rdev->family >= CHIP_CAYMAN) {
4431 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4432 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4433 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4434 } else
4435 WREG32(CP_INT_CNTL, cp_int_cntl);
4436
4437 WREG32(DMA_CNTL, dma_cntl);
4438
4439 if (rdev->family >= CHIP_CAYMAN)
4440 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4441
4442 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4443
4444 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4445 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4446 if (rdev->num_crtc >= 4) {
4447 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4448 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4449 }
4450 if (rdev->num_crtc >= 6) {
4451 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4452 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4453 }
4454
4455 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4456 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4457 if (rdev->num_crtc >= 4) {
4458 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4459 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4460 }
4461 if (rdev->num_crtc >= 6) {
4462 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4463 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4464 }
4465
4466 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4467 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4468 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4469 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4470 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4471 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4472 if (rdev->family == CHIP_ARUBA)
4473 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4474 else
4475 WREG32(CG_THERMAL_INT, thermal_int);
4476
4477 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4478 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4479 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4480 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4481 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4482 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4483
4484 return 0;
4485 }
4486
4487 static void evergreen_irq_ack(struct radeon_device *rdev)
4488 {
4489 u32 tmp;
4490
4491 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4492 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4493 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4494 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4495 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4496 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4497 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4498 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4499 if (rdev->num_crtc >= 4) {
4500 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4501 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4502 }
4503 if (rdev->num_crtc >= 6) {
4504 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4505 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4506 }
4507
4508 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4509 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4510 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4511 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4512 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4513 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4514
4515 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4516 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4517 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4518 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4519 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4520 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4521 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4522 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4523 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4524 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4525 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4526 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4527
4528 if (rdev->num_crtc >= 4) {
4529 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4530 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4531 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4532 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4533 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4534 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4535 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4536 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4537 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4538 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4539 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4540 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4541 }
4542
4543 if (rdev->num_crtc >= 6) {
4544 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4545 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4546 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4547 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4548 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4549 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4550 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4551 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4552 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4553 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4554 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4555 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4556 }
4557
4558 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4559 tmp = RREG32(DC_HPD1_INT_CONTROL);
4560 tmp |= DC_HPDx_INT_ACK;
4561 WREG32(DC_HPD1_INT_CONTROL, tmp);
4562 }
4563 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4564 tmp = RREG32(DC_HPD2_INT_CONTROL);
4565 tmp |= DC_HPDx_INT_ACK;
4566 WREG32(DC_HPD2_INT_CONTROL, tmp);
4567 }
4568 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4569 tmp = RREG32(DC_HPD3_INT_CONTROL);
4570 tmp |= DC_HPDx_INT_ACK;
4571 WREG32(DC_HPD3_INT_CONTROL, tmp);
4572 }
4573 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4574 tmp = RREG32(DC_HPD4_INT_CONTROL);
4575 tmp |= DC_HPDx_INT_ACK;
4576 WREG32(DC_HPD4_INT_CONTROL, tmp);
4577 }
4578 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4579 tmp = RREG32(DC_HPD5_INT_CONTROL);
4580 tmp |= DC_HPDx_INT_ACK;
4581 WREG32(DC_HPD5_INT_CONTROL, tmp);
4582 }
4583 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4584 tmp = RREG32(DC_HPD5_INT_CONTROL);
4585 tmp |= DC_HPDx_INT_ACK;
4586 WREG32(DC_HPD6_INT_CONTROL, tmp);
4587 }
4588 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4589 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4590 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4591 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4592 }
4593 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4594 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4595 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4596 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4597 }
4598 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4599 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4600 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4601 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4602 }
4603 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4604 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4605 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4606 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4607 }
4608 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4609 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4610 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4611 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4612 }
4613 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4614 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4615 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4616 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4617 }
4618 }
4619
4620 static void evergreen_irq_disable(struct radeon_device *rdev)
4621 {
4622 r600_disable_interrupts(rdev);
4623 /* Wait and acknowledge irq */
4624 mdelay(1);
4625 evergreen_irq_ack(rdev);
4626 evergreen_disable_interrupt_state(rdev);
4627 }
4628
4629 void evergreen_irq_suspend(struct radeon_device *rdev)
4630 {
4631 evergreen_irq_disable(rdev);
4632 r600_rlc_stop(rdev);
4633 }
4634
4635 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4636 {
4637 u32 wptr, tmp;
4638
4639 if (rdev->wb.enabled)
4640 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4641 else
4642 wptr = RREG32(IH_RB_WPTR);
4643
4644 if (wptr & RB_OVERFLOW) {
4645 /* When a ring buffer overflow happen start parsing interrupt
4646 * from the last not overwritten vector (wptr + 16). Hopefully
4647 * this should allow us to catchup.
4648 */
4649 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4650 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4651 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4652 tmp = RREG32(IH_RB_CNTL);
4653 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4654 WREG32(IH_RB_CNTL, tmp);
4655 }
4656 return (wptr & rdev->ih.ptr_mask);
4657 }
4658
4659 int evergreen_irq_process(struct radeon_device *rdev)
4660 {
4661 u32 wptr;
4662 u32 rptr;
4663 u32 src_id, src_data;
4664 u32 ring_index;
4665 bool queue_hotplug = false;
4666 bool queue_hdmi = false;
4667 bool queue_thermal = false;
4668 u32 status, addr;
4669
4670 if (!rdev->ih.enabled || rdev->shutdown)
4671 return IRQ_NONE;
4672
4673 wptr = evergreen_get_ih_wptr(rdev);
4674
4675 restart_ih:
4676 /* is somebody else already processing irqs? */
4677 if (atomic_xchg(&rdev->ih.lock, 1))
4678 return IRQ_NONE;
4679
4680 rptr = rdev->ih.rptr;
4681 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4682
4683 /* Order reading of wptr vs. reading of IH ring data */
4684 rmb();
4685
4686 /* display interrupts */
4687 evergreen_irq_ack(rdev);
4688
4689 while (rptr != wptr) {
4690 /* wptr/rptr are in bytes! */
4691 ring_index = rptr / 4;
4692 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4693 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4694
4695 switch (src_id) {
4696 case 1: /* D1 vblank/vline */
4697 switch (src_data) {
4698 case 0: /* D1 vblank */
4699 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4700 if (rdev->irq.crtc_vblank_int[0]) {
4701 drm_handle_vblank(rdev->ddev, 0);
4702 rdev->pm.vblank_sync = true;
4703 wake_up(&rdev->irq.vblank_queue);
4704 }
4705 if (atomic_read(&rdev->irq.pflip[0]))
4706 radeon_crtc_handle_flip(rdev, 0);
4707 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4708 DRM_DEBUG("IH: D1 vblank\n");
4709 }
4710 break;
4711 case 1: /* D1 vline */
4712 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4713 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4714 DRM_DEBUG("IH: D1 vline\n");
4715 }
4716 break;
4717 default:
4718 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4719 break;
4720 }
4721 break;
4722 case 2: /* D2 vblank/vline */
4723 switch (src_data) {
4724 case 0: /* D2 vblank */
4725 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4726 if (rdev->irq.crtc_vblank_int[1]) {
4727 drm_handle_vblank(rdev->ddev, 1);
4728 rdev->pm.vblank_sync = true;
4729 wake_up(&rdev->irq.vblank_queue);
4730 }
4731 if (atomic_read(&rdev->irq.pflip[1]))
4732 radeon_crtc_handle_flip(rdev, 1);
4733 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4734 DRM_DEBUG("IH: D2 vblank\n");
4735 }
4736 break;
4737 case 1: /* D2 vline */
4738 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4739 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4740 DRM_DEBUG("IH: D2 vline\n");
4741 }
4742 break;
4743 default:
4744 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4745 break;
4746 }
4747 break;
4748 case 3: /* D3 vblank/vline */
4749 switch (src_data) {
4750 case 0: /* D3 vblank */
4751 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4752 if (rdev->irq.crtc_vblank_int[2]) {
4753 drm_handle_vblank(rdev->ddev, 2);
4754 rdev->pm.vblank_sync = true;
4755 wake_up(&rdev->irq.vblank_queue);
4756 }
4757 if (atomic_read(&rdev->irq.pflip[2]))
4758 radeon_crtc_handle_flip(rdev, 2);
4759 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4760 DRM_DEBUG("IH: D3 vblank\n");
4761 }
4762 break;
4763 case 1: /* D3 vline */
4764 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4765 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4766 DRM_DEBUG("IH: D3 vline\n");
4767 }
4768 break;
4769 default:
4770 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4771 break;
4772 }
4773 break;
4774 case 4: /* D4 vblank/vline */
4775 switch (src_data) {
4776 case 0: /* D4 vblank */
4777 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4778 if (rdev->irq.crtc_vblank_int[3]) {
4779 drm_handle_vblank(rdev->ddev, 3);
4780 rdev->pm.vblank_sync = true;
4781 wake_up(&rdev->irq.vblank_queue);
4782 }
4783 if (atomic_read(&rdev->irq.pflip[3]))
4784 radeon_crtc_handle_flip(rdev, 3);
4785 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4786 DRM_DEBUG("IH: D4 vblank\n");
4787 }
4788 break;
4789 case 1: /* D4 vline */
4790 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4791 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4792 DRM_DEBUG("IH: D4 vline\n");
4793 }
4794 break;
4795 default:
4796 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4797 break;
4798 }
4799 break;
4800 case 5: /* D5 vblank/vline */
4801 switch (src_data) {
4802 case 0: /* D5 vblank */
4803 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4804 if (rdev->irq.crtc_vblank_int[4]) {
4805 drm_handle_vblank(rdev->ddev, 4);
4806 rdev->pm.vblank_sync = true;
4807 wake_up(&rdev->irq.vblank_queue);
4808 }
4809 if (atomic_read(&rdev->irq.pflip[4]))
4810 radeon_crtc_handle_flip(rdev, 4);
4811 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4812 DRM_DEBUG("IH: D5 vblank\n");
4813 }
4814 break;
4815 case 1: /* D5 vline */
4816 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4817 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4818 DRM_DEBUG("IH: D5 vline\n");
4819 }
4820 break;
4821 default:
4822 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4823 break;
4824 }
4825 break;
4826 case 6: /* D6 vblank/vline */
4827 switch (src_data) {
4828 case 0: /* D6 vblank */
4829 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4830 if (rdev->irq.crtc_vblank_int[5]) {
4831 drm_handle_vblank(rdev->ddev, 5);
4832 rdev->pm.vblank_sync = true;
4833 wake_up(&rdev->irq.vblank_queue);
4834 }
4835 if (atomic_read(&rdev->irq.pflip[5]))
4836 radeon_crtc_handle_flip(rdev, 5);
4837 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4838 DRM_DEBUG("IH: D6 vblank\n");
4839 }
4840 break;
4841 case 1: /* D6 vline */
4842 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4843 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4844 DRM_DEBUG("IH: D6 vline\n");
4845 }
4846 break;
4847 default:
4848 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4849 break;
4850 }
4851 break;
4852 case 42: /* HPD hotplug */
4853 switch (src_data) {
4854 case 0:
4855 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4856 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4857 queue_hotplug = true;
4858 DRM_DEBUG("IH: HPD1\n");
4859 }
4860 break;
4861 case 1:
4862 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4863 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4864 queue_hotplug = true;
4865 DRM_DEBUG("IH: HPD2\n");
4866 }
4867 break;
4868 case 2:
4869 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4870 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4871 queue_hotplug = true;
4872 DRM_DEBUG("IH: HPD3\n");
4873 }
4874 break;
4875 case 3:
4876 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4877 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4878 queue_hotplug = true;
4879 DRM_DEBUG("IH: HPD4\n");
4880 }
4881 break;
4882 case 4:
4883 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4884 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4885 queue_hotplug = true;
4886 DRM_DEBUG("IH: HPD5\n");
4887 }
4888 break;
4889 case 5:
4890 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4891 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4892 queue_hotplug = true;
4893 DRM_DEBUG("IH: HPD6\n");
4894 }
4895 break;
4896 default:
4897 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4898 break;
4899 }
4900 break;
4901 case 44: /* hdmi */
4902 switch (src_data) {
4903 case 0:
4904 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4905 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4906 queue_hdmi = true;
4907 DRM_DEBUG("IH: HDMI0\n");
4908 }
4909 break;
4910 case 1:
4911 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4912 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4913 queue_hdmi = true;
4914 DRM_DEBUG("IH: HDMI1\n");
4915 }
4916 break;
4917 case 2:
4918 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4919 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4920 queue_hdmi = true;
4921 DRM_DEBUG("IH: HDMI2\n");
4922 }
4923 break;
4924 case 3:
4925 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4926 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4927 queue_hdmi = true;
4928 DRM_DEBUG("IH: HDMI3\n");
4929 }
4930 break;
4931 case 4:
4932 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4933 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4934 queue_hdmi = true;
4935 DRM_DEBUG("IH: HDMI4\n");
4936 }
4937 break;
4938 case 5:
4939 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4940 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4941 queue_hdmi = true;
4942 DRM_DEBUG("IH: HDMI5\n");
4943 }
4944 break;
4945 default:
4946 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4947 break;
4948 }
4949 case 124: /* UVD */
4950 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4951 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4952 break;
4953 case 146:
4954 case 147:
4955 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4956 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4957 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4958 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4959 addr);
4960 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4961 status);
4962 cayman_vm_decode_fault(rdev, status, addr);
4963 /* reset addr and status */
4964 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4965 break;
4966 case 176: /* CP_INT in ring buffer */
4967 case 177: /* CP_INT in IB1 */
4968 case 178: /* CP_INT in IB2 */
4969 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4970 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4971 break;
4972 case 181: /* CP EOP event */
4973 DRM_DEBUG("IH: CP EOP\n");
4974 if (rdev->family >= CHIP_CAYMAN) {
4975 switch (src_data) {
4976 case 0:
4977 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4978 break;
4979 case 1:
4980 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4981 break;
4982 case 2:
4983 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4984 break;
4985 }
4986 } else
4987 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4988 break;
4989 case 224: /* DMA trap event */
4990 DRM_DEBUG("IH: DMA trap\n");
4991 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4992 break;
4993 case 230: /* thermal low to high */
4994 DRM_DEBUG("IH: thermal low to high\n");
4995 rdev->pm.dpm.thermal.high_to_low = false;
4996 queue_thermal = true;
4997 break;
4998 case 231: /* thermal high to low */
4999 DRM_DEBUG("IH: thermal high to low\n");
5000 rdev->pm.dpm.thermal.high_to_low = true;
5001 queue_thermal = true;
5002 break;
5003 case 233: /* GUI IDLE */
5004 DRM_DEBUG("IH: GUI idle\n");
5005 break;
5006 case 244: /* DMA trap event */
5007 if (rdev->family >= CHIP_CAYMAN) {
5008 DRM_DEBUG("IH: DMA1 trap\n");
5009 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5010 }
5011 break;
5012 default:
5013 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5014 break;
5015 }
5016
5017 /* wptr/rptr are in bytes! */
5018 rptr += 16;
5019 rptr &= rdev->ih.ptr_mask;
5020 }
5021 if (queue_hotplug)
5022 schedule_work(&rdev->hotplug_work);
5023 if (queue_hdmi)
5024 schedule_work(&rdev->audio_work);
5025 if (queue_thermal && rdev->pm.dpm_enabled)
5026 schedule_work(&rdev->pm.dpm.thermal.work);
5027 rdev->ih.rptr = rptr;
5028 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5029 atomic_set(&rdev->ih.lock, 0);
5030
5031 /* make sure wptr hasn't changed while processing */
5032 wptr = evergreen_get_ih_wptr(rdev);
5033 if (wptr != rptr)
5034 goto restart_ih;
5035
5036 return IRQ_HANDLED;
5037 }
5038
5039 static int evergreen_startup(struct radeon_device *rdev)
5040 {
5041 struct radeon_ring *ring;
5042 int r;
5043
5044 /* enable pcie gen2 link */
5045 evergreen_pcie_gen2_enable(rdev);
5046 /* enable aspm */
5047 evergreen_program_aspm(rdev);
5048
5049 /* scratch needs to be initialized before MC */
5050 r = r600_vram_scratch_init(rdev);
5051 if (r)
5052 return r;
5053
5054 evergreen_mc_program(rdev);
5055
5056 if (ASIC_IS_DCE5(rdev)) {
5057 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5058 r = ni_init_microcode(rdev);
5059 if (r) {
5060 DRM_ERROR("Failed to load firmware!\n");
5061 return r;
5062 }
5063 }
5064 r = ni_mc_load_microcode(rdev);
5065 if (r) {
5066 DRM_ERROR("Failed to load MC firmware!\n");
5067 return r;
5068 }
5069 } else {
5070 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5071 r = r600_init_microcode(rdev);
5072 if (r) {
5073 DRM_ERROR("Failed to load firmware!\n");
5074 return r;
5075 }
5076 }
5077 }
5078
5079 if (rdev->flags & RADEON_IS_AGP) {
5080 evergreen_agp_enable(rdev);
5081 } else {
5082 r = evergreen_pcie_gart_enable(rdev);
5083 if (r)
5084 return r;
5085 }
5086 evergreen_gpu_init(rdev);
5087
5088 /* allocate rlc buffers */
5089 if (rdev->flags & RADEON_IS_IGP) {
5090 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5091 rdev->rlc.reg_list_size =
5092 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5093 rdev->rlc.cs_data = evergreen_cs_data;
5094 r = sumo_rlc_init(rdev);
5095 if (r) {
5096 DRM_ERROR("Failed to init rlc BOs!\n");
5097 return r;
5098 }
5099 }
5100
5101 /* allocate wb buffer */
5102 r = radeon_wb_init(rdev);
5103 if (r)
5104 return r;
5105
5106 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5107 if (r) {
5108 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5109 return r;
5110 }
5111
5112 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5113 if (r) {
5114 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5115 return r;
5116 }
5117
5118 r = uvd_v2_2_resume(rdev);
5119 if (!r) {
5120 r = radeon_fence_driver_start_ring(rdev,
5121 R600_RING_TYPE_UVD_INDEX);
5122 if (r)
5123 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5124 }
5125
5126 if (r)
5127 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5128
5129 /* Enable IRQ */
5130 if (!rdev->irq.installed) {
5131 r = radeon_irq_kms_init(rdev);
5132 if (r)
5133 return r;
5134 }
5135
5136 r = r600_irq_init(rdev);
5137 if (r) {
5138 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5139 radeon_irq_kms_fini(rdev);
5140 return r;
5141 }
5142 evergreen_irq_set(rdev);
5143
5144 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5145 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5146 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5147 RADEON_CP_PACKET2);
5148 if (r)
5149 return r;
5150
5151 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5152 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5153 DMA_RB_RPTR, DMA_RB_WPTR,
5154 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5155 if (r)
5156 return r;
5157
5158 r = evergreen_cp_load_microcode(rdev);
5159 if (r)
5160 return r;
5161 r = evergreen_cp_resume(rdev);
5162 if (r)
5163 return r;
5164 r = r600_dma_resume(rdev);
5165 if (r)
5166 return r;
5167
5168 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5169 if (ring->ring_size) {
5170 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5171 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5172 RADEON_CP_PACKET2);
5173 if (!r)
5174 r = uvd_v1_0_init(rdev);
5175
5176 if (r)
5177 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5178 }
5179
5180 r = radeon_ib_pool_init(rdev);
5181 if (r) {
5182 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5183 return r;
5184 }
5185
5186 r = r600_audio_init(rdev);
5187 if (r) {
5188 DRM_ERROR("radeon: audio init failed\n");
5189 return r;
5190 }
5191
5192 return 0;
5193 }
5194
5195 int evergreen_resume(struct radeon_device *rdev)
5196 {
5197 int r;
5198
5199 /* reset the asic, the gfx blocks are often in a bad state
5200 * after the driver is unloaded or after a resume
5201 */
5202 if (radeon_asic_reset(rdev))
5203 dev_warn(rdev->dev, "GPU reset failed !\n");
5204 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5205 * posting will perform necessary task to bring back GPU into good
5206 * shape.
5207 */
5208 /* post card */
5209 atom_asic_init(rdev->mode_info.atom_context);
5210
5211 /* init golden registers */
5212 evergreen_init_golden_registers(rdev);
5213
5214 rdev->accel_working = true;
5215 r = evergreen_startup(rdev);
5216 if (r) {
5217 DRM_ERROR("evergreen startup failed on resume\n");
5218 rdev->accel_working = false;
5219 return r;
5220 }
5221
5222 return r;
5223
5224 }
5225
5226 int evergreen_suspend(struct radeon_device *rdev)
5227 {
5228 r600_audio_fini(rdev);
5229 uvd_v1_0_fini(rdev);
5230 radeon_uvd_suspend(rdev);
5231 r700_cp_stop(rdev);
5232 r600_dma_stop(rdev);
5233 evergreen_irq_suspend(rdev);
5234 radeon_wb_disable(rdev);
5235 evergreen_pcie_gart_disable(rdev);
5236
5237 return 0;
5238 }
5239
5240 /* Plan is to move initialization in that function and use
5241 * helper function so that radeon_device_init pretty much
5242 * do nothing more than calling asic specific function. This
5243 * should also allow to remove a bunch of callback function
5244 * like vram_info.
5245 */
5246 int evergreen_init(struct radeon_device *rdev)
5247 {
5248 int r;
5249
5250 /* Read BIOS */
5251 if (!radeon_get_bios(rdev)) {
5252 if (ASIC_IS_AVIVO(rdev))
5253 return -EINVAL;
5254 }
5255 /* Must be an ATOMBIOS */
5256 if (!rdev->is_atom_bios) {
5257 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5258 return -EINVAL;
5259 }
5260 r = radeon_atombios_init(rdev);
5261 if (r)
5262 return r;
5263 /* reset the asic, the gfx blocks are often in a bad state
5264 * after the driver is unloaded or after a resume
5265 */
5266 if (radeon_asic_reset(rdev))
5267 dev_warn(rdev->dev, "GPU reset failed !\n");
5268 /* Post card if necessary */
5269 if (!radeon_card_posted(rdev)) {
5270 if (!rdev->bios) {
5271 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5272 return -EINVAL;
5273 }
5274 DRM_INFO("GPU not posted. posting now...\n");
5275 atom_asic_init(rdev->mode_info.atom_context);
5276 }
5277 /* init golden registers */
5278 evergreen_init_golden_registers(rdev);
5279 /* Initialize scratch registers */
5280 r600_scratch_init(rdev);
5281 /* Initialize surface registers */
5282 radeon_surface_init(rdev);
5283 /* Initialize clocks */
5284 radeon_get_clock_info(rdev->ddev);
5285 /* Fence driver */
5286 r = radeon_fence_driver_init(rdev);
5287 if (r)
5288 return r;
5289 /* initialize AGP */
5290 if (rdev->flags & RADEON_IS_AGP) {
5291 r = radeon_agp_init(rdev);
5292 if (r)
5293 radeon_agp_disable(rdev);
5294 }
5295 /* initialize memory controller */
5296 r = evergreen_mc_init(rdev);
5297 if (r)
5298 return r;
5299 /* Memory manager */
5300 r = radeon_bo_init(rdev);
5301 if (r)
5302 return r;
5303
5304 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5305 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5306
5307 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5308 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5309
5310 r = radeon_uvd_init(rdev);
5311 if (!r) {
5312 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5313 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5314 4096);
5315 }
5316
5317 rdev->ih.ring_obj = NULL;
5318 r600_ih_ring_init(rdev, 64 * 1024);
5319
5320 r = r600_pcie_gart_init(rdev);
5321 if (r)
5322 return r;
5323
5324 rdev->accel_working = true;
5325 r = evergreen_startup(rdev);
5326 if (r) {
5327 dev_err(rdev->dev, "disabling GPU acceleration\n");
5328 r700_cp_fini(rdev);
5329 r600_dma_fini(rdev);
5330 r600_irq_fini(rdev);
5331 if (rdev->flags & RADEON_IS_IGP)
5332 sumo_rlc_fini(rdev);
5333 radeon_wb_fini(rdev);
5334 radeon_ib_pool_fini(rdev);
5335 radeon_irq_kms_fini(rdev);
5336 evergreen_pcie_gart_fini(rdev);
5337 rdev->accel_working = false;
5338 }
5339
5340 /* Don't start up if the MC ucode is missing on BTC parts.
5341 * The default clocks and voltages before the MC ucode
5342 * is loaded are not suffient for advanced operations.
5343 */
5344 if (ASIC_IS_DCE5(rdev)) {
5345 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5346 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5347 return -EINVAL;
5348 }
5349 }
5350
5351 return 0;
5352 }
5353
5354 void evergreen_fini(struct radeon_device *rdev)
5355 {
5356 r600_audio_fini(rdev);
5357 r700_cp_fini(rdev);
5358 r600_dma_fini(rdev);
5359 r600_irq_fini(rdev);
5360 if (rdev->flags & RADEON_IS_IGP)
5361 sumo_rlc_fini(rdev);
5362 radeon_wb_fini(rdev);
5363 radeon_ib_pool_fini(rdev);
5364 radeon_irq_kms_fini(rdev);
5365 evergreen_pcie_gart_fini(rdev);
5366 uvd_v1_0_fini(rdev);
5367 radeon_uvd_fini(rdev);
5368 r600_vram_scratch_fini(rdev);
5369 radeon_gem_fini(rdev);
5370 radeon_fence_driver_fini(rdev);
5371 radeon_agp_fini(rdev);
5372 radeon_bo_fini(rdev);
5373 radeon_atombios_fini(rdev);
5374 kfree(rdev->bios);
5375 rdev->bios = NULL;
5376 }
5377
5378 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5379 {
5380 u32 link_width_cntl, speed_cntl;
5381
5382 if (radeon_pcie_gen2 == 0)
5383 return;
5384
5385 if (rdev->flags & RADEON_IS_IGP)
5386 return;
5387
5388 if (!(rdev->flags & RADEON_IS_PCIE))
5389 return;
5390
5391 /* x2 cards have a special sequence */
5392 if (ASIC_IS_X2(rdev))
5393 return;
5394
5395 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5396 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5397 return;
5398
5399 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5400 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5401 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5402 return;
5403 }
5404
5405 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5406
5407 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5408 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5409
5410 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5411 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5412 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5413
5414 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5415 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5416 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5417
5418 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5419 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5420 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5421
5422 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5423 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5424 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5425
5426 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5427 speed_cntl |= LC_GEN2_EN_STRAP;
5428 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5429
5430 } else {
5431 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5432 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5433 if (1)
5434 link_width_cntl |= LC_UPCONFIGURE_DIS;
5435 else
5436 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5437 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5438 }
5439 }
5440
5441 void evergreen_program_aspm(struct radeon_device *rdev)
5442 {
5443 u32 data, orig;
5444 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5445 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5446 /* fusion_platform = true
5447 * if the system is a fusion system
5448 * (APU or DGPU in a fusion system).
5449 * todo: check if the system is a fusion platform.
5450 */
5451 bool fusion_platform = false;
5452
5453 if (radeon_aspm == 0)
5454 return;
5455
5456 if (!(rdev->flags & RADEON_IS_PCIE))
5457 return;
5458
5459 switch (rdev->family) {
5460 case CHIP_CYPRESS:
5461 case CHIP_HEMLOCK:
5462 case CHIP_JUNIPER:
5463 case CHIP_REDWOOD:
5464 case CHIP_CEDAR:
5465 case CHIP_SUMO:
5466 case CHIP_SUMO2:
5467 case CHIP_PALM:
5468 case CHIP_ARUBA:
5469 disable_l0s = true;
5470 break;
5471 default:
5472 disable_l0s = false;
5473 break;
5474 }
5475
5476 if (rdev->flags & RADEON_IS_IGP)
5477 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5478
5479 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5480 if (fusion_platform)
5481 data &= ~MULTI_PIF;
5482 else
5483 data |= MULTI_PIF;
5484 if (data != orig)
5485 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5486
5487 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5488 if (fusion_platform)
5489 data &= ~MULTI_PIF;
5490 else
5491 data |= MULTI_PIF;
5492 if (data != orig)
5493 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5494
5495 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5496 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5497 if (!disable_l0s) {
5498 if (rdev->family >= CHIP_BARTS)
5499 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5500 else
5501 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5502 }
5503
5504 if (!disable_l1) {
5505 if (rdev->family >= CHIP_BARTS)
5506 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5507 else
5508 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5509
5510 if (!disable_plloff_in_l1) {
5511 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5512 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5513 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5514 if (data != orig)
5515 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5516
5517 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5518 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5519 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5520 if (data != orig)
5521 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5522
5523 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5524 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5525 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5526 if (data != orig)
5527 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5528
5529 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5530 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5531 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5532 if (data != orig)
5533 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5534
5535 if (rdev->family >= CHIP_BARTS) {
5536 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5537 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5538 data |= PLL_RAMP_UP_TIME_0(4);
5539 if (data != orig)
5540 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5541
5542 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5543 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5544 data |= PLL_RAMP_UP_TIME_1(4);
5545 if (data != orig)
5546 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5547
5548 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5549 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5550 data |= PLL_RAMP_UP_TIME_0(4);
5551 if (data != orig)
5552 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5553
5554 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5555 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5556 data |= PLL_RAMP_UP_TIME_1(4);
5557 if (data != orig)
5558 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5559 }
5560
5561 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5562 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5563 data |= LC_DYN_LANES_PWR_STATE(3);
5564 if (data != orig)
5565 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5566
5567 if (rdev->family >= CHIP_BARTS) {
5568 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5569 data &= ~LS2_EXIT_TIME_MASK;
5570 data |= LS2_EXIT_TIME(1);
5571 if (data != orig)
5572 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5573
5574 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5575 data &= ~LS2_EXIT_TIME_MASK;
5576 data |= LS2_EXIT_TIME(1);
5577 if (data != orig)
5578 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5579 }
5580 }
5581 }
5582
5583 /* evergreen parts only */
5584 if (rdev->family < CHIP_BARTS)
5585 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5586
5587 if (pcie_lc_cntl != pcie_lc_cntl_old)
5588 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5589 }
Linus' kernel tree