?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/radeon: export reservation_object from dmabuf to ttm
[linux-2.6/btrfs-unstable.git] / drivers / gpu / drm / radeon / evergreen.c
blob8fe9f870fb5a318a44dbb90983fdafe01342ed3f
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include <drm/radeon_drm.h>
30 #include "evergreend.h"
31 #include "atom.h"
32 #include "avivod.h"
33 #include "evergreen_reg.h"
34 #include "evergreen_blit_shaders.h"
35 #include "radeon_ucode.h"
36
37 static const u32 crtc_offsets[6] =
38 {
39 EVERGREEN_CRTC0_REGISTER_OFFSET,
40 EVERGREEN_CRTC1_REGISTER_OFFSET,
41 EVERGREEN_CRTC2_REGISTER_OFFSET,
42 EVERGREEN_CRTC3_REGISTER_OFFSET,
43 EVERGREEN_CRTC4_REGISTER_OFFSET,
44 EVERGREEN_CRTC5_REGISTER_OFFSET
45 };
46
47 #include "clearstate_evergreen.h"
48
49 static const u32 sumo_rlc_save_restore_register_list[] =
50 {
51 0x98fc,
52 0x9830,
53 0x9834,
54 0x9838,
55 0x9870,
56 0x9874,
57 0x8a14,
58 0x8b24,
59 0x8bcc,
60 0x8b10,
61 0x8d00,
62 0x8d04,
63 0x8c00,
64 0x8c04,
65 0x8c08,
66 0x8c0c,
67 0x8d8c,
68 0x8c20,
69 0x8c24,
70 0x8c28,
71 0x8c18,
72 0x8c1c,
73 0x8cf0,
74 0x8e2c,
75 0x8e38,
76 0x8c30,
77 0x9508,
78 0x9688,
79 0x9608,
80 0x960c,
81 0x9610,
82 0x9614,
83 0x88c4,
84 0x88d4,
85 0xa008,
86 0x900c,
87 0x9100,
88 0x913c,
89 0x98f8,
90 0x98f4,
91 0x9b7c,
92 0x3f8c,
93 0x8950,
94 0x8954,
95 0x8a18,
96 0x8b28,
97 0x9144,
98 0x9148,
99 0x914c,
100 0x3f90,
101 0x3f94,
102 0x915c,
103 0x9160,
104 0x9178,
105 0x917c,
106 0x9180,
107 0x918c,
108 0x9190,
109 0x9194,
110 0x9198,
111 0x919c,
112 0x91a8,
113 0x91ac,
114 0x91b0,
115 0x91b4,
116 0x91b8,
117 0x91c4,
118 0x91c8,
119 0x91cc,
120 0x91d0,
121 0x91d4,
122 0x91e0,
123 0x91e4,
124 0x91ec,
125 0x91f0,
126 0x91f4,
127 0x9200,
128 0x9204,
129 0x929c,
130 0x9150,
131 0x802c,
132 };
133
134 static void evergreen_gpu_init(struct radeon_device *rdev);
135 void evergreen_fini(struct radeon_device *rdev);
136 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
137 void evergreen_program_aspm(struct radeon_device *rdev);
138 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
139 int ring, u32 cp_int_cntl);
140 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
141 u32 status, u32 addr);
142 void cik_init_cp_pg_table(struct radeon_device *rdev);
143
144 extern u32 si_get_csb_size(struct radeon_device *rdev);
145 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
146 extern u32 cik_get_csb_size(struct radeon_device *rdev);
147 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
148 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
149
150 static const u32 evergreen_golden_registers[] =
151 {
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
191 0x5c4, 0xffffffff, 0x00000001,
192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194 };
195
196 static const u32 evergreen_golden_registers2[] =
197 {
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212 };
213
214 static const u32 cypress_mgcg_init[] =
215 {
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365 };
366
367 static const u32 redwood_mgcg_init[] =
368 {
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437 };
438
439 static const u32 cedar_golden_registers[] =
440 {
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
478 0x5c4, 0xffffffff, 0x00000001,
479 0x9508, 0xffffffff, 0x00000002
480 };
481
482 static const u32 cedar_mgcg_init[] =
483 {
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534 };
535
536 static const u32 juniper_mgcg_init[] =
537 {
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632 };
633
634 static const u32 supersumo_golden_registers[] =
635 {
636 0x5eb4, 0xffffffff, 0x00000002,
637 0x5c4, 0xffffffff, 0x00000001,
638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707 };
708
709 static const u32 sumo_golden_registers[] =
710 {
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716 };
717
718 static const u32 wrestler_golden_registers[] =
719 {
720 0x5eb4, 0xffffffff, 0x00000002,
721 0x5c4, 0xffffffff, 0x00000001,
722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769 };
770
771 static const u32 barts_golden_registers[] =
772 {
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818 };
819
820 static const u32 turks_golden_registers[] =
821 {
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869 };
870
871 static const u32 caicos_golden_registers[] =
872 {
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920 };
921
922 static void evergreen_init_golden_registers(struct radeon_device *rdev)
923 {
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006 }
1007
1008 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011 {
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037 }
1038
1039 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041 {
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061 }
1062
1063 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064 {
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080 done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084 }
1085
1086 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087 {
1088 /* start off with something large */
1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1090 int r;
1091
1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
1140 if (fb_div < 307200)
1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173 }
1174
1175 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176 {
1177 int readrq;
1178 u16 v;
1179
1180 readrq = pcie_get_readrq(rdev->pdev);
1181 v = ffs(readrq) - 8;
1182 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183 * to avoid hangs or perfomance issues
1184 */
1185 if ((v == 0) || (v == 6) || (v == 7))
1186 pcie_set_readrq(rdev->pdev, 512);
1187 }
1188
1189 void dce4_program_fmt(struct drm_encoder *encoder)
1190 {
1191 struct drm_device *dev = encoder->dev;
1192 struct radeon_device *rdev = dev->dev_private;
1193 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1194 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1195 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1196 int bpc = 0;
1197 u32 tmp = 0;
1198 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1199
1200 if (connector) {
1201 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1202 bpc = radeon_get_monitor_bpc(connector);
1203 dither = radeon_connector->dither;
1204 }
1205
1206 /* LVDS/eDP FMT is set up by atom */
1207 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1208 return;
1209
1210 /* not needed for analog */
1211 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1212 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1213 return;
1214
1215 if (bpc == 0)
1216 return;
1217
1218 switch (bpc) {
1219 case 6:
1220 if (dither == RADEON_FMT_DITHER_ENABLE)
1221 /* XXX sort out optimal dither settings */
1222 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1223 FMT_SPATIAL_DITHER_EN);
1224 else
1225 tmp |= FMT_TRUNCATE_EN;
1226 break;
1227 case 8:
1228 if (dither == RADEON_FMT_DITHER_ENABLE)
1229 /* XXX sort out optimal dither settings */
1230 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1231 FMT_RGB_RANDOM_ENABLE |
1232 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1233 else
1234 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1235 break;
1236 case 10:
1237 default:
1238 /* not needed */
1239 break;
1240 }
1241
1242 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1243 }
1244
1245 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1246 {
1247 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1248 return true;
1249 else
1250 return false;
1251 }
1252
1253 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1254 {
1255 u32 pos1, pos2;
1256
1257 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1258 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259
1260 if (pos1 != pos2)
1261 return true;
1262 else
1263 return false;
1264 }
1265
1266 /**
1267 * dce4_wait_for_vblank - vblank wait asic callback.
1268 *
1269 * @rdev: radeon_device pointer
1270 * @crtc: crtc to wait for vblank on
1271 *
1272 * Wait for vblank on the requested crtc (evergreen+).
1273 */
1274 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1275 {
1276 unsigned i = 0;
1277
1278 if (crtc >= rdev->num_crtc)
1279 return;
1280
1281 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1282 return;
1283
1284 /* depending on when we hit vblank, we may be close to active; if so,
1285 * wait for another frame.
1286 */
1287 while (dce4_is_in_vblank(rdev, crtc)) {
1288 if (i++ % 100 == 0) {
1289 if (!dce4_is_counter_moving(rdev, crtc))
1290 break;
1291 }
1292 }
1293
1294 while (!dce4_is_in_vblank(rdev, crtc)) {
1295 if (i++ % 100 == 0) {
1296 if (!dce4_is_counter_moving(rdev, crtc))
1297 break;
1298 }
1299 }
1300 }
1301
1302 /**
1303 * evergreen_page_flip - pageflip callback.
1304 *
1305 * @rdev: radeon_device pointer
1306 * @crtc_id: crtc to cleanup pageflip on
1307 * @crtc_base: new address of the crtc (GPU MC address)
1308 *
1309 * Does the actual pageflip (evergreen+).
1310 * During vblank we take the crtc lock and wait for the update_pending
1311 * bit to go high, when it does, we release the lock, and allow the
1312 * double buffered update to take place.
1313 * Returns the current update pending status.
1314 */
1315 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1316 {
1317 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1318 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1319 int i;
1320
1321 /* Lock the graphics update lock */
1322 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1323 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1324
1325 /* update the scanout addresses */
1326 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1327 upper_32_bits(crtc_base));
1328 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1329 (u32)crtc_base);
1330
1331 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1332 upper_32_bits(crtc_base));
1333 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1334 (u32)crtc_base);
1335
1336 /* Wait for update_pending to go high. */
1337 for (i = 0; i < rdev->usec_timeout; i++) {
1338 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1339 break;
1340 udelay(1);
1341 }
1342 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1343
1344 /* Unlock the lock, so double-buffering can take place inside vblank */
1345 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1346 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1347 }
1348
1349 /**
1350 * evergreen_page_flip_pending - check if page flip is still pending
1351 *
1352 * @rdev: radeon_device pointer
1353 * @crtc_id: crtc to check
1354 *
1355 * Returns the current update pending status.
1356 */
1357 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1358 {
1359 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1360
1361 /* Return current update_pending status: */
1362 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1363 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1364 }
1365
1366 /* get temperature in millidegrees */
1367 int evergreen_get_temp(struct radeon_device *rdev)
1368 {
1369 u32 temp, toffset;
1370 int actual_temp = 0;
1371
1372 if (rdev->family == CHIP_JUNIPER) {
1373 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1374 TOFFSET_SHIFT;
1375 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1376 TS0_ADC_DOUT_SHIFT;
1377
1378 if (toffset & 0x100)
1379 actual_temp = temp / 2 - (0x200 - toffset);
1380 else
1381 actual_temp = temp / 2 + toffset;
1382
1383 actual_temp = actual_temp * 1000;
1384
1385 } else {
1386 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1387 ASIC_T_SHIFT;
1388
1389 if (temp & 0x400)
1390 actual_temp = -256;
1391 else if (temp & 0x200)
1392 actual_temp = 255;
1393 else if (temp & 0x100) {
1394 actual_temp = temp & 0x1ff;
1395 actual_temp |= ~0x1ff;
1396 } else
1397 actual_temp = temp & 0xff;
1398
1399 actual_temp = (actual_temp * 1000) / 2;
1400 }
1401
1402 return actual_temp;
1403 }
1404
1405 int sumo_get_temp(struct radeon_device *rdev)
1406 {
1407 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1408 int actual_temp = temp - 49;
1409
1410 return actual_temp * 1000;
1411 }
1412
1413 /**
1414 * sumo_pm_init_profile - Initialize power profiles callback.
1415 *
1416 * @rdev: radeon_device pointer
1417 *
1418 * Initialize the power states used in profile mode
1419 * (sumo, trinity, SI).
1420 * Used for profile mode only.
1421 */
1422 void sumo_pm_init_profile(struct radeon_device *rdev)
1423 {
1424 int idx;
1425
1426 /* default */
1427 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1428 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1431
1432 /* low,mid sh/mh */
1433 if (rdev->flags & RADEON_IS_MOBILITY)
1434 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1435 else
1436 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1437
1438 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1442
1443 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1444 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1445 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1446 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1447
1448 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1449 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1450 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1451 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1452
1453 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1457
1458 /* high sh/mh */
1459 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1460 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1464 rdev->pm.power_state[idx].num_clock_modes - 1;
1465
1466 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1469 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1470 rdev->pm.power_state[idx].num_clock_modes - 1;
1471 }
1472
1473 /**
1474 * btc_pm_init_profile - Initialize power profiles callback.
1475 *
1476 * @rdev: radeon_device pointer
1477 *
1478 * Initialize the power states used in profile mode
1479 * (BTC, cayman).
1480 * Used for profile mode only.
1481 */
1482 void btc_pm_init_profile(struct radeon_device *rdev)
1483 {
1484 int idx;
1485
1486 /* default */
1487 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1488 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1491 /* starting with BTC, there is one state that is used for both
1492 * MH and SH. Difference is that we always use the high clock index for
1493 * mclk.
1494 */
1495 if (rdev->flags & RADEON_IS_MOBILITY)
1496 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1497 else
1498 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1499 /* low sh */
1500 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1504 /* mid sh */
1505 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1506 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1507 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1508 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1509 /* high sh */
1510 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1511 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1512 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1513 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1514 /* low mh */
1515 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1516 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1519 /* mid mh */
1520 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1524 /* high mh */
1525 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1529 }
1530
1531 /**
1532 * evergreen_pm_misc - set additional pm hw parameters callback.
1533 *
1534 * @rdev: radeon_device pointer
1535 *
1536 * Set non-clock parameters associated with a power state
1537 * (voltage, etc.) (evergreen+).
1538 */
1539 void evergreen_pm_misc(struct radeon_device *rdev)
1540 {
1541 int req_ps_idx = rdev->pm.requested_power_state_index;
1542 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1543 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1544 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1545
1546 if (voltage->type == VOLTAGE_SW) {
1547 /* 0xff0x are flags rather then an actual voltage */
1548 if ((voltage->voltage & 0xff00) == 0xff00)
1549 return;
1550 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1551 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1552 rdev->pm.current_vddc = voltage->voltage;
1553 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1554 }
1555
1556 /* starting with BTC, there is one state that is used for both
1557 * MH and SH. Difference is that we always use the high clock index for
1558 * mclk and vddci.
1559 */
1560 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1561 (rdev->family >= CHIP_BARTS) &&
1562 rdev->pm.active_crtc_count &&
1563 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1564 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1565 voltage = &rdev->pm.power_state[req_ps_idx].
1566 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1567
1568 /* 0xff0x are flags rather then an actual voltage */
1569 if ((voltage->vddci & 0xff00) == 0xff00)
1570 return;
1571 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1572 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1573 rdev->pm.current_vddci = voltage->vddci;
1574 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1575 }
1576 }
1577 }
1578
1579 /**
1580 * evergreen_pm_prepare - pre-power state change callback.
1581 *
1582 * @rdev: radeon_device pointer
1583 *
1584 * Prepare for a power state change (evergreen+).
1585 */
1586 void evergreen_pm_prepare(struct radeon_device *rdev)
1587 {
1588 struct drm_device *ddev = rdev->ddev;
1589 struct drm_crtc *crtc;
1590 struct radeon_crtc *radeon_crtc;
1591 u32 tmp;
1592
1593 /* disable any active CRTCs */
1594 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1595 radeon_crtc = to_radeon_crtc(crtc);
1596 if (radeon_crtc->enabled) {
1597 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1598 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1599 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1600 }
1601 }
1602 }
1603
1604 /**
1605 * evergreen_pm_finish - post-power state change callback.
1606 *
1607 * @rdev: radeon_device pointer
1608 *
1609 * Clean up after a power state change (evergreen+).
1610 */
1611 void evergreen_pm_finish(struct radeon_device *rdev)
1612 {
1613 struct drm_device *ddev = rdev->ddev;
1614 struct drm_crtc *crtc;
1615 struct radeon_crtc *radeon_crtc;
1616 u32 tmp;
1617
1618 /* enable any active CRTCs */
1619 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1620 radeon_crtc = to_radeon_crtc(crtc);
1621 if (radeon_crtc->enabled) {
1622 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1623 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1624 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1625 }
1626 }
1627 }
1628
1629 /**
1630 * evergreen_hpd_sense - hpd sense callback.
1631 *
1632 * @rdev: radeon_device pointer
1633 * @hpd: hpd (hotplug detect) pin
1634 *
1635 * Checks if a digital monitor is connected (evergreen+).
1636 * Returns true if connected, false if not connected.
1637 */
1638 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1639 {
1640 bool connected = false;
1641
1642 switch (hpd) {
1643 case RADEON_HPD_1:
1644 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1645 connected = true;
1646 break;
1647 case RADEON_HPD_2:
1648 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1649 connected = true;
1650 break;
1651 case RADEON_HPD_3:
1652 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1653 connected = true;
1654 break;
1655 case RADEON_HPD_4:
1656 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1657 connected = true;
1658 break;
1659 case RADEON_HPD_5:
1660 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1661 connected = true;
1662 break;
1663 case RADEON_HPD_6:
1664 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1665 connected = true;
1666 break;
1667 default:
1668 break;
1669 }
1670
1671 return connected;
1672 }
1673
1674 /**
1675 * evergreen_hpd_set_polarity - hpd set polarity callback.
1676 *
1677 * @rdev: radeon_device pointer
1678 * @hpd: hpd (hotplug detect) pin
1679 *
1680 * Set the polarity of the hpd pin (evergreen+).
1681 */
1682 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1683 enum radeon_hpd_id hpd)
1684 {
1685 u32 tmp;
1686 bool connected = evergreen_hpd_sense(rdev, hpd);
1687
1688 switch (hpd) {
1689 case RADEON_HPD_1:
1690 tmp = RREG32(DC_HPD1_INT_CONTROL);
1691 if (connected)
1692 tmp &= ~DC_HPDx_INT_POLARITY;
1693 else
1694 tmp |= DC_HPDx_INT_POLARITY;
1695 WREG32(DC_HPD1_INT_CONTROL, tmp);
1696 break;
1697 case RADEON_HPD_2:
1698 tmp = RREG32(DC_HPD2_INT_CONTROL);
1699 if (connected)
1700 tmp &= ~DC_HPDx_INT_POLARITY;
1701 else
1702 tmp |= DC_HPDx_INT_POLARITY;
1703 WREG32(DC_HPD2_INT_CONTROL, tmp);
1704 break;
1705 case RADEON_HPD_3:
1706 tmp = RREG32(DC_HPD3_INT_CONTROL);
1707 if (connected)
1708 tmp &= ~DC_HPDx_INT_POLARITY;
1709 else
1710 tmp |= DC_HPDx_INT_POLARITY;
1711 WREG32(DC_HPD3_INT_CONTROL, tmp);
1712 break;
1713 case RADEON_HPD_4:
1714 tmp = RREG32(DC_HPD4_INT_CONTROL);
1715 if (connected)
1716 tmp &= ~DC_HPDx_INT_POLARITY;
1717 else
1718 tmp |= DC_HPDx_INT_POLARITY;
1719 WREG32(DC_HPD4_INT_CONTROL, tmp);
1720 break;
1721 case RADEON_HPD_5:
1722 tmp = RREG32(DC_HPD5_INT_CONTROL);
1723 if (connected)
1724 tmp &= ~DC_HPDx_INT_POLARITY;
1725 else
1726 tmp |= DC_HPDx_INT_POLARITY;
1727 WREG32(DC_HPD5_INT_CONTROL, tmp);
1728 break;
1729 case RADEON_HPD_6:
1730 tmp = RREG32(DC_HPD6_INT_CONTROL);
1731 if (connected)
1732 tmp &= ~DC_HPDx_INT_POLARITY;
1733 else
1734 tmp |= DC_HPDx_INT_POLARITY;
1735 WREG32(DC_HPD6_INT_CONTROL, tmp);
1736 break;
1737 default:
1738 break;
1739 }
1740 }
1741
1742 /**
1743 * evergreen_hpd_init - hpd setup callback.
1744 *
1745 * @rdev: radeon_device pointer
1746 *
1747 * Setup the hpd pins used by the card (evergreen+).
1748 * Enable the pin, set the polarity, and enable the hpd interrupts.
1749 */
1750 void evergreen_hpd_init(struct radeon_device *rdev)
1751 {
1752 struct drm_device *dev = rdev->ddev;
1753 struct drm_connector *connector;
1754 unsigned enabled = 0;
1755 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1756 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1757
1758 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1759 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1760
1761 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1762 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1763 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1764 * aux dp channel on imac and help (but not completely fix)
1765 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1766 * also avoid interrupt storms during dpms.
1767 */
1768 continue;
1769 }
1770 switch (radeon_connector->hpd.hpd) {
1771 case RADEON_HPD_1:
1772 WREG32(DC_HPD1_CONTROL, tmp);
1773 break;
1774 case RADEON_HPD_2:
1775 WREG32(DC_HPD2_CONTROL, tmp);
1776 break;
1777 case RADEON_HPD_3:
1778 WREG32(DC_HPD3_CONTROL, tmp);
1779 break;
1780 case RADEON_HPD_4:
1781 WREG32(DC_HPD4_CONTROL, tmp);
1782 break;
1783 case RADEON_HPD_5:
1784 WREG32(DC_HPD5_CONTROL, tmp);
1785 break;
1786 case RADEON_HPD_6:
1787 WREG32(DC_HPD6_CONTROL, tmp);
1788 break;
1789 default:
1790 break;
1791 }
1792 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1793 enabled |= 1 << radeon_connector->hpd.hpd;
1794 }
1795 radeon_irq_kms_enable_hpd(rdev, enabled);
1796 }
1797
1798 /**
1799 * evergreen_hpd_fini - hpd tear down callback.
1800 *
1801 * @rdev: radeon_device pointer
1802 *
1803 * Tear down the hpd pins used by the card (evergreen+).
1804 * Disable the hpd interrupts.
1805 */
1806 void evergreen_hpd_fini(struct radeon_device *rdev)
1807 {
1808 struct drm_device *dev = rdev->ddev;
1809 struct drm_connector *connector;
1810 unsigned disabled = 0;
1811
1812 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1813 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1814 switch (radeon_connector->hpd.hpd) {
1815 case RADEON_HPD_1:
1816 WREG32(DC_HPD1_CONTROL, 0);
1817 break;
1818 case RADEON_HPD_2:
1819 WREG32(DC_HPD2_CONTROL, 0);
1820 break;
1821 case RADEON_HPD_3:
1822 WREG32(DC_HPD3_CONTROL, 0);
1823 break;
1824 case RADEON_HPD_4:
1825 WREG32(DC_HPD4_CONTROL, 0);
1826 break;
1827 case RADEON_HPD_5:
1828 WREG32(DC_HPD5_CONTROL, 0);
1829 break;
1830 case RADEON_HPD_6:
1831 WREG32(DC_HPD6_CONTROL, 0);
1832 break;
1833 default:
1834 break;
1835 }
1836 disabled |= 1 << radeon_connector->hpd.hpd;
1837 }
1838 radeon_irq_kms_disable_hpd(rdev, disabled);
1839 }
1840
1841 /* watermark setup */
1842
1843 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1844 struct radeon_crtc *radeon_crtc,
1845 struct drm_display_mode *mode,
1846 struct drm_display_mode *other_mode)
1847 {
1848 u32 tmp, buffer_alloc, i;
1849 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1850 /*
1851 * Line Buffer Setup
1852 * There are 3 line buffers, each one shared by 2 display controllers.
1853 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1854 * the display controllers. The paritioning is done via one of four
1855 * preset allocations specified in bits 2:0:
1856 * first display controller
1857 * 0 - first half of lb (3840 * 2)
1858 * 1 - first 3/4 of lb (5760 * 2)
1859 * 2 - whole lb (7680 * 2), other crtc must be disabled
1860 * 3 - first 1/4 of lb (1920 * 2)
1861 * second display controller
1862 * 4 - second half of lb (3840 * 2)
1863 * 5 - second 3/4 of lb (5760 * 2)
1864 * 6 - whole lb (7680 * 2), other crtc must be disabled
1865 * 7 - last 1/4 of lb (1920 * 2)
1866 */
1867 /* this can get tricky if we have two large displays on a paired group
1868 * of crtcs. Ideally for multiple large displays we'd assign them to
1869 * non-linked crtcs for maximum line buffer allocation.
1870 */
1871 if (radeon_crtc->base.enabled && mode) {
1872 if (other_mode) {
1873 tmp = 0; /* 1/2 */
1874 buffer_alloc = 1;
1875 } else {
1876 tmp = 2; /* whole */
1877 buffer_alloc = 2;
1878 }
1879 } else {
1880 tmp = 0;
1881 buffer_alloc = 0;
1882 }
1883
1884 /* second controller of the pair uses second half of the lb */
1885 if (radeon_crtc->crtc_id % 2)
1886 tmp += 4;
1887 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1888
1889 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1890 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1891 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1892 for (i = 0; i < rdev->usec_timeout; i++) {
1893 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1894 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1895 break;
1896 udelay(1);
1897 }
1898 }
1899
1900 if (radeon_crtc->base.enabled && mode) {
1901 switch (tmp) {
1902 case 0:
1903 case 4:
1904 default:
1905 if (ASIC_IS_DCE5(rdev))
1906 return 4096 * 2;
1907 else
1908 return 3840 * 2;
1909 case 1:
1910 case 5:
1911 if (ASIC_IS_DCE5(rdev))
1912 return 6144 * 2;
1913 else
1914 return 5760 * 2;
1915 case 2:
1916 case 6:
1917 if (ASIC_IS_DCE5(rdev))
1918 return 8192 * 2;
1919 else
1920 return 7680 * 2;
1921 case 3:
1922 case 7:
1923 if (ASIC_IS_DCE5(rdev))
1924 return 2048 * 2;
1925 else
1926 return 1920 * 2;
1927 }
1928 }
1929
1930 /* controller not enabled, so no lb used */
1931 return 0;
1932 }
1933
1934 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1935 {
1936 u32 tmp = RREG32(MC_SHARED_CHMAP);
1937
1938 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1939 case 0:
1940 default:
1941 return 1;
1942 case 1:
1943 return 2;
1944 case 2:
1945 return 4;
1946 case 3:
1947 return 8;
1948 }
1949 }
1950
1951 struct evergreen_wm_params {
1952 u32 dram_channels; /* number of dram channels */
1953 u32 yclk; /* bandwidth per dram data pin in kHz */
1954 u32 sclk; /* engine clock in kHz */
1955 u32 disp_clk; /* display clock in kHz */
1956 u32 src_width; /* viewport width */
1957 u32 active_time; /* active display time in ns */
1958 u32 blank_time; /* blank time in ns */
1959 bool interlaced; /* mode is interlaced */
1960 fixed20_12 vsc; /* vertical scale ratio */
1961 u32 num_heads; /* number of active crtcs */
1962 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1963 u32 lb_size; /* line buffer allocated to pipe */
1964 u32 vtaps; /* vertical scaler taps */
1965 };
1966
1967 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1968 {
1969 /* Calculate DRAM Bandwidth and the part allocated to display. */
1970 fixed20_12 dram_efficiency; /* 0.7 */
1971 fixed20_12 yclk, dram_channels, bandwidth;
1972 fixed20_12 a;
1973
1974 a.full = dfixed_const(1000);
1975 yclk.full = dfixed_const(wm->yclk);
1976 yclk.full = dfixed_div(yclk, a);
1977 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978 a.full = dfixed_const(10);
1979 dram_efficiency.full = dfixed_const(7);
1980 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1981 bandwidth.full = dfixed_mul(dram_channels, yclk);
1982 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1983
1984 return dfixed_trunc(bandwidth);
1985 }
1986
1987 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1988 {
1989 /* Calculate DRAM Bandwidth and the part allocated to display. */
1990 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1991 fixed20_12 yclk, dram_channels, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 yclk.full = dfixed_const(wm->yclk);
1996 yclk.full = dfixed_div(yclk, a);
1997 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1998 a.full = dfixed_const(10);
1999 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2000 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2001 bandwidth.full = dfixed_mul(dram_channels, yclk);
2002 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2003
2004 return dfixed_trunc(bandwidth);
2005 }
2006
2007 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2008 {
2009 /* Calculate the display Data return Bandwidth */
2010 fixed20_12 return_efficiency; /* 0.8 */
2011 fixed20_12 sclk, bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 sclk.full = dfixed_const(wm->sclk);
2016 sclk.full = dfixed_div(sclk, a);
2017 a.full = dfixed_const(10);
2018 return_efficiency.full = dfixed_const(8);
2019 return_efficiency.full = dfixed_div(return_efficiency, a);
2020 a.full = dfixed_const(32);
2021 bandwidth.full = dfixed_mul(a, sclk);
2022 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2023
2024 return dfixed_trunc(bandwidth);
2025 }
2026
2027 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2028 {
2029 /* Calculate the DMIF Request Bandwidth */
2030 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2031 fixed20_12 disp_clk, bandwidth;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1000);
2035 disp_clk.full = dfixed_const(wm->disp_clk);
2036 disp_clk.full = dfixed_div(disp_clk, a);
2037 a.full = dfixed_const(10);
2038 disp_clk_request_efficiency.full = dfixed_const(8);
2039 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2040 a.full = dfixed_const(32);
2041 bandwidth.full = dfixed_mul(a, disp_clk);
2042 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2043
2044 return dfixed_trunc(bandwidth);
2045 }
2046
2047 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2048 {
2049 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2050 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2051 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2052 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2053
2054 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2055 }
2056
2057 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2058 {
2059 /* Calculate the display mode Average Bandwidth
2060 * DisplayMode should contain the source and destination dimensions,
2061 * timing, etc.
2062 */
2063 fixed20_12 bpp;
2064 fixed20_12 line_time;
2065 fixed20_12 src_width;
2066 fixed20_12 bandwidth;
2067 fixed20_12 a;
2068
2069 a.full = dfixed_const(1000);
2070 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2071 line_time.full = dfixed_div(line_time, a);
2072 bpp.full = dfixed_const(wm->bytes_per_pixel);
2073 src_width.full = dfixed_const(wm->src_width);
2074 bandwidth.full = dfixed_mul(src_width, bpp);
2075 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2076 bandwidth.full = dfixed_div(bandwidth, line_time);
2077
2078 return dfixed_trunc(bandwidth);
2079 }
2080
2081 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2082 {
2083 /* First calcualte the latency in ns */
2084 u32 mc_latency = 2000; /* 2000 ns. */
2085 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2086 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2087 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2088 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2089 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2090 (wm->num_heads * cursor_line_pair_return_time);
2091 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2092 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2093 fixed20_12 a, b, c;
2094
2095 if (wm->num_heads == 0)
2096 return 0;
2097
2098 a.full = dfixed_const(2);
2099 b.full = dfixed_const(1);
2100 if ((wm->vsc.full > a.full) ||
2101 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2102 (wm->vtaps >= 5) ||
2103 ((wm->vsc.full >= a.full) && wm->interlaced))
2104 max_src_lines_per_dst_line = 4;
2105 else
2106 max_src_lines_per_dst_line = 2;
2107
2108 a.full = dfixed_const(available_bandwidth);
2109 b.full = dfixed_const(wm->num_heads);
2110 a.full = dfixed_div(a, b);
2111
2112 b.full = dfixed_const(1000);
2113 c.full = dfixed_const(wm->disp_clk);
2114 b.full = dfixed_div(c, b);
2115 c.full = dfixed_const(wm->bytes_per_pixel);
2116 b.full = dfixed_mul(b, c);
2117
2118 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2119
2120 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2121 b.full = dfixed_const(1000);
2122 c.full = dfixed_const(lb_fill_bw);
2123 b.full = dfixed_div(c, b);
2124 a.full = dfixed_div(a, b);
2125 line_fill_time = dfixed_trunc(a);
2126
2127 if (line_fill_time < wm->active_time)
2128 return latency;
2129 else
2130 return latency + (line_fill_time - wm->active_time);
2131
2132 }
2133
2134 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2135 {
2136 if (evergreen_average_bandwidth(wm) <=
2137 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2138 return true;
2139 else
2140 return false;
2141 };
2142
2143 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2144 {
2145 if (evergreen_average_bandwidth(wm) <=
2146 (evergreen_available_bandwidth(wm) / wm->num_heads))
2147 return true;
2148 else
2149 return false;
2150 };
2151
2152 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2153 {
2154 u32 lb_partitions = wm->lb_size / wm->src_width;
2155 u32 line_time = wm->active_time + wm->blank_time;
2156 u32 latency_tolerant_lines;
2157 u32 latency_hiding;
2158 fixed20_12 a;
2159
2160 a.full = dfixed_const(1);
2161 if (wm->vsc.full > a.full)
2162 latency_tolerant_lines = 1;
2163 else {
2164 if (lb_partitions <= (wm->vtaps + 1))
2165 latency_tolerant_lines = 1;
2166 else
2167 latency_tolerant_lines = 2;
2168 }
2169
2170 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2171
2172 if (evergreen_latency_watermark(wm) <= latency_hiding)
2173 return true;
2174 else
2175 return false;
2176 }
2177
2178 static void evergreen_program_watermarks(struct radeon_device *rdev,
2179 struct radeon_crtc *radeon_crtc,
2180 u32 lb_size, u32 num_heads)
2181 {
2182 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2183 struct evergreen_wm_params wm_low, wm_high;
2184 u32 dram_channels;
2185 u32 pixel_period;
2186 u32 line_time = 0;
2187 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2188 u32 priority_a_mark = 0, priority_b_mark = 0;
2189 u32 priority_a_cnt = PRIORITY_OFF;
2190 u32 priority_b_cnt = PRIORITY_OFF;
2191 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2192 u32 tmp, arb_control3;
2193 fixed20_12 a, b, c;
2194
2195 if (radeon_crtc->base.enabled && num_heads && mode) {
2196 pixel_period = 1000000 / (u32)mode->clock;
2197 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2198 priority_a_cnt = 0;
2199 priority_b_cnt = 0;
2200 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2201
2202 /* watermark for high clocks */
2203 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2204 wm_high.yclk =
2205 radeon_dpm_get_mclk(rdev, false) * 10;
2206 wm_high.sclk =
2207 radeon_dpm_get_sclk(rdev, false) * 10;
2208 } else {
2209 wm_high.yclk = rdev->pm.current_mclk * 10;
2210 wm_high.sclk = rdev->pm.current_sclk * 10;
2211 }
2212
2213 wm_high.disp_clk = mode->clock;
2214 wm_high.src_width = mode->crtc_hdisplay;
2215 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2216 wm_high.blank_time = line_time - wm_high.active_time;
2217 wm_high.interlaced = false;
2218 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2219 wm_high.interlaced = true;
2220 wm_high.vsc = radeon_crtc->vsc;
2221 wm_high.vtaps = 1;
2222 if (radeon_crtc->rmx_type != RMX_OFF)
2223 wm_high.vtaps = 2;
2224 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2225 wm_high.lb_size = lb_size;
2226 wm_high.dram_channels = dram_channels;
2227 wm_high.num_heads = num_heads;
2228
2229 /* watermark for low clocks */
2230 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2231 wm_low.yclk =
2232 radeon_dpm_get_mclk(rdev, true) * 10;
2233 wm_low.sclk =
2234 radeon_dpm_get_sclk(rdev, true) * 10;
2235 } else {
2236 wm_low.yclk = rdev->pm.current_mclk * 10;
2237 wm_low.sclk = rdev->pm.current_sclk * 10;
2238 }
2239
2240 wm_low.disp_clk = mode->clock;
2241 wm_low.src_width = mode->crtc_hdisplay;
2242 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2243 wm_low.blank_time = line_time - wm_low.active_time;
2244 wm_low.interlaced = false;
2245 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2246 wm_low.interlaced = true;
2247 wm_low.vsc = radeon_crtc->vsc;
2248 wm_low.vtaps = 1;
2249 if (radeon_crtc->rmx_type != RMX_OFF)
2250 wm_low.vtaps = 2;
2251 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2252 wm_low.lb_size = lb_size;
2253 wm_low.dram_channels = dram_channels;
2254 wm_low.num_heads = num_heads;
2255
2256 /* set for high clocks */
2257 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2258 /* set for low clocks */
2259 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2260
2261 /* possibly force display priority to high */
2262 /* should really do this at mode validation time... */
2263 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2264 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2265 !evergreen_check_latency_hiding(&wm_high) ||
2266 (rdev->disp_priority == 2)) {
2267 DRM_DEBUG_KMS("force priority a to high\n");
2268 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2269 }
2270 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2271 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2272 !evergreen_check_latency_hiding(&wm_low) ||
2273 (rdev->disp_priority == 2)) {
2274 DRM_DEBUG_KMS("force priority b to high\n");
2275 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2276 }
2277
2278 a.full = dfixed_const(1000);
2279 b.full = dfixed_const(mode->clock);
2280 b.full = dfixed_div(b, a);
2281 c.full = dfixed_const(latency_watermark_a);
2282 c.full = dfixed_mul(c, b);
2283 c.full = dfixed_mul(c, radeon_crtc->hsc);
2284 c.full = dfixed_div(c, a);
2285 a.full = dfixed_const(16);
2286 c.full = dfixed_div(c, a);
2287 priority_a_mark = dfixed_trunc(c);
2288 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2289
2290 a.full = dfixed_const(1000);
2291 b.full = dfixed_const(mode->clock);
2292 b.full = dfixed_div(b, a);
2293 c.full = dfixed_const(latency_watermark_b);
2294 c.full = dfixed_mul(c, b);
2295 c.full = dfixed_mul(c, radeon_crtc->hsc);
2296 c.full = dfixed_div(c, a);
2297 a.full = dfixed_const(16);
2298 c.full = dfixed_div(c, a);
2299 priority_b_mark = dfixed_trunc(c);
2300 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2301 }
2302
2303 /* select wm A */
2304 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2305 tmp = arb_control3;
2306 tmp &= ~LATENCY_WATERMARK_MASK(3);
2307 tmp |= LATENCY_WATERMARK_MASK(1);
2308 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2309 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2310 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2311 LATENCY_HIGH_WATERMARK(line_time)));
2312 /* select wm B */
2313 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2314 tmp &= ~LATENCY_WATERMARK_MASK(3);
2315 tmp |= LATENCY_WATERMARK_MASK(2);
2316 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2317 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2318 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2319 LATENCY_HIGH_WATERMARK(line_time)));
2320 /* restore original selection */
2321 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2322
2323 /* write the priority marks */
2324 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2325 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2326
2327 /* save values for DPM */
2328 radeon_crtc->line_time = line_time;
2329 radeon_crtc->wm_high = latency_watermark_a;
2330 radeon_crtc->wm_low = latency_watermark_b;
2331 }
2332
2333 /**
2334 * evergreen_bandwidth_update - update display watermarks callback.
2335 *
2336 * @rdev: radeon_device pointer
2337 *
2338 * Update the display watermarks based on the requested mode(s)
2339 * (evergreen+).
2340 */
2341 void evergreen_bandwidth_update(struct radeon_device *rdev)
2342 {
2343 struct drm_display_mode *mode0 = NULL;
2344 struct drm_display_mode *mode1 = NULL;
2345 u32 num_heads = 0, lb_size;
2346 int i;
2347
2348 radeon_update_display_priority(rdev);
2349
2350 for (i = 0; i < rdev->num_crtc; i++) {
2351 if (rdev->mode_info.crtcs[i]->base.enabled)
2352 num_heads++;
2353 }
2354 for (i = 0; i < rdev->num_crtc; i += 2) {
2355 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2356 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2357 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2358 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2359 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2360 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2361 }
2362 }
2363
2364 /**
2365 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2366 *
2367 * @rdev: radeon_device pointer
2368 *
2369 * Wait for the MC (memory controller) to be idle.
2370 * (evergreen+).
2371 * Returns 0 if the MC is idle, -1 if not.
2372 */
2373 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2374 {
2375 unsigned i;
2376 u32 tmp;
2377
2378 for (i = 0; i < rdev->usec_timeout; i++) {
2379 /* read MC_STATUS */
2380 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2381 if (!tmp)
2382 return 0;
2383 udelay(1);
2384 }
2385 return -1;
2386 }
2387
2388 /*
2389 * GART
2390 */
2391 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2392 {
2393 unsigned i;
2394 u32 tmp;
2395
2396 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2397
2398 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2399 for (i = 0; i < rdev->usec_timeout; i++) {
2400 /* read MC_STATUS */
2401 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2402 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2403 if (tmp == 2) {
2404 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2405 return;
2406 }
2407 if (tmp) {
2408 return;
2409 }
2410 udelay(1);
2411 }
2412 }
2413
2414 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2415 {
2416 u32 tmp;
2417 int r;
2418
2419 if (rdev->gart.robj == NULL) {
2420 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2421 return -EINVAL;
2422 }
2423 r = radeon_gart_table_vram_pin(rdev);
2424 if (r)
2425 return r;
2426 /* Setup L2 cache */
2427 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2428 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2429 EFFECTIVE_L2_QUEUE_SIZE(7));
2430 WREG32(VM_L2_CNTL2, 0);
2431 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2432 /* Setup TLB control */
2433 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2434 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2435 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2436 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2437 if (rdev->flags & RADEON_IS_IGP) {
2438 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2439 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2440 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2441 } else {
2442 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2443 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2444 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2445 if ((rdev->family == CHIP_JUNIPER) ||
2446 (rdev->family == CHIP_CYPRESS) ||
2447 (rdev->family == CHIP_HEMLOCK) ||
2448 (rdev->family == CHIP_BARTS))
2449 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2450 }
2451 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2452 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2453 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2454 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2455 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2456 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2457 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2458 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2459 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2460 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2461 (u32)(rdev->dummy_page.addr >> 12));
2462 WREG32(VM_CONTEXT1_CNTL, 0);
2463
2464 evergreen_pcie_gart_tlb_flush(rdev);
2465 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2466 (unsigned)(rdev->mc.gtt_size >> 20),
2467 (unsigned long long)rdev->gart.table_addr);
2468 rdev->gart.ready = true;
2469 return 0;
2470 }
2471
2472 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2473 {
2474 u32 tmp;
2475
2476 /* Disable all tables */
2477 WREG32(VM_CONTEXT0_CNTL, 0);
2478 WREG32(VM_CONTEXT1_CNTL, 0);
2479
2480 /* Setup L2 cache */
2481 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2482 EFFECTIVE_L2_QUEUE_SIZE(7));
2483 WREG32(VM_L2_CNTL2, 0);
2484 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2485 /* Setup TLB control */
2486 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2487 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2488 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2489 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2490 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2491 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2492 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2493 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2494 radeon_gart_table_vram_unpin(rdev);
2495 }
2496
2497 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2498 {
2499 evergreen_pcie_gart_disable(rdev);
2500 radeon_gart_table_vram_free(rdev);
2501 radeon_gart_fini(rdev);
2502 }
2503
2504
2505 static void evergreen_agp_enable(struct radeon_device *rdev)
2506 {
2507 u32 tmp;
2508
2509 /* Setup L2 cache */
2510 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2511 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2512 EFFECTIVE_L2_QUEUE_SIZE(7));
2513 WREG32(VM_L2_CNTL2, 0);
2514 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2515 /* Setup TLB control */
2516 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2517 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2518 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2519 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2520 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2521 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2522 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2523 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2524 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2525 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2526 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2527 WREG32(VM_CONTEXT0_CNTL, 0);
2528 WREG32(VM_CONTEXT1_CNTL, 0);
2529 }
2530
2531 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2532 {
2533 u32 crtc_enabled, tmp, frame_count, blackout;
2534 int i, j;
2535
2536 if (!ASIC_IS_NODCE(rdev)) {
2537 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2538 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2539
2540 /* disable VGA render */
2541 WREG32(VGA_RENDER_CONTROL, 0);
2542 }
2543 /* blank the display controllers */
2544 for (i = 0; i < rdev->num_crtc; i++) {
2545 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2546 if (crtc_enabled) {
2547 save->crtc_enabled[i] = true;
2548 if (ASIC_IS_DCE6(rdev)) {
2549 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2550 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2551 radeon_wait_for_vblank(rdev, i);
2552 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2553 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2554 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2555 }
2556 } else {
2557 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2558 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2559 radeon_wait_for_vblank(rdev, i);
2560 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2561 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2562 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2563 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2564 }
2565 }
2566 /* wait for the next frame */
2567 frame_count = radeon_get_vblank_counter(rdev, i);
2568 for (j = 0; j < rdev->usec_timeout; j++) {
2569 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2570 break;
2571 udelay(1);
2572 }
2573
2574 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2575 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2576 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2577 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2578 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2579 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2580 save->crtc_enabled[i] = false;
2581 /* ***** */
2582 } else {
2583 save->crtc_enabled[i] = false;
2584 }
2585 }
2586
2587 radeon_mc_wait_for_idle(rdev);
2588
2589 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2590 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2591 /* Block CPU access */
2592 WREG32(BIF_FB_EN, 0);
2593 /* blackout the MC */
2594 blackout &= ~BLACKOUT_MODE_MASK;
2595 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2596 }
2597 /* wait for the MC to settle */
2598 udelay(100);
2599
2600 /* lock double buffered regs */
2601 for (i = 0; i < rdev->num_crtc; i++) {
2602 if (save->crtc_enabled[i]) {
2603 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2604 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2605 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2606 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2607 }
2608 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2609 if (!(tmp & 1)) {
2610 tmp |= 1;
2611 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2612 }
2613 }
2614 }
2615 }
2616
2617 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2618 {
2619 u32 tmp, frame_count;
2620 int i, j;
2621
2622 /* update crtc base addresses */
2623 for (i = 0; i < rdev->num_crtc; i++) {
2624 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2625 upper_32_bits(rdev->mc.vram_start));
2626 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2627 upper_32_bits(rdev->mc.vram_start));
2628 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2629 (u32)rdev->mc.vram_start);
2630 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2631 (u32)rdev->mc.vram_start);
2632 }
2633
2634 if (!ASIC_IS_NODCE(rdev)) {
2635 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2636 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2637 }
2638
2639 /* unlock regs and wait for update */
2640 for (i = 0; i < rdev->num_crtc; i++) {
2641 if (save->crtc_enabled[i]) {
2642 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2643 if ((tmp & 0x7) != 3) {
2644 tmp &= ~0x7;
2645 tmp |= 0x3;
2646 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2647 }
2648 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2649 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2650 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2651 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2652 }
2653 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2654 if (tmp & 1) {
2655 tmp &= ~1;
2656 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2657 }
2658 for (j = 0; j < rdev->usec_timeout; j++) {
2659 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2660 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2661 break;
2662 udelay(1);
2663 }
2664 }
2665 }
2666
2667 /* unblackout the MC */
2668 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2669 tmp &= ~BLACKOUT_MODE_MASK;
2670 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2671 /* allow CPU access */
2672 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2673
2674 for (i = 0; i < rdev->num_crtc; i++) {
2675 if (save->crtc_enabled[i]) {
2676 if (ASIC_IS_DCE6(rdev)) {
2677 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2678 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2679 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2680 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2681 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2682 } else {
2683 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2684 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2685 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2686 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2687 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2688 }
2689 /* wait for the next frame */
2690 frame_count = radeon_get_vblank_counter(rdev, i);
2691 for (j = 0; j < rdev->usec_timeout; j++) {
2692 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2693 break;
2694 udelay(1);
2695 }
2696 }
2697 }
2698 if (!ASIC_IS_NODCE(rdev)) {
2699 /* Unlock vga access */
2700 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2701 mdelay(1);
2702 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2703 }
2704 }
2705
2706 void evergreen_mc_program(struct radeon_device *rdev)
2707 {
2708 struct evergreen_mc_save save;
2709 u32 tmp;
2710 int i, j;
2711
2712 /* Initialize HDP */
2713 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2714 WREG32((0x2c14 + j), 0x00000000);
2715 WREG32((0x2c18 + j), 0x00000000);
2716 WREG32((0x2c1c + j), 0x00000000);
2717 WREG32((0x2c20 + j), 0x00000000);
2718 WREG32((0x2c24 + j), 0x00000000);
2719 }
2720 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2721
2722 evergreen_mc_stop(rdev, &save);
2723 if (evergreen_mc_wait_for_idle(rdev)) {
2724 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2725 }
2726 /* Lockout access through VGA aperture*/
2727 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2728 /* Update configuration */
2729 if (rdev->flags & RADEON_IS_AGP) {
2730 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2731 /* VRAM before AGP */
2732 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2733 rdev->mc.vram_start >> 12);
2734 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2735 rdev->mc.gtt_end >> 12);
2736 } else {
2737 /* VRAM after AGP */
2738 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2739 rdev->mc.gtt_start >> 12);
2740 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2741 rdev->mc.vram_end >> 12);
2742 }
2743 } else {
2744 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2745 rdev->mc.vram_start >> 12);
2746 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2747 rdev->mc.vram_end >> 12);
2748 }
2749 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2750 /* llano/ontario only */
2751 if ((rdev->family == CHIP_PALM) ||
2752 (rdev->family == CHIP_SUMO) ||
2753 (rdev->family == CHIP_SUMO2)) {
2754 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2755 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2756 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2757 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2758 }
2759 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2760 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2761 WREG32(MC_VM_FB_LOCATION, tmp);
2762 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2763 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2764 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2765 if (rdev->flags & RADEON_IS_AGP) {
2766 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2767 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2768 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2769 } else {
2770 WREG32(MC_VM_AGP_BASE, 0);
2771 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2772 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2773 }
2774 if (evergreen_mc_wait_for_idle(rdev)) {
2775 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2776 }
2777 evergreen_mc_resume(rdev, &save);
2778 /* we need to own VRAM, so turn off the VGA renderer here
2779 * to stop it overwriting our objects */
2780 rv515_vga_render_disable(rdev);
2781 }
2782
2783 /*
2784 * CP.
2785 */
2786 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2787 {
2788 struct radeon_ring *ring = &rdev->ring[ib->ring];
2789 u32 next_rptr;
2790
2791 /* set to DX10/11 mode */
2792 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2793 radeon_ring_write(ring, 1);
2794
2795 if (ring->rptr_save_reg) {
2796 next_rptr = ring->wptr + 3 + 4;
2797 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2798 radeon_ring_write(ring, ((ring->rptr_save_reg -
2799 PACKET3_SET_CONFIG_REG_START) >> 2));
2800 radeon_ring_write(ring, next_rptr);
2801 } else if (rdev->wb.enabled) {
2802 next_rptr = ring->wptr + 5 + 4;
2803 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2804 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2805 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2806 radeon_ring_write(ring, next_rptr);
2807 radeon_ring_write(ring, 0);
2808 }
2809
2810 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2811 radeon_ring_write(ring,
2812 #ifdef __BIG_ENDIAN
2813 (2 << 0) |
2814 #endif
2815 (ib->gpu_addr & 0xFFFFFFFC));
2816 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2817 radeon_ring_write(ring, ib->length_dw);
2818 }
2819
2820
2821 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2822 {
2823 const __be32 *fw_data;
2824 int i;
2825
2826 if (!rdev->me_fw || !rdev->pfp_fw)
2827 return -EINVAL;
2828
2829 r700_cp_stop(rdev);
2830 WREG32(CP_RB_CNTL,
2831 #ifdef __BIG_ENDIAN
2832 BUF_SWAP_32BIT |
2833 #endif
2834 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2835
2836 fw_data = (const __be32 *)rdev->pfp_fw->data;
2837 WREG32(CP_PFP_UCODE_ADDR, 0);
2838 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2839 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2840 WREG32(CP_PFP_UCODE_ADDR, 0);
2841
2842 fw_data = (const __be32 *)rdev->me_fw->data;
2843 WREG32(CP_ME_RAM_WADDR, 0);
2844 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2845 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2846
2847 WREG32(CP_PFP_UCODE_ADDR, 0);
2848 WREG32(CP_ME_RAM_WADDR, 0);
2849 WREG32(CP_ME_RAM_RADDR, 0);
2850 return 0;
2851 }
2852
2853 static int evergreen_cp_start(struct radeon_device *rdev)
2854 {
2855 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2856 int r, i;
2857 uint32_t cp_me;
2858
2859 r = radeon_ring_lock(rdev, ring, 7);
2860 if (r) {
2861 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2862 return r;
2863 }
2864 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2865 radeon_ring_write(ring, 0x1);
2866 radeon_ring_write(ring, 0x0);
2867 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2868 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2869 radeon_ring_write(ring, 0);
2870 radeon_ring_write(ring, 0);
2871 radeon_ring_unlock_commit(rdev, ring, false);
2872
2873 cp_me = 0xff;
2874 WREG32(CP_ME_CNTL, cp_me);
2875
2876 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2877 if (r) {
2878 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2879 return r;
2880 }
2881
2882 /* setup clear context state */
2883 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2884 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2885
2886 for (i = 0; i < evergreen_default_size; i++)
2887 radeon_ring_write(ring, evergreen_default_state[i]);
2888
2889 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2890 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2891
2892 /* set clear context state */
2893 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2894 radeon_ring_write(ring, 0);
2895
2896 /* SQ_VTX_BASE_VTX_LOC */
2897 radeon_ring_write(ring, 0xc0026f00);
2898 radeon_ring_write(ring, 0x00000000);
2899 radeon_ring_write(ring, 0x00000000);
2900 radeon_ring_write(ring, 0x00000000);
2901
2902 /* Clear consts */
2903 radeon_ring_write(ring, 0xc0036f00);
2904 radeon_ring_write(ring, 0x00000bc4);
2905 radeon_ring_write(ring, 0xffffffff);
2906 radeon_ring_write(ring, 0xffffffff);
2907 radeon_ring_write(ring, 0xffffffff);
2908
2909 radeon_ring_write(ring, 0xc0026900);
2910 radeon_ring_write(ring, 0x00000316);
2911 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2912 radeon_ring_write(ring, 0x00000010); /* */
2913
2914 radeon_ring_unlock_commit(rdev, ring, false);
2915
2916 return 0;
2917 }
2918
2919 static int evergreen_cp_resume(struct radeon_device *rdev)
2920 {
2921 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2922 u32 tmp;
2923 u32 rb_bufsz;
2924 int r;
2925
2926 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2927 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2928 SOFT_RESET_PA |
2929 SOFT_RESET_SH |
2930 SOFT_RESET_VGT |
2931 SOFT_RESET_SPI |
2932 SOFT_RESET_SX));
2933 RREG32(GRBM_SOFT_RESET);
2934 mdelay(15);
2935 WREG32(GRBM_SOFT_RESET, 0);
2936 RREG32(GRBM_SOFT_RESET);
2937
2938 /* Set ring buffer size */
2939 rb_bufsz = order_base_2(ring->ring_size / 8);
2940 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2941 #ifdef __BIG_ENDIAN
2942 tmp |= BUF_SWAP_32BIT;
2943 #endif
2944 WREG32(CP_RB_CNTL, tmp);
2945 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2946 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2947
2948 /* Set the write pointer delay */
2949 WREG32(CP_RB_WPTR_DELAY, 0);
2950
2951 /* Initialize the ring buffer's read and write pointers */
2952 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2953 WREG32(CP_RB_RPTR_WR, 0);
2954 ring->wptr = 0;
2955 WREG32(CP_RB_WPTR, ring->wptr);
2956
2957 /* set the wb address whether it's enabled or not */
2958 WREG32(CP_RB_RPTR_ADDR,
2959 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2960 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2961 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2962
2963 if (rdev->wb.enabled)
2964 WREG32(SCRATCH_UMSK, 0xff);
2965 else {
2966 tmp |= RB_NO_UPDATE;
2967 WREG32(SCRATCH_UMSK, 0);
2968 }
2969
2970 mdelay(1);
2971 WREG32(CP_RB_CNTL, tmp);
2972
2973 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2974 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2975
2976 evergreen_cp_start(rdev);
2977 ring->ready = true;
2978 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2979 if (r) {
2980 ring->ready = false;
2981 return r;
2982 }
2983 return 0;
2984 }
2985
2986 /*
2987 * Core functions
2988 */
2989 static void evergreen_gpu_init(struct radeon_device *rdev)
2990 {
2991 u32 gb_addr_config;
2992 u32 mc_shared_chmap, mc_arb_ramcfg;
2993 u32 sx_debug_1;
2994 u32 smx_dc_ctl0;
2995 u32 sq_config;
2996 u32 sq_lds_resource_mgmt;
2997 u32 sq_gpr_resource_mgmt_1;
2998 u32 sq_gpr_resource_mgmt_2;
2999 u32 sq_gpr_resource_mgmt_3;
3000 u32 sq_thread_resource_mgmt;
3001 u32 sq_thread_resource_mgmt_2;
3002 u32 sq_stack_resource_mgmt_1;
3003 u32 sq_stack_resource_mgmt_2;
3004 u32 sq_stack_resource_mgmt_3;
3005 u32 vgt_cache_invalidation;
3006 u32 hdp_host_path_cntl, tmp;
3007 u32 disabled_rb_mask;
3008 int i, j, num_shader_engines, ps_thread_count;
3009
3010 switch (rdev->family) {
3011 case CHIP_CYPRESS:
3012 case CHIP_HEMLOCK:
3013 rdev->config.evergreen.num_ses = 2;
3014 rdev->config.evergreen.max_pipes = 4;
3015 rdev->config.evergreen.max_tile_pipes = 8;
3016 rdev->config.evergreen.max_simds = 10;
3017 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3018 rdev->config.evergreen.max_gprs = 256;
3019 rdev->config.evergreen.max_threads = 248;
3020 rdev->config.evergreen.max_gs_threads = 32;
3021 rdev->config.evergreen.max_stack_entries = 512;
3022 rdev->config.evergreen.sx_num_of_sets = 4;
3023 rdev->config.evergreen.sx_max_export_size = 256;
3024 rdev->config.evergreen.sx_max_export_pos_size = 64;
3025 rdev->config.evergreen.sx_max_export_smx_size = 192;
3026 rdev->config.evergreen.max_hw_contexts = 8;
3027 rdev->config.evergreen.sq_num_cf_insts = 2;
3028
3029 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3030 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3031 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3032 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3033 break;
3034 case CHIP_JUNIPER:
3035 rdev->config.evergreen.num_ses = 1;
3036 rdev->config.evergreen.max_pipes = 4;
3037 rdev->config.evergreen.max_tile_pipes = 4;
3038 rdev->config.evergreen.max_simds = 10;
3039 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3040 rdev->config.evergreen.max_gprs = 256;
3041 rdev->config.evergreen.max_threads = 248;
3042 rdev->config.evergreen.max_gs_threads = 32;
3043 rdev->config.evergreen.max_stack_entries = 512;
3044 rdev->config.evergreen.sx_num_of_sets = 4;
3045 rdev->config.evergreen.sx_max_export_size = 256;
3046 rdev->config.evergreen.sx_max_export_pos_size = 64;
3047 rdev->config.evergreen.sx_max_export_smx_size = 192;
3048 rdev->config.evergreen.max_hw_contexts = 8;
3049 rdev->config.evergreen.sq_num_cf_insts = 2;
3050
3051 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3052 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3053 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3054 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3055 break;
3056 case CHIP_REDWOOD:
3057 rdev->config.evergreen.num_ses = 1;
3058 rdev->config.evergreen.max_pipes = 4;
3059 rdev->config.evergreen.max_tile_pipes = 4;
3060 rdev->config.evergreen.max_simds = 5;
3061 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3062 rdev->config.evergreen.max_gprs = 256;
3063 rdev->config.evergreen.max_threads = 248;
3064 rdev->config.evergreen.max_gs_threads = 32;
3065 rdev->config.evergreen.max_stack_entries = 256;
3066 rdev->config.evergreen.sx_num_of_sets = 4;
3067 rdev->config.evergreen.sx_max_export_size = 256;
3068 rdev->config.evergreen.sx_max_export_pos_size = 64;
3069 rdev->config.evergreen.sx_max_export_smx_size = 192;
3070 rdev->config.evergreen.max_hw_contexts = 8;
3071 rdev->config.evergreen.sq_num_cf_insts = 2;
3072
3073 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3074 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3075 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3076 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3077 break;
3078 case CHIP_CEDAR:
3079 default:
3080 rdev->config.evergreen.num_ses = 1;
3081 rdev->config.evergreen.max_pipes = 2;
3082 rdev->config.evergreen.max_tile_pipes = 2;
3083 rdev->config.evergreen.max_simds = 2;
3084 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3085 rdev->config.evergreen.max_gprs = 256;
3086 rdev->config.evergreen.max_threads = 192;
3087 rdev->config.evergreen.max_gs_threads = 16;
3088 rdev->config.evergreen.max_stack_entries = 256;
3089 rdev->config.evergreen.sx_num_of_sets = 4;
3090 rdev->config.evergreen.sx_max_export_size = 128;
3091 rdev->config.evergreen.sx_max_export_pos_size = 32;
3092 rdev->config.evergreen.sx_max_export_smx_size = 96;
3093 rdev->config.evergreen.max_hw_contexts = 4;
3094 rdev->config.evergreen.sq_num_cf_insts = 1;
3095
3096 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3097 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3098 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3099 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3100 break;
3101 case CHIP_PALM:
3102 rdev->config.evergreen.num_ses = 1;
3103 rdev->config.evergreen.max_pipes = 2;
3104 rdev->config.evergreen.max_tile_pipes = 2;
3105 rdev->config.evergreen.max_simds = 2;
3106 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3107 rdev->config.evergreen.max_gprs = 256;
3108 rdev->config.evergreen.max_threads = 192;
3109 rdev->config.evergreen.max_gs_threads = 16;
3110 rdev->config.evergreen.max_stack_entries = 256;
3111 rdev->config.evergreen.sx_num_of_sets = 4;
3112 rdev->config.evergreen.sx_max_export_size = 128;
3113 rdev->config.evergreen.sx_max_export_pos_size = 32;
3114 rdev->config.evergreen.sx_max_export_smx_size = 96;
3115 rdev->config.evergreen.max_hw_contexts = 4;
3116 rdev->config.evergreen.sq_num_cf_insts = 1;
3117
3118 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3119 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3120 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3121 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3122 break;
3123 case CHIP_SUMO:
3124 rdev->config.evergreen.num_ses = 1;
3125 rdev->config.evergreen.max_pipes = 4;
3126 rdev->config.evergreen.max_tile_pipes = 4;
3127 if (rdev->pdev->device == 0x9648)
3128 rdev->config.evergreen.max_simds = 3;
3129 else if ((rdev->pdev->device == 0x9647) ||
3130 (rdev->pdev->device == 0x964a))
3131 rdev->config.evergreen.max_simds = 4;
3132 else
3133 rdev->config.evergreen.max_simds = 5;
3134 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3135 rdev->config.evergreen.max_gprs = 256;
3136 rdev->config.evergreen.max_threads = 248;
3137 rdev->config.evergreen.max_gs_threads = 32;
3138 rdev->config.evergreen.max_stack_entries = 256;
3139 rdev->config.evergreen.sx_num_of_sets = 4;
3140 rdev->config.evergreen.sx_max_export_size = 256;
3141 rdev->config.evergreen.sx_max_export_pos_size = 64;
3142 rdev->config.evergreen.sx_max_export_smx_size = 192;
3143 rdev->config.evergreen.max_hw_contexts = 8;
3144 rdev->config.evergreen.sq_num_cf_insts = 2;
3145
3146 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3147 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3148 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3149 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3150 break;
3151 case CHIP_SUMO2:
3152 rdev->config.evergreen.num_ses = 1;
3153 rdev->config.evergreen.max_pipes = 4;
3154 rdev->config.evergreen.max_tile_pipes = 4;
3155 rdev->config.evergreen.max_simds = 2;
3156 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3157 rdev->config.evergreen.max_gprs = 256;
3158 rdev->config.evergreen.max_threads = 248;
3159 rdev->config.evergreen.max_gs_threads = 32;
3160 rdev->config.evergreen.max_stack_entries = 512;
3161 rdev->config.evergreen.sx_num_of_sets = 4;
3162 rdev->config.evergreen.sx_max_export_size = 256;
3163 rdev->config.evergreen.sx_max_export_pos_size = 64;
3164 rdev->config.evergreen.sx_max_export_smx_size = 192;
3165 rdev->config.evergreen.max_hw_contexts = 4;
3166 rdev->config.evergreen.sq_num_cf_insts = 2;
3167
3168 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3169 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3170 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3171 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3172 break;
3173 case CHIP_BARTS:
3174 rdev->config.evergreen.num_ses = 2;
3175 rdev->config.evergreen.max_pipes = 4;
3176 rdev->config.evergreen.max_tile_pipes = 8;
3177 rdev->config.evergreen.max_simds = 7;
3178 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3179 rdev->config.evergreen.max_gprs = 256;
3180 rdev->config.evergreen.max_threads = 248;
3181 rdev->config.evergreen.max_gs_threads = 32;
3182 rdev->config.evergreen.max_stack_entries = 512;
3183 rdev->config.evergreen.sx_num_of_sets = 4;
3184 rdev->config.evergreen.sx_max_export_size = 256;
3185 rdev->config.evergreen.sx_max_export_pos_size = 64;
3186 rdev->config.evergreen.sx_max_export_smx_size = 192;
3187 rdev->config.evergreen.max_hw_contexts = 8;
3188 rdev->config.evergreen.sq_num_cf_insts = 2;
3189
3190 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3191 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3192 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3193 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3194 break;
3195 case CHIP_TURKS:
3196 rdev->config.evergreen.num_ses = 1;
3197 rdev->config.evergreen.max_pipes = 4;
3198 rdev->config.evergreen.max_tile_pipes = 4;
3199 rdev->config.evergreen.max_simds = 6;
3200 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3201 rdev->config.evergreen.max_gprs = 256;
3202 rdev->config.evergreen.max_threads = 248;
3203 rdev->config.evergreen.max_gs_threads = 32;
3204 rdev->config.evergreen.max_stack_entries = 256;
3205 rdev->config.evergreen.sx_num_of_sets = 4;
3206 rdev->config.evergreen.sx_max_export_size = 256;
3207 rdev->config.evergreen.sx_max_export_pos_size = 64;
3208 rdev->config.evergreen.sx_max_export_smx_size = 192;
3209 rdev->config.evergreen.max_hw_contexts = 8;
3210 rdev->config.evergreen.sq_num_cf_insts = 2;
3211
3212 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3213 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3214 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3215 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3216 break;
3217 case CHIP_CAICOS:
3218 rdev->config.evergreen.num_ses = 1;
3219 rdev->config.evergreen.max_pipes = 2;
3220 rdev->config.evergreen.max_tile_pipes = 2;
3221 rdev->config.evergreen.max_simds = 2;
3222 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3223 rdev->config.evergreen.max_gprs = 256;
3224 rdev->config.evergreen.max_threads = 192;
3225 rdev->config.evergreen.max_gs_threads = 16;
3226 rdev->config.evergreen.max_stack_entries = 256;
3227 rdev->config.evergreen.sx_num_of_sets = 4;
3228 rdev->config.evergreen.sx_max_export_size = 128;
3229 rdev->config.evergreen.sx_max_export_pos_size = 32;
3230 rdev->config.evergreen.sx_max_export_smx_size = 96;
3231 rdev->config.evergreen.max_hw_contexts = 4;
3232 rdev->config.evergreen.sq_num_cf_insts = 1;
3233
3234 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3235 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3236 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3237 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3238 break;
3239 }
3240
3241 /* Initialize HDP */
3242 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3243 WREG32((0x2c14 + j), 0x00000000);
3244 WREG32((0x2c18 + j), 0x00000000);
3245 WREG32((0x2c1c + j), 0x00000000);
3246 WREG32((0x2c20 + j), 0x00000000);
3247 WREG32((0x2c24 + j), 0x00000000);
3248 }
3249
3250 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3251
3252 evergreen_fix_pci_max_read_req_size(rdev);
3253
3254 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3255 if ((rdev->family == CHIP_PALM) ||
3256 (rdev->family == CHIP_SUMO) ||
3257 (rdev->family == CHIP_SUMO2))
3258 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3259 else
3260 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3261
3262 /* setup tiling info dword. gb_addr_config is not adequate since it does
3263 * not have bank info, so create a custom tiling dword.
3264 * bits 3:0 num_pipes
3265 * bits 7:4 num_banks
3266 * bits 11:8 group_size
3267 * bits 15:12 row_size
3268 */
3269 rdev->config.evergreen.tile_config = 0;
3270 switch (rdev->config.evergreen.max_tile_pipes) {
3271 case 1:
3272 default:
3273 rdev->config.evergreen.tile_config |= (0 << 0);
3274 break;
3275 case 2:
3276 rdev->config.evergreen.tile_config |= (1 << 0);
3277 break;
3278 case 4:
3279 rdev->config.evergreen.tile_config |= (2 << 0);
3280 break;
3281 case 8:
3282 rdev->config.evergreen.tile_config |= (3 << 0);
3283 break;
3284 }
3285 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3286 if (rdev->flags & RADEON_IS_IGP)
3287 rdev->config.evergreen.tile_config |= 1 << 4;
3288 else {
3289 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3290 case 0: /* four banks */
3291 rdev->config.evergreen.tile_config |= 0 << 4;
3292 break;
3293 case 1: /* eight banks */
3294 rdev->config.evergreen.tile_config |= 1 << 4;
3295 break;
3296 case 2: /* sixteen banks */
3297 default:
3298 rdev->config.evergreen.tile_config |= 2 << 4;
3299 break;
3300 }
3301 }
3302 rdev->config.evergreen.tile_config |= 0 << 8;
3303 rdev->config.evergreen.tile_config |=
3304 ((gb_addr_config & 0x30000000) >> 28) << 12;
3305
3306 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3307
3308 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3309 u32 efuse_straps_4;
3310 u32 efuse_straps_3;
3311
3312 efuse_straps_4 = RREG32_RCU(0x204);
3313 efuse_straps_3 = RREG32_RCU(0x203);
3314 tmp = (((efuse_straps_4 & 0xf) << 4) |
3315 ((efuse_straps_3 & 0xf0000000) >> 28));
3316 } else {
3317 tmp = 0;
3318 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3319 u32 rb_disable_bitmap;
3320
3321 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3322 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3323 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3324 tmp <<= 4;
3325 tmp |= rb_disable_bitmap;
3326 }
3327 }
3328 /* enabled rb are just the one not disabled :) */
3329 disabled_rb_mask = tmp;
3330 tmp = 0;
3331 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3332 tmp |= (1 << i);
3333 /* if all the backends are disabled, fix it up here */
3334 if ((disabled_rb_mask & tmp) == tmp) {
3335 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3336 disabled_rb_mask &= ~(1 << i);
3337 }
3338
3339 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3340 u32 simd_disable_bitmap;
3341
3342 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3343 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3344 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3345 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3346 tmp <<= 16;
3347 tmp |= simd_disable_bitmap;
3348 }
3349 rdev->config.evergreen.active_simds = hweight32(~tmp);
3350
3351 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3352 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3353
3354 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3355 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3356 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3357 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3358 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3359 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3360 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3361
3362 if ((rdev->config.evergreen.max_backends == 1) &&
3363 (rdev->flags & RADEON_IS_IGP)) {
3364 if ((disabled_rb_mask & 3) == 1) {
3365 /* RB0 disabled, RB1 enabled */
3366 tmp = 0x11111111;
3367 } else {
3368 /* RB1 disabled, RB0 enabled */
3369 tmp = 0x00000000;
3370 }
3371 } else {
3372 tmp = gb_addr_config & NUM_PIPES_MASK;
3373 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3374 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3375 }
3376 WREG32(GB_BACKEND_MAP, tmp);
3377
3378 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3379 WREG32(CGTS_TCC_DISABLE, 0);
3380 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3381 WREG32(CGTS_USER_TCC_DISABLE, 0);
3382
3383 /* set HW defaults for 3D engine */
3384 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3385 ROQ_IB2_START(0x2b)));
3386
3387 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3388
3389 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3390 SYNC_GRADIENT |
3391 SYNC_WALKER |
3392 SYNC_ALIGNER));
3393
3394 sx_debug_1 = RREG32(SX_DEBUG_1);
3395 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3396 WREG32(SX_DEBUG_1, sx_debug_1);
3397
3398
3399 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3400 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3401 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3402 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3403
3404 if (rdev->family <= CHIP_SUMO2)
3405 WREG32(SMX_SAR_CTL0, 0x00010000);
3406
3407 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3408 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3409 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3410
3411 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3412 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3413 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3414
3415 WREG32(VGT_NUM_INSTANCES, 1);
3416 WREG32(SPI_CONFIG_CNTL, 0);
3417 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3418 WREG32(CP_PERFMON_CNTL, 0);
3419
3420 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3421 FETCH_FIFO_HIWATER(0x4) |
3422 DONE_FIFO_HIWATER(0xe0) |
3423 ALU_UPDATE_FIFO_HIWATER(0x8)));
3424
3425 sq_config = RREG32(SQ_CONFIG);
3426 sq_config &= ~(PS_PRIO(3) |
3427 VS_PRIO(3) |
3428 GS_PRIO(3) |
3429 ES_PRIO(3));
3430 sq_config |= (VC_ENABLE |
3431 EXPORT_SRC_C |
3432 PS_PRIO(0) |
3433 VS_PRIO(1) |
3434 GS_PRIO(2) |
3435 ES_PRIO(3));
3436
3437 switch (rdev->family) {
3438 case CHIP_CEDAR:
3439 case CHIP_PALM:
3440 case CHIP_SUMO:
3441 case CHIP_SUMO2:
3442 case CHIP_CAICOS:
3443 /* no vertex cache */
3444 sq_config &= ~VC_ENABLE;
3445 break;
3446 default:
3447 break;
3448 }
3449
3450 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3451
3452 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3453 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3454 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3455 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3456 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3457 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3458 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3459
3460 switch (rdev->family) {
3461 case CHIP_CEDAR:
3462 case CHIP_PALM:
3463 case CHIP_SUMO:
3464 case CHIP_SUMO2:
3465 ps_thread_count = 96;
3466 break;
3467 default:
3468 ps_thread_count = 128;
3469 break;
3470 }
3471
3472 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3473 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3474 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3475 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3476 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3477 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3478
3479 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3480 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3481 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3482 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3483 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3484 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3485
3486 WREG32(SQ_CONFIG, sq_config);
3487 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3488 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3489 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3490 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3491 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3492 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3493 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3494 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3495 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3496 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3497
3498 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3499 FORCE_EOV_MAX_REZ_CNT(255)));
3500
3501 switch (rdev->family) {
3502 case CHIP_CEDAR:
3503 case CHIP_PALM:
3504 case CHIP_SUMO:
3505 case CHIP_SUMO2:
3506 case CHIP_CAICOS:
3507 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3508 break;
3509 default:
3510 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3511 break;
3512 }
3513 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3514 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3515
3516 WREG32(VGT_GS_VERTEX_REUSE, 16);
3517 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3518 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3519
3520 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3521 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3522
3523 WREG32(CB_PERF_CTR0_SEL_0, 0);
3524 WREG32(CB_PERF_CTR0_SEL_1, 0);
3525 WREG32(CB_PERF_CTR1_SEL_0, 0);
3526 WREG32(CB_PERF_CTR1_SEL_1, 0);
3527 WREG32(CB_PERF_CTR2_SEL_0, 0);
3528 WREG32(CB_PERF_CTR2_SEL_1, 0);
3529 WREG32(CB_PERF_CTR3_SEL_0, 0);
3530 WREG32(CB_PERF_CTR3_SEL_1, 0);
3531
3532 /* clear render buffer base addresses */
3533 WREG32(CB_COLOR0_BASE, 0);
3534 WREG32(CB_COLOR1_BASE, 0);
3535 WREG32(CB_COLOR2_BASE, 0);
3536 WREG32(CB_COLOR3_BASE, 0);
3537 WREG32(CB_COLOR4_BASE, 0);
3538 WREG32(CB_COLOR5_BASE, 0);
3539 WREG32(CB_COLOR6_BASE, 0);
3540 WREG32(CB_COLOR7_BASE, 0);
3541 WREG32(CB_COLOR8_BASE, 0);
3542 WREG32(CB_COLOR9_BASE, 0);
3543 WREG32(CB_COLOR10_BASE, 0);
3544 WREG32(CB_COLOR11_BASE, 0);
3545
3546 /* set the shader const cache sizes to 0 */
3547 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3548 WREG32(i, 0);
3549 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3550 WREG32(i, 0);
3551
3552 tmp = RREG32(HDP_MISC_CNTL);
3553 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3554 WREG32(HDP_MISC_CNTL, tmp);
3555
3556 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3557 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3558
3559 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3560
3561 udelay(50);
3562
3563 }
3564
3565 int evergreen_mc_init(struct radeon_device *rdev)
3566 {
3567 u32 tmp;
3568 int chansize, numchan;
3569
3570 /* Get VRAM informations */
3571 rdev->mc.vram_is_ddr = true;
3572 if ((rdev->family == CHIP_PALM) ||
3573 (rdev->family == CHIP_SUMO) ||
3574 (rdev->family == CHIP_SUMO2))
3575 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3576 else
3577 tmp = RREG32(MC_ARB_RAMCFG);
3578 if (tmp & CHANSIZE_OVERRIDE) {
3579 chansize = 16;
3580 } else if (tmp & CHANSIZE_MASK) {
3581 chansize = 64;
3582 } else {
3583 chansize = 32;
3584 }
3585 tmp = RREG32(MC_SHARED_CHMAP);
3586 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3587 case 0:
3588 default:
3589 numchan = 1;
3590 break;
3591 case 1:
3592 numchan = 2;
3593 break;
3594 case 2:
3595 numchan = 4;
3596 break;
3597 case 3:
3598 numchan = 8;
3599 break;
3600 }
3601 rdev->mc.vram_width = numchan * chansize;
3602 /* Could aper size report 0 ? */
3603 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3604 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3605 /* Setup GPU memory space */
3606 if ((rdev->family == CHIP_PALM) ||
3607 (rdev->family == CHIP_SUMO) ||
3608 (rdev->family == CHIP_SUMO2)) {
3609 /* size in bytes on fusion */
3610 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3611 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3612 } else {
3613 /* size in MB on evergreen/cayman/tn */
3614 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3615 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3616 }
3617 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3618 r700_vram_gtt_location(rdev, &rdev->mc);
3619 radeon_update_bandwidth_info(rdev);
3620
3621 return 0;
3622 }
3623
3624 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3625 {
3626 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3627 RREG32(GRBM_STATUS));
3628 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3629 RREG32(GRBM_STATUS_SE0));
3630 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3631 RREG32(GRBM_STATUS_SE1));
3632 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3633 RREG32(SRBM_STATUS));
3634 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3635 RREG32(SRBM_STATUS2));
3636 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3637 RREG32(CP_STALLED_STAT1));
3638 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3639 RREG32(CP_STALLED_STAT2));
3640 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3641 RREG32(CP_BUSY_STAT));
3642 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3643 RREG32(CP_STAT));
3644 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3645 RREG32(DMA_STATUS_REG));
3646 if (rdev->family >= CHIP_CAYMAN) {
3647 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3648 RREG32(DMA_STATUS_REG + 0x800));
3649 }
3650 }
3651
3652 bool evergreen_is_display_hung(struct radeon_device *rdev)
3653 {
3654 u32 crtc_hung = 0;
3655 u32 crtc_status[6];
3656 u32 i, j, tmp;
3657
3658 for (i = 0; i < rdev->num_crtc; i++) {
3659 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3660 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3661 crtc_hung |= (1 << i);
3662 }
3663 }
3664
3665 for (j = 0; j < 10; j++) {
3666 for (i = 0; i < rdev->num_crtc; i++) {
3667 if (crtc_hung & (1 << i)) {
3668 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3669 if (tmp != crtc_status[i])
3670 crtc_hung &= ~(1 << i);
3671 }
3672 }
3673 if (crtc_hung == 0)
3674 return false;
3675 udelay(100);
3676 }
3677
3678 return true;
3679 }
3680
3681 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3682 {
3683 u32 reset_mask = 0;
3684 u32 tmp;
3685
3686 /* GRBM_STATUS */
3687 tmp = RREG32(GRBM_STATUS);
3688 if (tmp & (PA_BUSY | SC_BUSY |
3689 SH_BUSY | SX_BUSY |
3690 TA_BUSY | VGT_BUSY |
3691 DB_BUSY | CB_BUSY |
3692 SPI_BUSY | VGT_BUSY_NO_DMA))
3693 reset_mask |= RADEON_RESET_GFX;
3694
3695 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3696 CP_BUSY | CP_COHERENCY_BUSY))
3697 reset_mask |= RADEON_RESET_CP;
3698
3699 if (tmp & GRBM_EE_BUSY)
3700 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3701
3702 /* DMA_STATUS_REG */
3703 tmp = RREG32(DMA_STATUS_REG);
3704 if (!(tmp & DMA_IDLE))
3705 reset_mask |= RADEON_RESET_DMA;
3706
3707 /* SRBM_STATUS2 */
3708 tmp = RREG32(SRBM_STATUS2);
3709 if (tmp & DMA_BUSY)
3710 reset_mask |= RADEON_RESET_DMA;
3711
3712 /* SRBM_STATUS */
3713 tmp = RREG32(SRBM_STATUS);
3714 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3715 reset_mask |= RADEON_RESET_RLC;
3716
3717 if (tmp & IH_BUSY)
3718 reset_mask |= RADEON_RESET_IH;
3719
3720 if (tmp & SEM_BUSY)
3721 reset_mask |= RADEON_RESET_SEM;
3722
3723 if (tmp & GRBM_RQ_PENDING)
3724 reset_mask |= RADEON_RESET_GRBM;
3725
3726 if (tmp & VMC_BUSY)
3727 reset_mask |= RADEON_RESET_VMC;
3728
3729 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3730 MCC_BUSY | MCD_BUSY))
3731 reset_mask |= RADEON_RESET_MC;
3732
3733 if (evergreen_is_display_hung(rdev))
3734 reset_mask |= RADEON_RESET_DISPLAY;
3735
3736 /* VM_L2_STATUS */
3737 tmp = RREG32(VM_L2_STATUS);
3738 if (tmp & L2_BUSY)
3739 reset_mask |= RADEON_RESET_VMC;
3740
3741 /* Skip MC reset as it's mostly likely not hung, just busy */
3742 if (reset_mask & RADEON_RESET_MC) {
3743 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3744 reset_mask &= ~RADEON_RESET_MC;
3745 }
3746
3747 return reset_mask;
3748 }
3749
3750 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3751 {
3752 struct evergreen_mc_save save;
3753 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3754 u32 tmp;
3755
3756 if (reset_mask == 0)
3757 return;
3758
3759 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3760
3761 evergreen_print_gpu_status_regs(rdev);
3762
3763 /* Disable CP parsing/prefetching */
3764 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3765
3766 if (reset_mask & RADEON_RESET_DMA) {
3767 /* Disable DMA */
3768 tmp = RREG32(DMA_RB_CNTL);
3769 tmp &= ~DMA_RB_ENABLE;
3770 WREG32(DMA_RB_CNTL, tmp);
3771 }
3772
3773 udelay(50);
3774
3775 evergreen_mc_stop(rdev, &save);
3776 if (evergreen_mc_wait_for_idle(rdev)) {
3777 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3778 }
3779
3780 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3781 grbm_soft_reset |= SOFT_RESET_DB |
3782 SOFT_RESET_CB |
3783 SOFT_RESET_PA |
3784 SOFT_RESET_SC |
3785 SOFT_RESET_SPI |
3786 SOFT_RESET_SX |
3787 SOFT_RESET_SH |
3788 SOFT_RESET_TC |
3789 SOFT_RESET_TA |
3790 SOFT_RESET_VC |
3791 SOFT_RESET_VGT;
3792 }
3793
3794 if (reset_mask & RADEON_RESET_CP) {
3795 grbm_soft_reset |= SOFT_RESET_CP |
3796 SOFT_RESET_VGT;
3797
3798 srbm_soft_reset |= SOFT_RESET_GRBM;
3799 }
3800
3801 if (reset_mask & RADEON_RESET_DMA)
3802 srbm_soft_reset |= SOFT_RESET_DMA;
3803
3804 if (reset_mask & RADEON_RESET_DISPLAY)
3805 srbm_soft_reset |= SOFT_RESET_DC;
3806
3807 if (reset_mask & RADEON_RESET_RLC)
3808 srbm_soft_reset |= SOFT_RESET_RLC;
3809
3810 if (reset_mask & RADEON_RESET_SEM)
3811 srbm_soft_reset |= SOFT_RESET_SEM;
3812
3813 if (reset_mask & RADEON_RESET_IH)
3814 srbm_soft_reset |= SOFT_RESET_IH;
3815
3816 if (reset_mask & RADEON_RESET_GRBM)
3817 srbm_soft_reset |= SOFT_RESET_GRBM;
3818
3819 if (reset_mask & RADEON_RESET_VMC)
3820 srbm_soft_reset |= SOFT_RESET_VMC;
3821
3822 if (!(rdev->flags & RADEON_IS_IGP)) {
3823 if (reset_mask & RADEON_RESET_MC)
3824 srbm_soft_reset |= SOFT_RESET_MC;
3825 }
3826
3827 if (grbm_soft_reset) {
3828 tmp = RREG32(GRBM_SOFT_RESET);
3829 tmp |= grbm_soft_reset;
3830 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3831 WREG32(GRBM_SOFT_RESET, tmp);
3832 tmp = RREG32(GRBM_SOFT_RESET);
3833
3834 udelay(50);
3835
3836 tmp &= ~grbm_soft_reset;
3837 WREG32(GRBM_SOFT_RESET, tmp);
3838 tmp = RREG32(GRBM_SOFT_RESET);
3839 }
3840
3841 if (srbm_soft_reset) {
3842 tmp = RREG32(SRBM_SOFT_RESET);
3843 tmp |= srbm_soft_reset;
3844 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3845 WREG32(SRBM_SOFT_RESET, tmp);
3846 tmp = RREG32(SRBM_SOFT_RESET);
3847
3848 udelay(50);
3849
3850 tmp &= ~srbm_soft_reset;
3851 WREG32(SRBM_SOFT_RESET, tmp);
3852 tmp = RREG32(SRBM_SOFT_RESET);
3853 }
3854
3855 /* Wait a little for things to settle down */
3856 udelay(50);
3857
3858 evergreen_mc_resume(rdev, &save);
3859 udelay(50);
3860
3861 evergreen_print_gpu_status_regs(rdev);
3862 }
3863
3864 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3865 {
3866 struct evergreen_mc_save save;
3867 u32 tmp, i;
3868
3869 dev_info(rdev->dev, "GPU pci config reset\n");
3870
3871 /* disable dpm? */
3872
3873 /* Disable CP parsing/prefetching */
3874 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3875 udelay(50);
3876 /* Disable DMA */
3877 tmp = RREG32(DMA_RB_CNTL);
3878 tmp &= ~DMA_RB_ENABLE;
3879 WREG32(DMA_RB_CNTL, tmp);
3880 /* XXX other engines? */
3881
3882 /* halt the rlc */
3883 r600_rlc_stop(rdev);
3884
3885 udelay(50);
3886
3887 /* set mclk/sclk to bypass */
3888 rv770_set_clk_bypass_mode(rdev);
3889 /* disable BM */
3890 pci_clear_master(rdev->pdev);
3891 /* disable mem access */
3892 evergreen_mc_stop(rdev, &save);
3893 if (evergreen_mc_wait_for_idle(rdev)) {
3894 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3895 }
3896 /* reset */
3897 radeon_pci_config_reset(rdev);
3898 /* wait for asic to come out of reset */
3899 for (i = 0; i < rdev->usec_timeout; i++) {
3900 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3901 break;
3902 udelay(1);
3903 }
3904 }
3905
3906 int evergreen_asic_reset(struct radeon_device *rdev)
3907 {
3908 u32 reset_mask;
3909
3910 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3911
3912 if (reset_mask)
3913 r600_set_bios_scratch_engine_hung(rdev, true);
3914
3915 /* try soft reset */
3916 evergreen_gpu_soft_reset(rdev, reset_mask);
3917
3918 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3919
3920 /* try pci config reset */
3921 if (reset_mask && radeon_hard_reset)
3922 evergreen_gpu_pci_config_reset(rdev);
3923
3924 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3925
3926 if (!reset_mask)
3927 r600_set_bios_scratch_engine_hung(rdev, false);
3928
3929 return 0;
3930 }
3931
3932 /**
3933 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3934 *
3935 * @rdev: radeon_device pointer
3936 * @ring: radeon_ring structure holding ring information
3937 *
3938 * Check if the GFX engine is locked up.
3939 * Returns true if the engine appears to be locked up, false if not.
3940 */
3941 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3942 {
3943 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3944
3945 if (!(reset_mask & (RADEON_RESET_GFX |
3946 RADEON_RESET_COMPUTE |
3947 RADEON_RESET_CP))) {
3948 radeon_ring_lockup_update(rdev, ring);
3949 return false;
3950 }
3951 return radeon_ring_test_lockup(rdev, ring);
3952 }
3953
3954 /*
3955 * RLC
3956 */
3957 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3958 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3959
3960 void sumo_rlc_fini(struct radeon_device *rdev)
3961 {
3962 int r;
3963
3964 /* save restore block */
3965 if (rdev->rlc.save_restore_obj) {
3966 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3967 if (unlikely(r != 0))
3968 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3969 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3970 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3971
3972 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3973 rdev->rlc.save_restore_obj = NULL;
3974 }
3975
3976 /* clear state block */
3977 if (rdev->rlc.clear_state_obj) {
3978 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3979 if (unlikely(r != 0))
3980 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3981 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3982 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3983
3984 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3985 rdev->rlc.clear_state_obj = NULL;
3986 }
3987
3988 /* clear state block */
3989 if (rdev->rlc.cp_table_obj) {
3990 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3991 if (unlikely(r != 0))
3992 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3993 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3994 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3995
3996 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3997 rdev->rlc.cp_table_obj = NULL;
3998 }
3999 }
4000
4001 #define CP_ME_TABLE_SIZE 96
4002
4003 int sumo_rlc_init(struct radeon_device *rdev)
4004 {
4005 const u32 *src_ptr;
4006 volatile u32 *dst_ptr;
4007 u32 dws, data, i, j, k, reg_num;
4008 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4009 u64 reg_list_mc_addr;
4010 const struct cs_section_def *cs_data;
4011 int r;
4012
4013 src_ptr = rdev->rlc.reg_list;
4014 dws = rdev->rlc.reg_list_size;
4015 if (rdev->family >= CHIP_BONAIRE) {
4016 dws += (5 * 16) + 48 + 48 + 64;
4017 }
4018 cs_data = rdev->rlc.cs_data;
4019
4020 if (src_ptr) {
4021 /* save restore block */
4022 if (rdev->rlc.save_restore_obj == NULL) {
4023 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4024 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4025 NULL, &rdev->rlc.save_restore_obj);
4026 if (r) {
4027 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4028 return r;
4029 }
4030 }
4031
4032 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4033 if (unlikely(r != 0)) {
4034 sumo_rlc_fini(rdev);
4035 return r;
4036 }
4037 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4038 &rdev->rlc.save_restore_gpu_addr);
4039 if (r) {
4040 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4041 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4042 sumo_rlc_fini(rdev);
4043 return r;
4044 }
4045
4046 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4047 if (r) {
4048 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4049 sumo_rlc_fini(rdev);
4050 return r;
4051 }
4052 /* write the sr buffer */
4053 dst_ptr = rdev->rlc.sr_ptr;
4054 if (rdev->family >= CHIP_TAHITI) {
4055 /* SI */
4056 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4057 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4058 } else {
4059 /* ON/LN/TN */
4060 /* format:
4061 * dw0: (reg2 << 16) | reg1
4062 * dw1: reg1 save space
4063 * dw2: reg2 save space
4064 */
4065 for (i = 0; i < dws; i++) {
4066 data = src_ptr[i] >> 2;
4067 i++;
4068 if (i < dws)
4069 data |= (src_ptr[i] >> 2) << 16;
4070 j = (((i - 1) * 3) / 2);
4071 dst_ptr[j] = cpu_to_le32(data);
4072 }
4073 j = ((i * 3) / 2);
4074 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4075 }
4076 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4077 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4078 }
4079
4080 if (cs_data) {
4081 /* clear state block */
4082 if (rdev->family >= CHIP_BONAIRE) {
4083 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4084 } else if (rdev->family >= CHIP_TAHITI) {
4085 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4086 dws = rdev->rlc.clear_state_size + (256 / 4);
4087 } else {
4088 reg_list_num = 0;
4089 dws = 0;
4090 for (i = 0; cs_data[i].section != NULL; i++) {
4091 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4092 reg_list_num++;
4093 dws += cs_data[i].section[j].reg_count;
4094 }
4095 }
4096 reg_list_blk_index = (3 * reg_list_num + 2);
4097 dws += reg_list_blk_index;
4098 rdev->rlc.clear_state_size = dws;
4099 }
4100
4101 if (rdev->rlc.clear_state_obj == NULL) {
4102 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4103 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4104 NULL, &rdev->rlc.clear_state_obj);
4105 if (r) {
4106 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4107 sumo_rlc_fini(rdev);
4108 return r;
4109 }
4110 }
4111 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4112 if (unlikely(r != 0)) {
4113 sumo_rlc_fini(rdev);
4114 return r;
4115 }
4116 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4117 &rdev->rlc.clear_state_gpu_addr);
4118 if (r) {
4119 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4120 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4121 sumo_rlc_fini(rdev);
4122 return r;
4123 }
4124
4125 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4126 if (r) {
4127 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4128 sumo_rlc_fini(rdev);
4129 return r;
4130 }
4131 /* set up the cs buffer */
4132 dst_ptr = rdev->rlc.cs_ptr;
4133 if (rdev->family >= CHIP_BONAIRE) {
4134 cik_get_csb_buffer(rdev, dst_ptr);
4135 } else if (rdev->family >= CHIP_TAHITI) {
4136 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4137 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4138 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4139 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4140 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4141 } else {
4142 reg_list_hdr_blk_index = 0;
4143 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4144 data = upper_32_bits(reg_list_mc_addr);
4145 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4146 reg_list_hdr_blk_index++;
4147 for (i = 0; cs_data[i].section != NULL; i++) {
4148 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4149 reg_num = cs_data[i].section[j].reg_count;
4150 data = reg_list_mc_addr & 0xffffffff;
4151 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4152 reg_list_hdr_blk_index++;
4153
4154 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4155 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4156 reg_list_hdr_blk_index++;
4157
4158 data = 0x08000000 | (reg_num * 4);
4159 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4160 reg_list_hdr_blk_index++;
4161
4162 for (k = 0; k < reg_num; k++) {
4163 data = cs_data[i].section[j].extent[k];
4164 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4165 }
4166 reg_list_mc_addr += reg_num * 4;
4167 reg_list_blk_index += reg_num;
4168 }
4169 }
4170 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4171 }
4172 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4173 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4174 }
4175
4176 if (rdev->rlc.cp_table_size) {
4177 if (rdev->rlc.cp_table_obj == NULL) {
4178 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4179 PAGE_SIZE, true,
4180 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4181 NULL, &rdev->rlc.cp_table_obj);
4182 if (r) {
4183 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4184 sumo_rlc_fini(rdev);
4185 return r;
4186 }
4187 }
4188
4189 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4190 if (unlikely(r != 0)) {
4191 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4192 sumo_rlc_fini(rdev);
4193 return r;
4194 }
4195 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4196 &rdev->rlc.cp_table_gpu_addr);
4197 if (r) {
4198 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4199 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4200 sumo_rlc_fini(rdev);
4201 return r;
4202 }
4203 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4204 if (r) {
4205 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4206 sumo_rlc_fini(rdev);
4207 return r;
4208 }
4209
4210 cik_init_cp_pg_table(rdev);
4211
4212 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4213 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4214
4215 }
4216
4217 return 0;
4218 }
4219
4220 static void evergreen_rlc_start(struct radeon_device *rdev)
4221 {
4222 u32 mask = RLC_ENABLE;
4223
4224 if (rdev->flags & RADEON_IS_IGP) {
4225 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4226 }
4227
4228 WREG32(RLC_CNTL, mask);
4229 }
4230
4231 int evergreen_rlc_resume(struct radeon_device *rdev)
4232 {
4233 u32 i;
4234 const __be32 *fw_data;
4235
4236 if (!rdev->rlc_fw)
4237 return -EINVAL;
4238
4239 r600_rlc_stop(rdev);
4240
4241 WREG32(RLC_HB_CNTL, 0);
4242
4243 if (rdev->flags & RADEON_IS_IGP) {
4244 if (rdev->family == CHIP_ARUBA) {
4245 u32 always_on_bitmap =
4246 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4247 /* find out the number of active simds */
4248 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4249 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4250 tmp = hweight32(~tmp);
4251 if (tmp == rdev->config.cayman.max_simds_per_se) {
4252 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4253 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4254 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4255 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4256 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4257 }
4258 } else {
4259 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4260 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4261 }
4262 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4263 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4264 } else {
4265 WREG32(RLC_HB_BASE, 0);
4266 WREG32(RLC_HB_RPTR, 0);
4267 WREG32(RLC_HB_WPTR, 0);
4268 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4269 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4270 }
4271 WREG32(RLC_MC_CNTL, 0);
4272 WREG32(RLC_UCODE_CNTL, 0);
4273
4274 fw_data = (const __be32 *)rdev->rlc_fw->data;
4275 if (rdev->family >= CHIP_ARUBA) {
4276 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4277 WREG32(RLC_UCODE_ADDR, i);
4278 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4279 }
4280 } else if (rdev->family >= CHIP_CAYMAN) {
4281 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4282 WREG32(RLC_UCODE_ADDR, i);
4283 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4284 }
4285 } else {
4286 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4287 WREG32(RLC_UCODE_ADDR, i);
4288 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4289 }
4290 }
4291 WREG32(RLC_UCODE_ADDR, 0);
4292
4293 evergreen_rlc_start(rdev);
4294
4295 return 0;
4296 }
4297
4298 /* Interrupts */
4299
4300 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4301 {
4302 if (crtc >= rdev->num_crtc)
4303 return 0;
4304 else
4305 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4306 }
4307
4308 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4309 {
4310 u32 tmp;
4311
4312 if (rdev->family >= CHIP_CAYMAN) {
4313 cayman_cp_int_cntl_setup(rdev, 0,
4314 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4315 cayman_cp_int_cntl_setup(rdev, 1, 0);
4316 cayman_cp_int_cntl_setup(rdev, 2, 0);
4317 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4318 WREG32(CAYMAN_DMA1_CNTL, tmp);
4319 } else
4320 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4321 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4322 WREG32(DMA_CNTL, tmp);
4323 WREG32(GRBM_INT_CNTL, 0);
4324 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4325 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4326 if (rdev->num_crtc >= 4) {
4327 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4328 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4329 }
4330 if (rdev->num_crtc >= 6) {
4331 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4332 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4333 }
4334
4335 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4336 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4337 if (rdev->num_crtc >= 4) {
4338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4339 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4340 }
4341 if (rdev->num_crtc >= 6) {
4342 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4343 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4344 }
4345
4346 /* only one DAC on DCE5 */
4347 if (!ASIC_IS_DCE5(rdev))
4348 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4349 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4350
4351 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4352 WREG32(DC_HPD1_INT_CONTROL, tmp);
4353 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4354 WREG32(DC_HPD2_INT_CONTROL, tmp);
4355 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4356 WREG32(DC_HPD3_INT_CONTROL, tmp);
4357 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4358 WREG32(DC_HPD4_INT_CONTROL, tmp);
4359 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4360 WREG32(DC_HPD5_INT_CONTROL, tmp);
4361 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4362 WREG32(DC_HPD6_INT_CONTROL, tmp);
4363
4364 }
4365
4366 int evergreen_irq_set(struct radeon_device *rdev)
4367 {
4368 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4369 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4370 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4371 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4372 u32 grbm_int_cntl = 0;
4373 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4374 u32 dma_cntl, dma_cntl1 = 0;
4375 u32 thermal_int = 0;
4376
4377 if (!rdev->irq.installed) {
4378 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4379 return -EINVAL;
4380 }
4381 /* don't enable anything if the ih is disabled */
4382 if (!rdev->ih.enabled) {
4383 r600_disable_interrupts(rdev);
4384 /* force the active interrupt state to all disabled */
4385 evergreen_disable_interrupt_state(rdev);
4386 return 0;
4387 }
4388
4389 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4390 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4391 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4394 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4395 if (rdev->family == CHIP_ARUBA)
4396 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4397 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4398 else
4399 thermal_int = RREG32(CG_THERMAL_INT) &
4400 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4401
4402 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4403 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4404 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4405 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4406 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4407 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4408
4409 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4410
4411 if (rdev->family >= CHIP_CAYMAN) {
4412 /* enable CP interrupts on all rings */
4413 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4414 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4415 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4416 }
4417 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4418 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4419 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4420 }
4421 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4422 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4423 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4424 }
4425 } else {
4426 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4427 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4428 cp_int_cntl |= RB_INT_ENABLE;
4429 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4430 }
4431 }
4432
4433 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4434 DRM_DEBUG("r600_irq_set: sw int dma\n");
4435 dma_cntl |= TRAP_ENABLE;
4436 }
4437
4438 if (rdev->family >= CHIP_CAYMAN) {
4439 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4440 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4441 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4442 dma_cntl1 |= TRAP_ENABLE;
4443 }
4444 }
4445
4446 if (rdev->irq.dpm_thermal) {
4447 DRM_DEBUG("dpm thermal\n");
4448 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4449 }
4450
4451 if (rdev->irq.crtc_vblank_int[0] ||
4452 atomic_read(&rdev->irq.pflip[0])) {
4453 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4454 crtc1 |= VBLANK_INT_MASK;
4455 }
4456 if (rdev->irq.crtc_vblank_int[1] ||
4457 atomic_read(&rdev->irq.pflip[1])) {
4458 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4459 crtc2 |= VBLANK_INT_MASK;
4460 }
4461 if (rdev->irq.crtc_vblank_int[2] ||
4462 atomic_read(&rdev->irq.pflip[2])) {
4463 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4464 crtc3 |= VBLANK_INT_MASK;
4465 }
4466 if (rdev->irq.crtc_vblank_int[3] ||
4467 atomic_read(&rdev->irq.pflip[3])) {
4468 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4469 crtc4 |= VBLANK_INT_MASK;
4470 }
4471 if (rdev->irq.crtc_vblank_int[4] ||
4472 atomic_read(&rdev->irq.pflip[4])) {
4473 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4474 crtc5 |= VBLANK_INT_MASK;
4475 }
4476 if (rdev->irq.crtc_vblank_int[5] ||
4477 atomic_read(&rdev->irq.pflip[5])) {
4478 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4479 crtc6 |= VBLANK_INT_MASK;
4480 }
4481 if (rdev->irq.hpd[0]) {
4482 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4483 hpd1 |= DC_HPDx_INT_EN;
4484 }
4485 if (rdev->irq.hpd[1]) {
4486 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4487 hpd2 |= DC_HPDx_INT_EN;
4488 }
4489 if (rdev->irq.hpd[2]) {
4490 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4491 hpd3 |= DC_HPDx_INT_EN;
4492 }
4493 if (rdev->irq.hpd[3]) {
4494 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4495 hpd4 |= DC_HPDx_INT_EN;
4496 }
4497 if (rdev->irq.hpd[4]) {
4498 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4499 hpd5 |= DC_HPDx_INT_EN;
4500 }
4501 if (rdev->irq.hpd[5]) {
4502 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4503 hpd6 |= DC_HPDx_INT_EN;
4504 }
4505 if (rdev->irq.afmt[0]) {
4506 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4507 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4508 }
4509 if (rdev->irq.afmt[1]) {
4510 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4511 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4512 }
4513 if (rdev->irq.afmt[2]) {
4514 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4515 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4516 }
4517 if (rdev->irq.afmt[3]) {
4518 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4519 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4520 }
4521 if (rdev->irq.afmt[4]) {
4522 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4523 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4524 }
4525 if (rdev->irq.afmt[5]) {
4526 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4527 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4528 }
4529
4530 if (rdev->family >= CHIP_CAYMAN) {
4531 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4532 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4533 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4534 } else
4535 WREG32(CP_INT_CNTL, cp_int_cntl);
4536
4537 WREG32(DMA_CNTL, dma_cntl);
4538
4539 if (rdev->family >= CHIP_CAYMAN)
4540 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4541
4542 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4543
4544 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4545 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4546 if (rdev->num_crtc >= 4) {
4547 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4548 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4549 }
4550 if (rdev->num_crtc >= 6) {
4551 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4552 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4553 }
4554
4555 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4556 GRPH_PFLIP_INT_MASK);
4557 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4558 GRPH_PFLIP_INT_MASK);
4559 if (rdev->num_crtc >= 4) {
4560 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4561 GRPH_PFLIP_INT_MASK);
4562 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4563 GRPH_PFLIP_INT_MASK);
4564 }
4565 if (rdev->num_crtc >= 6) {
4566 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4567 GRPH_PFLIP_INT_MASK);
4568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4569 GRPH_PFLIP_INT_MASK);
4570 }
4571
4572 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4573 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4574 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4575 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4576 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4577 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4578 if (rdev->family == CHIP_ARUBA)
4579 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4580 else
4581 WREG32(CG_THERMAL_INT, thermal_int);
4582
4583 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4584 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4585 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4587 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4588 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4589
4590 return 0;
4591 }
4592
4593 static void evergreen_irq_ack(struct radeon_device *rdev)
4594 {
4595 u32 tmp;
4596
4597 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4598 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4599 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4600 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4601 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4602 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4603 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4604 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4605 if (rdev->num_crtc >= 4) {
4606 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4607 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4608 }
4609 if (rdev->num_crtc >= 6) {
4610 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4611 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4612 }
4613
4614 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4615 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4616 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4617 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4618 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4619 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4620
4621 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4623 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4624 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4625 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4626 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4627 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4628 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4629 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4630 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4631 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4632 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4633
4634 if (rdev->num_crtc >= 4) {
4635 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4637 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4638 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4639 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4640 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4642 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4643 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4644 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4645 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4646 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4647 }
4648
4649 if (rdev->num_crtc >= 6) {
4650 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4651 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4652 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4653 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4655 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4656 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4657 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4659 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4660 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4661 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4662 }
4663
4664 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4665 tmp = RREG32(DC_HPD1_INT_CONTROL);
4666 tmp |= DC_HPDx_INT_ACK;
4667 WREG32(DC_HPD1_INT_CONTROL, tmp);
4668 }
4669 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4670 tmp = RREG32(DC_HPD2_INT_CONTROL);
4671 tmp |= DC_HPDx_INT_ACK;
4672 WREG32(DC_HPD2_INT_CONTROL, tmp);
4673 }
4674 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4675 tmp = RREG32(DC_HPD3_INT_CONTROL);
4676 tmp |= DC_HPDx_INT_ACK;
4677 WREG32(DC_HPD3_INT_CONTROL, tmp);
4678 }
4679 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4680 tmp = RREG32(DC_HPD4_INT_CONTROL);
4681 tmp |= DC_HPDx_INT_ACK;
4682 WREG32(DC_HPD4_INT_CONTROL, tmp);
4683 }
4684 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4685 tmp = RREG32(DC_HPD5_INT_CONTROL);
4686 tmp |= DC_HPDx_INT_ACK;
4687 WREG32(DC_HPD5_INT_CONTROL, tmp);
4688 }
4689 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4690 tmp = RREG32(DC_HPD5_INT_CONTROL);
4691 tmp |= DC_HPDx_INT_ACK;
4692 WREG32(DC_HPD6_INT_CONTROL, tmp);
4693 }
4694 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4695 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4696 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4697 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4698 }
4699 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4700 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4701 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4702 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4703 }
4704 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4705 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4706 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4707 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4708 }
4709 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4710 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4711 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4712 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4713 }
4714 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4715 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4716 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4717 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4718 }
4719 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4720 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4721 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4722 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4723 }
4724 }
4725
4726 static void evergreen_irq_disable(struct radeon_device *rdev)
4727 {
4728 r600_disable_interrupts(rdev);
4729 /* Wait and acknowledge irq */
4730 mdelay(1);
4731 evergreen_irq_ack(rdev);
4732 evergreen_disable_interrupt_state(rdev);
4733 }
4734
4735 void evergreen_irq_suspend(struct radeon_device *rdev)
4736 {
4737 evergreen_irq_disable(rdev);
4738 r600_rlc_stop(rdev);
4739 }
4740
4741 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4742 {
4743 u32 wptr, tmp;
4744
4745 if (rdev->wb.enabled)
4746 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4747 else
4748 wptr = RREG32(IH_RB_WPTR);
4749
4750 if (wptr & RB_OVERFLOW) {
4751 /* When a ring buffer overflow happen start parsing interrupt
4752 * from the last not overwritten vector (wptr + 16). Hopefully
4753 * this should allow us to catchup.
4754 */
4755 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4756 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4757 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4758 tmp = RREG32(IH_RB_CNTL);
4759 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4760 WREG32(IH_RB_CNTL, tmp);
4761 wptr &= ~RB_OVERFLOW;
4762 }
4763 return (wptr & rdev->ih.ptr_mask);
4764 }
4765
4766 int evergreen_irq_process(struct radeon_device *rdev)
4767 {
4768 u32 wptr;
4769 u32 rptr;
4770 u32 src_id, src_data;
4771 u32 ring_index;
4772 bool queue_hotplug = false;
4773 bool queue_hdmi = false;
4774 bool queue_thermal = false;
4775 u32 status, addr;
4776
4777 if (!rdev->ih.enabled || rdev->shutdown)
4778 return IRQ_NONE;
4779
4780 wptr = evergreen_get_ih_wptr(rdev);
4781
4782 restart_ih:
4783 /* is somebody else already processing irqs? */
4784 if (atomic_xchg(&rdev->ih.lock, 1))
4785 return IRQ_NONE;
4786
4787 rptr = rdev->ih.rptr;
4788 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4789
4790 /* Order reading of wptr vs. reading of IH ring data */
4791 rmb();
4792
4793 /* display interrupts */
4794 evergreen_irq_ack(rdev);
4795
4796 while (rptr != wptr) {
4797 /* wptr/rptr are in bytes! */
4798 ring_index = rptr / 4;
4799 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4800 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4801
4802 switch (src_id) {
4803 case 1: /* D1 vblank/vline */
4804 switch (src_data) {
4805 case 0: /* D1 vblank */
4806 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4807 if (rdev->irq.crtc_vblank_int[0]) {
4808 drm_handle_vblank(rdev->ddev, 0);
4809 rdev->pm.vblank_sync = true;
4810 wake_up(&rdev->irq.vblank_queue);
4811 }
4812 if (atomic_read(&rdev->irq.pflip[0]))
4813 radeon_crtc_handle_vblank(rdev, 0);
4814 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4815 DRM_DEBUG("IH: D1 vblank\n");
4816 }
4817 break;
4818 case 1: /* D1 vline */
4819 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4820 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4821 DRM_DEBUG("IH: D1 vline\n");
4822 }
4823 break;
4824 default:
4825 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4826 break;
4827 }
4828 break;
4829 case 2: /* D2 vblank/vline */
4830 switch (src_data) {
4831 case 0: /* D2 vblank */
4832 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4833 if (rdev->irq.crtc_vblank_int[1]) {
4834 drm_handle_vblank(rdev->ddev, 1);
4835 rdev->pm.vblank_sync = true;
4836 wake_up(&rdev->irq.vblank_queue);
4837 }
4838 if (atomic_read(&rdev->irq.pflip[1]))
4839 radeon_crtc_handle_vblank(rdev, 1);
4840 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4841 DRM_DEBUG("IH: D2 vblank\n");
4842 }
4843 break;
4844 case 1: /* D2 vline */
4845 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4846 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4847 DRM_DEBUG("IH: D2 vline\n");
4848 }
4849 break;
4850 default:
4851 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4852 break;
4853 }
4854 break;
4855 case 3: /* D3 vblank/vline */
4856 switch (src_data) {
4857 case 0: /* D3 vblank */
4858 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4859 if (rdev->irq.crtc_vblank_int[2]) {
4860 drm_handle_vblank(rdev->ddev, 2);
4861 rdev->pm.vblank_sync = true;
4862 wake_up(&rdev->irq.vblank_queue);
4863 }
4864 if (atomic_read(&rdev->irq.pflip[2]))
4865 radeon_crtc_handle_vblank(rdev, 2);
4866 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4867 DRM_DEBUG("IH: D3 vblank\n");
4868 }
4869 break;
4870 case 1: /* D3 vline */
4871 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4872 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4873 DRM_DEBUG("IH: D3 vline\n");
4874 }
4875 break;
4876 default:
4877 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4878 break;
4879 }
4880 break;
4881 case 4: /* D4 vblank/vline */
4882 switch (src_data) {
4883 case 0: /* D4 vblank */
4884 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4885 if (rdev->irq.crtc_vblank_int[3]) {
4886 drm_handle_vblank(rdev->ddev, 3);
4887 rdev->pm.vblank_sync = true;
4888 wake_up(&rdev->irq.vblank_queue);
4889 }
4890 if (atomic_read(&rdev->irq.pflip[3]))
4891 radeon_crtc_handle_vblank(rdev, 3);
4892 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4893 DRM_DEBUG("IH: D4 vblank\n");
4894 }
4895 break;
4896 case 1: /* D4 vline */
4897 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4898 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4899 DRM_DEBUG("IH: D4 vline\n");
4900 }
4901 break;
4902 default:
4903 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4904 break;
4905 }
4906 break;
4907 case 5: /* D5 vblank/vline */
4908 switch (src_data) {
4909 case 0: /* D5 vblank */
4910 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4911 if (rdev->irq.crtc_vblank_int[4]) {
4912 drm_handle_vblank(rdev->ddev, 4);
4913 rdev->pm.vblank_sync = true;
4914 wake_up(&rdev->irq.vblank_queue);
4915 }
4916 if (atomic_read(&rdev->irq.pflip[4]))
4917 radeon_crtc_handle_vblank(rdev, 4);
4918 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4919 DRM_DEBUG("IH: D5 vblank\n");
4920 }
4921 break;
4922 case 1: /* D5 vline */
4923 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4924 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4925 DRM_DEBUG("IH: D5 vline\n");
4926 }
4927 break;
4928 default:
4929 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4930 break;
4931 }
4932 break;
4933 case 6: /* D6 vblank/vline */
4934 switch (src_data) {
4935 case 0: /* D6 vblank */
4936 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4937 if (rdev->irq.crtc_vblank_int[5]) {
4938 drm_handle_vblank(rdev->ddev, 5);
4939 rdev->pm.vblank_sync = true;
4940 wake_up(&rdev->irq.vblank_queue);
4941 }
4942 if (atomic_read(&rdev->irq.pflip[5]))
4943 radeon_crtc_handle_vblank(rdev, 5);
4944 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4945 DRM_DEBUG("IH: D6 vblank\n");
4946 }
4947 break;
4948 case 1: /* D6 vline */
4949 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4950 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4951 DRM_DEBUG("IH: D6 vline\n");
4952 }
4953 break;
4954 default:
4955 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4956 break;
4957 }
4958 break;
4959 case 8: /* D1 page flip */
4960 case 10: /* D2 page flip */
4961 case 12: /* D3 page flip */
4962 case 14: /* D4 page flip */
4963 case 16: /* D5 page flip */
4964 case 18: /* D6 page flip */
4965 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4966 if (radeon_use_pflipirq > 0)
4967 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4968 break;
4969 case 42: /* HPD hotplug */
4970 switch (src_data) {
4971 case 0:
4972 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4973 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4974 queue_hotplug = true;
4975 DRM_DEBUG("IH: HPD1\n");
4976 }
4977 break;
4978 case 1:
4979 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4980 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4981 queue_hotplug = true;
4982 DRM_DEBUG("IH: HPD2\n");
4983 }
4984 break;
4985 case 2:
4986 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4987 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4988 queue_hotplug = true;
4989 DRM_DEBUG("IH: HPD3\n");
4990 }
4991 break;
4992 case 3:
4993 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4994 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4995 queue_hotplug = true;
4996 DRM_DEBUG("IH: HPD4\n");
4997 }
4998 break;
4999 case 4:
5000 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5001 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5002 queue_hotplug = true;
5003 DRM_DEBUG("IH: HPD5\n");
5004 }
5005 break;
5006 case 5:
5007 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5008 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5009 queue_hotplug = true;
5010 DRM_DEBUG("IH: HPD6\n");
5011 }
5012 break;
5013 default:
5014 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5015 break;
5016 }
5017 break;
5018 case 44: /* hdmi */
5019 switch (src_data) {
5020 case 0:
5021 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5022 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5023 queue_hdmi = true;
5024 DRM_DEBUG("IH: HDMI0\n");
5025 }
5026 break;
5027 case 1:
5028 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5029 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5030 queue_hdmi = true;
5031 DRM_DEBUG("IH: HDMI1\n");
5032 }
5033 break;
5034 case 2:
5035 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5036 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5037 queue_hdmi = true;
5038 DRM_DEBUG("IH: HDMI2\n");
5039 }
5040 break;
5041 case 3:
5042 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5043 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5044 queue_hdmi = true;
5045 DRM_DEBUG("IH: HDMI3\n");
5046 }
5047 break;
5048 case 4:
5049 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5050 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5051 queue_hdmi = true;
5052 DRM_DEBUG("IH: HDMI4\n");
5053 }
5054 break;
5055 case 5:
5056 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5057 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5058 queue_hdmi = true;
5059 DRM_DEBUG("IH: HDMI5\n");
5060 }
5061 break;
5062 default:
5063 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5064 break;
5065 }
5066 case 124: /* UVD */
5067 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5068 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5069 break;
5070 case 146:
5071 case 147:
5072 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5073 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5074 /* reset addr and status */
5075 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5076 if (addr == 0x0 && status == 0x0)
5077 break;
5078 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5079 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5080 addr);
5081 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5082 status);
5083 cayman_vm_decode_fault(rdev, status, addr);
5084 break;
5085 case 176: /* CP_INT in ring buffer */
5086 case 177: /* CP_INT in IB1 */
5087 case 178: /* CP_INT in IB2 */
5088 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5089 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5090 break;
5091 case 181: /* CP EOP event */
5092 DRM_DEBUG("IH: CP EOP\n");
5093 if (rdev->family >= CHIP_CAYMAN) {
5094 switch (src_data) {
5095 case 0:
5096 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5097 break;
5098 case 1:
5099 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5100 break;
5101 case 2:
5102 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5103 break;
5104 }
5105 } else
5106 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5107 break;
5108 case 224: /* DMA trap event */
5109 DRM_DEBUG("IH: DMA trap\n");
5110 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5111 break;
5112 case 230: /* thermal low to high */
5113 DRM_DEBUG("IH: thermal low to high\n");
5114 rdev->pm.dpm.thermal.high_to_low = false;
5115 queue_thermal = true;
5116 break;
5117 case 231: /* thermal high to low */
5118 DRM_DEBUG("IH: thermal high to low\n");
5119 rdev->pm.dpm.thermal.high_to_low = true;
5120 queue_thermal = true;
5121 break;
5122 case 233: /* GUI IDLE */
5123 DRM_DEBUG("IH: GUI idle\n");
5124 break;
5125 case 244: /* DMA trap event */
5126 if (rdev->family >= CHIP_CAYMAN) {
5127 DRM_DEBUG("IH: DMA1 trap\n");
5128 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5129 }
5130 break;
5131 default:
5132 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5133 break;
5134 }
5135
5136 /* wptr/rptr are in bytes! */
5137 rptr += 16;
5138 rptr &= rdev->ih.ptr_mask;
5139 }
5140 if (queue_hotplug)
5141 schedule_work(&rdev->hotplug_work);
5142 if (queue_hdmi)
5143 schedule_work(&rdev->audio_work);
5144 if (queue_thermal && rdev->pm.dpm_enabled)
5145 schedule_work(&rdev->pm.dpm.thermal.work);
5146 rdev->ih.rptr = rptr;
5147 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5148 atomic_set(&rdev->ih.lock, 0);
5149
5150 /* make sure wptr hasn't changed while processing */
5151 wptr = evergreen_get_ih_wptr(rdev);
5152 if (wptr != rptr)
5153 goto restart_ih;
5154
5155 return IRQ_HANDLED;
5156 }
5157
5158 static int evergreen_startup(struct radeon_device *rdev)
5159 {
5160 struct radeon_ring *ring;
5161 int r;
5162
5163 /* enable pcie gen2 link */
5164 evergreen_pcie_gen2_enable(rdev);
5165 /* enable aspm */
5166 evergreen_program_aspm(rdev);
5167
5168 /* scratch needs to be initialized before MC */
5169 r = r600_vram_scratch_init(rdev);
5170 if (r)
5171 return r;
5172
5173 evergreen_mc_program(rdev);
5174
5175 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5176 r = ni_mc_load_microcode(rdev);
5177 if (r) {
5178 DRM_ERROR("Failed to load MC firmware!\n");
5179 return r;
5180 }
5181 }
5182
5183 if (rdev->flags & RADEON_IS_AGP) {
5184 evergreen_agp_enable(rdev);
5185 } else {
5186 r = evergreen_pcie_gart_enable(rdev);
5187 if (r)
5188 return r;
5189 }
5190 evergreen_gpu_init(rdev);
5191
5192 /* allocate rlc buffers */
5193 if (rdev->flags & RADEON_IS_IGP) {
5194 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5195 rdev->rlc.reg_list_size =
5196 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5197 rdev->rlc.cs_data = evergreen_cs_data;
5198 r = sumo_rlc_init(rdev);
5199 if (r) {
5200 DRM_ERROR("Failed to init rlc BOs!\n");
5201 return r;
5202 }
5203 }
5204
5205 /* allocate wb buffer */
5206 r = radeon_wb_init(rdev);
5207 if (r)
5208 return r;
5209
5210 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5211 if (r) {
5212 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5213 return r;
5214 }
5215
5216 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5217 if (r) {
5218 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5219 return r;
5220 }
5221
5222 r = uvd_v2_2_resume(rdev);
5223 if (!r) {
5224 r = radeon_fence_driver_start_ring(rdev,
5225 R600_RING_TYPE_UVD_INDEX);
5226 if (r)
5227 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5228 }
5229
5230 if (r)
5231 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5232
5233 /* Enable IRQ */
5234 if (!rdev->irq.installed) {
5235 r = radeon_irq_kms_init(rdev);
5236 if (r)
5237 return r;
5238 }
5239
5240 r = r600_irq_init(rdev);
5241 if (r) {
5242 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5243 radeon_irq_kms_fini(rdev);
5244 return r;
5245 }
5246 evergreen_irq_set(rdev);
5247
5248 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5249 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5250 RADEON_CP_PACKET2);
5251 if (r)
5252 return r;
5253
5254 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5255 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5256 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5257 if (r)
5258 return r;
5259
5260 r = evergreen_cp_load_microcode(rdev);
5261 if (r)
5262 return r;
5263 r = evergreen_cp_resume(rdev);
5264 if (r)
5265 return r;
5266 r = r600_dma_resume(rdev);
5267 if (r)
5268 return r;
5269
5270 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5271 if (ring->ring_size) {
5272 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5273 RADEON_CP_PACKET2);
5274 if (!r)
5275 r = uvd_v1_0_init(rdev);
5276
5277 if (r)
5278 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5279 }
5280
5281 r = radeon_ib_pool_init(rdev);
5282 if (r) {
5283 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5284 return r;
5285 }
5286
5287 r = r600_audio_init(rdev);
5288 if (r) {
5289 DRM_ERROR("radeon: audio init failed\n");
5290 return r;
5291 }
5292
5293 return 0;
5294 }
5295
5296 int evergreen_resume(struct radeon_device *rdev)
5297 {
5298 int r;
5299
5300 /* reset the asic, the gfx blocks are often in a bad state
5301 * after the driver is unloaded or after a resume
5302 */
5303 if (radeon_asic_reset(rdev))
5304 dev_warn(rdev->dev, "GPU reset failed !\n");
5305 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5306 * posting will perform necessary task to bring back GPU into good
5307 * shape.
5308 */
5309 /* post card */
5310 atom_asic_init(rdev->mode_info.atom_context);
5311
5312 /* init golden registers */
5313 evergreen_init_golden_registers(rdev);
5314
5315 if (rdev->pm.pm_method == PM_METHOD_DPM)
5316 radeon_pm_resume(rdev);
5317
5318 rdev->accel_working = true;
5319 r = evergreen_startup(rdev);
5320 if (r) {
5321 DRM_ERROR("evergreen startup failed on resume\n");
5322 rdev->accel_working = false;
5323 return r;
5324 }
5325
5326 return r;
5327
5328 }
5329
5330 int evergreen_suspend(struct radeon_device *rdev)
5331 {
5332 radeon_pm_suspend(rdev);
5333 r600_audio_fini(rdev);
5334 uvd_v1_0_fini(rdev);
5335 radeon_uvd_suspend(rdev);
5336 r700_cp_stop(rdev);
5337 r600_dma_stop(rdev);
5338 evergreen_irq_suspend(rdev);
5339 radeon_wb_disable(rdev);
5340 evergreen_pcie_gart_disable(rdev);
5341
5342 return 0;
5343 }
5344
5345 /* Plan is to move initialization in that function and use
5346 * helper function so that radeon_device_init pretty much
5347 * do nothing more than calling asic specific function. This
5348 * should also allow to remove a bunch of callback function
5349 * like vram_info.
5350 */
5351 int evergreen_init(struct radeon_device *rdev)
5352 {
5353 int r;
5354
5355 /* Read BIOS */
5356 if (!radeon_get_bios(rdev)) {
5357 if (ASIC_IS_AVIVO(rdev))
5358 return -EINVAL;
5359 }
5360 /* Must be an ATOMBIOS */
5361 if (!rdev->is_atom_bios) {
5362 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5363 return -EINVAL;
5364 }
5365 r = radeon_atombios_init(rdev);
5366 if (r)
5367 return r;
5368 /* reset the asic, the gfx blocks are often in a bad state
5369 * after the driver is unloaded or after a resume
5370 */
5371 if (radeon_asic_reset(rdev))
5372 dev_warn(rdev->dev, "GPU reset failed !\n");
5373 /* Post card if necessary */
5374 if (!radeon_card_posted(rdev)) {
5375 if (!rdev->bios) {
5376 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5377 return -EINVAL;
5378 }
5379 DRM_INFO("GPU not posted. posting now...\n");
5380 atom_asic_init(rdev->mode_info.atom_context);
5381 }
5382 /* init golden registers */
5383 evergreen_init_golden_registers(rdev);
5384 /* Initialize scratch registers */
5385 r600_scratch_init(rdev);
5386 /* Initialize surface registers */
5387 radeon_surface_init(rdev);
5388 /* Initialize clocks */
5389 radeon_get_clock_info(rdev->ddev);
5390 /* Fence driver */
5391 r = radeon_fence_driver_init(rdev);
5392 if (r)
5393 return r;
5394 /* initialize AGP */
5395 if (rdev->flags & RADEON_IS_AGP) {
5396 r = radeon_agp_init(rdev);
5397 if (r)
5398 radeon_agp_disable(rdev);
5399 }
5400 /* initialize memory controller */
5401 r = evergreen_mc_init(rdev);
5402 if (r)
5403 return r;
5404 /* Memory manager */
5405 r = radeon_bo_init(rdev);
5406 if (r)
5407 return r;
5408
5409 if (ASIC_IS_DCE5(rdev)) {
5410 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5411 r = ni_init_microcode(rdev);
5412 if (r) {
5413 DRM_ERROR("Failed to load firmware!\n");
5414 return r;
5415 }
5416 }
5417 } else {
5418 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5419 r = r600_init_microcode(rdev);
5420 if (r) {
5421 DRM_ERROR("Failed to load firmware!\n");
5422 return r;
5423 }
5424 }
5425 }
5426
5427 /* Initialize power management */
5428 radeon_pm_init(rdev);
5429
5430 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5431 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5432
5433 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5434 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5435
5436 r = radeon_uvd_init(rdev);
5437 if (!r) {
5438 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5439 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5440 4096);
5441 }
5442
5443 rdev->ih.ring_obj = NULL;
5444 r600_ih_ring_init(rdev, 64 * 1024);
5445
5446 r = r600_pcie_gart_init(rdev);
5447 if (r)
5448 return r;
5449
5450 rdev->accel_working = true;
5451 r = evergreen_startup(rdev);
5452 if (r) {
5453 dev_err(rdev->dev, "disabling GPU acceleration\n");
5454 r700_cp_fini(rdev);
5455 r600_dma_fini(rdev);
5456 r600_irq_fini(rdev);
5457 if (rdev->flags & RADEON_IS_IGP)
5458 sumo_rlc_fini(rdev);
5459 radeon_wb_fini(rdev);
5460 radeon_ib_pool_fini(rdev);
5461 radeon_irq_kms_fini(rdev);
5462 evergreen_pcie_gart_fini(rdev);
5463 rdev->accel_working = false;
5464 }
5465
5466 /* Don't start up if the MC ucode is missing on BTC parts.
5467 * The default clocks and voltages before the MC ucode
5468 * is loaded are not suffient for advanced operations.
5469 */
5470 if (ASIC_IS_DCE5(rdev)) {
5471 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5472 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5473 return -EINVAL;
5474 }
5475 }
5476
5477 return 0;
5478 }
5479
5480 void evergreen_fini(struct radeon_device *rdev)
5481 {
5482 radeon_pm_fini(rdev);
5483 r600_audio_fini(rdev);
5484 r700_cp_fini(rdev);
5485 r600_dma_fini(rdev);
5486 r600_irq_fini(rdev);
5487 if (rdev->flags & RADEON_IS_IGP)
5488 sumo_rlc_fini(rdev);
5489 radeon_wb_fini(rdev);
5490 radeon_ib_pool_fini(rdev);
5491 radeon_irq_kms_fini(rdev);
5492 uvd_v1_0_fini(rdev);
5493 radeon_uvd_fini(rdev);
5494 evergreen_pcie_gart_fini(rdev);
5495 r600_vram_scratch_fini(rdev);
5496 radeon_gem_fini(rdev);
5497 radeon_fence_driver_fini(rdev);
5498 radeon_agp_fini(rdev);
5499 radeon_bo_fini(rdev);
5500 radeon_atombios_fini(rdev);
5501 kfree(rdev->bios);
5502 rdev->bios = NULL;
5503 }
5504
5505 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5506 {
5507 u32 link_width_cntl, speed_cntl;
5508
5509 if (radeon_pcie_gen2 == 0)
5510 return;
5511
5512 if (rdev->flags & RADEON_IS_IGP)
5513 return;
5514
5515 if (!(rdev->flags & RADEON_IS_PCIE))
5516 return;
5517
5518 /* x2 cards have a special sequence */
5519 if (ASIC_IS_X2(rdev))
5520 return;
5521
5522 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5523 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5524 return;
5525
5526 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5527 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5528 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5529 return;
5530 }
5531
5532 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5533
5534 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5535 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5536
5537 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5538 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5539 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5540
5541 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5542 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5543 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5544
5545 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5546 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5547 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5548
5549 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5550 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5551 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5552
5553 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5554 speed_cntl |= LC_GEN2_EN_STRAP;
5555 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5556
5557 } else {
5558 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5559 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5560 if (1)
5561 link_width_cntl |= LC_UPCONFIGURE_DIS;
5562 else
5563 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5564 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5565 }
5566 }
5567
5568 void evergreen_program_aspm(struct radeon_device *rdev)
5569 {
5570 u32 data, orig;
5571 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5572 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5573 /* fusion_platform = true
5574 * if the system is a fusion system
5575 * (APU or DGPU in a fusion system).
5576 * todo: check if the system is a fusion platform.
5577 */
5578 bool fusion_platform = false;
5579
5580 if (radeon_aspm == 0)
5581 return;
5582
5583 if (!(rdev->flags & RADEON_IS_PCIE))
5584 return;
5585
5586 switch (rdev->family) {
5587 case CHIP_CYPRESS:
5588 case CHIP_HEMLOCK:
5589 case CHIP_JUNIPER:
5590 case CHIP_REDWOOD:
5591 case CHIP_CEDAR:
5592 case CHIP_SUMO:
5593 case CHIP_SUMO2:
5594 case CHIP_PALM:
5595 case CHIP_ARUBA:
5596 disable_l0s = true;
5597 break;
5598 default:
5599 disable_l0s = false;
5600 break;
5601 }
5602
5603 if (rdev->flags & RADEON_IS_IGP)
5604 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5605
5606 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5607 if (fusion_platform)
5608 data &= ~MULTI_PIF;
5609 else
5610 data |= MULTI_PIF;
5611 if (data != orig)
5612 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5613
5614 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5615 if (fusion_platform)
5616 data &= ~MULTI_PIF;
5617 else
5618 data |= MULTI_PIF;
5619 if (data != orig)
5620 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5621
5622 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5623 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5624 if (!disable_l0s) {
5625 if (rdev->family >= CHIP_BARTS)
5626 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5627 else
5628 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5629 }
5630
5631 if (!disable_l1) {
5632 if (rdev->family >= CHIP_BARTS)
5633 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5634 else
5635 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5636
5637 if (!disable_plloff_in_l1) {
5638 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5639 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5640 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5641 if (data != orig)
5642 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5643
5644 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5645 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5646 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5647 if (data != orig)
5648 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5649
5650 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5651 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5652 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5653 if (data != orig)
5654 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5655
5656 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5657 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5658 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5659 if (data != orig)
5660 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5661
5662 if (rdev->family >= CHIP_BARTS) {
5663 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5664 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5665 data |= PLL_RAMP_UP_TIME_0(4);
5666 if (data != orig)
5667 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5668
5669 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5670 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5671 data |= PLL_RAMP_UP_TIME_1(4);
5672 if (data != orig)
5673 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5674
5675 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5676 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5677 data |= PLL_RAMP_UP_TIME_0(4);
5678 if (data != orig)
5679 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5680
5681 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5682 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5683 data |= PLL_RAMP_UP_TIME_1(4);
5684 if (data != orig)
5685 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5686 }
5687
5688 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5689 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5690 data |= LC_DYN_LANES_PWR_STATE(3);
5691 if (data != orig)
5692 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5693
5694 if (rdev->family >= CHIP_BARTS) {
5695 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5696 data &= ~LS2_EXIT_TIME_MASK;
5697 data |= LS2_EXIT_TIME(1);
5698 if (data != orig)
5699 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5700
5701 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5702 data &= ~LS2_EXIT_TIME_MASK;
5703 data |= LS2_EXIT_TIME(1);
5704 if (data != orig)
5705 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5706 }
5707 }
5708 }
5709
5710 /* evergreen parts only */
5711 if (rdev->family < CHIP_BARTS)
5712 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5713
5714 if (pcie_lc_cntl != pcie_lc_cntl_old)
5715 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5716 }
(deprecated) Btrfs devel tree