2 * Copyright (C) 2007 Ben Skeggs.
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 #include "nouveau_drv.h"
32 struct nv50_instmem_priv
{
33 uint32_t save1700
[5]; /* 0x1700->0x1710 */
35 struct nouveau_gpuobj_ref
*pramin_pt
;
36 struct nouveau_gpuobj_ref
*pramin_bar
;
37 struct nouveau_gpuobj_ref
*fb_bar
;
42 #define NV50_INSTMEM_PAGE_SHIFT 12
43 #define NV50_INSTMEM_PAGE_SIZE (1 << NV50_INSTMEM_PAGE_SHIFT)
44 #define NV50_INSTMEM_PT_SIZE(a) (((a) >> 12) << 3)
46 /*NOTE: - Assumes 0x1700 already covers the correct MiB of PRAMIN
48 #define BAR0_WI32(g, o, v) do { \
50 if ((g)->im_backing) { \
51 offset = (g)->im_backing_start; \
53 offset = chan->ramin->gpuobj->im_backing_start; \
54 offset += (g)->im_pramin->start; \
57 nv_wr32(dev, NV_RAMIN + (offset & 0xfffff), (v)); \
61 nv50_instmem_init(struct drm_device
*dev
)
63 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
64 struct nouveau_channel
*chan
;
65 uint32_t c_offset
, c_size
, c_ramfc
, c_vmpd
, c_base
, pt_size
;
66 uint32_t save_nv001700
;
68 struct nv50_instmem_priv
*priv
;
71 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
74 dev_priv
->engine
.instmem
.priv
= priv
;
76 /* Save state, will restore at takedown. */
77 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
78 priv
->save1700
[(i
-0x1700)/4] = nv_rd32(dev
, i
);
80 /* Reserve the last MiB of VRAM, we should probably try to avoid
81 * setting up the below tables over the top of the VBIOS image at
84 dev_priv
->ramin_rsvd_vram
= 1 << 20;
85 c_offset
= dev_priv
->vram_size
- dev_priv
->ramin_rsvd_vram
;
87 c_vmpd
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x1400 : 0x200;
88 c_ramfc
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x0 : 0x20;
89 c_base
= c_vmpd
+ 0x4000;
90 pt_size
= NV50_INSTMEM_PT_SIZE(dev_priv
->ramin_size
);
92 NV_DEBUG(dev
, " Rsvd VRAM base: 0x%08x\n", c_offset
);
93 NV_DEBUG(dev
, " VBIOS image: 0x%08x\n",
94 (nv_rd32(dev
, 0x619f04) & ~0xff) << 8);
95 NV_DEBUG(dev
, " Aperture size: %d MiB\n", dev_priv
->ramin_size
>> 20);
96 NV_DEBUG(dev
, " PT size: %d KiB\n", pt_size
>> 10);
98 /* Determine VM layout, we need to do this first to make sure
99 * we allocate enough memory for all the page tables.
101 dev_priv
->vm_gart_base
= roundup(NV50_VM_BLOCK
, NV50_VM_BLOCK
);
102 dev_priv
->vm_gart_size
= NV50_VM_BLOCK
;
104 dev_priv
->vm_vram_base
= dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
;
105 dev_priv
->vm_vram_size
= dev_priv
->vram_size
;
106 if (dev_priv
->vm_vram_size
> NV50_VM_MAX_VRAM
)
107 dev_priv
->vm_vram_size
= NV50_VM_MAX_VRAM
;
108 dev_priv
->vm_vram_size
= roundup(dev_priv
->vm_vram_size
, NV50_VM_BLOCK
);
109 dev_priv
->vm_vram_pt_nr
= dev_priv
->vm_vram_size
/ NV50_VM_BLOCK
;
111 dev_priv
->vm_end
= dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
;
113 NV_DEBUG(dev
, "NV50VM: GART 0x%016llx-0x%016llx\n",
114 dev_priv
->vm_gart_base
,
115 dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
- 1);
116 NV_DEBUG(dev
, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
117 dev_priv
->vm_vram_base
,
118 dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
- 1);
120 c_size
+= dev_priv
->vm_vram_pt_nr
* (NV50_VM_BLOCK
/ 65536 * 8);
122 /* Map BAR0 PRAMIN aperture over the memory we want to use */
123 save_nv001700
= nv_rd32(dev
, NV50_PUNK_BAR0_PRAMIN
);
124 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (c_offset
>> 16));
126 /* Create a fake channel, and use it as our "dummy" channels 0/127.
127 * The main reason for creating a channel is so we can use the gpuobj
128 * code. However, it's probably worth noting that NVIDIA also setup
129 * their channels 0/127 with the same values they configure here.
130 * So, there may be some other reason for doing this.
132 * Have to create the entire channel manually, as the real channel
133 * creation code assumes we have PRAMIN access, and we don't until
136 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
141 chan
->file_priv
= (struct drm_file
*)-2;
142 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = chan
;
144 /* Channel's PRAMIN object + heap */
145 ret
= nouveau_gpuobj_new_fake(dev
, 0, c_offset
, c_size
, 0,
150 if (drm_mm_init(&chan
->ramin_heap
, c_base
, c_size
- c_base
))
153 /* RAMFC + zero channel's PRAMIN up to start of VM pagedir */
154 ret
= nouveau_gpuobj_new_fake(dev
, c_ramfc
, c_offset
+ c_ramfc
,
155 0x4000, 0, NULL
, &chan
->ramfc
);
159 for (i
= 0; i
< c_vmpd
; i
+= 4)
160 BAR0_WI32(chan
->ramin
->gpuobj
, i
, 0);
162 /* VM page directory */
163 ret
= nouveau_gpuobj_new_fake(dev
, c_vmpd
, c_offset
+ c_vmpd
,
164 0x4000, 0, &chan
->vm_pd
, NULL
);
167 for (i
= 0; i
< 0x4000; i
+= 8) {
168 BAR0_WI32(chan
->vm_pd
, i
+ 0x00, 0x00000000);
169 BAR0_WI32(chan
->vm_pd
, i
+ 0x04, 0x00000000);
172 /* PRAMIN page table, cheat and map into VM at 0x0000000000.
173 * We map the entire fake channel into the start of the PRAMIN BAR
175 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0, pt_size
, 0x1000,
176 0, &priv
->pramin_pt
);
181 if (dev_priv
->vram_sys_base
) {
182 v
+= dev_priv
->vram_sys_base
;
187 while (v
< dev_priv
->vram_sys_base
+ c_offset
+ c_size
) {
188 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, lower_32_bits(v
));
189 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, upper_32_bits(v
));
194 while (i
< pt_size
) {
195 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, 0x00000000);
196 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, 0x00000000);
200 BAR0_WI32(chan
->vm_pd
, 0x00, priv
->pramin_pt
->instance
| 0x63);
201 BAR0_WI32(chan
->vm_pd
, 0x04, 0x00000000);
203 /* VRAM page table(s), mapped into VM at +1GiB */
204 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
205 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0,
206 NV50_VM_BLOCK
/65536*8, 0, 0,
207 &chan
->vm_vram_pt
[i
]);
209 NV_ERROR(dev
, "Error creating VRAM page tables: %d\n",
211 dev_priv
->vm_vram_pt_nr
= i
;
214 dev_priv
->vm_vram_pt
[i
] = chan
->vm_vram_pt
[i
]->gpuobj
;
216 for (v
= 0; v
< dev_priv
->vm_vram_pt
[i
]->im_pramin
->size
;
218 BAR0_WI32(dev_priv
->vm_vram_pt
[i
], v
, 0);
220 BAR0_WI32(chan
->vm_pd
, 0x10 + (i
*8),
221 chan
->vm_vram_pt
[i
]->instance
| 0x61);
222 BAR0_WI32(chan
->vm_pd
, 0x14 + (i
*8), 0);
225 /* DMA object for PRAMIN BAR */
226 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
230 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x00, 0x7fc00000);
231 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x04, dev_priv
->ramin_size
- 1);
232 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x08, 0x00000000);
233 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x0c, 0x00000000);
234 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x10, 0x00000000);
235 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x14, 0x00000000);
237 /* DMA object for FB BAR */
238 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
242 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x00, 0x7fc00000);
243 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x04, 0x40000000 +
244 pci_resource_len(dev
->pdev
, 1) - 1);
245 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x08, 0x40000000);
246 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x0c, 0x00000000);
247 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x10, 0x00000000);
248 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x14, 0x00000000);
250 /* Poke the relevant regs, and pray it works :) */
251 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
252 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
253 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
254 NV50_PUNK_BAR_CFG_BASE_VALID
);
255 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
256 NV50_PUNK_BAR1_CTXDMA_VALID
);
257 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
258 NV50_PUNK_BAR3_CTXDMA_VALID
);
260 for (i
= 0; i
< 8; i
++)
261 nv_wr32(dev
, 0x1900 + (i
*4), 0);
263 /* Assume that praying isn't enough, check that we can re-read the
264 * entire fake channel back from the PRAMIN BAR */
265 dev_priv
->engine
.instmem
.prepare_access(dev
, false);
266 for (i
= 0; i
< c_size
; i
+= 4) {
267 if (nv_rd32(dev
, NV_RAMIN
+ i
) != nv_ri32(dev
, i
)) {
268 NV_ERROR(dev
, "Error reading back PRAMIN at 0x%08x\n",
270 dev_priv
->engine
.instmem
.finish_access(dev
);
274 dev_priv
->engine
.instmem
.finish_access(dev
);
276 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, save_nv001700
);
278 /* Global PRAMIN heap */
279 if (drm_mm_init(&dev_priv
->ramin_heap
, c_size
, dev_priv
->ramin_size
- c_size
)) {
280 NV_ERROR(dev
, "Failed to init RAMIN heap\n");
283 /*XXX: incorrect, but needed to make hash func "work" */
284 dev_priv
->ramht_offset
= 0x10000;
285 dev_priv
->ramht_bits
= 9;
286 dev_priv
->ramht_size
= (1 << dev_priv
->ramht_bits
);
291 nv50_instmem_takedown(struct drm_device
*dev
)
293 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
294 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
295 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
303 /* Restore state from before init */
304 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
305 nv_wr32(dev
, i
, priv
->save1700
[(i
- 0x1700) / 4]);
307 nouveau_gpuobj_ref_del(dev
, &priv
->fb_bar
);
308 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_bar
);
309 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_pt
);
311 /* Destroy dummy channel */
313 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
314 nouveau_gpuobj_ref_del(dev
, &chan
->vm_vram_pt
[i
]);
315 dev_priv
->vm_vram_pt
[i
] = NULL
;
317 dev_priv
->vm_vram_pt_nr
= 0;
319 nouveau_gpuobj_del(dev
, &chan
->vm_pd
);
320 nouveau_gpuobj_ref_del(dev
, &chan
->ramfc
);
321 nouveau_gpuobj_ref_del(dev
, &chan
->ramin
);
322 drm_mm_takedown(&chan
->ramin_heap
);
324 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = NULL
;
328 dev_priv
->engine
.instmem
.priv
= NULL
;
333 nv50_instmem_suspend(struct drm_device
*dev
)
335 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
336 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
337 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
340 ramin
->im_backing_suspend
= vmalloc(ramin
->im_pramin
->size
);
341 if (!ramin
->im_backing_suspend
)
344 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
345 ramin
->im_backing_suspend
[i
/4] = nv_ri32(dev
, i
);
350 nv50_instmem_resume(struct drm_device
*dev
)
352 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
353 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
354 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
355 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
358 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (ramin
->im_backing_start
>> 16));
359 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
360 BAR0_WI32(ramin
, i
, ramin
->im_backing_suspend
[i
/4]);
361 vfree(ramin
->im_backing_suspend
);
362 ramin
->im_backing_suspend
= NULL
;
364 /* Poke the relevant regs, and pray it works :) */
365 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
366 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
367 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
368 NV50_PUNK_BAR_CFG_BASE_VALID
);
369 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
370 NV50_PUNK_BAR1_CTXDMA_VALID
);
371 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
372 NV50_PUNK_BAR3_CTXDMA_VALID
);
374 for (i
= 0; i
< 8; i
++)
375 nv_wr32(dev
, 0x1900 + (i
*4), 0);
379 nv50_instmem_populate(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
,
384 if (gpuobj
->im_backing
)
387 *sz
= ALIGN(*sz
, NV50_INSTMEM_PAGE_SIZE
);
391 ret
= nouveau_bo_new(dev
, NULL
, *sz
, 0, TTM_PL_FLAG_VRAM
, 0, 0x0000,
392 true, false, &gpuobj
->im_backing
);
394 NV_ERROR(dev
, "error getting PRAMIN backing pages: %d\n", ret
);
398 ret
= nouveau_bo_pin(gpuobj
->im_backing
, TTM_PL_FLAG_VRAM
);
400 NV_ERROR(dev
, "error pinning PRAMIN backing VRAM: %d\n", ret
);
401 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
405 gpuobj
->im_backing_start
= gpuobj
->im_backing
->bo
.mem
.mm_node
->start
;
406 gpuobj
->im_backing_start
<<= PAGE_SHIFT
;
412 nv50_instmem_clear(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
414 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
416 if (gpuobj
&& gpuobj
->im_backing
) {
417 if (gpuobj
->im_bound
)
418 dev_priv
->engine
.instmem
.unbind(dev
, gpuobj
);
419 nouveau_bo_unpin(gpuobj
->im_backing
);
420 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
421 gpuobj
->im_backing
= NULL
;
426 nv50_instmem_bind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
428 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
429 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
430 struct nouveau_gpuobj
*pramin_pt
= priv
->pramin_pt
->gpuobj
;
431 uint32_t pte
, pte_end
;
434 if (!gpuobj
->im_backing
|| !gpuobj
->im_pramin
|| gpuobj
->im_bound
)
437 NV_DEBUG(dev
, "st=0x%lx sz=0x%lx\n",
438 gpuobj
->im_pramin
->start
, gpuobj
->im_pramin
->size
);
440 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
441 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
442 vram
= gpuobj
->im_backing_start
;
444 NV_DEBUG(dev
, "pramin=0x%lx, pte=%d, pte_end=%d\n",
445 gpuobj
->im_pramin
->start
, pte
, pte_end
);
446 NV_DEBUG(dev
, "first vram page: 0x%08x\n", gpuobj
->im_backing_start
);
449 if (dev_priv
->vram_sys_base
) {
450 vram
+= dev_priv
->vram_sys_base
;
454 dev_priv
->engine
.instmem
.prepare_access(dev
, true);
455 while (pte
< pte_end
) {
456 nv_wo32(dev
, pramin_pt
, pte
++, lower_32_bits(vram
));
457 nv_wo32(dev
, pramin_pt
, pte
++, upper_32_bits(vram
));
458 vram
+= NV50_INSTMEM_PAGE_SIZE
;
460 dev_priv
->engine
.instmem
.finish_access(dev
);
462 nv_wr32(dev
, 0x100c80, 0x00040001);
463 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
464 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (1)\n");
465 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
469 nv_wr32(dev
, 0x100c80, 0x00060001);
470 if (!nv_wait(0x100c80, 0x00000001, 0x00000000)) {
471 NV_ERROR(dev
, "timeout: (0x100c80 & 1) == 0 (2)\n");
472 NV_ERROR(dev
, "0x100c80 = 0x%08x\n", nv_rd32(dev
, 0x100c80));
476 gpuobj
->im_bound
= 1;
481 nv50_instmem_unbind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
483 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
484 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
485 uint32_t pte
, pte_end
;
487 if (gpuobj
->im_bound
== 0)
490 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
491 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
493 dev_priv
->engine
.instmem
.prepare_access(dev
, true);
494 while (pte
< pte_end
) {
495 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
496 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
498 dev_priv
->engine
.instmem
.finish_access(dev
);
500 gpuobj
->im_bound
= 0;
505 nv50_instmem_prepare_access(struct drm_device
*dev
, bool write
)
507 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
508 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
510 priv
->last_access_wr
= write
;
514 nv50_instmem_finish_access(struct drm_device
*dev
)
516 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
517 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
519 if (priv
->last_access_wr
) {
520 nv_wr32(dev
, 0x070000, 0x00000001);
521 if (!nv_wait(0x070000, 0x00000001, 0x00000000))
522 NV_ERROR(dev
, "PRAMIN flush timeout\n");