2 * Copyright (c) 2007, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
19 * Alan Cox <alan@linux.intel.com>
27 * GTT resource allocator - manage page mappings in GTT space
31 * psb_gtt_mask_pte - generate GTT pte entry
32 * @pfn: page number to encode
33 * @type: type of memory in the GTT
35 * Set the GTT entry for the appropriate memory type.
37 static inline uint32_t psb_gtt_mask_pte(uint32_t pfn
, int type
)
39 uint32_t mask
= PSB_PTE_VALID
;
41 if (type
& PSB_MMU_CACHED_MEMORY
)
42 mask
|= PSB_PTE_CACHED
;
43 if (type
& PSB_MMU_RO_MEMORY
)
45 if (type
& PSB_MMU_WO_MEMORY
)
48 return (pfn
<< PAGE_SHIFT
) | mask
;
52 * psb_gtt_entry - find the GTT entries for a gtt_range
53 * @dev: our DRM device
56 * Given a gtt_range object return the GTT offset of the page table
57 * entries for this gtt_range
59 u32
*psb_gtt_entry(struct drm_device
*dev
, struct gtt_range
*r
)
61 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
64 offset
= r
->resource
.start
- dev_priv
->gtt_mem
->start
;
66 return dev_priv
->gtt_map
+ (offset
>> PAGE_SHIFT
);
70 * psb_gtt_insert - put an object into the GTT
71 * @dev: our DRM device
74 * Take our preallocated GTT range and insert the GEM object into
79 static int psb_gtt_insert(struct drm_device
*dev
, struct gtt_range
*r
)
85 if (r
->pages
== NULL
) {
90 WARN_ON(r
->stolen
); /* refcount these maybe ? */
92 gtt_slot
= psb_gtt_entry(dev
, r
);
95 /* Make sure changes are visible to the GPU */
96 set_pages_array_uc(pages
, r
->npage
);
98 /* Write our page entries into the GTT itself */
99 for (i
= 0; i
< r
->npage
; i
++) {
100 pte
= psb_gtt_mask_pte(page_to_pfn(*pages
++), 0/*type*/);
101 iowrite32(pte
, gtt_slot
++);
103 /* Make sure all the entries are set before we return */
104 ioread32(gtt_slot
- 1);
109 * psb_gtt_remove - remove an object from the GTT
110 * @dev: our DRM device
113 * Remove a preallocated GTT range from the GTT. Overwrite all the
114 * page table entries with the dummy page
117 static void psb_gtt_remove(struct drm_device
*dev
, struct gtt_range
*r
)
119 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
125 gtt_slot
= psb_gtt_entry(dev
, r
);
126 pte
= psb_gtt_mask_pte(page_to_pfn(dev_priv
->scratch_page
), 0);
128 for (i
= 0; i
< r
->npage
; i
++)
129 iowrite32(pte
, gtt_slot
++);
130 ioread32(gtt_slot
- 1);
131 set_pages_array_wb(r
->pages
, r
->npage
);
135 * psb_gtt_attach_pages - attach and pin GEM pages
138 * Pin and build an in kernel list of the pages that back our GEM object.
139 * While we hold this the pages cannot be swapped out
141 static int psb_gtt_attach_pages(struct gtt_range
*gt
)
144 struct address_space
*mapping
;
147 int pages
= gt
->gem
.size
/ PAGE_SIZE
;
151 /* This is the shared memory object that backs the GEM resource */
152 inode
= gt
->gem
.filp
->f_path
.dentry
->d_inode
;
153 mapping
= inode
->i_mapping
;
155 gt
->pages
= kmalloc(pages
* sizeof(struct page
*), GFP_KERNEL
);
156 if (gt
->pages
== NULL
)
160 for (i
= 0; i
< pages
; i
++) {
161 /* FIXME: review flags later */
162 p
= read_cache_page_gfp(mapping
, i
,
163 __GFP_COLD
| GFP_KERNEL
);
172 page_cache_release(gt
->pages
[i
]);
179 * psb_gtt_detach_pages - attach and pin GEM pages
182 * Undo the effect of psb_gtt_attach_pages. At this point the pages
183 * must have been removed from the GTT as they could now be paged out
184 * and move bus address.
186 static void psb_gtt_detach_pages(struct gtt_range
*gt
)
189 for (i
= 0; i
< gt
->npage
; i
++) {
190 /* FIXME: do we need to force dirty */
191 set_page_dirty(gt
->pages
[i
]);
192 page_cache_release(gt
->pages
[i
]);
199 * psb_gtt_pin - pin pages into the GTT
202 * Pin a set of pages into the GTT. The pins are refcounted so that
203 * multiple pins need multiple unpins to undo.
205 * Non GEM backed objects treat this as a no-op as they are always GTT
208 int psb_gtt_pin(struct gtt_range
*gt
)
211 struct drm_device
*dev
= gt
->gem
.dev
;
212 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
214 mutex_lock(&dev_priv
->gtt_mutex
);
216 if (gt
->in_gart
== 0 && gt
->stolen
== 0) {
217 ret
= psb_gtt_attach_pages(gt
);
220 ret
= psb_gtt_insert(dev
, gt
);
222 psb_gtt_detach_pages(gt
);
228 mutex_unlock(&dev_priv
->gtt_mutex
);
233 * psb_gtt_unpin - Drop a GTT pin requirement
236 * Undoes the effect of psb_gtt_pin. On the last drop the GEM object
237 * will be removed from the GTT which will also drop the page references
238 * and allow the VM to clean up or page stuff.
240 * Non GEM backed objects treat this as a no-op as they are always GTT
243 void psb_gtt_unpin(struct gtt_range
*gt
)
245 struct drm_device
*dev
= gt
->gem
.dev
;
246 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
248 mutex_lock(&dev_priv
->gtt_mutex
);
250 WARN_ON(!gt
->in_gart
);
253 if (gt
->in_gart
== 0 && gt
->stolen
== 0) {
254 psb_gtt_remove(dev
, gt
);
255 psb_gtt_detach_pages(gt
);
257 mutex_unlock(&dev_priv
->gtt_mutex
);
261 * GTT resource allocator - allocate and manage GTT address space
265 * psb_gtt_alloc_range - allocate GTT address space
266 * @dev: Our DRM device
267 * @len: length (bytes) of address space required
268 * @name: resource name
269 * @backed: resource should be backed by stolen pages
271 * Ask the kernel core to find us a suitable range of addresses
272 * to use for a GTT mapping.
274 * Returns a gtt_range structure describing the object, or NULL on
275 * error. On successful return the resource is both allocated and marked
278 struct gtt_range
*psb_gtt_alloc_range(struct drm_device
*dev
, int len
,
279 const char *name
, int backed
)
281 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
282 struct gtt_range
*gt
;
283 struct resource
*r
= dev_priv
->gtt_mem
;
285 unsigned long start
, end
;
288 /* The start of the GTT is the stolen pages */
290 end
= r
->start
+ dev_priv
->pg
->stolen_size
- 1;
292 /* The rest we will use for GEM backed objects */
293 start
= r
->start
+ dev_priv
->pg
->stolen_size
;
297 gt
= kzalloc(sizeof(struct gtt_range
), GFP_KERNEL
);
300 gt
->resource
.name
= name
;
302 gt
->in_gart
= backed
;
303 /* Ensure this is set for non GEM objects */
305 ret
= allocate_resource(dev_priv
->gtt_mem
, >
->resource
,
306 len
, start
, end
, PAGE_SIZE
, NULL
, NULL
);
308 gt
->offset
= gt
->resource
.start
- r
->start
;
316 * psb_gtt_free_range - release GTT address space
317 * @dev: our DRM device
318 * @gt: a mapping created with psb_gtt_alloc_range
320 * Release a resource that was allocated with psb_gtt_alloc_range. If the
321 * object has been pinned by mmap users we clean this up here currently.
323 void psb_gtt_free_range(struct drm_device
*dev
, struct gtt_range
*gt
)
325 /* Undo the mmap pin if we are destroying the object */
330 WARN_ON(gt
->in_gart
&& !gt
->stolen
);
331 release_resource(>
->resource
);
335 struct psb_gtt
*psb_gtt_alloc(struct drm_device
*dev
)
337 struct psb_gtt
*tmp
= kzalloc(sizeof(*tmp
), GFP_KERNEL
);
342 init_rwsem(&tmp
->sem
);
348 void psb_gtt_takedown(struct drm_device
*dev
)
350 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
352 if (dev_priv
->gtt_map
) {
353 iounmap(dev_priv
->gtt_map
);
354 dev_priv
->gtt_map
= NULL
;
356 if (dev_priv
->gtt_initialized
) {
357 pci_write_config_word(dev
->pdev
, PSB_GMCH_CTRL
,
358 dev_priv
->gmch_ctrl
);
359 PSB_WVDC32(dev_priv
->pge_ctl
, PSB_PGETBL_CTL
);
360 (void) PSB_RVDC32(PSB_PGETBL_CTL
);
362 if (dev_priv
->vram_addr
)
363 iounmap(dev_priv
->gtt_map
);
368 int psb_gtt_init(struct drm_device
*dev
, int resume
)
370 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
372 unsigned long stolen_size
, vram_stolen_size
;
373 unsigned i
, num_pages
;
376 uint32_t dvmt_mode
= 0;
382 mutex_init(&dev_priv
->gtt_mutex
);
384 dev_priv
->pg
= pg
= psb_gtt_alloc(dev
);
389 pci_read_config_word(dev
->pdev
, PSB_GMCH_CTRL
, &dev_priv
->gmch_ctrl
);
390 pci_write_config_word(dev
->pdev
, PSB_GMCH_CTRL
,
391 dev_priv
->gmch_ctrl
| _PSB_GMCH_ENABLED
);
393 dev_priv
->pge_ctl
= PSB_RVDC32(PSB_PGETBL_CTL
);
394 PSB_WVDC32(dev_priv
->pge_ctl
| _PSB_PGETBL_ENABLED
, PSB_PGETBL_CTL
);
395 (void) PSB_RVDC32(PSB_PGETBL_CTL
);
397 /* The root resource we allocate address space from */
398 dev_priv
->gtt_initialized
= 1;
400 pg
->gtt_phys_start
= dev_priv
->pge_ctl
& PAGE_MASK
;
403 * FIXME: video mmu has hw bug to access 0x0D0000000,
404 * then make gatt start at 0x0e000,0000
406 pg
->mmu_gatt_start
= 0xE0000000;
408 pg
->gtt_start
= pci_resource_start(dev
->pdev
, PSB_GTT_RESOURCE
);
409 gtt_pages
= pci_resource_len(dev
->pdev
, PSB_GTT_RESOURCE
)
412 if (pg
->gtt_start
== 0 || gtt_pages
== 0) {
413 dev_err(dev
->dev
, "GTT PCI BAR not initialized.\n");
415 pg
->gtt_start
= dev_priv
->pge_ctl
;
418 pg
->gatt_start
= pci_resource_start(dev
->pdev
, PSB_GATT_RESOURCE
);
419 pg
->gatt_pages
= pci_resource_len(dev
->pdev
, PSB_GATT_RESOURCE
)
421 dev_priv
->gtt_mem
= &dev
->pdev
->resource
[PSB_GATT_RESOURCE
];
423 if (pg
->gatt_pages
== 0 || pg
->gatt_start
== 0) {
424 static struct resource fudge
; /* Preferably peppermint */
426 /* This can occur on CDV SDV systems. Fudge it in this case.
427 We really don't care what imaginary space is being allocated
429 dev_err(dev
->dev
, "GATT PCI BAR not initialized.\n");
430 pg
->gatt_start
= 0x40000000;
431 pg
->gatt_pages
= (128 * 1024 * 1024) >> PAGE_SHIFT
;
432 fudge
.start
= 0x40000000;
433 fudge
.end
= 0x40000000 + 128 * 1024 * 1024 - 1;
434 fudge
.name
= "fudge";
435 fudge
.flags
= IORESOURCE_MEM
;
436 dev_priv
->gtt_mem
= &fudge
;
439 pci_read_config_dword(dev
->pdev
, PSB_BSM
, &dev_priv
->stolen_base
);
440 vram_stolen_size
= pg
->gtt_phys_start
- dev_priv
->stolen_base
443 stolen_size
= vram_stolen_size
;
445 printk(KERN_INFO
"Stolen memory information\n");
446 printk(KERN_INFO
" base in RAM: 0x%x\n", dev_priv
->stolen_base
);
447 printk(KERN_INFO
" size: %luK, calculated by (GTT RAM base) - (Stolen base), seems wrong\n",
448 vram_stolen_size
/1024);
449 dvmt_mode
= (dev_priv
->gmch_ctrl
>> 4) & 0x7;
450 printk(KERN_INFO
" the correct size should be: %dM(dvmt mode=%d)\n",
451 (dvmt_mode
== 1) ? 1 : (2 << (dvmt_mode
- 1)), dvmt_mode
);
453 if (resume
&& (gtt_pages
!= pg
->gtt_pages
) &&
454 (stolen_size
!= pg
->stolen_size
)) {
455 dev_err(dev
->dev
, "GTT resume error.\n");
460 pg
->gtt_pages
= gtt_pages
;
461 pg
->stolen_size
= stolen_size
;
462 dev_priv
->vram_stolen_size
= vram_stolen_size
;
465 * Map the GTT and the stolen memory area
467 dev_priv
->gtt_map
= ioremap_nocache(pg
->gtt_phys_start
,
468 gtt_pages
<< PAGE_SHIFT
);
469 if (!dev_priv
->gtt_map
) {
470 dev_err(dev
->dev
, "Failure to map gtt.\n");
475 dev_priv
->vram_addr
= ioremap_wc(dev_priv
->stolen_base
, stolen_size
);
476 if (!dev_priv
->vram_addr
) {
477 dev_err(dev
->dev
, "Failure to map stolen base.\n");
483 * Insert vram stolen pages into the GTT
486 pfn_base
= dev_priv
->stolen_base
>> PAGE_SHIFT
;
487 vram_pages
= num_pages
= vram_stolen_size
>> PAGE_SHIFT
;
488 printk(KERN_INFO
"Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
489 num_pages
, pfn_base
<< PAGE_SHIFT
, 0);
490 for (i
= 0; i
< num_pages
; ++i
) {
491 pte
= psb_gtt_mask_pte(pfn_base
+ i
, 0);
492 iowrite32(pte
, dev_priv
->gtt_map
+ i
);
496 * Init rest of GTT to the scratch page to avoid accidents or scribbles
499 pfn_base
= page_to_pfn(dev_priv
->scratch_page
);
500 pte
= psb_gtt_mask_pte(pfn_base
, 0);
501 for (; i
< gtt_pages
; ++i
)
502 iowrite32(pte
, dev_priv
->gtt_map
+ i
);
504 (void) ioread32(dev_priv
->gtt_map
+ i
- 1);
508 psb_gtt_takedown(dev
);