2 * Copyright (c) 2007, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
19 * Alan Cox <alan@linux.intel.com>
27 * GTT resource allocator - manage page mappings in GTT space
31 * psb_gtt_mask_pte - generate GTT pte entry
32 * @pfn: page number to encode
33 * @type: type of memory in the GTT
35 * Set the GTT entry for the appropriate memory type.
37 static inline uint32_t psb_gtt_mask_pte(uint32_t pfn
, int type
)
39 uint32_t mask
= PSB_PTE_VALID
;
41 if (type
& PSB_MMU_CACHED_MEMORY
)
42 mask
|= PSB_PTE_CACHED
;
43 if (type
& PSB_MMU_RO_MEMORY
)
45 if (type
& PSB_MMU_WO_MEMORY
)
48 return (pfn
<< PAGE_SHIFT
) | mask
;
52 * psb_gtt_entry - find the GTT entries for a gtt_range
53 * @dev: our DRM device
56 * Given a gtt_range object return the GTT offset of the page table
57 * entries for this gtt_range
59 u32
*psb_gtt_entry(struct drm_device
*dev
, struct gtt_range
*r
)
61 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
64 offset
= r
->resource
.start
- dev_priv
->gtt_mem
->start
;
66 return dev_priv
->gtt_map
+ (offset
>> PAGE_SHIFT
);
70 * psb_gtt_insert - put an object into the GTT
71 * @dev: our DRM device
74 * Take our preallocated GTT range and insert the GEM object into
79 static int psb_gtt_insert(struct drm_device
*dev
, struct gtt_range
*r
)
85 if (r
->pages
== NULL
) {
90 WARN_ON(r
->stolen
); /* refcount these maybe ? */
92 gtt_slot
= psb_gtt_entry(dev
, r
);
95 /* Make sure changes are visible to the GPU */
96 set_pages_array_uc(pages
, r
->npage
);
98 /* Write our page entries into the GTT itself */
99 for (i
= 0; i
< r
->npage
; i
++) {
100 pte
= psb_gtt_mask_pte(page_to_pfn(*pages
++), 0/*type*/);
101 iowrite32(pte
, gtt_slot
++);
103 /* Make sure all the entries are set before we return */
104 ioread32(gtt_slot
- 1);
110 * psb_gtt_remove - remove an object from the GTT
111 * @dev: our DRM device
114 * Remove a preallocated GTT range from the GTT. Overwrite all the
115 * page table entries with the dummy page
118 static void psb_gtt_remove(struct drm_device
*dev
, struct gtt_range
*r
)
120 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
126 gtt_slot
= psb_gtt_entry(dev
, r
);
127 pte
= psb_gtt_mask_pte(page_to_pfn(dev_priv
->scratch_page
), 0);
129 for (i
= 0; i
< r
->npage
; i
++)
130 iowrite32(pte
, gtt_slot
++);
131 ioread32(gtt_slot
- 1);
132 set_pages_array_wb(r
->pages
, r
->npage
);
136 * psb_gtt_attach_pages - attach and pin GEM pages
139 * Pin and build an in kernel list of the pages that back our GEM object.
140 * While we hold this the pages cannot be swapped out
142 static int psb_gtt_attach_pages(struct gtt_range
*gt
)
145 struct address_space
*mapping
;
148 int pages
= gt
->gem
.size
/ PAGE_SIZE
;
152 /* This is the shared memory object that backs the GEM resource */
153 inode
= gt
->gem
.filp
->f_path
.dentry
->d_inode
;
154 mapping
= inode
->i_mapping
;
156 gt
->pages
= kmalloc(pages
* sizeof(struct page
*), GFP_KERNEL
);
157 if (gt
->pages
== NULL
)
161 for (i
= 0; i
< pages
; i
++) {
162 /* FIXME: review flags later */
163 p
= read_cache_page_gfp(mapping
, i
,
164 __GFP_COLD
| GFP_KERNEL
);
173 page_cache_release(gt
->pages
[i
]);
180 * psb_gtt_detach_pages - attach and pin GEM pages
183 * Undo the effect of psb_gtt_attach_pages. At this point the pages
184 * must have been removed from the GTT as they could now be paged out
185 * and move bus address.
187 static void psb_gtt_detach_pages(struct gtt_range
*gt
)
190 for (i
= 0; i
< gt
->npage
; i
++) {
191 /* FIXME: do we need to force dirty */
192 set_page_dirty(gt
->pages
[i
]);
193 page_cache_release(gt
->pages
[i
]);
200 * psb_gtt_pin - pin pages into the GTT
203 * Pin a set of pages into the GTT. The pins are refcounted so that
204 * multiple pins need multiple unpins to undo.
206 * Non GEM backed objects treat this as a no-op as they are always GTT
209 int psb_gtt_pin(struct gtt_range
*gt
)
212 struct drm_device
*dev
= gt
->gem
.dev
;
213 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
215 mutex_lock(&dev_priv
->gtt_mutex
);
217 if (gt
->in_gart
== 0 && gt
->stolen
== 0) {
218 ret
= psb_gtt_attach_pages(gt
);
221 ret
= psb_gtt_insert(dev
, gt
);
223 psb_gtt_detach_pages(gt
);
229 mutex_unlock(&dev_priv
->gtt_mutex
);
234 * psb_gtt_unpin - Drop a GTT pin requirement
237 * Undoes the effect of psb_gtt_pin. On the last drop the GEM object
238 * will be removed from the GTT which will also drop the page references
239 * and allow the VM to clean up or page stuff.
241 * Non GEM backed objects treat this as a no-op as they are always GTT
244 void psb_gtt_unpin(struct gtt_range
*gt
)
246 struct drm_device
*dev
= gt
->gem
.dev
;
247 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
249 mutex_lock(&dev_priv
->gtt_mutex
);
251 WARN_ON(!gt
->in_gart
);
254 if (gt
->in_gart
== 0 && gt
->stolen
== 0) {
255 psb_gtt_remove(dev
, gt
);
256 psb_gtt_detach_pages(gt
);
258 mutex_unlock(&dev_priv
->gtt_mutex
);
262 * GTT resource allocator - allocate and manage GTT address space
266 * psb_gtt_alloc_range - allocate GTT address space
267 * @dev: Our DRM device
268 * @len: length (bytes) of address space required
269 * @name: resource name
270 * @backed: resource should be backed by stolen pages
272 * Ask the kernel core to find us a suitable range of addresses
273 * to use for a GTT mapping.
275 * Returns a gtt_range structure describing the object, or NULL on
276 * error. On successful return the resource is both allocated and marked
279 struct gtt_range
*psb_gtt_alloc_range(struct drm_device
*dev
, int len
,
280 const char *name
, int backed
)
282 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
283 struct gtt_range
*gt
;
284 struct resource
*r
= dev_priv
->gtt_mem
;
286 unsigned long start
, end
;
289 /* The start of the GTT is the stolen pages */
291 end
= r
->start
+ dev_priv
->pg
->stolen_size
- 1;
293 /* The rest we will use for GEM backed objects */
294 start
= r
->start
+ dev_priv
->pg
->stolen_size
;
298 gt
= kzalloc(sizeof(struct gtt_range
), GFP_KERNEL
);
301 gt
->resource
.name
= name
;
303 gt
->in_gart
= backed
;
304 /* Ensure this is set for non GEM objects */
306 kref_init(>
->kref
);
308 ret
= allocate_resource(dev_priv
->gtt_mem
, >
->resource
,
309 len
, start
, end
, PAGE_SIZE
, NULL
, NULL
);
311 gt
->offset
= gt
->resource
.start
- r
->start
;
319 * psb_gtt_destroy - final free up of a gtt
320 * @kref: the kref of the gtt
322 * Called from the kernel kref put when the final reference to our
323 * GTT object is dropped. At that point we can free up the resources.
325 * For now we handle mmap clean up here to work around limits in GEM
327 static void psb_gtt_destroy(struct kref
*kref
)
329 struct gtt_range
*gt
= container_of(kref
, struct gtt_range
, kref
);
331 /* Undo the mmap pin if we are destroying the object */
336 WARN_ON(gt
->in_gart
&& !gt
->stolen
);
337 release_resource(>
->resource
);
342 * psb_gtt_kref_put - drop reference to a GTT object
343 * @gt: the GT being dropped
345 * Drop a reference to a psb gtt
347 void psb_gtt_kref_put(struct gtt_range
*gt
)
349 kref_put(>
->kref
, psb_gtt_destroy
);
353 * psb_gtt_free_range - release GTT address space
354 * @dev: our DRM device
355 * @gt: a mapping created with psb_gtt_alloc_range
357 * Release a resource that was allocated with psb_gtt_alloc_range
359 void psb_gtt_free_range(struct drm_device
*dev
, struct gtt_range
*gt
)
361 psb_gtt_kref_put(gt
);
365 struct psb_gtt
*psb_gtt_alloc(struct drm_device
*dev
)
367 struct psb_gtt
*tmp
= kzalloc(sizeof(*tmp
), GFP_KERNEL
);
372 init_rwsem(&tmp
->sem
);
378 void psb_gtt_takedown(struct drm_device
*dev
)
380 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
382 if (dev_priv
->gtt_map
) {
383 iounmap(dev_priv
->gtt_map
);
384 dev_priv
->gtt_map
= NULL
;
386 if (dev_priv
->gtt_initialized
) {
387 pci_write_config_word(dev
->pdev
, PSB_GMCH_CTRL
,
388 dev_priv
->gmch_ctrl
);
389 PSB_WVDC32(dev_priv
->pge_ctl
, PSB_PGETBL_CTL
);
390 (void) PSB_RVDC32(PSB_PGETBL_CTL
);
392 if (dev_priv
->vram_addr
)
393 iounmap(dev_priv
->gtt_map
);
398 int psb_gtt_init(struct drm_device
*dev
, int resume
)
400 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
402 unsigned long stolen_size
, vram_stolen_size
;
403 unsigned i
, num_pages
;
406 uint32_t dvmt_mode
= 0;
412 mutex_init(&dev_priv
->gtt_mutex
);
414 dev_priv
->pg
= pg
= psb_gtt_alloc(dev
);
419 pci_read_config_word(dev
->pdev
, PSB_GMCH_CTRL
, &dev_priv
->gmch_ctrl
);
420 pci_write_config_word(dev
->pdev
, PSB_GMCH_CTRL
,
421 dev_priv
->gmch_ctrl
| _PSB_GMCH_ENABLED
);
423 dev_priv
->pge_ctl
= PSB_RVDC32(PSB_PGETBL_CTL
);
424 PSB_WVDC32(dev_priv
->pge_ctl
| _PSB_PGETBL_ENABLED
, PSB_PGETBL_CTL
);
425 (void) PSB_RVDC32(PSB_PGETBL_CTL
);
427 /* The root resource we allocate address space from */
428 dev_priv
->gtt_mem
= &dev
->pdev
->resource
[PSB_GATT_RESOURCE
];
429 dev_priv
->gtt_initialized
= 1;
431 pg
->gtt_phys_start
= dev_priv
->pge_ctl
& PAGE_MASK
;
433 pg
->gatt_start
= pci_resource_start(dev
->pdev
, PSB_GATT_RESOURCE
);
435 * FIXME: video mmu has hw bug to access 0x0D0000000,
436 * then make gatt start at 0x0e000,0000
438 pg
->mmu_gatt_start
= 0xE0000000;
440 pg
->gtt_start
= pci_resource_start(dev
->pdev
, PSB_GTT_RESOURCE
);
441 gtt_pages
= pci_resource_len(dev
->pdev
, PSB_GTT_RESOURCE
) >> PAGE_SHIFT
;
442 pg
->gatt_pages
= pci_resource_len(dev
->pdev
, PSB_GATT_RESOURCE
) >> PAGE_SHIFT
;
444 pci_read_config_dword(dev
->pdev
, PSB_BSM
, &dev_priv
->stolen_base
);
445 vram_stolen_size
= pg
->gtt_phys_start
- dev_priv
->stolen_base
- PAGE_SIZE
;
447 stolen_size
= vram_stolen_size
;
449 printk(KERN_INFO
"Stolen memory information\n");
450 printk(KERN_INFO
" base in RAM: 0x%x\n", dev_priv
->stolen_base
);
451 printk(KERN_INFO
" size: %luK, calculated by (GTT RAM base) - (Stolen base), seems wrong\n",
452 vram_stolen_size
/1024);
453 dvmt_mode
= (dev_priv
->gmch_ctrl
>> 4) & 0x7;
454 printk(KERN_INFO
" the correct size should be: %dM(dvmt mode=%d)\n",
455 (dvmt_mode
== 1) ? 1 : (2 << (dvmt_mode
- 1)), dvmt_mode
);
457 if (resume
&& (gtt_pages
!= pg
->gtt_pages
) &&
458 (stolen_size
!= pg
->stolen_size
)) {
459 dev_err(dev
->dev
, "GTT resume error.\n");
464 pg
->gtt_pages
= gtt_pages
;
465 pg
->stolen_size
= stolen_size
;
466 dev_priv
->vram_stolen_size
= vram_stolen_size
;
469 * Map the GTT and the stolen memory area
471 dev_priv
->gtt_map
= ioremap_nocache(pg
->gtt_phys_start
, gtt_pages
<< PAGE_SHIFT
);
472 if (!dev_priv
->gtt_map
) {
473 dev_err(dev
->dev
, "Failure to map gtt.\n");
478 dev_priv
->vram_addr
= ioremap_wc(dev_priv
->stolen_base
, stolen_size
);
479 if (!dev_priv
->vram_addr
) {
480 dev_err(dev
->dev
, "Failure to map stolen base.\n");
486 * Insert vram stolen pages into the GTT
489 pfn_base
= dev_priv
->stolen_base
>> PAGE_SHIFT
;
490 vram_pages
= num_pages
= vram_stolen_size
>> PAGE_SHIFT
;
491 printk(KERN_INFO
"Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
492 num_pages
, pfn_base
, 0);
493 for (i
= 0; i
< num_pages
; ++i
) {
494 pte
= psb_gtt_mask_pte(pfn_base
+ i
, 0);
495 iowrite32(pte
, dev_priv
->gtt_map
+ i
);
499 * Init rest of GTT to the scratch page to avoid accidents or scribbles
502 pfn_base
= page_to_pfn(dev_priv
->scratch_page
);
503 pte
= psb_gtt_mask_pte(pfn_base
, 0);
504 for (; i
< gtt_pages
; ++i
)
505 iowrite32(pte
, dev_priv
->gtt_map
+ i
);
507 (void) ioread32(dev_priv
->gtt_map
+ i
- 1);
511 psb_gtt_takedown(dev
);