2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
6 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
7 * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
9 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 #include <linux/init.h>
28 #include <linux/types.h>
29 #include <linux/slab.h>
31 #include <linux/spinlock.h>
32 #include <linux/string.h>
33 #include <linux/pci.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/crash_dump.h>
39 #include <asm/iommu.h>
40 #include <asm/pci-bridge.h>
41 #include <asm/machdep.h>
42 #include <asm/abs_addr.h>
43 #include <asm/pSeries_reconfig.h>
44 #include <asm/firmware.h>
46 #include <asm/ppc-pci.h>
49 #include "plpar_wrappers.h"
52 static int tce_build_pSeries(struct iommu_table
*tbl
, long index
,
53 long npages
, unsigned long uaddr
,
54 enum dma_data_direction direction
,
55 struct dma_attrs
*attrs
)
61 proto_tce
= TCE_PCI_READ
; // Read allowed
63 if (direction
!= DMA_TO_DEVICE
)
64 proto_tce
|= TCE_PCI_WRITE
;
66 tcep
= ((u64
*)tbl
->it_base
) + index
;
69 /* can't move this out since we might cross LMB boundary */
70 rpn
= (virt_to_abs(uaddr
)) >> TCE_SHIFT
;
71 *tcep
= proto_tce
| (rpn
& TCE_RPN_MASK
) << TCE_RPN_SHIFT
;
73 uaddr
+= TCE_PAGE_SIZE
;
80 static void tce_free_pSeries(struct iommu_table
*tbl
, long index
, long npages
)
84 tcep
= ((u64
*)tbl
->it_base
) + index
;
90 static unsigned long tce_get_pseries(struct iommu_table
*tbl
, long index
)
94 tcep
= ((u64
*)tbl
->it_base
) + index
;
99 static void tce_free_pSeriesLP(struct iommu_table
*, long, long);
100 static void tce_freemulti_pSeriesLP(struct iommu_table
*, long, long);
102 static int tce_build_pSeriesLP(struct iommu_table
*tbl
, long tcenum
,
103 long npages
, unsigned long uaddr
,
104 enum dma_data_direction direction
,
105 struct dma_attrs
*attrs
)
111 long tcenum_start
= tcenum
, npages_start
= npages
;
113 rpn
= (virt_to_abs(uaddr
)) >> TCE_SHIFT
;
114 proto_tce
= TCE_PCI_READ
;
115 if (direction
!= DMA_TO_DEVICE
)
116 proto_tce
|= TCE_PCI_WRITE
;
119 tce
= proto_tce
| (rpn
& TCE_RPN_MASK
) << TCE_RPN_SHIFT
;
120 rc
= plpar_tce_put((u64
)tbl
->it_index
, (u64
)tcenum
<< 12, tce
);
122 if (unlikely(rc
== H_NOT_ENOUGH_RESOURCES
)) {
124 tce_free_pSeriesLP(tbl
, tcenum_start
,
125 (npages_start
- (npages
+ 1)));
129 if (rc
&& printk_ratelimit()) {
130 printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc
);
131 printk("\tindex = 0x%llx\n", (u64
)tbl
->it_index
);
132 printk("\ttcenum = 0x%llx\n", (u64
)tcenum
);
133 printk("\ttce val = 0x%llx\n", tce
);
134 show_stack(current
, (unsigned long *)__get_SP());
143 static DEFINE_PER_CPU_LOCKED(u64
*, tce_page
) = NULL
;
145 static int tce_buildmulti_pSeriesLP(struct iommu_table
*tbl
, long tcenum
,
146 long npages
, unsigned long uaddr
,
147 enum dma_data_direction direction
,
148 struct dma_attrs
*attrs
)
155 long tcenum_start
= tcenum
, npages_start
= npages
;
160 return tce_build_pSeriesLP(tbl
, tcenum
, npages
, uaddr
,
164 tcep
= get_cpu_var_locked(tce_page
, &cpu
);
166 /* This is safe to do since interrupts are off when we're called
167 * from iommu_alloc{,_sg}()
170 tcep
= (u64
*)__get_free_page(GFP_ATOMIC
);
171 /* If allocation fails, fall back to the loop implementation */
173 put_cpu_var_locked(tce_page
, cpu
);
174 return tce_build_pSeriesLP(tbl
, tcenum
, npages
, uaddr
,
177 per_cpu_var_locked(tce_page
, cpu
) = tcep
;
180 rpn
= (virt_to_abs(uaddr
)) >> TCE_SHIFT
;
181 proto_tce
= TCE_PCI_READ
;
182 if (direction
!= DMA_TO_DEVICE
)
183 proto_tce
|= TCE_PCI_WRITE
;
185 /* We can map max one pageful of TCEs at a time */
188 * Set up the page with TCE data, looping through and setting
191 limit
= min_t(long, npages
, 4096/TCE_ENTRY_SIZE
);
193 for (l
= 0; l
< limit
; l
++) {
194 tcep
[l
] = proto_tce
| (rpn
& TCE_RPN_MASK
) << TCE_RPN_SHIFT
;
198 rc
= plpar_tce_put_indirect((u64
)tbl
->it_index
,
200 (u64
)virt_to_abs(tcep
),
205 } while (npages
> 0 && !rc
);
207 if (unlikely(rc
== H_NOT_ENOUGH_RESOURCES
)) {
209 tce_freemulti_pSeriesLP(tbl
, tcenum_start
,
210 (npages_start
- (npages
+ limit
)));
214 if (rc
&& printk_ratelimit()) {
215 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc
);
216 printk("\tindex = 0x%llx\n", (u64
)tbl
->it_index
);
217 printk("\tnpages = 0x%llx\n", (u64
)npages
);
218 printk("\ttce[0] val = 0x%llx\n", tcep
[0]);
219 show_stack(current
, (unsigned long *)__get_SP());
221 put_cpu_var_locked(tce_page
, cpu
);
225 static void tce_free_pSeriesLP(struct iommu_table
*tbl
, long tcenum
, long npages
)
230 rc
= plpar_tce_put((u64
)tbl
->it_index
, (u64
)tcenum
<< 12, 0);
232 if (rc
&& printk_ratelimit()) {
233 printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc
);
234 printk("\tindex = 0x%llx\n", (u64
)tbl
->it_index
);
235 printk("\ttcenum = 0x%llx\n", (u64
)tcenum
);
236 show_stack(current
, (unsigned long *)__get_SP());
244 static void tce_freemulti_pSeriesLP(struct iommu_table
*tbl
, long tcenum
, long npages
)
248 rc
= plpar_tce_stuff((u64
)tbl
->it_index
, (u64
)tcenum
<< 12, 0, npages
);
250 if (rc
&& printk_ratelimit()) {
251 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
252 printk("\trc = %lld\n", rc
);
253 printk("\tindex = 0x%llx\n", (u64
)tbl
->it_index
);
254 printk("\tnpages = 0x%llx\n", (u64
)npages
);
255 show_stack(current
, (unsigned long *)__get_SP());
259 static unsigned long tce_get_pSeriesLP(struct iommu_table
*tbl
, long tcenum
)
262 unsigned long tce_ret
;
264 rc
= plpar_tce_get((u64
)tbl
->it_index
, (u64
)tcenum
<< 12, &tce_ret
);
266 if (rc
&& printk_ratelimit()) {
267 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc
);
268 printk("\tindex = 0x%llx\n", (u64
)tbl
->it_index
);
269 printk("\ttcenum = 0x%llx\n", (u64
)tcenum
);
270 show_stack(current
, (unsigned long *)__get_SP());
277 static void iommu_table_setparms(struct pci_controller
*phb
,
278 struct device_node
*dn
,
279 struct iommu_table
*tbl
)
281 struct device_node
*node
;
282 const unsigned long *basep
;
287 basep
= of_get_property(node
, "linux,tce-base", NULL
);
288 sizep
= of_get_property(node
, "linux,tce-size", NULL
);
289 if (basep
== NULL
|| sizep
== NULL
) {
290 printk(KERN_ERR
"PCI_DMA: iommu_table_setparms: %s has "
291 "missing tce entries !\n", dn
->full_name
);
295 tbl
->it_base
= (unsigned long)__va(*basep
);
297 if (!is_kdump_kernel())
298 memset((void *)tbl
->it_base
, 0, *sizep
);
300 tbl
->it_busno
= phb
->bus
->number
;
302 /* Units of tce entries */
303 tbl
->it_offset
= phb
->dma_window_base_cur
>> IOMMU_PAGE_SHIFT
;
305 /* Test if we are going over 2GB of DMA space */
306 if (phb
->dma_window_base_cur
+ phb
->dma_window_size
> 0x80000000ul
) {
307 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
308 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
311 phb
->dma_window_base_cur
+= phb
->dma_window_size
;
313 /* Set the tce table size - measured in entries */
314 tbl
->it_size
= phb
->dma_window_size
>> IOMMU_PAGE_SHIFT
;
317 tbl
->it_blocksize
= 16;
318 tbl
->it_type
= TCE_PCI
;
322 * iommu_table_setparms_lpar
324 * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
326 static void iommu_table_setparms_lpar(struct pci_controller
*phb
,
327 struct device_node
*dn
,
328 struct iommu_table
*tbl
,
329 const void *dma_window
,
332 unsigned long offset
, size
;
334 tbl
->it_busno
= bussubno
;
335 of_parse_dma_window(dn
, dma_window
, &tbl
->it_index
, &offset
, &size
);
338 tbl
->it_blocksize
= 16;
339 tbl
->it_type
= TCE_PCI
;
340 tbl
->it_offset
= offset
>> IOMMU_PAGE_SHIFT
;
341 tbl
->it_size
= size
>> IOMMU_PAGE_SHIFT
;
344 static void pci_dma_bus_setup_pSeries(struct pci_bus
*bus
)
346 struct device_node
*dn
;
347 struct iommu_table
*tbl
;
348 struct device_node
*isa_dn
, *isa_dn_orig
;
349 struct device_node
*tmp
;
353 dn
= pci_bus_to_OF_node(bus
);
355 pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn
->full_name
);
358 /* This is not a root bus, any setup will be done for the
359 * device-side of the bridge in iommu_dev_setup_pSeries().
365 /* Check if the ISA bus on the system is under
368 isa_dn
= isa_dn_orig
= of_find_node_by_type(NULL
, "isa");
370 while (isa_dn
&& isa_dn
!= dn
)
371 isa_dn
= isa_dn
->parent
;
374 of_node_put(isa_dn_orig
);
376 /* Count number of direct PCI children of the PHB. */
377 for (children
= 0, tmp
= dn
->child
; tmp
; tmp
= tmp
->sibling
)
380 pr_debug("Children: %d\n", children
);
382 /* Calculate amount of DMA window per slot. Each window must be
383 * a power of two (due to pci_alloc_consistent requirements).
385 * Keep 256MB aside for PHBs with ISA.
389 /* No ISA/IDE - just set window size and return */
390 pci
->phb
->dma_window_size
= 0x80000000ul
; /* To be divided */
392 while (pci
->phb
->dma_window_size
* children
> 0x80000000ul
)
393 pci
->phb
->dma_window_size
>>= 1;
394 pr_debug("No ISA/IDE, window size is 0x%lx\n",
395 pci
->phb
->dma_window_size
);
396 pci
->phb
->dma_window_base_cur
= 0;
401 /* If we have ISA, then we probably have an IDE
402 * controller too. Allocate a 128MB table but
403 * skip the first 128MB to avoid stepping on ISA
406 pci
->phb
->dma_window_size
= 0x8000000ul
;
407 pci
->phb
->dma_window_base_cur
= 0x8000000ul
;
409 tbl
= kmalloc_node(sizeof(struct iommu_table
), GFP_KERNEL
,
412 iommu_table_setparms(pci
->phb
, dn
, tbl
);
413 pci
->iommu_table
= iommu_init_table(tbl
, pci
->phb
->node
);
415 /* Divide the rest (1.75GB) among the children */
416 pci
->phb
->dma_window_size
= 0x80000000ul
;
417 while (pci
->phb
->dma_window_size
* children
> 0x70000000ul
)
418 pci
->phb
->dma_window_size
>>= 1;
420 pr_debug("ISA/IDE, window size is 0x%lx\n", pci
->phb
->dma_window_size
);
424 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus
*bus
)
426 struct iommu_table
*tbl
;
427 struct device_node
*dn
, *pdn
;
429 const void *dma_window
= NULL
;
431 dn
= pci_bus_to_OF_node(bus
);
433 pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n",
436 /* Find nearest ibm,dma-window, walking up the device tree */
437 for (pdn
= dn
; pdn
!= NULL
; pdn
= pdn
->parent
) {
438 dma_window
= of_get_property(pdn
, "ibm,dma-window", NULL
);
439 if (dma_window
!= NULL
)
443 if (dma_window
== NULL
) {
444 pr_debug(" no ibm,dma-window property !\n");
450 pr_debug(" parent is %s, iommu_table: 0x%p\n",
451 pdn
->full_name
, ppci
->iommu_table
);
453 if (!ppci
->iommu_table
) {
454 tbl
= kmalloc_node(sizeof(struct iommu_table
), GFP_KERNEL
,
456 iommu_table_setparms_lpar(ppci
->phb
, pdn
, tbl
, dma_window
,
458 ppci
->iommu_table
= iommu_init_table(tbl
, ppci
->phb
->node
);
459 pr_debug(" created table: %p\n", ppci
->iommu_table
);
463 PCI_DN(dn
)->iommu_table
= ppci
->iommu_table
;
467 static void pci_dma_dev_setup_pSeries(struct pci_dev
*dev
)
469 struct device_node
*dn
;
470 struct iommu_table
*tbl
;
472 pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev
));
474 dn
= dev
->dev
.archdata
.of_node
;
476 /* If we're the direct child of a root bus, then we need to allocate
477 * an iommu table ourselves. The bus setup code should have setup
478 * the window sizes already.
480 if (!dev
->bus
->self
) {
481 struct pci_controller
*phb
= PCI_DN(dn
)->phb
;
483 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
484 tbl
= kmalloc_node(sizeof(struct iommu_table
), GFP_KERNEL
,
486 iommu_table_setparms(phb
, dn
, tbl
);
487 PCI_DN(dn
)->iommu_table
= iommu_init_table(tbl
, phb
->node
);
488 dev
->dev
.archdata
.dma_data
= PCI_DN(dn
)->iommu_table
;
492 /* If this device is further down the bus tree, search upwards until
493 * an already allocated iommu table is found and use that.
496 while (dn
&& PCI_DN(dn
) && PCI_DN(dn
)->iommu_table
== NULL
)
499 if (dn
&& PCI_DN(dn
))
500 dev
->dev
.archdata
.dma_data
= PCI_DN(dn
)->iommu_table
;
502 printk(KERN_WARNING
"iommu: Device %s has no iommu table\n",
506 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev
*dev
)
508 struct device_node
*pdn
, *dn
;
509 struct iommu_table
*tbl
;
510 const void *dma_window
= NULL
;
513 pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev
));
515 /* dev setup for LPAR is a little tricky, since the device tree might
516 * contain the dma-window properties per-device and not neccesarily
517 * for the bus. So we need to search upwards in the tree until we
518 * either hit a dma-window property, OR find a parent with a table
521 dn
= pci_device_to_OF_node(dev
);
522 pr_debug(" node is %s\n", dn
->full_name
);
524 for (pdn
= dn
; pdn
&& PCI_DN(pdn
) && !PCI_DN(pdn
)->iommu_table
;
526 dma_window
= of_get_property(pdn
, "ibm,dma-window", NULL
);
531 if (!pdn
|| !PCI_DN(pdn
)) {
532 printk(KERN_WARNING
"pci_dma_dev_setup_pSeriesLP: "
533 "no DMA window found for pci dev=%s dn=%s\n",
534 pci_name(dev
), dn
? dn
->full_name
: "<null>");
537 pr_debug(" parent is %s\n", pdn
->full_name
);
539 /* Check for parent == NULL so we don't try to setup the empty EADS
540 * slots on POWER4 machines.
542 if (dma_window
== NULL
|| pdn
->parent
== NULL
) {
543 pr_debug(" no dma window for device, linking to parent\n");
544 dev
->dev
.archdata
.dma_data
= PCI_DN(pdn
)->iommu_table
;
549 if (!pci
->iommu_table
) {
550 tbl
= kmalloc_node(sizeof(struct iommu_table
), GFP_KERNEL
,
552 iommu_table_setparms_lpar(pci
->phb
, pdn
, tbl
, dma_window
,
553 pci
->phb
->bus
->number
);
554 pci
->iommu_table
= iommu_init_table(tbl
, pci
->phb
->node
);
555 pr_debug(" created table: %p\n", pci
->iommu_table
);
557 pr_debug(" found DMA window, table: %p\n", pci
->iommu_table
);
560 dev
->dev
.archdata
.dma_data
= pci
->iommu_table
;
562 #else /* CONFIG_PCI */
563 #define pci_dma_bus_setup_pSeries NULL
564 #define pci_dma_dev_setup_pSeries NULL
565 #define pci_dma_bus_setup_pSeriesLP NULL
566 #define pci_dma_dev_setup_pSeriesLP NULL
567 #endif /* !CONFIG_PCI */
569 static int iommu_reconfig_notifier(struct notifier_block
*nb
, unsigned long action
, void *node
)
572 struct device_node
*np
= node
;
573 struct pci_dn
*pci
= PCI_DN(np
);
576 case PSERIES_RECONFIG_REMOVE
:
577 if (pci
&& pci
->iommu_table
&&
578 of_get_property(np
, "ibm,dma-window", NULL
))
579 iommu_free_table(pci
->iommu_table
, np
->full_name
);
588 static struct notifier_block iommu_reconfig_nb
= {
589 .notifier_call
= iommu_reconfig_notifier
,
592 /* These are called very early. */
593 void iommu_init_early_pSeries(void)
595 if (of_chosen
&& of_get_property(of_chosen
, "linux,iommu-off", NULL
)) {
596 /* Direct I/O, IOMMU off */
597 ppc_md
.pci_dma_dev_setup
= NULL
;
598 ppc_md
.pci_dma_bus_setup
= NULL
;
599 set_pci_dma_ops(&dma_direct_ops
);
603 if (firmware_has_feature(FW_FEATURE_LPAR
)) {
604 if (firmware_has_feature(FW_FEATURE_MULTITCE
)) {
605 ppc_md
.tce_build
= tce_buildmulti_pSeriesLP
;
606 ppc_md
.tce_free
= tce_freemulti_pSeriesLP
;
608 ppc_md
.tce_build
= tce_build_pSeriesLP
;
609 ppc_md
.tce_free
= tce_free_pSeriesLP
;
611 ppc_md
.tce_get
= tce_get_pSeriesLP
;
612 ppc_md
.pci_dma_bus_setup
= pci_dma_bus_setup_pSeriesLP
;
613 ppc_md
.pci_dma_dev_setup
= pci_dma_dev_setup_pSeriesLP
;
615 ppc_md
.tce_build
= tce_build_pSeries
;
616 ppc_md
.tce_free
= tce_free_pSeries
;
617 ppc_md
.tce_get
= tce_get_pseries
;
618 ppc_md
.pci_dma_bus_setup
= pci_dma_bus_setup_pSeries
;
619 ppc_md
.pci_dma_dev_setup
= pci_dma_dev_setup_pSeries
;
623 pSeries_reconfig_notifier_register(&iommu_reconfig_nb
);
625 set_pci_dma_ops(&dma_iommu_ops
);