2 * Derived from arch/powerpc/kernel/iommu.c
4 * Copyright (C) 2006 Jon Mason <jdmason@us.ibm.com>, IBM Corporation
5 * Copyright (C) 2006 Muli Ben-Yehuda <muli@il.ibm.com>, IBM Corporation
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #include <linux/config.h>
23 #include <linux/kernel.h>
24 #include <linux/init.h>
25 #include <linux/types.h>
26 #include <linux/slab.h>
28 #include <linux/spinlock.h>
29 #include <linux/string.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/init.h>
32 #include <linux/bitops.h>
33 #include <linux/pci_ids.h>
34 #include <linux/pci.h>
35 #include <linux/delay.h>
36 #include <asm/proto.h>
37 #include <asm/calgary.h>
39 #include <asm/pci-direct.h>
40 #include <asm/system.h>
43 #define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
44 #define PCI_VENDOR_DEVICE_ID_CALGARY \
45 (PCI_VENDOR_ID_IBM | PCI_DEVICE_ID_IBM_CALGARY << 16)
47 /* we need these for register space address calculation */
48 #define START_ADDRESS 0xfe000000
49 #define CHASSIS_BASE 0
50 #define ONE_BASED_CHASSIS_NUM 1
52 /* register offsets inside the host bridge space */
53 #define PHB_CSR_OFFSET 0x0110
54 #define PHB_PLSSR_OFFSET 0x0120
55 #define PHB_CONFIG_RW_OFFSET 0x0160
56 #define PHB_IOBASE_BAR_LOW 0x0170
57 #define PHB_IOBASE_BAR_HIGH 0x0180
58 #define PHB_MEM_1_LOW 0x0190
59 #define PHB_MEM_1_HIGH 0x01A0
60 #define PHB_IO_ADDR_SIZE 0x01B0
61 #define PHB_MEM_1_SIZE 0x01C0
62 #define PHB_MEM_ST_OFFSET 0x01D0
63 #define PHB_AER_OFFSET 0x0200
64 #define PHB_CONFIG_0_HIGH 0x0220
65 #define PHB_CONFIG_0_LOW 0x0230
66 #define PHB_CONFIG_0_END 0x0240
67 #define PHB_MEM_2_LOW 0x02B0
68 #define PHB_MEM_2_HIGH 0x02C0
69 #define PHB_MEM_2_SIZE_HIGH 0x02D0
70 #define PHB_MEM_2_SIZE_LOW 0x02E0
71 #define PHB_DOSHOLE_OFFSET 0x08E0
74 #define PHB_TCE_ENABLE 0x20000000
75 #define PHB_SLOT_DISABLE 0x1C000000
76 #define PHB_DAC_DISABLE 0x01000000
77 #define PHB_MEM2_ENABLE 0x00400000
78 #define PHB_MCSR_ENABLE 0x00100000
79 /* TAR (Table Address Register) */
80 #define TAR_SW_BITS 0x0000ffffffff800fUL
81 #define TAR_VALID 0x0000000000000008UL
82 /* CSR (Channel/DMA Status Register) */
83 #define CSR_AGENT_MASK 0xffe0ffff
85 #define MAX_NUM_OF_PHBS 8 /* how many PHBs in total? */
86 #define MAX_PHB_BUS_NUM (MAX_NUM_OF_PHBS * 2) /* max dev->bus->number */
87 #define PHBS_PER_CALGARY 4
89 /* register offsets in Calgary's internal register space */
90 static const unsigned long tar_offsets
[] = {
97 static const unsigned long split_queue_offsets
[] = {
98 0x4870 /* SPLIT QUEUE 0 */,
99 0x5870 /* SPLIT QUEUE 1 */,
100 0x6870 /* SPLIT QUEUE 2 */,
101 0x7870 /* SPLIT QUEUE 3 */
104 static const unsigned long phb_offsets
[] = {
111 void* tce_table_kva
[MAX_NUM_OF_PHBS
* MAX_NUMNODES
];
112 unsigned int specified_table_size
= TCE_TABLE_SIZE_UNSPECIFIED
;
113 static int translate_empty_slots __read_mostly
= 0;
114 static int calgary_detected __read_mostly
= 0;
117 * the bitmap of PHBs the user requested that we disable
120 static DECLARE_BITMAP(translation_disabled
, MAX_NUMNODES
* MAX_PHB_BUS_NUM
);
122 static void tce_cache_blast(struct iommu_table
*tbl
);
124 /* enable this to stress test the chip's TCE cache */
125 #ifdef CONFIG_IOMMU_DEBUG
126 static inline void tce_cache_blast_stress(struct iommu_table
*tbl
)
128 tce_cache_blast(tbl
);
131 static inline void tce_cache_blast_stress(struct iommu_table
*tbl
)
134 #endif /* BLAST_TCE_CACHE_ON_UNMAP */
136 static inline unsigned int num_dma_pages(unsigned long dma
, unsigned int dmalen
)
140 npages
= PAGE_ALIGN(dma
+ dmalen
) - (dma
& PAGE_MASK
);
141 npages
>>= PAGE_SHIFT
;
146 static inline int translate_phb(struct pci_dev
* dev
)
148 int disabled
= test_bit(dev
->bus
->number
, translation_disabled
);
152 static void iommu_range_reserve(struct iommu_table
*tbl
,
153 unsigned long start_addr
, unsigned int npages
)
158 index
= start_addr
>> PAGE_SHIFT
;
160 /* bail out if we're asked to reserve a region we don't cover */
161 if (index
>= tbl
->it_size
)
164 end
= index
+ npages
;
165 if (end
> tbl
->it_size
) /* don't go off the table */
168 while (index
< end
) {
169 if (test_bit(index
, tbl
->it_map
))
170 printk(KERN_ERR
"Calgary: entry already allocated at "
171 "0x%lx tbl %p dma 0x%lx npages %u\n",
172 index
, tbl
, start_addr
, npages
);
175 set_bit_string(tbl
->it_map
, start_addr
>> PAGE_SHIFT
, npages
);
178 static unsigned long iommu_range_alloc(struct iommu_table
*tbl
,
181 unsigned long offset
;
185 offset
= find_next_zero_string(tbl
->it_map
, tbl
->it_hint
,
186 tbl
->it_size
, npages
);
187 if (offset
== ~0UL) {
188 tce_cache_blast(tbl
);
189 offset
= find_next_zero_string(tbl
->it_map
, 0,
190 tbl
->it_size
, npages
);
191 if (offset
== ~0UL) {
192 printk(KERN_WARNING
"Calgary: IOMMU full.\n");
193 if (panic_on_overflow
)
194 panic("Calgary: fix the allocator.\n");
196 return bad_dma_address
;
200 set_bit_string(tbl
->it_map
, offset
, npages
);
201 tbl
->it_hint
= offset
+ npages
;
202 BUG_ON(tbl
->it_hint
> tbl
->it_size
);
207 static dma_addr_t
iommu_alloc(struct iommu_table
*tbl
, void *vaddr
,
208 unsigned int npages
, int direction
)
210 unsigned long entry
, flags
;
211 dma_addr_t ret
= bad_dma_address
;
213 spin_lock_irqsave(&tbl
->it_lock
, flags
);
215 entry
= iommu_range_alloc(tbl
, npages
);
217 if (unlikely(entry
== bad_dma_address
))
220 /* set the return dma address */
221 ret
= (entry
<< PAGE_SHIFT
) | ((unsigned long)vaddr
& ~PAGE_MASK
);
223 /* put the TCEs in the HW table */
224 tce_build(tbl
, entry
, npages
, (unsigned long)vaddr
& PAGE_MASK
,
227 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
232 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
233 printk(KERN_WARNING
"Calgary: failed to allocate %u pages in "
234 "iommu %p\n", npages
, tbl
);
235 return bad_dma_address
;
238 static void __iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
244 entry
= dma_addr
>> PAGE_SHIFT
;
246 BUG_ON(entry
+ npages
> tbl
->it_size
);
248 tce_free(tbl
, entry
, npages
);
250 for (i
= 0; i
< npages
; ++i
) {
251 if (!test_bit(entry
+ i
, tbl
->it_map
))
252 printk(KERN_ERR
"Calgary: bit is off at 0x%lx "
253 "tbl %p dma 0x%Lx entry 0x%lx npages %u\n",
254 entry
+ i
, tbl
, dma_addr
, entry
, npages
);
257 __clear_bit_string(tbl
->it_map
, entry
, npages
);
259 tce_cache_blast_stress(tbl
);
262 static void iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
267 spin_lock_irqsave(&tbl
->it_lock
, flags
);
269 __iommu_free(tbl
, dma_addr
, npages
);
271 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
274 static void __calgary_unmap_sg(struct iommu_table
*tbl
,
275 struct scatterlist
*sglist
, int nelems
, int direction
)
279 dma_addr_t dma
= sglist
->dma_address
;
280 unsigned int dmalen
= sglist
->dma_length
;
285 npages
= num_dma_pages(dma
, dmalen
);
286 __iommu_free(tbl
, dma
, npages
);
291 void calgary_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
292 int nelems
, int direction
)
295 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
297 if (!translate_phb(to_pci_dev(dev
)))
300 spin_lock_irqsave(&tbl
->it_lock
, flags
);
302 __calgary_unmap_sg(tbl
, sglist
, nelems
, direction
);
304 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
307 static int calgary_nontranslate_map_sg(struct device
* dev
,
308 struct scatterlist
*sg
, int nelems
, int direction
)
312 for (i
= 0; i
< nelems
; i
++ ) {
313 struct scatterlist
*s
= &sg
[i
];
315 s
->dma_address
= virt_to_bus(page_address(s
->page
) +s
->offset
);
316 s
->dma_length
= s
->length
;
321 int calgary_map_sg(struct device
*dev
, struct scatterlist
*sg
,
322 int nelems
, int direction
)
324 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
331 if (!translate_phb(to_pci_dev(dev
)))
332 return calgary_nontranslate_map_sg(dev
, sg
, nelems
, direction
);
334 spin_lock_irqsave(&tbl
->it_lock
, flags
);
336 for (i
= 0; i
< nelems
; i
++ ) {
337 struct scatterlist
*s
= &sg
[i
];
340 vaddr
= (unsigned long)page_address(s
->page
) + s
->offset
;
341 npages
= num_dma_pages(vaddr
, s
->length
);
343 entry
= iommu_range_alloc(tbl
, npages
);
344 if (entry
== bad_dma_address
) {
345 /* makes sure unmap knows to stop */
350 s
->dma_address
= (entry
<< PAGE_SHIFT
) | s
->offset
;
352 /* insert into HW table */
353 tce_build(tbl
, entry
, npages
, vaddr
& PAGE_MASK
,
356 s
->dma_length
= s
->length
;
359 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
363 __calgary_unmap_sg(tbl
, sg
, nelems
, direction
);
364 for (i
= 0; i
< nelems
; i
++) {
365 sg
[i
].dma_address
= bad_dma_address
;
366 sg
[i
].dma_length
= 0;
368 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
372 dma_addr_t
calgary_map_single(struct device
*dev
, void *vaddr
,
373 size_t size
, int direction
)
375 dma_addr_t dma_handle
= bad_dma_address
;
378 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
380 uaddr
= (unsigned long)vaddr
;
381 npages
= num_dma_pages(uaddr
, size
);
383 if (translate_phb(to_pci_dev(dev
)))
384 dma_handle
= iommu_alloc(tbl
, vaddr
, npages
, direction
);
386 dma_handle
= virt_to_bus(vaddr
);
391 void calgary_unmap_single(struct device
*dev
, dma_addr_t dma_handle
,
392 size_t size
, int direction
)
394 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
397 if (!translate_phb(to_pci_dev(dev
)))
400 npages
= num_dma_pages(dma_handle
, size
);
401 iommu_free(tbl
, dma_handle
, npages
);
404 void* calgary_alloc_coherent(struct device
*dev
, size_t size
,
405 dma_addr_t
*dma_handle
, gfp_t flag
)
409 unsigned int npages
, order
;
410 struct iommu_table
*tbl
;
412 tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
414 size
= PAGE_ALIGN(size
); /* size rounded up to full pages */
415 npages
= size
>> PAGE_SHIFT
;
416 order
= get_order(size
);
418 /* alloc enough pages (and possibly more) */
419 ret
= (void *)__get_free_pages(flag
, order
);
422 memset(ret
, 0, size
);
424 if (translate_phb(to_pci_dev(dev
))) {
425 /* set up tces to cover the allocated range */
426 mapping
= iommu_alloc(tbl
, ret
, npages
, DMA_BIDIRECTIONAL
);
427 if (mapping
== bad_dma_address
)
430 *dma_handle
= mapping
;
431 } else /* non translated slot */
432 *dma_handle
= virt_to_bus(ret
);
437 free_pages((unsigned long)ret
, get_order(size
));
443 static struct dma_mapping_ops calgary_dma_ops
= {
444 .alloc_coherent
= calgary_alloc_coherent
,
445 .map_single
= calgary_map_single
,
446 .unmap_single
= calgary_unmap_single
,
447 .map_sg
= calgary_map_sg
,
448 .unmap_sg
= calgary_unmap_sg
,
451 static inline int busno_to_phbid(unsigned char num
)
453 return bus_to_phb(num
) % PHBS_PER_CALGARY
;
456 static inline unsigned long split_queue_offset(unsigned char num
)
458 size_t idx
= busno_to_phbid(num
);
460 return split_queue_offsets
[idx
];
463 static inline unsigned long tar_offset(unsigned char num
)
465 size_t idx
= busno_to_phbid(num
);
467 return tar_offsets
[idx
];
470 static inline unsigned long phb_offset(unsigned char num
)
472 size_t idx
= busno_to_phbid(num
);
474 return phb_offsets
[idx
];
477 static inline void __iomem
* calgary_reg(void __iomem
*bar
, unsigned long offset
)
479 unsigned long target
= ((unsigned long)bar
) | offset
;
480 return (void __iomem
*)target
;
483 static void tce_cache_blast(struct iommu_table
*tbl
)
488 void __iomem
*bbar
= tbl
->bbar
;
489 void __iomem
*target
;
491 /* disable arbitration on the bus */
492 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_AER_OFFSET
);
496 /* read plssr to ensure it got there */
497 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_PLSSR_OFFSET
);
500 /* poll split queues until all DMA activity is done */
501 target
= calgary_reg(bbar
, split_queue_offset(tbl
->it_busno
));
505 } while ((val
& 0xff) != 0xff && i
< 100);
507 printk(KERN_WARNING
"Calgary: PCI bus not quiesced, "
508 "continuing anyway\n");
510 /* invalidate TCE cache */
511 target
= calgary_reg(bbar
, tar_offset(tbl
->it_busno
));
512 writeq(tbl
->tar_val
, target
);
514 /* enable arbitration */
515 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_AER_OFFSET
);
517 (void)readl(target
); /* flush */
520 static void __init
calgary_reserve_mem_region(struct pci_dev
*dev
, u64 start
,
523 unsigned int numpages
;
525 limit
= limit
| 0xfffff;
528 numpages
= ((limit
- start
) >> PAGE_SHIFT
);
529 iommu_range_reserve(dev
->sysdata
, start
, numpages
);
532 static void __init
calgary_reserve_peripheral_mem_1(struct pci_dev
*dev
)
534 void __iomem
*target
;
535 u64 low
, high
, sizelow
;
537 struct iommu_table
*tbl
= dev
->sysdata
;
538 unsigned char busnum
= dev
->bus
->number
;
539 void __iomem
*bbar
= tbl
->bbar
;
541 /* peripheral MEM_1 region */
542 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_LOW
);
543 low
= be32_to_cpu(readl(target
));
544 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_HIGH
);
545 high
= be32_to_cpu(readl(target
));
546 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_SIZE
);
547 sizelow
= be32_to_cpu(readl(target
));
549 start
= (high
<< 32) | low
;
552 calgary_reserve_mem_region(dev
, start
, limit
);
555 static void __init
calgary_reserve_peripheral_mem_2(struct pci_dev
*dev
)
557 void __iomem
*target
;
559 u64 low
, high
, sizelow
, sizehigh
;
561 struct iommu_table
*tbl
= dev
->sysdata
;
562 unsigned char busnum
= dev
->bus
->number
;
563 void __iomem
*bbar
= tbl
->bbar
;
566 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
567 val32
= be32_to_cpu(readl(target
));
568 if (!(val32
& PHB_MEM2_ENABLE
))
571 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_LOW
);
572 low
= be32_to_cpu(readl(target
));
573 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_HIGH
);
574 high
= be32_to_cpu(readl(target
));
575 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_SIZE_LOW
);
576 sizelow
= be32_to_cpu(readl(target
));
577 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_SIZE_HIGH
);
578 sizehigh
= be32_to_cpu(readl(target
));
580 start
= (high
<< 32) | low
;
581 limit
= (sizehigh
<< 32) | sizelow
;
583 calgary_reserve_mem_region(dev
, start
, limit
);
587 * some regions of the IO address space do not get translated, so we
588 * must not give devices IO addresses in those regions. The regions
589 * are the 640KB-1MB region and the two PCI peripheral memory holes.
590 * Reserve all of them in the IOMMU bitmap to avoid giving them out
593 static void __init
calgary_reserve_regions(struct pci_dev
*dev
)
597 unsigned char busnum
;
599 struct iommu_table
*tbl
= dev
->sysdata
;
602 busnum
= dev
->bus
->number
;
604 /* reserve bad_dma_address in case it's a legal address */
605 iommu_range_reserve(tbl
, bad_dma_address
, 1);
607 /* avoid the BIOS/VGA first 640KB-1MB region */
608 start
= (640 * 1024);
609 npages
= ((1024 - 640) * 1024) >> PAGE_SHIFT
;
610 iommu_range_reserve(tbl
, start
, npages
);
612 /* reserve the two PCI peripheral memory regions in IO space */
613 calgary_reserve_peripheral_mem_1(dev
);
614 calgary_reserve_peripheral_mem_2(dev
);
617 static int __init
calgary_setup_tar(struct pci_dev
*dev
, void __iomem
*bbar
)
621 void __iomem
*target
;
623 struct iommu_table
*tbl
;
625 /* build TCE tables for each PHB */
626 ret
= build_tce_table(dev
, bbar
);
630 calgary_reserve_regions(dev
);
632 /* set TARs for each PHB */
633 target
= calgary_reg(bbar
, tar_offset(dev
->bus
->number
));
634 val64
= be64_to_cpu(readq(target
));
636 /* zero out all TAR bits under sw control */
637 val64
&= ~TAR_SW_BITS
;
640 table_phys
= (u64
)__pa(tbl
->it_base
);
643 BUG_ON(specified_table_size
> TCE_TABLE_SIZE_8M
);
644 val64
|= (u64
) specified_table_size
;
646 tbl
->tar_val
= cpu_to_be64(val64
);
647 writeq(tbl
->tar_val
, target
);
648 readq(target
); /* flush */
653 static void __init
calgary_free_tar(struct pci_dev
*dev
)
656 struct iommu_table
*tbl
= dev
->sysdata
;
657 void __iomem
*target
;
659 target
= calgary_reg(tbl
->bbar
, tar_offset(dev
->bus
->number
));
660 val64
= be64_to_cpu(readq(target
));
661 val64
&= ~TAR_SW_BITS
;
662 writeq(cpu_to_be64(val64
), target
);
663 readq(target
); /* flush */
669 static void calgary_watchdog(unsigned long data
)
671 struct pci_dev
*dev
= (struct pci_dev
*)data
;
672 struct iommu_table
*tbl
= dev
->sysdata
;
673 void __iomem
*bbar
= tbl
->bbar
;
675 void __iomem
*target
;
677 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_CSR_OFFSET
);
678 val32
= be32_to_cpu(readl(target
));
680 /* If no error, the agent ID in the CSR is not valid */
681 if (val32
& CSR_AGENT_MASK
) {
682 printk(KERN_EMERG
"calgary_watchdog: DMA error on bus %d, "
683 "CSR = %#x\n", dev
->bus
->number
, val32
);
686 /* Disable bus that caused the error */
687 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) |
688 PHB_CONFIG_RW_OFFSET
);
689 val32
= be32_to_cpu(readl(target
));
690 val32
|= PHB_SLOT_DISABLE
;
691 writel(cpu_to_be32(val32
), target
);
692 readl(target
); /* flush */
694 /* Reset the timer */
695 mod_timer(&tbl
->watchdog_timer
, jiffies
+ 2 * HZ
);
699 static void __init
calgary_enable_translation(struct pci_dev
*dev
)
702 unsigned char busnum
;
703 void __iomem
*target
;
705 struct iommu_table
*tbl
;
707 busnum
= dev
->bus
->number
;
711 /* enable TCE in PHB Config Register */
712 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
713 val32
= be32_to_cpu(readl(target
));
714 val32
|= PHB_TCE_ENABLE
| PHB_DAC_DISABLE
| PHB_MCSR_ENABLE
;
716 printk(KERN_INFO
"Calgary: enabling translation on PHB %d\n", busnum
);
717 printk(KERN_INFO
"Calgary: errant DMAs will now be prevented on this "
720 writel(cpu_to_be32(val32
), target
);
721 readl(target
); /* flush */
723 init_timer(&tbl
->watchdog_timer
);
724 tbl
->watchdog_timer
.function
= &calgary_watchdog
;
725 tbl
->watchdog_timer
.data
= (unsigned long)dev
;
726 mod_timer(&tbl
->watchdog_timer
, jiffies
);
729 static void __init
calgary_disable_translation(struct pci_dev
*dev
)
732 unsigned char busnum
;
733 void __iomem
*target
;
735 struct iommu_table
*tbl
;
737 busnum
= dev
->bus
->number
;
741 /* disable TCE in PHB Config Register */
742 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
743 val32
= be32_to_cpu(readl(target
));
744 val32
&= ~(PHB_TCE_ENABLE
| PHB_DAC_DISABLE
| PHB_MCSR_ENABLE
);
746 printk(KERN_INFO
"Calgary: disabling translation on PHB %d!\n", busnum
);
747 writel(cpu_to_be32(val32
), target
);
748 readl(target
); /* flush */
750 del_timer_sync(&tbl
->watchdog_timer
);
753 static inline unsigned int __init
locate_register_space(struct pci_dev
*dev
)
758 rionodeid
= (dev
->bus
->number
% 15 > 4) ? 3 : 2;
760 * register space address calculation as follows:
761 * FE0MB-8MB*OneBasedChassisNumber+1MB*(RioNodeId-ChassisBase)
762 * ChassisBase is always zero for x366/x260/x460
763 * RioNodeId is 2 for first Calgary, 3 for second Calgary
765 address
= START_ADDRESS
-
766 (0x800000 * (ONE_BASED_CHASSIS_NUM
+ dev
->bus
->number
/ 15)) +
767 (0x100000) * (rionodeid
- CHASSIS_BASE
);
771 static int __init
calgary_init_one_nontraslated(struct pci_dev
*dev
)
774 dev
->bus
->self
= dev
;
779 static int __init
calgary_init_one(struct pci_dev
*dev
)
785 address
= locate_register_space(dev
);
786 /* map entire 1MB of Calgary config space */
787 bbar
= ioremap_nocache(address
, 1024 * 1024);
793 ret
= calgary_setup_tar(dev
, bbar
);
797 dev
->bus
->self
= dev
;
798 calgary_enable_translation(dev
);
808 static int __init
calgary_init(void)
810 int i
, ret
= -ENODEV
;
811 struct pci_dev
*dev
= NULL
;
813 for (i
= 0; i
<= num_online_nodes() * MAX_NUM_OF_PHBS
; i
++) {
814 dev
= pci_get_device(PCI_VENDOR_ID_IBM
,
815 PCI_DEVICE_ID_IBM_CALGARY
,
819 if (!translate_phb(dev
)) {
820 calgary_init_one_nontraslated(dev
);
823 if (!tce_table_kva
[i
] && !translate_empty_slots
) {
827 ret
= calgary_init_one(dev
);
835 for (i
--; i
>= 0; i
--) {
836 dev
= pci_find_device_reverse(PCI_VENDOR_ID_IBM
,
837 PCI_DEVICE_ID_IBM_CALGARY
,
839 if (!translate_phb(dev
)) {
843 if (!tce_table_kva
[i
] && !translate_empty_slots
)
845 calgary_disable_translation(dev
);
846 calgary_free_tar(dev
);
853 static inline int __init
determine_tce_table_size(u64 ram
)
857 if (specified_table_size
!= TCE_TABLE_SIZE_UNSPECIFIED
)
858 return specified_table_size
;
861 * Table sizes are from 0 to 7 (TCE_TABLE_SIZE_64K to
862 * TCE_TABLE_SIZE_8M). Table size 0 has 8K entries and each
863 * larger table size has twice as many entries, so shift the
864 * max ram address by 13 to divide by 8K and then look at the
865 * order of the result to choose between 0-7.
867 ret
= get_order(ram
>> 13);
868 if (ret
> TCE_TABLE_SIZE_8M
)
869 ret
= TCE_TABLE_SIZE_8M
;
874 void __init
detect_calgary(void)
882 * if the user specified iommu=off or iommu=soft or we found
883 * another HW IOMMU already, bail out.
885 if (swiotlb
|| no_iommu
|| iommu_detected
)
888 specified_table_size
= determine_tce_table_size(end_pfn
* PAGE_SIZE
);
890 for (bus
= 0, table_idx
= 0;
891 bus
<= num_online_nodes() * MAX_PHB_BUS_NUM
;
893 BUG_ON(bus
> MAX_NUMNODES
* MAX_PHB_BUS_NUM
);
894 if (read_pci_config(bus
, 0, 0, 0) != PCI_VENDOR_DEVICE_ID_CALGARY
)
896 if (test_bit(bus
, translation_disabled
)) {
897 printk(KERN_INFO
"Calgary: translation is disabled for "
899 /* skip this phb, don't allocate a tbl for it */
900 tce_table_kva
[table_idx
] = NULL
;
905 * scan the first slot of the PCI bus to see if there
906 * are any devices present
908 val
= read_pci_config(bus
, 1, 0, 0);
909 if (val
!= 0xffffffff || translate_empty_slots
) {
910 tbl
= alloc_tce_table();
917 tce_table_kva
[table_idx
] = tbl
;
923 calgary_detected
= 1;
924 printk(KERN_INFO
"PCI-DMA: Calgary IOMMU detected. "
925 "TCE table spec is %d.\n", specified_table_size
);
930 for (--table_idx
; table_idx
>= 0; --table_idx
)
931 if (tce_table_kva
[table_idx
])
932 free_tce_table(tce_table_kva
[table_idx
]);
935 int __init
calgary_iommu_init(void)
939 if (no_iommu
|| swiotlb
)
942 if (!calgary_detected
)
945 /* ok, we're trying to use Calgary - let's roll */
946 printk(KERN_INFO
"PCI-DMA: Using Calgary IOMMU\n");
948 ret
= calgary_init();
950 printk(KERN_ERR
"PCI-DMA: Calgary init failed %d, "
951 "falling back to no_iommu\n", ret
);
952 if (end_pfn
> MAX_DMA32_PFN
)
953 printk(KERN_ERR
"WARNING more than 4GB of memory, "
954 "32bit PCI may malfunction.\n");
959 dma_ops
= &calgary_dma_ops
;
964 static int __init
calgary_parse_options(char *p
)
971 if (!strncmp(p
, "64k", 3))
972 specified_table_size
= TCE_TABLE_SIZE_64K
;
973 else if (!strncmp(p
, "128k", 4))
974 specified_table_size
= TCE_TABLE_SIZE_128K
;
975 else if (!strncmp(p
, "256k", 4))
976 specified_table_size
= TCE_TABLE_SIZE_256K
;
977 else if (!strncmp(p
, "512k", 4))
978 specified_table_size
= TCE_TABLE_SIZE_512K
;
979 else if (!strncmp(p
, "1M", 2))
980 specified_table_size
= TCE_TABLE_SIZE_1M
;
981 else if (!strncmp(p
, "2M", 2))
982 specified_table_size
= TCE_TABLE_SIZE_2M
;
983 else if (!strncmp(p
, "4M", 2))
984 specified_table_size
= TCE_TABLE_SIZE_4M
;
985 else if (!strncmp(p
, "8M", 2))
986 specified_table_size
= TCE_TABLE_SIZE_8M
;
988 len
= strlen("translate_empty_slots");
989 if (!strncmp(p
, "translate_empty_slots", len
))
990 translate_empty_slots
= 1;
992 len
= strlen("disable");
993 if (!strncmp(p
, "disable", len
)) {
999 bridge
= simple_strtol(p
, &endp
, 0);
1003 if (bridge
<= (num_online_nodes() * MAX_PHB_BUS_NUM
)) {
1004 printk(KERN_INFO
"Calgary: disabling "
1005 "translation for PHB 0x%x\n", bridge
);
1006 set_bit(bridge
, translation_disabled
);
1010 p
= strpbrk(p
, ",");
1018 __setup("calgary=", calgary_parse_options
);