2 * Derived from arch/powerpc/kernel/iommu.c
4 * Copyright (C) IBM Corporation, 2006
6 * Author: Jon Mason <jdmason@us.ibm.com>
7 * Author: Muli Ben-Yehuda <muli@il.ibm.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include <linux/config.h>
25 #include <linux/kernel.h>
26 #include <linux/init.h>
27 #include <linux/types.h>
28 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/string.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/init.h>
34 #include <linux/bitops.h>
35 #include <linux/pci_ids.h>
36 #include <linux/pci.h>
37 #include <linux/delay.h>
38 #include <asm/proto.h>
39 #include <asm/calgary.h>
41 #include <asm/pci-direct.h>
42 #include <asm/system.h>
45 #define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
46 #define PCI_VENDOR_DEVICE_ID_CALGARY \
47 (PCI_VENDOR_ID_IBM | PCI_DEVICE_ID_IBM_CALGARY << 16)
49 /* we need these for register space address calculation */
50 #define START_ADDRESS 0xfe000000
51 #define CHASSIS_BASE 0
52 #define ONE_BASED_CHASSIS_NUM 1
54 /* register offsets inside the host bridge space */
55 #define PHB_CSR_OFFSET 0x0110
56 #define PHB_PLSSR_OFFSET 0x0120
57 #define PHB_CONFIG_RW_OFFSET 0x0160
58 #define PHB_IOBASE_BAR_LOW 0x0170
59 #define PHB_IOBASE_BAR_HIGH 0x0180
60 #define PHB_MEM_1_LOW 0x0190
61 #define PHB_MEM_1_HIGH 0x01A0
62 #define PHB_IO_ADDR_SIZE 0x01B0
63 #define PHB_MEM_1_SIZE 0x01C0
64 #define PHB_MEM_ST_OFFSET 0x01D0
65 #define PHB_AER_OFFSET 0x0200
66 #define PHB_CONFIG_0_HIGH 0x0220
67 #define PHB_CONFIG_0_LOW 0x0230
68 #define PHB_CONFIG_0_END 0x0240
69 #define PHB_MEM_2_LOW 0x02B0
70 #define PHB_MEM_2_HIGH 0x02C0
71 #define PHB_MEM_2_SIZE_HIGH 0x02D0
72 #define PHB_MEM_2_SIZE_LOW 0x02E0
73 #define PHB_DOSHOLE_OFFSET 0x08E0
76 #define PHB_TCE_ENABLE 0x20000000
77 #define PHB_SLOT_DISABLE 0x1C000000
78 #define PHB_DAC_DISABLE 0x01000000
79 #define PHB_MEM2_ENABLE 0x00400000
80 #define PHB_MCSR_ENABLE 0x00100000
81 /* TAR (Table Address Register) */
82 #define TAR_SW_BITS 0x0000ffffffff800fUL
83 #define TAR_VALID 0x0000000000000008UL
84 /* CSR (Channel/DMA Status Register) */
85 #define CSR_AGENT_MASK 0xffe0ffff
87 #define MAX_NUM_OF_PHBS 8 /* how many PHBs in total? */
88 #define MAX_PHB_BUS_NUM (MAX_NUM_OF_PHBS * 2) /* max dev->bus->number */
89 #define PHBS_PER_CALGARY 4
91 /* register offsets in Calgary's internal register space */
92 static const unsigned long tar_offsets
[] = {
99 static const unsigned long split_queue_offsets
[] = {
100 0x4870 /* SPLIT QUEUE 0 */,
101 0x5870 /* SPLIT QUEUE 1 */,
102 0x6870 /* SPLIT QUEUE 2 */,
103 0x7870 /* SPLIT QUEUE 3 */
106 static const unsigned long phb_offsets
[] = {
113 void* tce_table_kva
[MAX_NUM_OF_PHBS
* MAX_NUMNODES
];
114 unsigned int specified_table_size
= TCE_TABLE_SIZE_UNSPECIFIED
;
115 static int translate_empty_slots __read_mostly
= 0;
116 static int calgary_detected __read_mostly
= 0;
119 * the bitmap of PHBs the user requested that we disable
122 static DECLARE_BITMAP(translation_disabled
, MAX_NUMNODES
* MAX_PHB_BUS_NUM
);
124 static void tce_cache_blast(struct iommu_table
*tbl
);
126 /* enable this to stress test the chip's TCE cache */
127 #ifdef CONFIG_IOMMU_DEBUG
128 static inline void tce_cache_blast_stress(struct iommu_table
*tbl
)
130 tce_cache_blast(tbl
);
133 static inline void tce_cache_blast_stress(struct iommu_table
*tbl
)
136 #endif /* BLAST_TCE_CACHE_ON_UNMAP */
138 static inline unsigned int num_dma_pages(unsigned long dma
, unsigned int dmalen
)
142 npages
= PAGE_ALIGN(dma
+ dmalen
) - (dma
& PAGE_MASK
);
143 npages
>>= PAGE_SHIFT
;
148 static inline int translate_phb(struct pci_dev
* dev
)
150 int disabled
= test_bit(dev
->bus
->number
, translation_disabled
);
154 static void iommu_range_reserve(struct iommu_table
*tbl
,
155 unsigned long start_addr
, unsigned int npages
)
160 index
= start_addr
>> PAGE_SHIFT
;
162 /* bail out if we're asked to reserve a region we don't cover */
163 if (index
>= tbl
->it_size
)
166 end
= index
+ npages
;
167 if (end
> tbl
->it_size
) /* don't go off the table */
170 while (index
< end
) {
171 if (test_bit(index
, tbl
->it_map
))
172 printk(KERN_ERR
"Calgary: entry already allocated at "
173 "0x%lx tbl %p dma 0x%lx npages %u\n",
174 index
, tbl
, start_addr
, npages
);
177 set_bit_string(tbl
->it_map
, start_addr
>> PAGE_SHIFT
, npages
);
180 static unsigned long iommu_range_alloc(struct iommu_table
*tbl
,
183 unsigned long offset
;
187 offset
= find_next_zero_string(tbl
->it_map
, tbl
->it_hint
,
188 tbl
->it_size
, npages
);
189 if (offset
== ~0UL) {
190 tce_cache_blast(tbl
);
191 offset
= find_next_zero_string(tbl
->it_map
, 0,
192 tbl
->it_size
, npages
);
193 if (offset
== ~0UL) {
194 printk(KERN_WARNING
"Calgary: IOMMU full.\n");
195 if (panic_on_overflow
)
196 panic("Calgary: fix the allocator.\n");
198 return bad_dma_address
;
202 set_bit_string(tbl
->it_map
, offset
, npages
);
203 tbl
->it_hint
= offset
+ npages
;
204 BUG_ON(tbl
->it_hint
> tbl
->it_size
);
209 static dma_addr_t
iommu_alloc(struct iommu_table
*tbl
, void *vaddr
,
210 unsigned int npages
, int direction
)
212 unsigned long entry
, flags
;
213 dma_addr_t ret
= bad_dma_address
;
215 spin_lock_irqsave(&tbl
->it_lock
, flags
);
217 entry
= iommu_range_alloc(tbl
, npages
);
219 if (unlikely(entry
== bad_dma_address
))
222 /* set the return dma address */
223 ret
= (entry
<< PAGE_SHIFT
) | ((unsigned long)vaddr
& ~PAGE_MASK
);
225 /* put the TCEs in the HW table */
226 tce_build(tbl
, entry
, npages
, (unsigned long)vaddr
& PAGE_MASK
,
229 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
234 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
235 printk(KERN_WARNING
"Calgary: failed to allocate %u pages in "
236 "iommu %p\n", npages
, tbl
);
237 return bad_dma_address
;
240 static void __iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
246 entry
= dma_addr
>> PAGE_SHIFT
;
248 BUG_ON(entry
+ npages
> tbl
->it_size
);
250 tce_free(tbl
, entry
, npages
);
252 for (i
= 0; i
< npages
; ++i
) {
253 if (!test_bit(entry
+ i
, tbl
->it_map
))
254 printk(KERN_ERR
"Calgary: bit is off at 0x%lx "
255 "tbl %p dma 0x%Lx entry 0x%lx npages %u\n",
256 entry
+ i
, tbl
, dma_addr
, entry
, npages
);
259 __clear_bit_string(tbl
->it_map
, entry
, npages
);
261 tce_cache_blast_stress(tbl
);
264 static void iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
269 spin_lock_irqsave(&tbl
->it_lock
, flags
);
271 __iommu_free(tbl
, dma_addr
, npages
);
273 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
276 static void __calgary_unmap_sg(struct iommu_table
*tbl
,
277 struct scatterlist
*sglist
, int nelems
, int direction
)
281 dma_addr_t dma
= sglist
->dma_address
;
282 unsigned int dmalen
= sglist
->dma_length
;
287 npages
= num_dma_pages(dma
, dmalen
);
288 __iommu_free(tbl
, dma
, npages
);
293 void calgary_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
294 int nelems
, int direction
)
297 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
299 if (!translate_phb(to_pci_dev(dev
)))
302 spin_lock_irqsave(&tbl
->it_lock
, flags
);
304 __calgary_unmap_sg(tbl
, sglist
, nelems
, direction
);
306 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
309 static int calgary_nontranslate_map_sg(struct device
* dev
,
310 struct scatterlist
*sg
, int nelems
, int direction
)
314 for (i
= 0; i
< nelems
; i
++ ) {
315 struct scatterlist
*s
= &sg
[i
];
317 s
->dma_address
= virt_to_bus(page_address(s
->page
) +s
->offset
);
318 s
->dma_length
= s
->length
;
323 int calgary_map_sg(struct device
*dev
, struct scatterlist
*sg
,
324 int nelems
, int direction
)
326 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
333 if (!translate_phb(to_pci_dev(dev
)))
334 return calgary_nontranslate_map_sg(dev
, sg
, nelems
, direction
);
336 spin_lock_irqsave(&tbl
->it_lock
, flags
);
338 for (i
= 0; i
< nelems
; i
++ ) {
339 struct scatterlist
*s
= &sg
[i
];
342 vaddr
= (unsigned long)page_address(s
->page
) + s
->offset
;
343 npages
= num_dma_pages(vaddr
, s
->length
);
345 entry
= iommu_range_alloc(tbl
, npages
);
346 if (entry
== bad_dma_address
) {
347 /* makes sure unmap knows to stop */
352 s
->dma_address
= (entry
<< PAGE_SHIFT
) | s
->offset
;
354 /* insert into HW table */
355 tce_build(tbl
, entry
, npages
, vaddr
& PAGE_MASK
,
358 s
->dma_length
= s
->length
;
361 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
365 __calgary_unmap_sg(tbl
, sg
, nelems
, direction
);
366 for (i
= 0; i
< nelems
; i
++) {
367 sg
[i
].dma_address
= bad_dma_address
;
368 sg
[i
].dma_length
= 0;
370 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
374 dma_addr_t
calgary_map_single(struct device
*dev
, void *vaddr
,
375 size_t size
, int direction
)
377 dma_addr_t dma_handle
= bad_dma_address
;
380 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
382 uaddr
= (unsigned long)vaddr
;
383 npages
= num_dma_pages(uaddr
, size
);
385 if (translate_phb(to_pci_dev(dev
)))
386 dma_handle
= iommu_alloc(tbl
, vaddr
, npages
, direction
);
388 dma_handle
= virt_to_bus(vaddr
);
393 void calgary_unmap_single(struct device
*dev
, dma_addr_t dma_handle
,
394 size_t size
, int direction
)
396 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
399 if (!translate_phb(to_pci_dev(dev
)))
402 npages
= num_dma_pages(dma_handle
, size
);
403 iommu_free(tbl
, dma_handle
, npages
);
406 void* calgary_alloc_coherent(struct device
*dev
, size_t size
,
407 dma_addr_t
*dma_handle
, gfp_t flag
)
411 unsigned int npages
, order
;
412 struct iommu_table
*tbl
;
414 tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
416 size
= PAGE_ALIGN(size
); /* size rounded up to full pages */
417 npages
= size
>> PAGE_SHIFT
;
418 order
= get_order(size
);
420 /* alloc enough pages (and possibly more) */
421 ret
= (void *)__get_free_pages(flag
, order
);
424 memset(ret
, 0, size
);
426 if (translate_phb(to_pci_dev(dev
))) {
427 /* set up tces to cover the allocated range */
428 mapping
= iommu_alloc(tbl
, ret
, npages
, DMA_BIDIRECTIONAL
);
429 if (mapping
== bad_dma_address
)
432 *dma_handle
= mapping
;
433 } else /* non translated slot */
434 *dma_handle
= virt_to_bus(ret
);
439 free_pages((unsigned long)ret
, get_order(size
));
445 static struct dma_mapping_ops calgary_dma_ops
= {
446 .alloc_coherent
= calgary_alloc_coherent
,
447 .map_single
= calgary_map_single
,
448 .unmap_single
= calgary_unmap_single
,
449 .map_sg
= calgary_map_sg
,
450 .unmap_sg
= calgary_unmap_sg
,
453 static inline int busno_to_phbid(unsigned char num
)
455 return bus_to_phb(num
) % PHBS_PER_CALGARY
;
458 static inline unsigned long split_queue_offset(unsigned char num
)
460 size_t idx
= busno_to_phbid(num
);
462 return split_queue_offsets
[idx
];
465 static inline unsigned long tar_offset(unsigned char num
)
467 size_t idx
= busno_to_phbid(num
);
469 return tar_offsets
[idx
];
472 static inline unsigned long phb_offset(unsigned char num
)
474 size_t idx
= busno_to_phbid(num
);
476 return phb_offsets
[idx
];
479 static inline void __iomem
* calgary_reg(void __iomem
*bar
, unsigned long offset
)
481 unsigned long target
= ((unsigned long)bar
) | offset
;
482 return (void __iomem
*)target
;
485 static void tce_cache_blast(struct iommu_table
*tbl
)
490 void __iomem
*bbar
= tbl
->bbar
;
491 void __iomem
*target
;
493 /* disable arbitration on the bus */
494 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_AER_OFFSET
);
498 /* read plssr to ensure it got there */
499 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_PLSSR_OFFSET
);
502 /* poll split queues until all DMA activity is done */
503 target
= calgary_reg(bbar
, split_queue_offset(tbl
->it_busno
));
507 } while ((val
& 0xff) != 0xff && i
< 100);
509 printk(KERN_WARNING
"Calgary: PCI bus not quiesced, "
510 "continuing anyway\n");
512 /* invalidate TCE cache */
513 target
= calgary_reg(bbar
, tar_offset(tbl
->it_busno
));
514 writeq(tbl
->tar_val
, target
);
516 /* enable arbitration */
517 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_AER_OFFSET
);
519 (void)readl(target
); /* flush */
522 static void __init
calgary_reserve_mem_region(struct pci_dev
*dev
, u64 start
,
525 unsigned int numpages
;
527 limit
= limit
| 0xfffff;
530 numpages
= ((limit
- start
) >> PAGE_SHIFT
);
531 iommu_range_reserve(dev
->sysdata
, start
, numpages
);
534 static void __init
calgary_reserve_peripheral_mem_1(struct pci_dev
*dev
)
536 void __iomem
*target
;
537 u64 low
, high
, sizelow
;
539 struct iommu_table
*tbl
= dev
->sysdata
;
540 unsigned char busnum
= dev
->bus
->number
;
541 void __iomem
*bbar
= tbl
->bbar
;
543 /* peripheral MEM_1 region */
544 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_LOW
);
545 low
= be32_to_cpu(readl(target
));
546 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_HIGH
);
547 high
= be32_to_cpu(readl(target
));
548 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_SIZE
);
549 sizelow
= be32_to_cpu(readl(target
));
551 start
= (high
<< 32) | low
;
554 calgary_reserve_mem_region(dev
, start
, limit
);
557 static void __init
calgary_reserve_peripheral_mem_2(struct pci_dev
*dev
)
559 void __iomem
*target
;
561 u64 low
, high
, sizelow
, sizehigh
;
563 struct iommu_table
*tbl
= dev
->sysdata
;
564 unsigned char busnum
= dev
->bus
->number
;
565 void __iomem
*bbar
= tbl
->bbar
;
568 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
569 val32
= be32_to_cpu(readl(target
));
570 if (!(val32
& PHB_MEM2_ENABLE
))
573 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_LOW
);
574 low
= be32_to_cpu(readl(target
));
575 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_HIGH
);
576 high
= be32_to_cpu(readl(target
));
577 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_SIZE_LOW
);
578 sizelow
= be32_to_cpu(readl(target
));
579 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_SIZE_HIGH
);
580 sizehigh
= be32_to_cpu(readl(target
));
582 start
= (high
<< 32) | low
;
583 limit
= (sizehigh
<< 32) | sizelow
;
585 calgary_reserve_mem_region(dev
, start
, limit
);
589 * some regions of the IO address space do not get translated, so we
590 * must not give devices IO addresses in those regions. The regions
591 * are the 640KB-1MB region and the two PCI peripheral memory holes.
592 * Reserve all of them in the IOMMU bitmap to avoid giving them out
595 static void __init
calgary_reserve_regions(struct pci_dev
*dev
)
599 unsigned char busnum
;
601 struct iommu_table
*tbl
= dev
->sysdata
;
604 busnum
= dev
->bus
->number
;
606 /* reserve bad_dma_address in case it's a legal address */
607 iommu_range_reserve(tbl
, bad_dma_address
, 1);
609 /* avoid the BIOS/VGA first 640KB-1MB region */
610 start
= (640 * 1024);
611 npages
= ((1024 - 640) * 1024) >> PAGE_SHIFT
;
612 iommu_range_reserve(tbl
, start
, npages
);
614 /* reserve the two PCI peripheral memory regions in IO space */
615 calgary_reserve_peripheral_mem_1(dev
);
616 calgary_reserve_peripheral_mem_2(dev
);
619 static int __init
calgary_setup_tar(struct pci_dev
*dev
, void __iomem
*bbar
)
623 void __iomem
*target
;
625 struct iommu_table
*tbl
;
627 /* build TCE tables for each PHB */
628 ret
= build_tce_table(dev
, bbar
);
632 calgary_reserve_regions(dev
);
634 /* set TARs for each PHB */
635 target
= calgary_reg(bbar
, tar_offset(dev
->bus
->number
));
636 val64
= be64_to_cpu(readq(target
));
638 /* zero out all TAR bits under sw control */
639 val64
&= ~TAR_SW_BITS
;
642 table_phys
= (u64
)__pa(tbl
->it_base
);
645 BUG_ON(specified_table_size
> TCE_TABLE_SIZE_8M
);
646 val64
|= (u64
) specified_table_size
;
648 tbl
->tar_val
= cpu_to_be64(val64
);
649 writeq(tbl
->tar_val
, target
);
650 readq(target
); /* flush */
655 static void __init
calgary_free_tar(struct pci_dev
*dev
)
658 struct iommu_table
*tbl
= dev
->sysdata
;
659 void __iomem
*target
;
661 target
= calgary_reg(tbl
->bbar
, tar_offset(dev
->bus
->number
));
662 val64
= be64_to_cpu(readq(target
));
663 val64
&= ~TAR_SW_BITS
;
664 writeq(cpu_to_be64(val64
), target
);
665 readq(target
); /* flush */
671 static void calgary_watchdog(unsigned long data
)
673 struct pci_dev
*dev
= (struct pci_dev
*)data
;
674 struct iommu_table
*tbl
= dev
->sysdata
;
675 void __iomem
*bbar
= tbl
->bbar
;
677 void __iomem
*target
;
679 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_CSR_OFFSET
);
680 val32
= be32_to_cpu(readl(target
));
682 /* If no error, the agent ID in the CSR is not valid */
683 if (val32
& CSR_AGENT_MASK
) {
684 printk(KERN_EMERG
"calgary_watchdog: DMA error on bus %d, "
685 "CSR = %#x\n", dev
->bus
->number
, val32
);
688 /* Disable bus that caused the error */
689 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) |
690 PHB_CONFIG_RW_OFFSET
);
691 val32
= be32_to_cpu(readl(target
));
692 val32
|= PHB_SLOT_DISABLE
;
693 writel(cpu_to_be32(val32
), target
);
694 readl(target
); /* flush */
696 /* Reset the timer */
697 mod_timer(&tbl
->watchdog_timer
, jiffies
+ 2 * HZ
);
701 static void __init
calgary_enable_translation(struct pci_dev
*dev
)
704 unsigned char busnum
;
705 void __iomem
*target
;
707 struct iommu_table
*tbl
;
709 busnum
= dev
->bus
->number
;
713 /* enable TCE in PHB Config Register */
714 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
715 val32
= be32_to_cpu(readl(target
));
716 val32
|= PHB_TCE_ENABLE
| PHB_DAC_DISABLE
| PHB_MCSR_ENABLE
;
718 printk(KERN_INFO
"Calgary: enabling translation on PHB %d\n", busnum
);
719 printk(KERN_INFO
"Calgary: errant DMAs will now be prevented on this "
722 writel(cpu_to_be32(val32
), target
);
723 readl(target
); /* flush */
725 init_timer(&tbl
->watchdog_timer
);
726 tbl
->watchdog_timer
.function
= &calgary_watchdog
;
727 tbl
->watchdog_timer
.data
= (unsigned long)dev
;
728 mod_timer(&tbl
->watchdog_timer
, jiffies
);
731 static void __init
calgary_disable_translation(struct pci_dev
*dev
)
734 unsigned char busnum
;
735 void __iomem
*target
;
737 struct iommu_table
*tbl
;
739 busnum
= dev
->bus
->number
;
743 /* disable TCE in PHB Config Register */
744 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
745 val32
= be32_to_cpu(readl(target
));
746 val32
&= ~(PHB_TCE_ENABLE
| PHB_DAC_DISABLE
| PHB_MCSR_ENABLE
);
748 printk(KERN_INFO
"Calgary: disabling translation on PHB %d!\n", busnum
);
749 writel(cpu_to_be32(val32
), target
);
750 readl(target
); /* flush */
752 del_timer_sync(&tbl
->watchdog_timer
);
755 static inline unsigned int __init
locate_register_space(struct pci_dev
*dev
)
760 rionodeid
= (dev
->bus
->number
% 15 > 4) ? 3 : 2;
762 * register space address calculation as follows:
763 * FE0MB-8MB*OneBasedChassisNumber+1MB*(RioNodeId-ChassisBase)
764 * ChassisBase is always zero for x366/x260/x460
765 * RioNodeId is 2 for first Calgary, 3 for second Calgary
767 address
= START_ADDRESS
-
768 (0x800000 * (ONE_BASED_CHASSIS_NUM
+ dev
->bus
->number
/ 15)) +
769 (0x100000) * (rionodeid
- CHASSIS_BASE
);
773 static int __init
calgary_init_one_nontraslated(struct pci_dev
*dev
)
776 dev
->bus
->self
= dev
;
781 static int __init
calgary_init_one(struct pci_dev
*dev
)
787 address
= locate_register_space(dev
);
788 /* map entire 1MB of Calgary config space */
789 bbar
= ioremap_nocache(address
, 1024 * 1024);
795 ret
= calgary_setup_tar(dev
, bbar
);
799 dev
->bus
->self
= dev
;
800 calgary_enable_translation(dev
);
810 static int __init
calgary_init(void)
812 int i
, ret
= -ENODEV
;
813 struct pci_dev
*dev
= NULL
;
815 for (i
= 0; i
<= num_online_nodes() * MAX_NUM_OF_PHBS
; i
++) {
816 dev
= pci_get_device(PCI_VENDOR_ID_IBM
,
817 PCI_DEVICE_ID_IBM_CALGARY
,
821 if (!translate_phb(dev
)) {
822 calgary_init_one_nontraslated(dev
);
825 if (!tce_table_kva
[i
] && !translate_empty_slots
) {
829 ret
= calgary_init_one(dev
);
837 for (i
--; i
>= 0; i
--) {
838 dev
= pci_find_device_reverse(PCI_VENDOR_ID_IBM
,
839 PCI_DEVICE_ID_IBM_CALGARY
,
841 if (!translate_phb(dev
)) {
845 if (!tce_table_kva
[i
] && !translate_empty_slots
)
847 calgary_disable_translation(dev
);
848 calgary_free_tar(dev
);
855 static inline int __init
determine_tce_table_size(u64 ram
)
859 if (specified_table_size
!= TCE_TABLE_SIZE_UNSPECIFIED
)
860 return specified_table_size
;
863 * Table sizes are from 0 to 7 (TCE_TABLE_SIZE_64K to
864 * TCE_TABLE_SIZE_8M). Table size 0 has 8K entries and each
865 * larger table size has twice as many entries, so shift the
866 * max ram address by 13 to divide by 8K and then look at the
867 * order of the result to choose between 0-7.
869 ret
= get_order(ram
>> 13);
870 if (ret
> TCE_TABLE_SIZE_8M
)
871 ret
= TCE_TABLE_SIZE_8M
;
876 void __init
detect_calgary(void)
884 * if the user specified iommu=off or iommu=soft or we found
885 * another HW IOMMU already, bail out.
887 if (swiotlb
|| no_iommu
|| iommu_detected
)
890 specified_table_size
= determine_tce_table_size(end_pfn
* PAGE_SIZE
);
892 for (bus
= 0, table_idx
= 0;
893 bus
<= num_online_nodes() * MAX_PHB_BUS_NUM
;
895 BUG_ON(bus
> MAX_NUMNODES
* MAX_PHB_BUS_NUM
);
896 if (read_pci_config(bus
, 0, 0, 0) != PCI_VENDOR_DEVICE_ID_CALGARY
)
898 if (test_bit(bus
, translation_disabled
)) {
899 printk(KERN_INFO
"Calgary: translation is disabled for "
901 /* skip this phb, don't allocate a tbl for it */
902 tce_table_kva
[table_idx
] = NULL
;
907 * scan the first slot of the PCI bus to see if there
908 * are any devices present
910 val
= read_pci_config(bus
, 1, 0, 0);
911 if (val
!= 0xffffffff || translate_empty_slots
) {
912 tbl
= alloc_tce_table();
919 tce_table_kva
[table_idx
] = tbl
;
925 calgary_detected
= 1;
926 printk(KERN_INFO
"PCI-DMA: Calgary IOMMU detected. "
927 "TCE table spec is %d.\n", specified_table_size
);
932 for (--table_idx
; table_idx
>= 0; --table_idx
)
933 if (tce_table_kva
[table_idx
])
934 free_tce_table(tce_table_kva
[table_idx
]);
937 int __init
calgary_iommu_init(void)
941 if (no_iommu
|| swiotlb
)
944 if (!calgary_detected
)
947 /* ok, we're trying to use Calgary - let's roll */
948 printk(KERN_INFO
"PCI-DMA: Using Calgary IOMMU\n");
950 ret
= calgary_init();
952 printk(KERN_ERR
"PCI-DMA: Calgary init failed %d, "
953 "falling back to no_iommu\n", ret
);
954 if (end_pfn
> MAX_DMA32_PFN
)
955 printk(KERN_ERR
"WARNING more than 4GB of memory, "
956 "32bit PCI may malfunction.\n");
961 dma_ops
= &calgary_dma_ops
;
966 static int __init
calgary_parse_options(char *p
)
973 if (!strncmp(p
, "64k", 3))
974 specified_table_size
= TCE_TABLE_SIZE_64K
;
975 else if (!strncmp(p
, "128k", 4))
976 specified_table_size
= TCE_TABLE_SIZE_128K
;
977 else if (!strncmp(p
, "256k", 4))
978 specified_table_size
= TCE_TABLE_SIZE_256K
;
979 else if (!strncmp(p
, "512k", 4))
980 specified_table_size
= TCE_TABLE_SIZE_512K
;
981 else if (!strncmp(p
, "1M", 2))
982 specified_table_size
= TCE_TABLE_SIZE_1M
;
983 else if (!strncmp(p
, "2M", 2))
984 specified_table_size
= TCE_TABLE_SIZE_2M
;
985 else if (!strncmp(p
, "4M", 2))
986 specified_table_size
= TCE_TABLE_SIZE_4M
;
987 else if (!strncmp(p
, "8M", 2))
988 specified_table_size
= TCE_TABLE_SIZE_8M
;
990 len
= strlen("translate_empty_slots");
991 if (!strncmp(p
, "translate_empty_slots", len
))
992 translate_empty_slots
= 1;
994 len
= strlen("disable");
995 if (!strncmp(p
, "disable", len
)) {
1001 bridge
= simple_strtol(p
, &endp
, 0);
1005 if (bridge
<= (num_online_nodes() * MAX_PHB_BUS_NUM
)) {
1006 printk(KERN_INFO
"Calgary: disabling "
1007 "translation for PHB 0x%x\n", bridge
);
1008 set_bit(bridge
, translation_disabled
);
1012 p
= strpbrk(p
, ",");
1020 __setup("calgary=", calgary_parse_options
);