2 * Derived from arch/powerpc/kernel/iommu.c
4 * Copyright (C) IBM Corporation, 2006
6 * Author: Jon Mason <jdmason@us.ibm.com>
7 * Author: Muli Ben-Yehuda <muli@il.ibm.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include <linux/config.h>
25 #include <linux/kernel.h>
26 #include <linux/init.h>
27 #include <linux/types.h>
28 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/string.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/init.h>
34 #include <linux/bitops.h>
35 #include <linux/pci_ids.h>
36 #include <linux/pci.h>
37 #include <linux/delay.h>
38 #include <asm/proto.h>
39 #include <asm/calgary.h>
41 #include <asm/pci-direct.h>
42 #include <asm/system.h>
45 #define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
46 #define PCI_VENDOR_DEVICE_ID_CALGARY \
47 (PCI_VENDOR_ID_IBM | PCI_DEVICE_ID_IBM_CALGARY << 16)
49 /* we need these for register space address calculation */
50 #define START_ADDRESS 0xfe000000
51 #define CHASSIS_BASE 0
52 #define ONE_BASED_CHASSIS_NUM 1
54 /* register offsets inside the host bridge space */
55 #define PHB_CSR_OFFSET 0x0110
56 #define PHB_PLSSR_OFFSET 0x0120
57 #define PHB_CONFIG_RW_OFFSET 0x0160
58 #define PHB_IOBASE_BAR_LOW 0x0170
59 #define PHB_IOBASE_BAR_HIGH 0x0180
60 #define PHB_MEM_1_LOW 0x0190
61 #define PHB_MEM_1_HIGH 0x01A0
62 #define PHB_IO_ADDR_SIZE 0x01B0
63 #define PHB_MEM_1_SIZE 0x01C0
64 #define PHB_MEM_ST_OFFSET 0x01D0
65 #define PHB_AER_OFFSET 0x0200
66 #define PHB_CONFIG_0_HIGH 0x0220
67 #define PHB_CONFIG_0_LOW 0x0230
68 #define PHB_CONFIG_0_END 0x0240
69 #define PHB_MEM_2_LOW 0x02B0
70 #define PHB_MEM_2_HIGH 0x02C0
71 #define PHB_MEM_2_SIZE_HIGH 0x02D0
72 #define PHB_MEM_2_SIZE_LOW 0x02E0
73 #define PHB_DOSHOLE_OFFSET 0x08E0
76 #define PHB_TCE_ENABLE 0x20000000
77 #define PHB_SLOT_DISABLE 0x1C000000
78 #define PHB_DAC_DISABLE 0x01000000
79 #define PHB_MEM2_ENABLE 0x00400000
80 #define PHB_MCSR_ENABLE 0x00100000
81 /* TAR (Table Address Register) */
82 #define TAR_SW_BITS 0x0000ffffffff800fUL
83 #define TAR_VALID 0x0000000000000008UL
84 /* CSR (Channel/DMA Status Register) */
85 #define CSR_AGENT_MASK 0xffe0ffff
87 #define MAX_NUM_OF_PHBS 8 /* how many PHBs in total? */
88 #define MAX_NUM_CHASSIS 8 /* max number of chassis */
89 #define MAX_PHB_BUS_NUM (MAX_NUM_OF_PHBS * MAX_NUM_CHASSIS * 2) /* max dev->bus->number */
90 #define PHBS_PER_CALGARY 4
92 /* register offsets in Calgary's internal register space */
93 static const unsigned long tar_offsets
[] = {
100 static const unsigned long split_queue_offsets
[] = {
101 0x4870 /* SPLIT QUEUE 0 */,
102 0x5870 /* SPLIT QUEUE 1 */,
103 0x6870 /* SPLIT QUEUE 2 */,
104 0x7870 /* SPLIT QUEUE 3 */
107 static const unsigned long phb_offsets
[] = {
114 unsigned int specified_table_size
= TCE_TABLE_SIZE_UNSPECIFIED
;
115 static int translate_empty_slots __read_mostly
= 0;
116 static int calgary_detected __read_mostly
= 0;
118 struct calgary_bus_info
{
120 unsigned char translation_disabled
;
124 static struct calgary_bus_info bus_info
[MAX_PHB_BUS_NUM
] = { { NULL
, 0, 0 }, };
126 static void tce_cache_blast(struct iommu_table
*tbl
);
128 /* enable this to stress test the chip's TCE cache */
129 #ifdef CONFIG_IOMMU_DEBUG
130 static inline void tce_cache_blast_stress(struct iommu_table
*tbl
)
132 tce_cache_blast(tbl
);
135 static inline void tce_cache_blast_stress(struct iommu_table
*tbl
)
138 #endif /* BLAST_TCE_CACHE_ON_UNMAP */
140 static inline unsigned int num_dma_pages(unsigned long dma
, unsigned int dmalen
)
144 npages
= PAGE_ALIGN(dma
+ dmalen
) - (dma
& PAGE_MASK
);
145 npages
>>= PAGE_SHIFT
;
150 static inline int translate_phb(struct pci_dev
* dev
)
152 int disabled
= bus_info
[dev
->bus
->number
].translation_disabled
;
156 static void iommu_range_reserve(struct iommu_table
*tbl
,
157 unsigned long start_addr
, unsigned int npages
)
162 index
= start_addr
>> PAGE_SHIFT
;
164 /* bail out if we're asked to reserve a region we don't cover */
165 if (index
>= tbl
->it_size
)
168 end
= index
+ npages
;
169 if (end
> tbl
->it_size
) /* don't go off the table */
172 while (index
< end
) {
173 if (test_bit(index
, tbl
->it_map
))
174 printk(KERN_ERR
"Calgary: entry already allocated at "
175 "0x%lx tbl %p dma 0x%lx npages %u\n",
176 index
, tbl
, start_addr
, npages
);
179 set_bit_string(tbl
->it_map
, start_addr
>> PAGE_SHIFT
, npages
);
182 static unsigned long iommu_range_alloc(struct iommu_table
*tbl
,
185 unsigned long offset
;
189 offset
= find_next_zero_string(tbl
->it_map
, tbl
->it_hint
,
190 tbl
->it_size
, npages
);
191 if (offset
== ~0UL) {
192 tce_cache_blast(tbl
);
193 offset
= find_next_zero_string(tbl
->it_map
, 0,
194 tbl
->it_size
, npages
);
195 if (offset
== ~0UL) {
196 printk(KERN_WARNING
"Calgary: IOMMU full.\n");
197 if (panic_on_overflow
)
198 panic("Calgary: fix the allocator.\n");
200 return bad_dma_address
;
204 set_bit_string(tbl
->it_map
, offset
, npages
);
205 tbl
->it_hint
= offset
+ npages
;
206 BUG_ON(tbl
->it_hint
> tbl
->it_size
);
211 static dma_addr_t
iommu_alloc(struct iommu_table
*tbl
, void *vaddr
,
212 unsigned int npages
, int direction
)
214 unsigned long entry
, flags
;
215 dma_addr_t ret
= bad_dma_address
;
217 spin_lock_irqsave(&tbl
->it_lock
, flags
);
219 entry
= iommu_range_alloc(tbl
, npages
);
221 if (unlikely(entry
== bad_dma_address
))
224 /* set the return dma address */
225 ret
= (entry
<< PAGE_SHIFT
) | ((unsigned long)vaddr
& ~PAGE_MASK
);
227 /* put the TCEs in the HW table */
228 tce_build(tbl
, entry
, npages
, (unsigned long)vaddr
& PAGE_MASK
,
231 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
236 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
237 printk(KERN_WARNING
"Calgary: failed to allocate %u pages in "
238 "iommu %p\n", npages
, tbl
);
239 return bad_dma_address
;
242 static void __iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
248 entry
= dma_addr
>> PAGE_SHIFT
;
250 BUG_ON(entry
+ npages
> tbl
->it_size
);
252 tce_free(tbl
, entry
, npages
);
254 for (i
= 0; i
< npages
; ++i
) {
255 if (!test_bit(entry
+ i
, tbl
->it_map
))
256 printk(KERN_ERR
"Calgary: bit is off at 0x%lx "
257 "tbl %p dma 0x%Lx entry 0x%lx npages %u\n",
258 entry
+ i
, tbl
, dma_addr
, entry
, npages
);
261 __clear_bit_string(tbl
->it_map
, entry
, npages
);
263 tce_cache_blast_stress(tbl
);
266 static void iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
271 spin_lock_irqsave(&tbl
->it_lock
, flags
);
273 __iommu_free(tbl
, dma_addr
, npages
);
275 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
278 static void __calgary_unmap_sg(struct iommu_table
*tbl
,
279 struct scatterlist
*sglist
, int nelems
, int direction
)
283 dma_addr_t dma
= sglist
->dma_address
;
284 unsigned int dmalen
= sglist
->dma_length
;
289 npages
= num_dma_pages(dma
, dmalen
);
290 __iommu_free(tbl
, dma
, npages
);
295 void calgary_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
296 int nelems
, int direction
)
299 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
301 if (!translate_phb(to_pci_dev(dev
)))
304 spin_lock_irqsave(&tbl
->it_lock
, flags
);
306 __calgary_unmap_sg(tbl
, sglist
, nelems
, direction
);
308 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
311 static int calgary_nontranslate_map_sg(struct device
* dev
,
312 struct scatterlist
*sg
, int nelems
, int direction
)
316 for (i
= 0; i
< nelems
; i
++ ) {
317 struct scatterlist
*s
= &sg
[i
];
319 s
->dma_address
= virt_to_bus(page_address(s
->page
) +s
->offset
);
320 s
->dma_length
= s
->length
;
325 int calgary_map_sg(struct device
*dev
, struct scatterlist
*sg
,
326 int nelems
, int direction
)
328 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
335 if (!translate_phb(to_pci_dev(dev
)))
336 return calgary_nontranslate_map_sg(dev
, sg
, nelems
, direction
);
338 spin_lock_irqsave(&tbl
->it_lock
, flags
);
340 for (i
= 0; i
< nelems
; i
++ ) {
341 struct scatterlist
*s
= &sg
[i
];
344 vaddr
= (unsigned long)page_address(s
->page
) + s
->offset
;
345 npages
= num_dma_pages(vaddr
, s
->length
);
347 entry
= iommu_range_alloc(tbl
, npages
);
348 if (entry
== bad_dma_address
) {
349 /* makes sure unmap knows to stop */
354 s
->dma_address
= (entry
<< PAGE_SHIFT
) | s
->offset
;
356 /* insert into HW table */
357 tce_build(tbl
, entry
, npages
, vaddr
& PAGE_MASK
,
360 s
->dma_length
= s
->length
;
363 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
367 __calgary_unmap_sg(tbl
, sg
, nelems
, direction
);
368 for (i
= 0; i
< nelems
; i
++) {
369 sg
[i
].dma_address
= bad_dma_address
;
370 sg
[i
].dma_length
= 0;
372 spin_unlock_irqrestore(&tbl
->it_lock
, flags
);
376 dma_addr_t
calgary_map_single(struct device
*dev
, void *vaddr
,
377 size_t size
, int direction
)
379 dma_addr_t dma_handle
= bad_dma_address
;
382 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
384 uaddr
= (unsigned long)vaddr
;
385 npages
= num_dma_pages(uaddr
, size
);
387 if (translate_phb(to_pci_dev(dev
)))
388 dma_handle
= iommu_alloc(tbl
, vaddr
, npages
, direction
);
390 dma_handle
= virt_to_bus(vaddr
);
395 void calgary_unmap_single(struct device
*dev
, dma_addr_t dma_handle
,
396 size_t size
, int direction
)
398 struct iommu_table
*tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
401 if (!translate_phb(to_pci_dev(dev
)))
404 npages
= num_dma_pages(dma_handle
, size
);
405 iommu_free(tbl
, dma_handle
, npages
);
408 void* calgary_alloc_coherent(struct device
*dev
, size_t size
,
409 dma_addr_t
*dma_handle
, gfp_t flag
)
413 unsigned int npages
, order
;
414 struct iommu_table
*tbl
;
416 tbl
= to_pci_dev(dev
)->bus
->self
->sysdata
;
418 size
= PAGE_ALIGN(size
); /* size rounded up to full pages */
419 npages
= size
>> PAGE_SHIFT
;
420 order
= get_order(size
);
422 /* alloc enough pages (and possibly more) */
423 ret
= (void *)__get_free_pages(flag
, order
);
426 memset(ret
, 0, size
);
428 if (translate_phb(to_pci_dev(dev
))) {
429 /* set up tces to cover the allocated range */
430 mapping
= iommu_alloc(tbl
, ret
, npages
, DMA_BIDIRECTIONAL
);
431 if (mapping
== bad_dma_address
)
434 *dma_handle
= mapping
;
435 } else /* non translated slot */
436 *dma_handle
= virt_to_bus(ret
);
441 free_pages((unsigned long)ret
, get_order(size
));
447 static struct dma_mapping_ops calgary_dma_ops
= {
448 .alloc_coherent
= calgary_alloc_coherent
,
449 .map_single
= calgary_map_single
,
450 .unmap_single
= calgary_unmap_single
,
451 .map_sg
= calgary_map_sg
,
452 .unmap_sg
= calgary_unmap_sg
,
455 static inline int busno_to_phbid(unsigned char num
)
457 return bus_info
[num
].phbid
;
460 static inline unsigned long split_queue_offset(unsigned char num
)
462 size_t idx
= busno_to_phbid(num
);
464 return split_queue_offsets
[idx
];
467 static inline unsigned long tar_offset(unsigned char num
)
469 size_t idx
= busno_to_phbid(num
);
471 return tar_offsets
[idx
];
474 static inline unsigned long phb_offset(unsigned char num
)
476 size_t idx
= busno_to_phbid(num
);
478 return phb_offsets
[idx
];
481 static inline void __iomem
* calgary_reg(void __iomem
*bar
, unsigned long offset
)
483 unsigned long target
= ((unsigned long)bar
) | offset
;
484 return (void __iomem
*)target
;
487 static void tce_cache_blast(struct iommu_table
*tbl
)
492 void __iomem
*bbar
= tbl
->bbar
;
493 void __iomem
*target
;
495 /* disable arbitration on the bus */
496 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_AER_OFFSET
);
500 /* read plssr to ensure it got there */
501 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_PLSSR_OFFSET
);
504 /* poll split queues until all DMA activity is done */
505 target
= calgary_reg(bbar
, split_queue_offset(tbl
->it_busno
));
509 } while ((val
& 0xff) != 0xff && i
< 100);
511 printk(KERN_WARNING
"Calgary: PCI bus not quiesced, "
512 "continuing anyway\n");
514 /* invalidate TCE cache */
515 target
= calgary_reg(bbar
, tar_offset(tbl
->it_busno
));
516 writeq(tbl
->tar_val
, target
);
518 /* enable arbitration */
519 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_AER_OFFSET
);
521 (void)readl(target
); /* flush */
524 static void __init
calgary_reserve_mem_region(struct pci_dev
*dev
, u64 start
,
527 unsigned int numpages
;
529 limit
= limit
| 0xfffff;
532 numpages
= ((limit
- start
) >> PAGE_SHIFT
);
533 iommu_range_reserve(dev
->sysdata
, start
, numpages
);
536 static void __init
calgary_reserve_peripheral_mem_1(struct pci_dev
*dev
)
538 void __iomem
*target
;
539 u64 low
, high
, sizelow
;
541 struct iommu_table
*tbl
= dev
->sysdata
;
542 unsigned char busnum
= dev
->bus
->number
;
543 void __iomem
*bbar
= tbl
->bbar
;
545 /* peripheral MEM_1 region */
546 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_LOW
);
547 low
= be32_to_cpu(readl(target
));
548 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_HIGH
);
549 high
= be32_to_cpu(readl(target
));
550 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_1_SIZE
);
551 sizelow
= be32_to_cpu(readl(target
));
553 start
= (high
<< 32) | low
;
556 calgary_reserve_mem_region(dev
, start
, limit
);
559 static void __init
calgary_reserve_peripheral_mem_2(struct pci_dev
*dev
)
561 void __iomem
*target
;
563 u64 low
, high
, sizelow
, sizehigh
;
565 struct iommu_table
*tbl
= dev
->sysdata
;
566 unsigned char busnum
= dev
->bus
->number
;
567 void __iomem
*bbar
= tbl
->bbar
;
570 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
571 val32
= be32_to_cpu(readl(target
));
572 if (!(val32
& PHB_MEM2_ENABLE
))
575 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_LOW
);
576 low
= be32_to_cpu(readl(target
));
577 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_HIGH
);
578 high
= be32_to_cpu(readl(target
));
579 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_SIZE_LOW
);
580 sizelow
= be32_to_cpu(readl(target
));
581 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_MEM_2_SIZE_HIGH
);
582 sizehigh
= be32_to_cpu(readl(target
));
584 start
= (high
<< 32) | low
;
585 limit
= (sizehigh
<< 32) | sizelow
;
587 calgary_reserve_mem_region(dev
, start
, limit
);
591 * some regions of the IO address space do not get translated, so we
592 * must not give devices IO addresses in those regions. The regions
593 * are the 640KB-1MB region and the two PCI peripheral memory holes.
594 * Reserve all of them in the IOMMU bitmap to avoid giving them out
597 static void __init
calgary_reserve_regions(struct pci_dev
*dev
)
601 unsigned char busnum
;
603 struct iommu_table
*tbl
= dev
->sysdata
;
606 busnum
= dev
->bus
->number
;
608 /* reserve bad_dma_address in case it's a legal address */
609 iommu_range_reserve(tbl
, bad_dma_address
, 1);
611 /* avoid the BIOS/VGA first 640KB-1MB region */
612 start
= (640 * 1024);
613 npages
= ((1024 - 640) * 1024) >> PAGE_SHIFT
;
614 iommu_range_reserve(tbl
, start
, npages
);
616 /* reserve the two PCI peripheral memory regions in IO space */
617 calgary_reserve_peripheral_mem_1(dev
);
618 calgary_reserve_peripheral_mem_2(dev
);
621 static int __init
calgary_setup_tar(struct pci_dev
*dev
, void __iomem
*bbar
)
625 void __iomem
*target
;
627 struct iommu_table
*tbl
;
629 /* build TCE tables for each PHB */
630 ret
= build_tce_table(dev
, bbar
);
635 tbl
->it_base
= (unsigned long)bus_info
[dev
->bus
->number
].tce_space
;
636 tce_free(tbl
, 0, tbl
->it_size
);
638 calgary_reserve_regions(dev
);
640 /* set TARs for each PHB */
641 target
= calgary_reg(bbar
, tar_offset(dev
->bus
->number
));
642 val64
= be64_to_cpu(readq(target
));
644 /* zero out all TAR bits under sw control */
645 val64
&= ~TAR_SW_BITS
;
648 table_phys
= (u64
)__pa(tbl
->it_base
);
651 BUG_ON(specified_table_size
> TCE_TABLE_SIZE_8M
);
652 val64
|= (u64
) specified_table_size
;
654 tbl
->tar_val
= cpu_to_be64(val64
);
655 writeq(tbl
->tar_val
, target
);
656 readq(target
); /* flush */
661 static void __init
calgary_free_bus(struct pci_dev
*dev
)
664 struct iommu_table
*tbl
= dev
->sysdata
;
665 void __iomem
*target
;
666 unsigned int bitmapsz
;
668 target
= calgary_reg(tbl
->bbar
, tar_offset(dev
->bus
->number
));
669 val64
= be64_to_cpu(readq(target
));
670 val64
&= ~TAR_SW_BITS
;
671 writeq(cpu_to_be64(val64
), target
);
672 readq(target
); /* flush */
674 bitmapsz
= tbl
->it_size
/ BITS_PER_BYTE
;
675 free_pages((unsigned long)tbl
->it_map
, get_order(bitmapsz
));
681 /* Can't free bootmem allocated memory after system is up :-( */
682 bus_info
[dev
->bus
->number
].tce_space
= NULL
;
685 static void calgary_watchdog(unsigned long data
)
687 struct pci_dev
*dev
= (struct pci_dev
*)data
;
688 struct iommu_table
*tbl
= dev
->sysdata
;
689 void __iomem
*bbar
= tbl
->bbar
;
691 void __iomem
*target
;
693 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) | PHB_CSR_OFFSET
);
694 val32
= be32_to_cpu(readl(target
));
696 /* If no error, the agent ID in the CSR is not valid */
697 if (val32
& CSR_AGENT_MASK
) {
698 printk(KERN_EMERG
"calgary_watchdog: DMA error on bus %d, "
699 "CSR = %#x\n", dev
->bus
->number
, val32
);
702 /* Disable bus that caused the error */
703 target
= calgary_reg(bbar
, phb_offset(tbl
->it_busno
) |
704 PHB_CONFIG_RW_OFFSET
);
705 val32
= be32_to_cpu(readl(target
));
706 val32
|= PHB_SLOT_DISABLE
;
707 writel(cpu_to_be32(val32
), target
);
708 readl(target
); /* flush */
710 /* Reset the timer */
711 mod_timer(&tbl
->watchdog_timer
, jiffies
+ 2 * HZ
);
715 static void __init
calgary_enable_translation(struct pci_dev
*dev
)
718 unsigned char busnum
;
719 void __iomem
*target
;
721 struct iommu_table
*tbl
;
723 busnum
= dev
->bus
->number
;
727 /* enable TCE in PHB Config Register */
728 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
729 val32
= be32_to_cpu(readl(target
));
730 val32
|= PHB_TCE_ENABLE
| PHB_DAC_DISABLE
| PHB_MCSR_ENABLE
;
732 printk(KERN_INFO
"Calgary: enabling translation on PHB %d\n", busnum
);
733 printk(KERN_INFO
"Calgary: errant DMAs will now be prevented on this "
736 writel(cpu_to_be32(val32
), target
);
737 readl(target
); /* flush */
739 init_timer(&tbl
->watchdog_timer
);
740 tbl
->watchdog_timer
.function
= &calgary_watchdog
;
741 tbl
->watchdog_timer
.data
= (unsigned long)dev
;
742 mod_timer(&tbl
->watchdog_timer
, jiffies
);
745 static void __init
calgary_disable_translation(struct pci_dev
*dev
)
748 unsigned char busnum
;
749 void __iomem
*target
;
751 struct iommu_table
*tbl
;
753 busnum
= dev
->bus
->number
;
757 /* disable TCE in PHB Config Register */
758 target
= calgary_reg(bbar
, phb_offset(busnum
) | PHB_CONFIG_RW_OFFSET
);
759 val32
= be32_to_cpu(readl(target
));
760 val32
&= ~(PHB_TCE_ENABLE
| PHB_DAC_DISABLE
| PHB_MCSR_ENABLE
);
762 printk(KERN_INFO
"Calgary: disabling translation on PHB %d!\n", busnum
);
763 writel(cpu_to_be32(val32
), target
);
764 readl(target
); /* flush */
766 del_timer_sync(&tbl
->watchdog_timer
);
769 static inline unsigned int __init
locate_register_space(struct pci_dev
*dev
)
774 rionodeid
= (dev
->bus
->number
% 15 > 4) ? 3 : 2;
776 * register space address calculation as follows:
777 * FE0MB-8MB*OneBasedChassisNumber+1MB*(RioNodeId-ChassisBase)
778 * ChassisBase is always zero for x366/x260/x460
779 * RioNodeId is 2 for first Calgary, 3 for second Calgary
781 address
= START_ADDRESS
-
782 (0x800000 * (ONE_BASED_CHASSIS_NUM
+ dev
->bus
->number
/ 15)) +
783 (0x100000) * (rionodeid
- CHASSIS_BASE
);
787 static void __init
calgary_init_one_nontraslated(struct pci_dev
*dev
)
791 dev
->bus
->self
= dev
;
794 static int __init
calgary_init_one(struct pci_dev
*dev
)
800 address
= locate_register_space(dev
);
801 /* map entire 1MB of Calgary config space */
802 bbar
= ioremap_nocache(address
, 1024 * 1024);
808 ret
= calgary_setup_tar(dev
, bbar
);
813 dev
->bus
->self
= dev
;
814 calgary_enable_translation(dev
);
824 static int __init
calgary_init(void)
826 int i
, ret
= -ENODEV
;
827 struct pci_dev
*dev
= NULL
;
829 for (i
= 0; i
< MAX_PHB_BUS_NUM
; i
++) {
830 dev
= pci_get_device(PCI_VENDOR_ID_IBM
,
831 PCI_DEVICE_ID_IBM_CALGARY
,
835 if (!translate_phb(dev
)) {
836 calgary_init_one_nontraslated(dev
);
839 if (!bus_info
[dev
->bus
->number
].tce_space
&& !translate_empty_slots
)
842 ret
= calgary_init_one(dev
);
850 for (i
--; i
>= 0; i
--) {
851 dev
= pci_find_device_reverse(PCI_VENDOR_ID_IBM
,
852 PCI_DEVICE_ID_IBM_CALGARY
,
856 if (!translate_phb(dev
)) {
860 if (!bus_info
[dev
->bus
->number
].tce_space
&& !translate_empty_slots
)
863 calgary_disable_translation(dev
);
864 calgary_free_bus(dev
);
865 pci_dev_put(dev
); /* Undo calgary_init_one()'s pci_dev_get() */
871 static inline int __init
determine_tce_table_size(u64 ram
)
875 if (specified_table_size
!= TCE_TABLE_SIZE_UNSPECIFIED
)
876 return specified_table_size
;
879 * Table sizes are from 0 to 7 (TCE_TABLE_SIZE_64K to
880 * TCE_TABLE_SIZE_8M). Table size 0 has 8K entries and each
881 * larger table size has twice as many entries, so shift the
882 * max ram address by 13 to divide by 8K and then look at the
883 * order of the result to choose between 0-7.
885 ret
= get_order(ram
>> 13);
886 if (ret
> TCE_TABLE_SIZE_8M
)
887 ret
= TCE_TABLE_SIZE_8M
;
892 void __init
detect_calgary(void)
897 int calgary_found
= 0;
901 * if the user specified iommu=off or iommu=soft or we found
902 * another HW IOMMU already, bail out.
904 if (swiotlb
|| no_iommu
|| iommu_detected
)
907 specified_table_size
= determine_tce_table_size(end_pfn
* PAGE_SIZE
);
909 for (bus
= 0; bus
< MAX_PHB_BUS_NUM
; bus
++) {
911 struct calgary_bus_info
*info
= &bus_info
[bus
];
914 if (read_pci_config(bus
, 0, 0, 0) != PCI_VENDOR_DEVICE_ID_CALGARY
)
918 * There are 4 PHBs per Calgary chip. Set phb to which phb (0-3)
919 * it is connected to releative to the clagary chip.
921 phb
= (phb
+ 1) % PHBS_PER_CALGARY
;
923 if (info
->translation_disabled
)
927 * Scan the slots of the PCI bus to see if there is a device present.
928 * The parent bus will be the zero-ith device, so start at 1.
930 for (dev
= 1; dev
< 8; dev
++) {
931 val
= read_pci_config(bus
, dev
, 0, 0);
932 if (val
!= 0xffffffff || translate_empty_slots
) {
933 tbl
= alloc_tce_table();
936 info
->tce_space
= tbl
;
946 calgary_detected
= 1;
947 printk(KERN_INFO
"PCI-DMA: Calgary IOMMU detected. "
948 "TCE table spec is %d.\n", specified_table_size
);
953 for (--bus
; bus
>= 0; --bus
) {
954 struct calgary_bus_info
*info
= &bus_info
[bus
];
957 free_tce_table(info
->tce_space
);
961 int __init
calgary_iommu_init(void)
965 if (no_iommu
|| swiotlb
)
968 if (!calgary_detected
)
971 /* ok, we're trying to use Calgary - let's roll */
972 printk(KERN_INFO
"PCI-DMA: Using Calgary IOMMU\n");
974 ret
= calgary_init();
976 printk(KERN_ERR
"PCI-DMA: Calgary init failed %d, "
977 "falling back to no_iommu\n", ret
);
978 if (end_pfn
> MAX_DMA32_PFN
)
979 printk(KERN_ERR
"WARNING more than 4GB of memory, "
980 "32bit PCI may malfunction.\n");
985 dma_ops
= &calgary_dma_ops
;
990 static int __init
calgary_parse_options(char *p
)
997 if (!strncmp(p
, "64k", 3))
998 specified_table_size
= TCE_TABLE_SIZE_64K
;
999 else if (!strncmp(p
, "128k", 4))
1000 specified_table_size
= TCE_TABLE_SIZE_128K
;
1001 else if (!strncmp(p
, "256k", 4))
1002 specified_table_size
= TCE_TABLE_SIZE_256K
;
1003 else if (!strncmp(p
, "512k", 4))
1004 specified_table_size
= TCE_TABLE_SIZE_512K
;
1005 else if (!strncmp(p
, "1M", 2))
1006 specified_table_size
= TCE_TABLE_SIZE_1M
;
1007 else if (!strncmp(p
, "2M", 2))
1008 specified_table_size
= TCE_TABLE_SIZE_2M
;
1009 else if (!strncmp(p
, "4M", 2))
1010 specified_table_size
= TCE_TABLE_SIZE_4M
;
1011 else if (!strncmp(p
, "8M", 2))
1012 specified_table_size
= TCE_TABLE_SIZE_8M
;
1014 len
= strlen("translate_empty_slots");
1015 if (!strncmp(p
, "translate_empty_slots", len
))
1016 translate_empty_slots
= 1;
1018 len
= strlen("disable");
1019 if (!strncmp(p
, "disable", len
)) {
1025 bridge
= simple_strtol(p
, &endp
, 0);
1029 if (bridge
< MAX_PHB_BUS_NUM
) {
1030 printk(KERN_INFO
"Calgary: disabling "
1031 "translation for PHB 0x%x\n", bridge
);
1032 bus_info
[bridge
].translation_disabled
= 1;
1036 p
= strpbrk(p
, ",");
1044 __setup("calgary=", calgary_parse_options
);