2 ** System Bus Adapter (SBA) I/O MMU manager
4 ** (c) Copyright 2000 Grant Grundler
5 ** (c) Copyright 2000 Hewlett-Packard Company
7 ** Portions (c) 1999 Dave S. Miller (from sparc64 I/O MMU code)
9 ** This program is free software; you can redistribute it and/or modify
10 ** it under the terms of the GNU General Public License as published by
11 ** the Free Software Foundation; either version 2 of the License, or
12 ** (at your option) any later version.
15 ** This module initializes the IOC (I/O Controller) found on B1000/C3000/
16 ** J5000/J7000/N-class/L-class machines and their successors.
18 ** FIXME: Multi-IOC support missing - depends on hp_device data
19 ** FIXME: add DMA hint support programming in both sba and lba modules.
22 #include <linux/config.h>
23 #include <linux/types.h>
24 #include <linux/kernel.h>
25 #include <linux/spinlock.h>
26 #include <linux/malloc.h>
27 #include <linux/init.h>
30 #include <linux/string.h>
31 #define PCI_DEBUG /* for ASSERT */
32 #include <linux/pci.h>
35 #include <asm/byteorder.h>
37 #include <asm/dma.h> /* for DMA_CHUNK_SIZE */
39 #include <asm/hardware.h> /* for register_driver() stuff */
40 #include <asm/gsc.h> /* FIXME: for gsc_read/gsc_write */
42 #include <linux/proc_fs.h>
43 #include <asm/runway.h> /* for proc_runway_root */
46 #define MODULE_NAME "SBA"
49 ** The number of debug flags is a clue - this code is fragile.
50 ** Don't even think about messing with it unless you have
51 ** plenty of 710's to sacrafice to the computer gods. :^)
55 #undef DEBUG_SBA_RUN_SG
56 #undef DEBUG_SBA_RESOURCE
57 #undef ASSERT_PDIR_SANITY
58 #undef DEBUG_LARGE_SG_ENTRIES
63 #define SBA_INLINE __inline__
67 #define DBG_INIT(x...) printk(x)
69 #define DBG_INIT(x...)
73 #define DBG_RUN(x...) printk(x)
78 #ifdef DEBUG_SBA_RUN_SG
79 #define DBG_RUN_SG(x...) printk(x)
81 #define DBG_RUN_SG(x...)
85 #ifdef DEBUG_SBA_RESOURCE
86 #define DBG_RES(x...) printk(x)
92 ** The number of pdir entries to "free" before issueing
93 ** a read to PCOM register to flush out PCOM writes.
94 ** Interacts with allocation granularity (ie 4 or 8 entries
95 ** allocated and free'd/purged at a time might make this
99 #define DELAYED_RESOURCE_CNT 16
101 #undef DELAYED_RESOURCE_CNT
104 #define DEFAULT_DMA_HINT_REG 0
106 #define ASTRO_RUNWAY_PORT 0x582
107 #define ASTRO_ROPES_PORT 0x780
109 #define IKE_MERCED_PORT 0x803
110 #define IKE_ROPES_PORT 0x781
112 int sba_driver_callback(struct hp_device
*, struct pa_iodc_driver
*);
114 static struct pa_iodc_driver sba_drivers_for
[] = {
116 /* FIXME: why is SVERSION checked? */
118 {HPHW_IOA
, ASTRO_RUNWAY_PORT
, 0x0, 0xb, 0, 0x10,
119 DRIVER_CHECK_HVERSION
+
120 DRIVER_CHECK_SVERSION
+ DRIVER_CHECK_HWTYPE
,
121 MODULE_NAME
, "I/O MMU", (void *) sba_driver_callback
},
123 {HPHW_BCPORT
, ASTRO_ROPES_PORT
, 0x0, 0xb, 0, 0x10,
124 DRIVER_CHECK_HVERSION
+
125 DRIVER_CHECK_SVERSION
+ DRIVER_CHECK_HWTYPE
,
126 MODULE_NAME
, "I/O MMU", (void *) sba_driver_callback
},
129 /* FIXME : N-class! Use a different "callback"? */
130 {HPHW_BCPORT
, IKE_MERCED_PORT
, 0x0, 0xb, 0, 0x10,
131 DRIVER_CHECK_HVERSION
+
132 DRIVER_CHECK_SVERSION
+ DRIVER_CHECK_HWTYPE
,
133 MODULE_NAME
, "I/O MMU", (void *) sba_driver_callback
},
135 {HPHW_BCPORT
, IKE_ROPES_PORT
, 0x0, 0xb, 0, 0x10,
136 DRIVER_CHECK_HVERSION
+
137 DRIVER_CHECK_SVERSION
+ DRIVER_CHECK_HWTYPE
,
138 MODULE_NAME
, "I/O MMU", (void *) sba_driver_callback
},
143 (char *) NULL
, (char *) NULL
, (void *) NULL
}
147 #define SBA_FUNC_ID 0x0000 /* function id */
148 #define SBA_FCLASS 0x0008 /* function class, bist, header, rev... */
150 #define IS_ASTRO(id) ( \
151 (((id)->hw_type == HPHW_IOA) && ((id)->hversion == ASTRO_RUNWAY_PORT)) || \
152 (((id)->hw_type == HPHW_BCPORT) && ((id)->hversion == ASTRO_ROPES_PORT)) \
155 #define CONFIG_FUNC_SIZE 4096 /* SBA configuration function reg set */
157 #define ASTRO_IOC_OFFSET 0x20000
158 /* Ike's IOC's occupy functions 2 and 3 (not 0 and 1) */
159 #define IKE_IOC_OFFSET(p) ((p+2)*CONFIG_FUNC_SIZE)
161 #define IOC_CTRL 0x8 /* IOC_CTRL offset */
162 #define IOC_CTRL_TE (0x1 << 0) /* TOC Enable */
163 #define IOC_CTRL_RM (0x1 << 8) /* Real Mode */
164 #define IOC_CTRL_NC (0x1 << 9) /* Non Coherent Mode */
166 #define MAX_IOC 2 /* per Ike. Astro only has 1 */
170 ** Offsets into MBIB (Function 0 on Ike and hopefully Astro)
171 ** Firmware programs this stuff. Don't touch it.
173 #define IOS_DIST_BASE 0x390
174 #define IOS_DIST_MASK 0x398
175 #define IOS_DIST_ROUTE 0x3A0
177 #define IOS_DIRECT_BASE 0x3C0
178 #define IOS_DIRECT_MASK 0x3C8
179 #define IOS_DIRECT_ROUTE 0x3D0
182 ** Offsets into I/O TLB (Function 2 and 3 on Ike)
184 #define ROPE0_CTL 0x200 /* "regbus pci0" */
185 #define ROPE1_CTL 0x208
186 #define ROPE2_CTL 0x210
187 #define ROPE3_CTL 0x218
188 #define ROPE4_CTL 0x220
189 #define ROPE5_CTL 0x228
190 #define ROPE6_CTL 0x230
191 #define ROPE7_CTL 0x238
193 #define HF_ENABLE 0x40
196 #define IOC_IBASE 0x300 /* IO TLB */
197 #define IOC_IMASK 0x308
198 #define IOC_PCOM 0x310
199 #define IOC_TCNFG 0x318
200 #define IOC_PDIR_BASE 0x320
202 #define IOC_IOVA_SPACE_BASE 0 /* IOVA ranges start at 0 */
205 ** IOC supports 4/8/16/64KB page sizes (see TCNFG register)
206 ** It's safer (avoid memory corruption) to keep DMA page mappings
207 ** equivalently sized to VM PAGE_SIZE.
209 ** We really can't avoid generating a new mapping for each
210 ** page since the Virtual Coherence Index has to be generated
211 ** and updated for each page.
213 ** IOVP_SIZE could only be greater than PAGE_SIZE if we are
214 ** confident the drivers really only touch the next physical
215 ** page iff that driver instance owns it.
217 #define IOVP_SIZE PAGE_SIZE
218 #define IOVP_SHIFT PAGE_SHIFT
219 #define IOVP_MASK PAGE_MASK
221 #define SBA_PERF_CFG 0x708 /* Performance Counter stuff */
222 #define SBA_PERF_MASK1 0x718
223 #define SBA_PERF_MASK2 0x730
227 ** Offsets into PCI Performance Counters (functions 12 and 13)
228 ** Controlled by PERF registers in function 2 & 3 respectively.
230 #define SBA_PERF_CNT1 0x200
231 #define SBA_PERF_CNT2 0x208
232 #define SBA_PERF_CNT3 0x210
236 char *ioc_hpa
; /* I/O MMU base address */
237 char *res_map
; /* resource map, bit == pdir entry */
238 u64
*pdir_base
; /* physical base address */
240 unsigned long *res_hint
; /* next available IOVP - circular search */
241 unsigned int res_bitshift
; /* from the LEFT! */
242 unsigned int res_size
; /* size of resource map in bytes */
243 unsigned int hint_shift_pdir
;
245 unsigned long hint_mask_pdir
; /* bits used for DMA hints */
246 #ifdef DELAYED_RESOURCE_CNT
247 dma_addr_t res_delay
[DELAYED_RESOURCE_CNT
];
250 #ifdef CONFIG_PROC_FS
251 #define SBA_SEARCH_SAMPLE 0x100
252 unsigned long avg_search
[SBA_SEARCH_SAMPLE
];
253 unsigned long avg_idx
; /* current index into avg_search */
254 unsigned long used_pages
;
255 unsigned long msingle_calls
;
256 unsigned long msingle_pages
;
257 unsigned long msg_calls
;
258 unsigned long msg_pages
;
259 unsigned long usingle_calls
;
260 unsigned long usingle_pages
;
261 unsigned long usg_calls
;
262 unsigned long usg_pages
;
265 /* STUFF We don't need in performance path */
266 unsigned int pdir_size
; /* in bytes, determined by IOV Space size */
267 unsigned long ibase
; /* pdir IOV Space base - shared w/lba_pci */
268 unsigned long imask
; /* pdir IOV Space mask - shared w/lba_pci */
272 struct sba_device
*next
; /* list of LBA's in system */
273 struct hp_device
*iodc
; /* data about dev from firmware */
274 char *sba_hpa
; /* base address */
276 unsigned int flags
; /* state/functionality enabled */
277 unsigned int hw_rev
; /* HW revision of chip */
279 unsigned int num_ioc
; /* number of on-board IOC's */
280 struct ioc ioc
[MAX_IOC
];
284 static struct sba_device
*sba_list
;
285 static int sba_count
;
287 /* Ratio of Host MEM to IOV Space size */
288 static unsigned long sba_mem_ratio
= 4;
290 /* Looks nice and keeps the compiler happy */
291 #define SBA_DEV(d) ((struct sba_device *) (d))
294 #define ROUNDUP(x,y) ((x + ((y)-1)) & ~((y)-1))
297 /************************************
298 ** SBA register read and write support
300 ** BE WARNED: register writes are posted.
301 ** (ie follow writes which must reach HW with a read)
303 #define READ_U8(addr) gsc_readb(addr)
304 #define READ_U16(addr) gsc_readw((u16 *) (addr))
305 #define READ_U32(addr) gsc_readl((u32 *) (addr))
306 #define WRITE_U8(value, addr) gsc_writeb(value, addr)
307 #define WRITE_U16(value, addr) gsc_writew(value, (u16 *) (addr))
308 #define WRITE_U32(value, addr) gsc_writel(value, (u32 *) (addr))
310 #define READ_REG8(addr) gsc_readb(addr)
311 #define READ_REG16(addr) le16_to_cpu(gsc_readw((u16 *) (addr)))
312 #define READ_REG32(addr) le32_to_cpu(gsc_readl((u32 *) (addr)))
313 #define READ_REG64(addr) le64_to_cpu(gsc_readq((u64 *) (addr)))
314 #define WRITE_REG8(value, addr) gsc_writeb(value, addr)
315 #define WRITE_REG16(value, addr) gsc_writew(cpu_to_le16(value), (u16 *) (addr))
316 #define WRITE_REG32(value, addr) gsc_writel(cpu_to_le32(value), (u32 *) (addr))
317 #define WRITE_REG64(value, addr) gsc_writeq(cpu_to_le64(value), (u64 *) (addr))
319 #ifdef DEBUG_SBA_INIT
322 sba_dump_ranges(char *hpa
)
324 printk("SBA at 0x%p\n", hpa
);
325 printk("IOS_DIST_BASE : %08x %08x\n",
326 READ_REG32(hpa
+IOS_DIST_BASE
+4),
327 READ_REG32(hpa
+IOS_DIST_BASE
));
328 printk("IOS_DIST_MASK : %08x %08x\n",
329 READ_REG32(hpa
+IOS_DIST_MASK
+4),
330 READ_REG32(hpa
+IOS_DIST_MASK
));
331 printk("IOS_DIST_ROUTE : %08x %08x\n",
332 READ_REG32(hpa
+IOS_DIST_ROUTE
+4),
333 READ_REG32(hpa
+IOS_DIST_ROUTE
));
335 printk("IOS_DIRECT_BASE : %08x %08x\n",
336 READ_REG32(hpa
+IOS_DIRECT_BASE
+4),
337 READ_REG32(hpa
+IOS_DIRECT_BASE
));
338 printk("IOS_DIRECT_MASK : %08x %08x\n",
339 READ_REG32(hpa
+IOS_DIRECT_MASK
+4),
340 READ_REG32(hpa
+IOS_DIRECT_MASK
));
341 printk("IOS_DIRECT_ROUTE: %08x %08x\n",
342 READ_REG32(hpa
+IOS_DIRECT_ROUTE
+4),
343 READ_REG32(hpa
+IOS_DIRECT_ROUTE
));
347 sba_dump_tlb(char *hpa
)
349 printk("IO TLB at 0x%p\n", hpa
);
350 printk("IOC_IBASE : %08x %08x\n",
351 READ_REG32(hpa
+IOC_IBASE
+4),
352 READ_REG32(hpa
+IOC_IBASE
));
353 printk("IOC_IMASK : %08x %08x\n",
354 READ_REG32(hpa
+IOC_IMASK
+4),
355 READ_REG32(hpa
+IOC_IMASK
));
356 printk("IOC_TCNFG : %08x %08x\n",
357 READ_REG32(hpa
+IOC_TCNFG
+4),
358 READ_REG32(hpa
+IOC_TCNFG
));
359 printk("IOC_PDIR_BASE: %08x %08x\n",
360 READ_REG32(hpa
+IOC_PDIR_BASE
+4),
361 READ_REG32(hpa
+IOC_PDIR_BASE
));
367 #ifdef ASSERT_PDIR_SANITY
370 sba_dump_pdir_entry(struct ioc
*ioc
, char *msg
, uint pide
)
372 /* start printing from lowest pde in rval */
373 u64
*ptr
= &(ioc
->pdir_base
[pide
& (~0U * BITS_PER_LONG
)]);
374 unsigned long *rptr
= (unsigned long *) &(ioc
->res_map
[(pide
>>3) & ~(sizeof(unsigned long) - 1)]);
377 printk("SBA: %s rp %p bit %d rval 0x%lx\n",
379 rptr
, pide
& (BITS_PER_LONG
- 1), *rptr
);
382 while (rcnt
< BITS_PER_LONG
) {
383 printk("%s %2d %p %016Lx\n",
384 (rcnt
== (pide
& (BITS_PER_LONG
- 1)))
394 /* Verify the resource map and pdir state is consistent */
396 sba_check_pdir(struct ioc
*ioc
, char *msg
)
398 u32
*rptr_end
= (u32
*) &(ioc
->res_map
[ioc
->res_size
]);
399 u32
*rptr
= (u32
*) ioc
->res_map
; /* resource map ptr */
400 u64
*pptr
= ioc
->pdir_base
; /* pdir ptr */
403 while (rptr
< rptr_end
) {
405 int rcnt
= 32; /* number of bits we might check */
408 /* Get last byte and highest bit from that */
409 u32 pde
= ((u32
) (((char *)pptr
)[7])) << 24;
410 if ((rval
^ pde
) & 0x80000000)
413 ** BUMMER! -- res_map != pdir --
414 ** Dump rval and matching pdir entries
416 sba_dump_pdir_entry(ioc
, msg
, pide
);
420 rval
<<= 1; /* try the next bit */
424 rptr
++; /* look at next word of res_map */
426 /* It'd be nice if we always got here :^) */
432 sba_dump_sg( struct ioc
*ioc
, struct scatterlist
*startsg
, int nents
)
434 while (nents
-- > 0) {
435 printk(" %d : %08lx/%05x %p/%05x\n",
437 (unsigned long) sg_dma_address(startsg
),
439 startsg
->address
, startsg
->length
);
444 #endif /* ASSERT_PDIR_SANITY */
449 ** One time initialization to let the world know the LBA was found.
450 ** This is the only routine which is NOT static.
451 ** Must be called exactly once before pci_init().
456 sba_list
= (struct sba_device
*) NULL
;
459 #ifdef DEBUG_SBA_INIT
460 sba_dump_ranges((char *) 0xFED00000L
);
463 register_driver(sba_drivers_for
);
468 /**************************************************************
470 * I/O Pdir Resource Management
472 * Bits set in the resource map are in use.
473 * Each bit can represent a number of pages.
474 * LSbs represent lower addresses (IOVA's).
476 ***************************************************************/
477 #define PAGES_PER_RANGE 1 /* could increase this to 4 or 8 if needed */
479 /* Convert from IOVP to IOVA and vice versa. */
480 #define SBA_IOVA(ioc,iovp,offset,hint_reg) ((iovp) | (offset) | ((hint_reg)<<(ioc->hint_shift_pdir)))
481 #define SBA_IOVP(ioc,iova) ((iova) & ioc->hint_mask_pdir)
483 /* FIXME : review these macros to verify correctness and usage */
484 #define PDIR_INDEX(iovp) ((iovp)>>IOVP_SHIFT)
485 #define MKIOVP(dma_hint,pide) (dma_addr_t)((long)(dma_hint) | ((long)(pide) << IOVP_SHIFT))
486 #define MKIOVA(iovp,offset) (dma_addr_t)((long)iovp | (long)offset)
488 #define RESMAP_MASK(n) (~0UL << (BITS_PER_LONG - (n)))
489 #define RESMAP_IDX_MASK (sizeof(unsigned long) - 1)
493 ** Perf optimizations:
494 ** o search for log2(size) bits at a time.
496 ** Search should use register width as "stride" to search the res_map.
499 static SBA_INLINE
unsigned long
500 sba_search_bitmap(struct ioc
*ioc
, unsigned long bits_wanted
)
502 unsigned long *res_ptr
= ioc
->res_hint
;
503 unsigned long *res_end
= (unsigned long *) &(ioc
->res_map
[ioc
->res_size
]);
504 unsigned long pide
= ~0UL;
506 ASSERT(((unsigned long) ioc
->res_hint
& (sizeof(unsigned long) - 1UL)) == 0);
507 ASSERT(res_ptr
< res_end
);
508 if (bits_wanted
> (BITS_PER_LONG
/2)) {
509 /* Search word at a time - no mask needed */
510 for(; res_ptr
< res_end
; ++res_ptr
) {
512 *res_ptr
= RESMAP_MASK(bits_wanted
);
513 pide
= ((unsigned long)res_ptr
- (unsigned long)ioc
->res_map
);
514 pide
<<= 3; /* convert to bit address */
519 /* point to the next word on next pass */
521 ioc
->res_bitshift
= 0;
524 ** Search the resource bit map on well-aligned values.
525 ** "o" is the alignment.
526 ** We need the alignment to invalidate I/O TLB using
527 ** SBA HW features in the unmap path.
529 unsigned long o
= 1 << get_order(bits_wanted
<< PAGE_SHIFT
);
530 uint bitshiftcnt
= ROUNDUP(ioc
->res_bitshift
, o
);
533 if (bitshiftcnt
>= BITS_PER_LONG
) {
537 mask
= RESMAP_MASK(bits_wanted
) >> bitshiftcnt
;
539 DBG_RES("sba_search_bitmap() o %ld %p", o
, res_ptr
);
540 while(res_ptr
< res_end
)
542 DBG_RES(" %p %lx %lx\n", res_ptr
, mask
, *res_ptr
);
544 if(0 == ((*res_ptr
) & mask
)) {
545 *res_ptr
|= mask
; /* mark resources busy! */
546 pide
= ((unsigned long)res_ptr
- (unsigned long)ioc
->res_map
);
547 pide
<<= 3; /* convert to bit address */
555 mask
= RESMAP_MASK(bits_wanted
);
560 /* look in the same word on the next pass */
561 ioc
->res_bitshift
= bitshiftcnt
+ bits_wanted
;
565 ioc
->res_hint
= (res_end
== res_ptr
) ? (unsigned long *) ioc
->res_map
: res_ptr
;
571 sba_alloc_range(struct ioc
*ioc
, size_t size
)
573 unsigned int pages_needed
= size
>> IOVP_SHIFT
;
574 #ifdef CONFIG_PROC_FS
575 unsigned long cr_start
= mfctl(16);
579 ASSERT(pages_needed
);
580 ASSERT((pages_needed
* IOVP_SIZE
) < DMA_CHUNK_SIZE
);
581 ASSERT(pages_needed
< BITS_PER_LONG
);
582 ASSERT(0 == (size
& ~IOVP_MASK
));
585 ** "seek and ye shall find"...praying never hurts either...
586 ** ggg sacrifices another 710 to the computer gods.
589 pide
= sba_search_bitmap(ioc
, pages_needed
);
590 if (pide
>= (ioc
->res_size
<< 3)) {
591 pide
= sba_search_bitmap(ioc
, pages_needed
);
592 if (pide
>= (ioc
->res_size
<< 3))
593 panic(__FILE__
": I/O MMU @ %p is out of mapping resources\n", ioc
->ioc_hpa
);
596 #ifdef ASSERT_PDIR_SANITY
597 /* verify the first enable bit is clear */
598 if(0x00 != ((u8
*) ioc
->pdir_base
)[pide
*sizeof(u64
) + 7]) {
599 sba_dump_pdir_entry(ioc
, "sba_search_bitmap() botched it?", pide
);
603 DBG_RES("sba_alloc_range(%x) %d -> %lx hint %x/%x\n",
604 size
, pages_needed
, pide
,
605 (uint
) ((unsigned long) ioc
->res_hint
- (unsigned long) ioc
->res_map
),
608 #ifdef CONFIG_PROC_FS
610 unsigned long cr_end
= mfctl(16);
611 unsigned long tmp
= cr_end
- cr_start
;
612 /* check for roll over */
613 cr_start
= (cr_end
< cr_start
) ? -(tmp
) : (tmp
);
615 ioc
->avg_search
[ioc
->avg_idx
++] = cr_start
;
616 ioc
->avg_idx
&= SBA_SEARCH_SAMPLE
- 1;
618 ioc
->used_pages
+= pages_needed
;
626 ** clear bits in the ioc's resource map
628 static SBA_INLINE
void
629 sba_free_range(struct ioc
*ioc
, dma_addr_t iova
, size_t size
)
631 unsigned long iovp
= SBA_IOVP(ioc
, iova
);
632 unsigned int pide
= PDIR_INDEX(iovp
);
633 unsigned int ridx
= pide
>> 3; /* convert bit to byte address */
634 unsigned long *res_ptr
= (unsigned long *) &((ioc
)->res_map
[ridx
& ~RESMAP_IDX_MASK
]);
636 int bits_not_wanted
= size
>> IOVP_SHIFT
;
638 /* 3-bits "bit" address plus 2 (or 3) bits for "byte" == bit in word */
639 unsigned long m
= RESMAP_MASK(bits_not_wanted
) >> (pide
& (BITS_PER_LONG
- 1));
641 DBG_RES("sba_free_range( ,%x,%x) %x/%lx %x %p %lx\n",
643 bits_not_wanted
, m
, pide
, res_ptr
, *res_ptr
);
645 #ifdef CONFIG_PROC_FS
646 ioc
->used_pages
-= bits_not_wanted
;
650 ASSERT(bits_not_wanted
);
651 ASSERT((bits_not_wanted
* IOVP_SIZE
) < DMA_CHUNK_SIZE
);
652 ASSERT(bits_not_wanted
< BITS_PER_LONG
);
653 ASSERT((*res_ptr
& m
) == m
); /* verify same bits are set */
658 /**************************************************************
660 * "Dynamic DMA Mapping" support (aka "Coherent I/O")
662 ***************************************************************/
664 #define SBA_DMA_HINT(ioc, val) ((val) << (ioc)->hint_shift_pdir)
667 typedef unsigned long space_t
;
668 #define KERNEL_SPACE 0
671 * SBA Mapping Routine
673 * Given a virtual address (vba, arg2) and space id, (sid, arg1)
674 * sba_io_pdir_entry() loads the I/O PDIR entry pointed to by
675 * pdir_ptr (arg0). Each IO Pdir entry consists of 8 bytes as
676 * shown below (MSB == bit 0):
679 * +-+---------------------+----------------------------------+----+--------+
680 * |V| U | PPN[43:12] | U | VI |
681 * +-+---------------------+----------------------------------+----+--------+
685 * PPN == Physical Page Number
686 * VI == Virtual Index (aka Coherent Index)
688 * The physical address fields are filled with the results of the LPA
689 * instruction. The virtual index field is filled with the results of
690 * of the LCI (Load Coherence Index) instruction. The 8 bits used for
691 * the virtual index are bits 12:19 of the value returned by LCI.
693 * We need to pre-swap the bytes since PCX-W is Big Endian.
697 sba_io_pdir_entry(u64
*pdir_ptr
, space_t sid
, unsigned long vba
)
699 u64 pa
; /* physical address */
700 register unsigned ci
; /* coherent index */
702 /* We currently only support kernel addresses */
704 ASSERT(((unsigned long) vba
& 0xc0000000UL
) == 0xc0000000UL
);
706 pa
= virt_to_phys(vba
);
707 pa
&= ~4095ULL; /* clear out offset bits */
710 asm("lci 0(%%sr1, %1), %0" : "=r" (ci
) : "r" (vba
));
711 pa
|= (ci
>> 12) & 0xff; /* move CI (8 bits) into lowest byte */
713 pa
|= 0x8000000000000000ULL
; /* set "valid" bit */
714 *pdir_ptr
= cpu_to_le64(pa
); /* swap and store into I/O Pdir */
718 /***********************************************************
719 * The Ike PCOM (Purge Command Register) is to purge
720 * stale entries in the IO TLB when unmapping entries.
722 * The PCOM register supports purging of multiple pages, with a minium
723 * of 1 page and a maximum of 2GB. Hardware requires the address be
724 * aligned to the size of the range being purged. The size of the range
725 * must be a power of 2.
726 ***********************************************************/
727 static SBA_INLINE
void
728 sba_mark_invalid(struct ioc
*ioc
, dma_addr_t iova
, size_t byte_cnt
)
730 u32 iovp
= (u32
) SBA_IOVP(ioc
,iova
);
732 /* Even though this is a big-endian machine, the entries
733 ** in the iopdir are swapped. That's why we clear the byte
734 ** at +7 instead of at +0.
736 int off
= PDIR_INDEX(iovp
)*sizeof(u64
)+7;
738 /* Must be non-zero and rounded up */
739 ASSERT(byte_cnt
> 0);
740 ASSERT(0 == (byte_cnt
& ~IOVP_MASK
));
742 #ifdef ASSERT_PDIR_SANITY
743 /* Assert first pdir entry is set */
744 if (0x80 != (((u8
*) ioc
->pdir_base
)[off
])) {
745 sba_dump_pdir_entry(ioc
,"sba_mark_invalid()", PDIR_INDEX(iovp
));
749 if (byte_cnt
<= IOVP_SIZE
)
751 ASSERT( off
< ioc
->pdir_size
);
753 iovp
|= IOVP_SHIFT
; /* set "size" field for PCOM */
756 ** clear I/O PDIR entry "valid" bit
757 ** Do NOT clear the rest - save it for debugging.
758 ** We should only clear bits that have previously
761 ((u8
*)(ioc
->pdir_base
))[off
] = 0;
763 u32 t
= get_order(byte_cnt
) + PAGE_SHIFT
;
766 ASSERT(t
<= 31); /* 2GB! Max value of "size" field */
769 /* verify this pdir entry is enabled */
770 ASSERT(0x80 == (((u8
*) ioc
->pdir_base
)[off
] & 0x80));
771 /* clear I/O Pdir entry "valid" bit first */
772 ((u8
*)(ioc
->pdir_base
))[off
] = 0;
774 byte_cnt
-= IOVP_SIZE
;
775 } while (byte_cnt
> 0);
778 WRITE_REG32(iovp
, ioc
->ioc_hpa
+IOC_PCOM
);
782 sba_dma_supported( struct pci_dev
*dev
, dma_addr_t mask
)
785 printk(MODULE_NAME
": EISA/ISA/et al not supported\n");
790 dev
->dma_mask
= mask
; /* save it */
792 /* only support PCI devices */
793 return((int) (mask
>= 0xffffffff));
798 ** map_single returns a fully formed IOVA
801 sba_map_single(struct pci_dev
*dev
, void *addr
, size_t size
, int direction
)
803 struct ioc
*ioc
= &sba_list
->ioc
[0]; /* FIXME : see Multi-IOC below */
812 /* save offset bits */
813 offset
= ((dma_addr_t
) addr
) & ~IOVP_MASK
;
815 /* round up to nearest IOVP_SIZE */
816 size
= (size
+ offset
+ ~IOVP_MASK
) & IOVP_MASK
;
818 spin_lock_irqsave(&ioc
->res_lock
, flags
);
819 #ifdef ASSERT_PDIR_SANITY
820 sba_check_pdir(ioc
,"Check before sba_map_single()");
823 #ifdef CONFIG_PROC_FS
824 ioc
->msingle_calls
++;
825 ioc
->msingle_pages
+= size
>> IOVP_SHIFT
;
827 pide
= sba_alloc_range(ioc
, size
);
828 iovp
= (dma_addr_t
) pide
<< IOVP_SHIFT
;
830 DBG_RUN("sba_map_single() 0x%p -> 0x%lx", addr
, (long) iovp
| offset
);
832 pdir_start
= &(ioc
->pdir_base
[pide
]);
835 ASSERT(((u8
*)pdir_start
)[7] == 0); /* verify availability */
836 sba_io_pdir_entry(pdir_start
, KERNEL_SPACE
, (unsigned long) addr
);
838 DBG_RUN(" pdir 0x%p %02x%02x%02x%02x%02x%02x%02x%02x\n",
840 (u8
) (((u8
*) pdir_start
)[7]),
841 (u8
) (((u8
*) pdir_start
)[6]),
842 (u8
) (((u8
*) pdir_start
)[5]),
843 (u8
) (((u8
*) pdir_start
)[4]),
844 (u8
) (((u8
*) pdir_start
)[3]),
845 (u8
) (((u8
*) pdir_start
)[2]),
846 (u8
) (((u8
*) pdir_start
)[1]),
847 (u8
) (((u8
*) pdir_start
)[0])
854 /* form complete address */
855 #ifdef ASSERT_PDIR_SANITY
856 sba_check_pdir(ioc
,"Check after sba_map_single()");
858 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
859 return SBA_IOVA(ioc
, iovp
, offset
, DEFAULT_DMA_HINT_REG
);
864 sba_unmap_single(struct pci_dev
*dev
, dma_addr_t iova
, size_t size
, int direction
)
867 /* Multi-IOC (ie N-class) : need to lookup IOC from dev
868 ** o If we can't know about lba PCI data structs, that eliminates ->sysdata.
869 ** o walking up pcidev->parent dead ends at elroy too
870 ** o leaves hashing dev->bus->number into some lookup.
871 ** (may only work for N-class)
872 ** o use (struct pci_hba) and put fields in there for DMA.
873 ** (ioc and per device dma_hint.)
875 ** Last one seems the clearest and most promising.
876 ** sba_dma_supported() fill in those fields when the driver queries
877 ** the system for support.
879 struct ioc
*ioc
= (struct ioc
*) ((struct pci_hba
*) (dev
->sysdata
))->dma_data
;
881 struct ioc
*ioc
= &sba_list
->ioc
[0];
886 offset
= iova
& ~IOVP_MASK
;
888 DBG_RUN("%s() iovp 0x%lx/%x\n", __FUNCTION__
, (long) iova
, size
);
890 iova
^= offset
; /* clear offset bits */
892 size
= ROUNDUP(size
, IOVP_SIZE
);
896 spin_lock_irqsave(&ioc
->res_lock
, flags
);
897 #ifdef CONFIG_PROC_FS
898 ioc
->usingle_calls
++;
899 ioc
->usingle_pages
+= size
>> IOVP_SHIFT
;
901 #ifdef DELAYED_RESOURCE_CNT
902 if (ioc
->saved_cnt
< DELAYED_RESOURCE_CNT
) {
903 ioc
->saved_iova
[ioc
->saved_cnt
] = iova
;
904 ioc
->saved_size
[ioc
->saved_cnt
] = size
;
909 sba_mark_invalid(ioc
, iova
, size
);
910 sba_free_range(ioc
, iova
, size
);
912 #ifdef DELAYED_RESOURCE_CNT
914 iova
= ioc
->saved_iova
[ioc
->saved_cnt
];
915 size
= ioc
->saved_size
[ioc
->saved_cnt
];
916 } while (ioc
->saved_cnt
)
919 (void) (volatile) READ_REG32(ioc
->ioc_hpa
+IOC_PCOM
);
923 READ_REG32(ioc
->ioc_hpa
+IOC_PCOM
);
925 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
930 sba_alloc_consistent(struct pci_dev
*hwdev
, size_t size
, dma_addr_t
*dma_handle
)
935 /* only support PCI */
940 ret
= (void *) __get_free_pages(GFP_ATOMIC
, get_order(size
));
943 memset(ret
, 0, size
);
944 *dma_handle
= sba_map_single(hwdev
, ret
, size
, 0);
952 sba_free_consistent(struct pci_dev
*hwdev
, size_t size
, void *vaddr
, dma_addr_t dma_handle
)
954 sba_unmap_single(hwdev
, dma_handle
, size
, 0);
955 free_pages((unsigned long) vaddr
, get_order(size
));
959 ** Two address ranges are "virtually contiguous" iff:
960 ** 1) end of prev == start of next, or... append case
961 ** 3) end of next == start of prev prepend case
963 ** and they are DMA contiguous *iff*:
964 ** 2) end of prev and start of next are both on a page boundry
966 ** (shift left is a quick trick to mask off upper bits)
968 #define DMA_CONTIG(__X, __Y) \
969 (((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - PAGE_SHIFT)) == 0UL)
972 ** Assumption is two transactions are mutually exclusive.
973 ** ie both go to different parts of memory.
974 ** If both are true, then both transaction are on the same page.
976 #define DMA_SAME_PAGE(s1,e1,s2,e2) \
977 ( ((((s1) ^ (s2)) >> PAGE_SHIFT) == 0) \
978 && ((((e1) ^ (e2)) >> PAGE_SHIFT) == 0) )
981 ** Since 0 is a valid pdir_base index value, can't use that
982 ** to determine if a value is valid or not. Use a flag to indicate
983 ** the SG list entry contains a valid pdir index.
985 #define PIDE_FLAG 0x80000000UL
987 #ifdef DEBUG_LARGE_SG_ENTRIES
991 static SBA_INLINE
int
994 struct scatterlist
*startsg
,
997 struct scatterlist
*dma_sg
= startsg
; /* pointer to current DMA */
1000 unsigned long dma_offset
= 0;
1003 while (nents
-- > 0) {
1004 int cnt
= sg_dma_len(startsg
);
1005 sg_dma_len(startsg
) = 0;
1007 #ifdef DEBUG_LARGE_SG_ENTRIES
1009 printk(" %d : %08lx/%05x %p/%05x\n",
1011 (unsigned long) sg_dma_address(startsg
), cnt
,
1012 startsg
->address
, startsg
->length
1015 DBG_RUN_SG(" %d : %08lx/%05x %p/%05x\n",
1017 (unsigned long) sg_dma_address(startsg
), cnt
,
1018 startsg
->address
, startsg
->length
1022 ** Look for the start of a new DMA stream
1024 if (sg_dma_address(startsg
) & PIDE_FLAG
) {
1025 u32 pide
= sg_dma_address(startsg
) & ~PIDE_FLAG
;
1026 dma_offset
= (unsigned long) pide
& ~IOVP_MASK
;
1027 pide
>>= IOVP_SHIFT
;
1028 pdirp
= &(ioc
->pdir_base
[pide
]);
1029 sg_dma_address(startsg
) = 0;
1031 sg_dma_address(dma_sg
) = (pide
<< IOVP_SHIFT
) + dma_offset
;
1036 ** Look for a VCONTIG chunk
1039 unsigned long vaddr
= (unsigned long) startsg
->address
;
1042 sg_dma_len(dma_sg
) += cnt
;
1044 dma_offset
=0; /* only want offset on first chunk */
1045 cnt
= ROUNDUP(cnt
, IOVP_SIZE
);
1046 #ifdef CONFIG_PROC_FS
1047 ioc
->msg_pages
+= cnt
>> IOVP_SHIFT
;
1050 sba_io_pdir_entry(pdirp
, KERNEL_SPACE
, vaddr
);
1058 #ifdef DEBUG_LARGE_SG_ENTRIES
1067 ** First pass is to walk the SG list and determine where the breaks are
1068 ** in the DMA stream. Allocates PDIR entries but does not fill them.
1069 ** Returns the number of DMA chunks.
1071 ** Doing the fill seperate from the coalescing/allocation keeps the
1072 ** code simpler. Future enhancement could make one pass through
1073 ** the sglist do both.
1075 static SBA_INLINE
int
1076 sba_coalesce_chunks( struct ioc
*ioc
,
1077 struct scatterlist
*startsg
,
1083 struct scatterlist
*dma_sg
; /* next DMA stream head */
1084 unsigned long dma_offset
, dma_len
; /* start/len of DMA stream */
1085 struct scatterlist
*chunksg
; /* virtually contig chunk head */
1086 unsigned long chunk_addr
, chunk_len
; /* start/len of VCONTIG chunk */
1089 ** Prepare for first/next DMA stream
1091 dma_sg
= chunksg
= startsg
;
1092 dma_len
= chunk_len
= startsg
->length
;
1093 chunk_addr
= (unsigned long) startsg
->address
;
1097 ** This loop terminates one iteration "early" since
1098 ** it's always looking one "ahead".
1100 while (--nents
> 0) {
1101 /* ptr to coalesce prev and next */
1102 struct scatterlist
*prev_sg
= startsg
;
1103 unsigned long prev_end
= (unsigned long) prev_sg
->address
+ prev_sg
->length
;
1104 unsigned long current_end
;
1106 /* PARANOID: clear entries */
1107 sg_dma_address(startsg
) = 0;
1108 sg_dma_len(startsg
) = 0;
1110 /* Now start looking ahead */
1112 current_end
= (unsigned long) startsg
->address
+ startsg
->length
;
1115 ** First look for virtually contiguous blocks.
1116 ** PARISC needs this since it's cache is virtually
1117 ** indexed and we need the associated virtual
1118 ** address for each I/O address we map.
1120 ** 1) can we *prepend* the next transaction?
1122 if (current_end
== (unsigned long) prev_sg
->address
)
1124 /* prepend : get new offset */
1126 chunk_addr
= (unsigned long) prev_sg
->address
;
1127 chunk_len
+= startsg
->length
;
1128 dma_len
+= startsg
->length
;
1133 ** 2) or append the next transaction?
1135 if (prev_end
== (unsigned long) startsg
->address
)
1137 chunk_len
+= startsg
->length
;
1138 dma_len
+= startsg
->length
;
1142 #ifdef DEBUG_LARGE_SG_ENTRIES
1143 dump_run_sg
= (chunk_len
> IOVP_SIZE
);
1146 ** Not virtually contigous.
1147 ** Terminate prev chunk.
1148 ** Start a new chunk.
1150 ** Once we start a new VCONTIG chunk, the offset
1151 ** can't change. And we need the offset from the first
1152 ** chunk - not the last one. Ergo Successive chunks
1153 ** must start on page boundaries and dove tail
1154 ** with it's predecessor.
1156 sg_dma_len(prev_sg
) = chunk_len
;
1158 chunk_len
= startsg
->length
;
1159 dma_offset
|= (chunk_addr
& ~IOVP_MASK
);
1160 ASSERT((0 == (chunk_addr
& ~IOVP_MASK
)) ||
1161 (dma_offset
== (chunk_addr
& ~IOVP_MASK
)));
1165 ** 4) do the chunks end/start on page boundaries?
1166 ** Easier than 3 since no offsets are involved.
1168 if (DMA_CONTIG(prev_end
, startsg
->address
))
1175 chunk_addr
= (unsigned long) startsg
->address
;
1186 ** End of DMA Stream
1188 ** Allocate space for DMA stream.
1190 sg_dma_len(startsg
) = chunk_len
;
1191 dma_len
= (dma_len
+ dma_offset
+ ~IOVP_MASK
) & IOVP_MASK
;
1192 sg_dma_address(dma_sg
) =
1194 | (sba_alloc_range(ioc
, dma_len
) << IOVP_SHIFT
)
1204 ** And this algorithm still generally only ends up coalescing entries
1205 ** that happens to be on the same page due to how sglists are assembled.
1208 sba_map_sg(struct pci_dev
*dev
, struct scatterlist
*sglist
, int nents
, int direction
)
1210 struct ioc
*ioc
= &sba_list
->ioc
[0]; /* FIXME : see Multi-IOC below */
1211 int coalesced
, filled
= 0;
1212 unsigned long flags
;
1214 DBG_RUN_SG("%s() START %d entries\n", __FUNCTION__
, nents
);
1216 /* Fast path single entry scatterlists. */
1218 sg_dma_address(sglist
)= sba_map_single(dev
, sglist
->address
,
1219 sglist
->length
, direction
);
1220 sg_dma_len(sglist
)= sglist
->length
;
1224 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1226 #ifdef ASSERT_PDIR_SANITY
1227 if (sba_check_pdir(ioc
,"Check before sba_map_sg()"))
1229 sba_dump_sg(ioc
, sglist
, nents
);
1230 panic("Check before sba_map_sg()");
1234 #ifdef CONFIG_PROC_FS
1239 ** First coalesce the chunks and allocate I/O pdir space
1241 ** If this is one DMA stream, we can properly map using the
1242 ** correct virtual address associated with each DMA page.
1243 ** w/o this association, we wouldn't have coherent DMA!
1244 ** Access to the virtual address is what forces a two pass algorithm.
1246 coalesced
= sba_coalesce_chunks(ioc
, sglist
, nents
);
1249 ** Program the I/O Pdir
1251 ** map the virtual addresses to the I/O Pdir
1252 ** o dma_address will contain the pdir index
1253 ** o dma_len will contain the number of bytes to map
1254 ** o address contains the virtual address.
1256 filled
= sba_fill_pdir(ioc
, sglist
, nents
);
1258 #ifdef ASSERT_PDIR_SANITY
1259 if (sba_check_pdir(ioc
,"Check after sba_map_sg()"))
1261 sba_dump_sg(ioc
, sglist
, nents
);
1262 panic("Check after sba_map_sg()\n");
1266 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1268 ASSERT(coalesced
== filled
);
1269 DBG_RUN_SG("%s() DONE %d mappings\n", __FUNCTION__
, filled
);
1276 sba_unmap_sg(struct pci_dev
*dev
, struct scatterlist
*sglist
, int nents
, int direction
)
1278 struct ioc
*ioc
= &sba_list
->ioc
[0]; /* FIXME : see Multi-IOC below */
1279 #ifdef ASSERT_PDIR_SANITY
1280 unsigned long flags
;
1283 DBG_RUN_SG("%s() START %d entries, %p,%x\n",
1284 __FUNCTION__
, nents
, sglist
->address
, sglist
->length
);
1286 #ifdef CONFIG_PROC_FS
1290 #ifdef ASSERT_PDIR_SANITY
1291 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1292 sba_check_pdir(ioc
,"Check before sba_unmap_sg()");
1293 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1296 while (sg_dma_len(sglist
) && nents
--) {
1298 #ifdef CONFIG_PROC_FS
1299 ioc
->usg_pages
+= sg_dma_len(sglist
) >> PAGE_SHIFT
;
1301 sba_unmap_single(dev
, sg_dma_address(sglist
), sg_dma_len(sglist
), direction
);
1305 DBG_RUN_SG("%s() DONE (nents %d)\n", __FUNCTION__
, nents
);
1307 #ifdef ASSERT_PDIR_SANITY
1308 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1309 sba_check_pdir(ioc
,"Check after sba_unmap_sg()");
1310 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1315 static struct pci_dma_ops sba_ops
= {
1317 sba_alloc_consistent
, /* allocate cacheable host mem */
1318 sba_free_consistent
, /* release cacheable host mem */
1323 NULL
, /* dma_sync_single */
1324 NULL
/* dma_sync_sg */
1328 /**************************************************************************
1330 ** SBA PAT PDC support
1332 ** o call pdc_pat_cell_module()
1333 ** o store ranges in PCI "resource" structures
1335 **************************************************************************/
1338 sba_get_pat_resources(struct sba_device
*sba_dev
)
1342 ** TODO/REVISIT/FIXME: support for directed ranges requires calls to
1343 ** PAT PDC to program the SBA/LBA directed range registers...this
1344 ** burden may fall on the LBA code since it directly supports the
1345 ** PCI subsystem. It's not clear yet. - ggg
1347 PAT_MOD(mod
)->mod_info
.mod_pages
= PAT_GET_MOD_PAGES(temp
);
1349 PAT_MOD(mod
)->mod_info
.dvi
= PAT_GET_DVI(temp
);
1350 Tells where the dvi bits are located in the address
.
1351 PAT_MOD(mod
)->mod_info
.ioc
= PAT_GET_IOC(temp
);
1357 /**************************************************************
1359 * Initialization and claim
1361 ***************************************************************/
1365 sba_ioc_init(struct ioc
*ioc
)
1367 extern unsigned long mem_max
; /* arch.../setup.c */
1368 extern void lba_init_iregs(void *, u32
, u32
); /* arch.../lba_pci.c */
1370 u32 iova_space_size
, iova_space_mask
;
1372 int pdir_size
, iov_order
;
1375 ** Determine IOVA Space size from memory size.
1376 ** Using "mem_max" is a kluge.
1378 ** Ideally, PCI drivers would register the maximum number
1379 ** of DMA they can have outstanding for each device they
1380 ** own. Next best thing would be to guess how much DMA
1381 ** can be outstanding based on PCI Class/sub-class. Both
1382 ** methods still require some "extra" to support PCI
1383 ** Hot-Plug/Removal of PCI cards. (aka PCI OLARD).
1385 ** While we have 32-bits "IOVA" space, top two 2 bits are used
1386 ** for DMA hints - ergo only 30 bits max.
1388 /* limit IOVA space size to 1MB-1GB */
1389 if (mem_max
< (sba_mem_ratio
*1024*1024)) {
1390 iova_space_size
= 1024*1024;
1392 } else if (mem_max
> (sba_mem_ratio
*512*1024*1024)) {
1393 iova_space_size
= 512*1024*1024;
1396 iova_space_size
= (u32
) (mem_max
/sba_mem_ratio
);
1400 ** iova space must be log2() in size.
1401 ** thus, pdir/res_map will also be log2().
1403 iov_order
= get_order(iova_space_size
>> (IOVP_SHIFT
-PAGE_SHIFT
));
1404 ASSERT(iov_order
<= (30 - IOVP_SHIFT
)); /* iova_space_size <= 1GB */
1405 ASSERT(iov_order
>= (20 - IOVP_SHIFT
)); /* iova_space_size >= 1MB */
1406 iova_space_size
= 1 << (iov_order
+ IOVP_SHIFT
);
1408 ioc
->pdir_size
= pdir_size
= (iova_space_size
/IOVP_SIZE
) * sizeof(u64
);
1410 ASSERT(pdir_size
< 4*1024*1024); /* max pdir size < 4MB */
1412 /* Verify it's a power of two */
1413 ASSERT((1 << get_order(pdir_size
)) == (pdir_size
>> PAGE_SHIFT
));
1415 DBG_INIT("%s() hpa 0x%p mem %dMBIOV %dMB (%d bits) PDIR size 0x%0x",
1416 __FUNCTION__
, ioc
->ioc_hpa
, (int) (mem_max
>>20),
1417 iova_space_size
>>20, iov_order
+ PAGE_SHIFT
, pdir_size
);
1419 /* FIXME : DMA HINTs not used */
1420 ioc
->hint_shift_pdir
= iov_order
+ PAGE_SHIFT
;
1421 ioc
->hint_mask_pdir
= ~(0x3 << (iov_order
+ PAGE_SHIFT
));
1424 pdir_base
= (void *) __get_free_pages(GFP_KERNEL
, get_order(pdir_size
));
1425 if (NULL
== pdir_base
)
1427 panic(__FILE__
":%s() could not allocate I/O Page Table\n", __FUNCTION__
);
1429 memset(pdir_base
, 0, pdir_size
);
1431 DBG_INIT("sba_ioc_init() pdir %p size %x hint_shift_pdir %x hint_mask_pdir %lx\n",
1432 pdir_base
, pdir_size
,
1433 ioc
->hint_shift_pdir
, ioc
->hint_mask_pdir
);
1435 ASSERT((((unsigned long) pdir_base
) & PAGE_MASK
) == (unsigned long) pdir_base
);
1436 WRITE_REG64(virt_to_phys(pdir_base
), (u64
*)(ioc
->ioc_hpa
+IOC_PDIR_BASE
));
1438 DBG_INIT(" base %p\n", pdir_base
);
1440 /* build IMASK for IOC and Elroy */
1441 iova_space_mask
= 0xffffffff;
1442 iova_space_mask
<<= (iov_order
+ PAGE_SHIFT
);
1445 ** On C3000 w/512MB mem, HP-UX 10.20 reports:
1446 ** ibase=0, imask=0xFE000000, size=0x2000000.
1448 ioc
->ibase
= IOC_IOVA_SPACE_BASE
| 1; /* bit 0 == enable bit */
1449 ioc
->imask
= iova_space_mask
; /* save it */
1451 DBG_INIT("%s() IOV base 0x%lx mask 0x%0lx\n", __FUNCTION__
,
1452 ioc
->ibase
, ioc
->imask
);
1455 ** FIXME: Hint registers are programmed with default hint
1456 ** values during boot, so hints should be sane even if we
1457 ** can't reprogram them the way drivers want.
1461 ** setup Elroy IBASE/IMASK registers as well.
1463 lba_init_iregs(ioc
->ioc_hpa
, ioc
->ibase
, ioc
->imask
);
1466 ** Program the IOC's ibase and enable IOVA translation
1468 WRITE_REG32(ioc
->ibase
, ioc
->ioc_hpa
+IOC_IBASE
);
1469 WRITE_REG32(ioc
->imask
, ioc
->ioc_hpa
+IOC_IMASK
);
1471 /* Set I/O PDIR Page size to 4K */
1472 WRITE_REG32(0, ioc
->ioc_hpa
+IOC_TCNFG
);
1475 ** Clear I/O TLB of any possible entries.
1476 ** (Yes. This is a it paranoid...but so what)
1478 WRITE_REG32(0 | 31, ioc
->ioc_hpa
+IOC_PCOM
);
1480 DBG_INIT("%s() DONE\n", __FUNCTION__
);
1485 /**************************************************************************
1487 ** SBA initialization code (HW and SW)
1489 ** o identify SBA chip itself
1490 ** o initialize SBA chip modes (HardFail)
1491 ** o initialize SBA chip modes (HardFail)
1492 ** o FIXME: initialize DMA hints for reasonable defaults
1494 **************************************************************************/
1497 sba_hw_init(struct sba_device
*sba_dev
)
1503 ioc_ctl
= READ_REG32(sba_dev
->sba_hpa
+IOC_CTRL
);
1504 DBG_INIT("%s() hpa 0x%p ioc_ctl 0x%x ->", __FUNCTION__
, sba_dev
->sba_hpa
, ioc_ctl
);
1505 ioc_ctl
&= ~(IOC_CTRL_RM
| IOC_CTRL_NC
);
1506 ASSERT(ioc_ctl
& IOC_CTRL_TE
); /* astro: firmware enables this */
1508 WRITE_REG32(ioc_ctl
, sba_dev
->sba_hpa
+IOC_CTRL
);
1510 #ifdef SBA_DEBUG_INIT
1511 ioc_ctl
= READ_REG32(sba_dev
->sba_hpa
+IOC_CTRL
);
1512 DBG_INIT(" 0x%x\n", ioc_ctl
);
1515 if (IS_ASTRO(sba_dev
->iodc
)) {
1516 /* PAT_PDC (L-class) also reports the same goofy base */
1517 sba_dev
->ioc
[0].ioc_hpa
= (char *) ASTRO_IOC_OFFSET
;
1520 sba_dev
->ioc
[0].ioc_hpa
= sba_dev
->ioc
[1].ioc_hpa
= 0;
1524 sba_dev
->num_ioc
= num_ioc
;
1525 for( i
= 0; i
< num_ioc
; i
++)
1527 (unsigned long) sba_dev
->ioc
[i
].ioc_hpa
+= (unsigned long) sba_dev
->sba_hpa
+ IKE_IOC_OFFSET(i
);
1530 ** Make sure the box crashes if we get any errors on a rope.
1532 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE0_CTL
);
1533 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE1_CTL
);
1534 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE2_CTL
);
1535 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE3_CTL
);
1536 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE4_CTL
);
1537 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE5_CTL
);
1538 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE6_CTL
);
1539 WRITE_REG32(HF_ENABLE
, sba_dev
->ioc
[i
].ioc_hpa
+ ROPE7_CTL
);
1541 /* flush out the writes */
1542 READ_REG32(sba_dev
->ioc
[i
].ioc_hpa
+ ROPE7_CTL
);
1544 sba_ioc_init(&(sba_dev
->ioc
[i
]));
1549 sba_common_init(struct sba_device
*sba_dev
)
1553 /* add this one to the head of the list (order doesn't matter)
1554 ** This will be useful for debugging - especially if we get coredumps
1556 sba_dev
->next
= sba_list
;
1560 for(i
=0; i
< sba_dev
->num_ioc
; i
++) {
1562 #ifdef CONFIG_DMB_TRAP
1563 extern void iterate_pages(unsigned long , unsigned long ,
1564 void (*)(pte_t
* , unsigned long),
1566 void set_data_memory_break(pte_t
* , unsigned long);
1568 /* resource map size dictated by pdir_size */
1569 res_size
= sba_dev
->ioc
[i
].pdir_size
/sizeof(u64
); /* entries */
1570 res_size
>>= 3; /* convert bit count to byte count */
1571 DBG_INIT("%s() res_size 0x%x\n", __FUNCTION__
, res_size
);
1573 sba_dev
->ioc
[i
].res_size
= res_size
;
1574 sba_dev
->ioc
[i
].res_map
= (char *) __get_free_pages(GFP_KERNEL
, get_order(res_size
));
1576 #ifdef CONFIG_DMB_TRAP
1577 iterate_pages( sba_dev
->ioc
[i
].res_map
, res_size
,
1578 set_data_memory_break
, 0);
1581 if (NULL
== sba_dev
->ioc
[i
].res_map
)
1583 panic(__FILE__
":%s() could not allocate resource map\n", __FUNCTION__
);
1586 memset(sba_dev
->ioc
[i
].res_map
, 0, res_size
);
1587 /* next available IOVP - circular search */
1588 sba_dev
->ioc
[i
].res_hint
= (unsigned long *)
1589 &(sba_dev
->ioc
[i
].res_map
[L1_CACHE_BYTES
]);
1591 #ifdef ASSERT_PDIR_SANITY
1592 /* Mark first bit busy - ie no IOVA 0 */
1593 sba_dev
->ioc
[i
].res_map
[0] = 0x80;
1594 sba_dev
->ioc
[i
].pdir_base
[0] = 0xeeffc0addbba0080ULL
;
1597 #ifdef CONFIG_DMB_TRAP
1598 iterate_pages( sba_dev
->ioc
[i
].res_map
, res_size
,
1599 set_data_memory_break
, 0);
1600 iterate_pages( sba_dev
->ioc
[i
].pdir_base
, sba_dev
->ioc
[i
].pdir_size
,
1601 set_data_memory_break
, 0);
1604 DBG_INIT("sba_common_init() %d res_map %x %p\n",
1605 i
, res_size
, sba_dev
->ioc
[i
].res_map
);
1608 sba_dev
->sba_lock
= SPIN_LOCK_UNLOCKED
;
1611 #ifdef CONFIG_PROC_FS
1612 static int sba_proc_info(char *buf
, char **start
, off_t offset
, int len
)
1614 struct sba_device
*sba_dev
= sba_list
;
1615 /* FIXME: Multi-IOC support broken! */
1616 struct ioc
*ioc
= &sba_dev
->ioc
[0];
1617 int total_pages
= (int) (ioc
->res_size
<< 3); /* 8 bits per byte */
1618 unsigned long i
= 0, avg
= 0, min
, max
;
1620 sprintf(buf
, "%s rev %d.%d\n",
1621 parisc_getHWdescription(sba_dev
->iodc
->hw_type
,
1622 sba_dev
->iodc
->hversion
, sba_dev
->iodc
->sversion
),
1623 (sba_dev
->hw_rev
& 0x7) + 1,
1624 (sba_dev
->hw_rev
& 0x18) >> 3
1626 sprintf(buf
, "%sIO PDIR size : %d bytes (%d entries)\n",
1628 ((ioc
->res_size
<< 3) * sizeof(u64
)), /* 8 bits per byte */
1629 total_pages
); /* 8 bits per byte */
1631 sprintf(buf
, "%sIO PDIR entries : %ld free %ld used (%d%%)\n", buf
,
1632 total_pages
- ioc
->used_pages
, ioc
->used_pages
,
1633 (int) (ioc
->used_pages
* 100 / total_pages
));
1635 sprintf(buf
, "%sResource bitmap : %d bytes (%d pages)\n",
1636 buf
, ioc
->res_size
, ioc
->res_size
<< 3); /* 8 bits per byte */
1638 min
= max
= ioc
->avg_search
[0];
1639 for (i
= 0; i
< SBA_SEARCH_SAMPLE
; i
++) {
1640 avg
+= ioc
->avg_search
[i
];
1641 if (ioc
->avg_search
[i
] > max
) max
= ioc
->avg_search
[i
];
1642 if (ioc
->avg_search
[i
] < min
) min
= ioc
->avg_search
[i
];
1644 avg
/= SBA_SEARCH_SAMPLE
;
1645 sprintf(buf
, "%s Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles)\n",
1646 buf
, min
, avg
, max
);
1648 sprintf(buf
, "%spci_map_single(): %8ld calls %8ld pages (avg %d/1000)\n",
1649 buf
, ioc
->msingle_calls
, ioc
->msingle_pages
,
1650 (int) ((ioc
->msingle_pages
* 1000)/ioc
->msingle_calls
));
1652 /* KLUGE - unmap_sg calls unmap_single for each mapped page */
1653 min
= ioc
->usingle_calls
- ioc
->usg_calls
;
1654 max
= ioc
->usingle_pages
- ioc
->usg_pages
;
1655 sprintf(buf
, "%spci_unmap_single: %8ld calls %8ld pages (avg %d/1000)\n",
1657 (int) ((max
* 1000)/min
));
1659 sprintf(buf
, "%spci_map_sg() : %8ld calls %8ld pages (avg %d/1000)\n",
1660 buf
, ioc
->msg_calls
, ioc
->msg_pages
,
1661 (int) ((ioc
->msg_pages
* 1000)/ioc
->msg_calls
));
1663 sprintf(buf
, "%spci_unmap_sg() : %8ld calls %8ld pages (avg %d/1000)\n",
1664 buf
, ioc
->usg_calls
, ioc
->usg_pages
,
1665 (int) ((ioc
->usg_pages
* 1000)/ioc
->usg_calls
));
1671 sba_resource_map(char *buf
, char **start
, off_t offset
, int len
)
1673 struct sba_device
*sba_dev
= sba_list
;
1674 struct ioc
*ioc
= &sba_dev
->ioc
[0];
1675 unsigned long *res_ptr
= (unsigned long *)ioc
->res_map
;
1678 for(i
= 0; i
< (ioc
->res_size
/ sizeof(unsigned long)); ++i
, ++res_ptr
) {
1681 sprintf(buf
, "%s %08lx", buf
, *res_ptr
);
1690 ** Determine if lba should claim this chip (return 0) or not (return 1).
1691 ** If so, initialize the chip and tell other partners in crime they
1695 sba_driver_callback(struct hp_device
*d
, struct pa_iodc_driver
*dri
)
1697 struct sba_device
*sba_dev
;
1702 static char astro_rev
[]="Astro ?.?";
1704 /* Read HW Rev First */
1705 func_class
= READ_REG32(d
->hpa
);
1707 astro_rev
[6] = '1' + (char) (func_class
& 0x7);
1708 astro_rev
[8] = '0' + (char) ((func_class
& 0x18) >> 3);
1709 dri
->version
= astro_rev
;
1711 static char ike_rev
[]="Ike rev ?";
1713 /* Read HW Rev First */
1714 func_class
= READ_REG32(d
->hpa
+ SBA_FCLASS
);
1716 ike_rev
[8] = '0' + (char) (func_class
& 0xff);
1717 dri
->version
= ike_rev
;
1720 printk("%s found %s at 0x%p\n", dri
->name
, dri
->version
, d
->hpa
);
1722 sba_dev
= kmalloc(sizeof(struct sba_device
), GFP_KERNEL
);
1723 if (NULL
== sba_dev
)
1725 printk(MODULE_NAME
" - couldn't alloc sba_device\n");
1728 memset(sba_dev
, 0, sizeof(struct sba_device
));
1729 for(i
=0; i
<MAX_IOC
; i
++)
1730 spin_lock_init(&(sba_dev
->ioc
[i
].res_lock
));
1733 sba_dev
->hw_rev
= func_class
;
1735 sba_dev
->sba_hpa
= d
->hpa
; /* faster access */
1737 sba_get_pat_resources(sba_dev
);
1738 sba_hw_init(sba_dev
);
1739 sba_common_init(sba_dev
);
1741 hppa_dma_ops
= &sba_ops
;
1743 #ifdef CONFIG_PROC_FS
1745 create_proc_info_entry("Astro", 0, proc_runway_root
, sba_proc_info
);
1747 create_proc_info_entry("Ike", 0, proc_runway_root
, sba_proc_info
);
1749 create_proc_info_entry("bitmap", 0, proc_runway_root
, sba_resource_map
);