2 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
3 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
4 * Copyright (C) 2000 SiByte, Inc.
5 * Copyright (C) 2005 Thiemo Seufer
7 * Written by Justin Carlson of SiByte, Inc.
8 * and Kip Walker of Broadcom Corp.
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version 2
14 * of the License, or (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #include <linux/config.h>
26 #include <linux/module.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
31 #include <asm/sibyte/sb1250.h>
32 #include <asm/sibyte/sb1250_regs.h>
33 #include <asm/sibyte/sb1250_dma.h>
35 #ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
36 #define SB1_PREF_LOAD_STREAMED_HINT "0"
37 #define SB1_PREF_STORE_STREAMED_HINT "1"
39 #define SB1_PREF_LOAD_STREAMED_HINT "4"
40 #define SB1_PREF_STORE_STREAMED_HINT "5"
43 static inline void clear_page_cpu(void *page
)
45 unsigned char *addr
= (unsigned char *) page
;
46 unsigned char *end
= addr
+ PAGE_SIZE
;
49 * JDCXXX - This should be bottlenecked by the write buffer, but these
50 * things tend to be mildly unpredictable...should check this on the
53 * We prefetch 4 lines ahead. We're also "cheating" slightly here...
54 * since we know we're on an SB1, we force the assembler to take
55 * 64-bit operands to speed things up
61 #ifdef CONFIG_CPU_HAS_PREFETCH
62 " daddiu %0, %0, 128 \n"
63 " pref " SB1_PREF_STORE_STREAMED_HINT
", -128(%0) \n" /* Prefetch the first 4 lines */
64 " pref " SB1_PREF_STORE_STREAMED_HINT
", -96(%0) \n"
65 " pref " SB1_PREF_STORE_STREAMED_HINT
", -64(%0) \n"
66 " pref " SB1_PREF_STORE_STREAMED_HINT
", -32(%0) \n"
67 "1: sd $0, -128(%0) \n" /* Throw out a cacheline of 0's */
71 " daddiu %0, %0, 32 \n"
73 " pref " SB1_PREF_STORE_STREAMED_HINT
", -32(%0) \n"
74 " daddiu %0, %0, -128 \n"
76 " sd $0, 0(%0) \n" /* Throw out a cacheline of 0's */
80 " daddiu %0, %0, 32 \n"
89 static inline void copy_page_cpu(void *to
, void *from
)
91 unsigned char *src
= (unsigned char *)from
;
92 unsigned char *dst
= (unsigned char *)to
;
93 unsigned char *end
= src
+ PAGE_SIZE
;
96 * The pref's used here are using "streaming" hints, which cause the
97 * copied data to be kicked out of the cache sooner. A page copy often
98 * ends up copying a lot more data than is commonly used, so this seems
99 * to make sense in terms of reducing cache pollution, but I've no real
100 * performance data to back this up
102 __asm__
__volatile__(
106 #ifdef CONFIG_CPU_HAS_PREFETCH
107 " daddiu %0, %0, 128 \n"
108 " daddiu %1, %1, 128 \n"
109 " pref " SB1_PREF_LOAD_STREAMED_HINT
", -128(%0)\n" /* Prefetch the first 4 lines */
110 " pref " SB1_PREF_STORE_STREAMED_HINT
", -128(%1)\n"
111 " pref " SB1_PREF_LOAD_STREAMED_HINT
", -96(%0)\n"
112 " pref " SB1_PREF_STORE_STREAMED_HINT
", -96(%1)\n"
113 " pref " SB1_PREF_LOAD_STREAMED_HINT
", -64(%0)\n"
114 " pref " SB1_PREF_STORE_STREAMED_HINT
", -64(%1)\n"
115 " pref " SB1_PREF_LOAD_STREAMED_HINT
", -32(%0)\n"
116 "1: pref " SB1_PREF_STORE_STREAMED_HINT
", -32(%1)\n"
118 " ld $8, -128(%0) \n" /* Block copy a cacheline */
119 " ld $9, -120(%0) \n"
120 " ld $10, -112(%0) \n"
121 " ld $11, -104(%0) \n"
122 " sd $8, -128(%1) \n"
123 " sd $9, -120(%1) \n"
124 " sd $10, -112(%1) \n"
125 " sd $11, -104(%1) \n"
127 " lw $2, -128(%0) \n" /* Block copy a cacheline */
128 " lw $3, -124(%0) \n"
129 " lw $6, -120(%0) \n"
130 " lw $7, -116(%0) \n"
131 " lw $8, -112(%0) \n"
132 " lw $9, -108(%0) \n"
133 " lw $10, -104(%0) \n"
134 " lw $11, -100(%0) \n"
135 " sw $2, -128(%1) \n"
136 " sw $3, -124(%1) \n"
137 " sw $6, -120(%1) \n"
138 " sw $7, -116(%1) \n"
139 " sw $8, -112(%1) \n"
140 " sw $9, -108(%1) \n"
141 " sw $10, -104(%1) \n"
142 " sw $11, -100(%1) \n"
144 " daddiu %0, %0, 32 \n"
145 " daddiu %1, %1, 32 \n"
146 " bnel %0, %2, 1b \n"
147 " pref " SB1_PREF_LOAD_STREAMED_HINT
", -32(%0)\n"
148 " daddiu %0, %0, -128 \n"
149 " daddiu %1, %1, -128 \n"
152 " ld $8, 0(%0) \n" /* Block copy a cacheline */
161 " lw $2, 0(%0) \n" /* Block copy a cacheline */
178 " daddiu %0, %0, 32 \n"
179 " daddiu %1, %1, 32 \n"
180 " bnel %0, %2, 1b \n"
187 : "+r" (src
), "+r" (dst
)
190 : "$8","$9","$10","$11","memory");
192 : "$2","$3","$6","$7","$8","$9","$10","$11","memory");
197 #ifdef CONFIG_SIBYTE_DMA_PAGEOPS
200 * Pad descriptors to cacheline, since each is exclusively owned by a
203 typedef struct dmadscr_s
{
210 static dmadscr_t page_descr
[DM_NUM_CHANNELS
] __attribute__((aligned(SMP_CACHE_BYTES
)));
212 void sb1_dma_init(void)
216 for (i
= 0; i
< DM_NUM_CHANNELS
; i
++) {
217 u64 base_val
= (u64
)CPHYSADDR(&page_descr
[i
]) | V_DM_DSCR_BASE_RINGSZ(1);
218 void *base_reg
= (void *)IOADDR(A_DM_REGISTER(i
, R_DM_DSCR_BASE
));
220 __raw_writeq(base_val
, base_reg
);
221 __raw_writeq(base_val
| M_DM_DSCR_BASE_RESET
, base_reg
);
222 __raw_writeq(base_val
| M_DM_DSCR_BASE_ENABL
, base_reg
);
226 void clear_page(void *page
)
228 u64 to_phys
= (u64
)CPHYSADDR(page
);
229 unsigned int cpu
= smp_processor_id();
231 /* if the page is not in KSEG0, use old way */
232 if ((long)KSEGX(page
) != (long)CKSEG0
)
233 return clear_page_cpu(page
);
235 page_descr
[cpu
].dscr_a
= to_phys
| M_DM_DSCRA_ZERO_MEM
| M_DM_DSCRA_L2C_DEST
| M_DM_DSCRA_INTERRUPT
;
236 page_descr
[cpu
].dscr_b
= V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE
);
237 __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu
, R_DM_DSCR_COUNT
)));
240 * Don't really want to do it this way, but there's no
241 * reliable way to delay completion detection.
243 while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu
, R_DM_DSCR_BASE_DEBUG
)))
244 & M_DM_DSCR_BASE_INTERRUPT
))
246 __raw_readq(IOADDR(A_DM_REGISTER(cpu
, R_DM_DSCR_BASE
)));
249 void copy_page(void *to
, void *from
)
251 u64 from_phys
= (u64
)CPHYSADDR(from
);
252 u64 to_phys
= (u64
)CPHYSADDR(to
);
253 unsigned int cpu
= smp_processor_id();
255 /* if any page is not in KSEG0, use old way */
256 if ((long)KSEGX(to
) != (long)CKSEG0
257 || (long)KSEGX(from
) != (long)CKSEG0
)
258 return copy_page_cpu(to
, from
);
260 page_descr
[cpu
].dscr_a
= to_phys
| M_DM_DSCRA_L2C_DEST
| M_DM_DSCRA_INTERRUPT
;
261 page_descr
[cpu
].dscr_b
= from_phys
| V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE
);
262 __raw_writeq(1, (void *)IOADDR(A_DM_REGISTER(cpu
, R_DM_DSCR_COUNT
)));
265 * Don't really want to do it this way, but there's no
266 * reliable way to delay completion detection.
268 while (!(__raw_readq((void *)IOADDR(A_DM_REGISTER(cpu
, R_DM_DSCR_BASE_DEBUG
)))
269 & M_DM_DSCR_BASE_INTERRUPT
))
271 __raw_readq(IOADDR(A_DM_REGISTER(cpu
, R_DM_DSCR_BASE
)));
274 #else /* !CONFIG_SIBYTE_DMA_PAGEOPS */
276 void clear_page(void *page
)
278 return clear_page_cpu(page
);
281 void copy_page(void *to
, void *from
)
283 return copy_page_cpu(to
, from
);
286 #endif /* !CONFIG_SIBYTE_DMA_PAGEOPS */
288 EXPORT_SYMBOL(clear_page
);
289 EXPORT_SYMBOL(copy_page
);