2 * xor offload engine api
4 * Copyright © 2006, Intel Corporation.
6 * Dan Williams <dan.j.williams@intel.com>
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
26 #include <linux/kernel.h>
27 #include <linux/interrupt.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/raid/xor.h>
31 #include <linux/async_tx.h>
33 /* do_async_xor - dma map the pages and perform the xor with an engine */
34 static __async_inline
struct dma_async_tx_descriptor
*
35 do_async_xor(struct dma_chan
*chan
, struct page
*dest
, struct page
**src_list
,
36 unsigned int offset
, int src_cnt
, size_t len
,
37 enum async_tx_flags flags
,
38 struct dma_async_tx_descriptor
*depend_tx
,
39 dma_async_tx_callback cb_fn
, void *cb_param
)
41 struct dma_device
*dma
= chan
->device
;
42 dma_addr_t
*dma_src
= (dma_addr_t
*) src_list
;
43 struct dma_async_tx_descriptor
*tx
= NULL
;
46 dma_async_tx_callback _cb_fn
;
48 enum async_tx_flags async_flags
;
49 enum dma_ctrl_flags dma_flags
;
53 /* map the dest bidrectional in case it is re-used as a source */
54 dma_dest
= dma_map_page(dma
->dev
, dest
, offset
, len
, DMA_BIDIRECTIONAL
);
55 for (i
= 0; i
< src_cnt
; i
++) {
56 /* only map the dest once */
57 if (unlikely(src_list
[i
] == dest
)) {
58 dma_src
[i
] = dma_dest
;
61 dma_src
[i
] = dma_map_page(dma
->dev
, src_list
[i
], offset
,
68 xor_src_cnt
= min(src_cnt
, dma
->max_xor
);
69 /* if we are submitting additional xors, leave the chain open,
70 * clear the callback parameters, and leave the destination
73 if (src_cnt
> xor_src_cnt
) {
74 async_flags
&= ~ASYNC_TX_ACK
;
75 dma_flags
= DMA_COMPL_SKIP_DEST_UNMAP
;
83 dma_flags
|= DMA_PREP_INTERRUPT
;
85 /* Since we have clobbered the src_list we are committed
86 * to doing this asynchronously. Drivers force forward progress
87 * in case they can not provide a descriptor
89 tx
= dma
->device_prep_dma_xor(chan
, dma_dest
, &dma_src
[src_off
],
90 xor_src_cnt
, len
, dma_flags
);
93 async_tx_quiesce(&depend_tx
);
95 /* spin wait for the preceeding transactions to complete */
96 while (unlikely(!tx
)) {
97 dma_async_issue_pending(chan
);
98 tx
= dma
->device_prep_dma_xor(chan
, dma_dest
,
104 async_tx_submit(chan
, tx
, async_flags
, depend_tx
, _cb_fn
,
109 if (src_cnt
> xor_src_cnt
) {
110 /* drop completed sources */
111 src_cnt
-= xor_src_cnt
;
112 src_off
+= xor_src_cnt
;
114 /* use the intermediate result a source */
115 dma_src
[--src_off
] = dma_dest
;
125 do_sync_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
126 int src_cnt
, size_t len
, enum async_tx_flags flags
,
127 dma_async_tx_callback cb_fn
, void *cb_param
)
133 void **srcs
= (void **) src_list
;
135 /* reuse the 'src_list' array to convert to buffer pointers */
136 for (i
= 0; i
< src_cnt
; i
++)
137 srcs
[i
] = page_address(src_list
[i
]) + offset
;
139 /* set destination address */
140 dest_buf
= page_address(dest
) + offset
;
142 if (flags
& ASYNC_TX_XOR_ZERO_DST
)
143 memset(dest_buf
, 0, len
);
145 while (src_cnt
> 0) {
146 /* process up to 'MAX_XOR_BLOCKS' sources */
147 xor_src_cnt
= min(src_cnt
, MAX_XOR_BLOCKS
);
148 xor_blocks(xor_src_cnt
, len
, dest_buf
, &srcs
[src_off
]);
150 /* drop completed sources */
151 src_cnt
-= xor_src_cnt
;
152 src_off
+= xor_src_cnt
;
155 async_tx_sync_epilog(cb_fn
, cb_param
);
159 * async_xor - attempt to xor a set of blocks with a dma engine.
160 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
161 * flag must be set to not include dest data in the calculation. The
162 * assumption with dma eninges is that they only use the destination
163 * buffer as a source when it is explicity specified in the source list.
164 * @dest: destination page
165 * @src_list: array of source pages (if the dest is also a source it must be
166 * at index zero). The contents of this array may be overwritten.
167 * @offset: offset in pages to start transaction
168 * @src_cnt: number of source pages
169 * @len: length in bytes
170 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST, ASYNC_TX_ACK
171 * @depend_tx: xor depends on the result of this transaction.
172 * @cb_fn: function to call when the xor completes
173 * @cb_param: parameter to pass to the callback routine
175 struct dma_async_tx_descriptor
*
176 async_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
177 int src_cnt
, size_t len
, enum async_tx_flags flags
,
178 struct dma_async_tx_descriptor
*depend_tx
,
179 dma_async_tx_callback cb_fn
, void *cb_param
)
181 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_XOR
,
184 BUG_ON(src_cnt
<= 1);
187 /* run the xor asynchronously */
188 pr_debug("%s (async): len: %zu\n", __func__
, len
);
190 return do_async_xor(chan
, dest
, src_list
, offset
, src_cnt
, len
,
191 flags
, depend_tx
, cb_fn
, cb_param
);
193 /* run the xor synchronously */
194 pr_debug("%s (sync): len: %zu\n", __func__
, len
);
196 /* in the sync case the dest is an implied source
197 * (assumes the dest is the first source)
199 if (flags
& ASYNC_TX_XOR_DROP_DST
) {
204 /* wait for any prerequisite operations */
205 async_tx_quiesce(&depend_tx
);
207 do_sync_xor(dest
, src_list
, offset
, src_cnt
, len
,
208 flags
, cb_fn
, cb_param
);
213 EXPORT_SYMBOL_GPL(async_xor
);
215 static int page_is_zero(struct page
*p
, unsigned int offset
, size_t len
)
217 char *a
= page_address(p
) + offset
;
218 return ((*(u32
*) a
) == 0 &&
219 memcmp(a
, a
+ 4, len
- 4) == 0);
223 * async_xor_val - attempt a xor parity check with a dma engine.
224 * @dest: destination page used if the xor is performed synchronously
225 * @src_list: array of source pages. The dest page must be listed as a source
226 * at index zero. The contents of this array may be overwritten.
227 * @offset: offset in pages to start transaction
228 * @src_cnt: number of source pages
229 * @len: length in bytes
230 * @result: 0 if sum == 0 else non-zero
231 * @flags: ASYNC_TX_ACK
232 * @depend_tx: xor depends on the result of this transaction.
233 * @cb_fn: function to call when the xor completes
234 * @cb_param: parameter to pass to the callback routine
236 struct dma_async_tx_descriptor
*
237 async_xor_val(struct page
*dest
, struct page
**src_list
,
238 unsigned int offset
, int src_cnt
, size_t len
,
239 u32
*result
, enum async_tx_flags flags
,
240 struct dma_async_tx_descriptor
*depend_tx
,
241 dma_async_tx_callback cb_fn
, void *cb_param
)
243 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_XOR_VAL
,
246 struct dma_device
*device
= chan
? chan
->device
: NULL
;
247 struct dma_async_tx_descriptor
*tx
= NULL
;
249 BUG_ON(src_cnt
<= 1);
251 if (device
&& src_cnt
<= device
->max_xor
) {
252 dma_addr_t
*dma_src
= (dma_addr_t
*) src_list
;
253 unsigned long dma_prep_flags
= cb_fn
? DMA_PREP_INTERRUPT
: 0;
256 pr_debug("%s: (async) len: %zu\n", __func__
, len
);
258 for (i
= 0; i
< src_cnt
; i
++)
259 dma_src
[i
] = dma_map_page(device
->dev
, src_list
[i
],
260 offset
, len
, DMA_TO_DEVICE
);
262 tx
= device
->device_prep_dma_xor_val(chan
, dma_src
, src_cnt
,
266 async_tx_quiesce(&depend_tx
);
269 dma_async_issue_pending(chan
);
270 tx
= device
->device_prep_dma_xor_val(chan
,
271 dma_src
, src_cnt
, len
, result
,
276 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
278 unsigned long xor_flags
= flags
;
280 pr_debug("%s: (sync) len: %zu\n", __func__
, len
);
282 xor_flags
|= ASYNC_TX_XOR_DROP_DST
;
283 xor_flags
&= ~ASYNC_TX_ACK
;
285 tx
= async_xor(dest
, src_list
, offset
, src_cnt
, len
, xor_flags
,
286 depend_tx
, NULL
, NULL
);
288 async_tx_quiesce(&tx
);
290 *result
= page_is_zero(dest
, offset
, len
) ? 0 : 1;
292 async_tx_sync_epilog(cb_fn
, cb_param
);
297 EXPORT_SYMBOL_GPL(async_xor_val
);
299 static int __init
async_xor_init(void)
301 #ifdef CONFIG_DMA_ENGINE
302 /* To conserve stack space the input src_list (array of page pointers)
303 * is reused to hold the array of dma addresses passed to the driver.
304 * This conversion is only possible when dma_addr_t is less than the
305 * the size of a pointer. HIGHMEM64G is known to violate this
308 BUILD_BUG_ON(sizeof(dma_addr_t
) > sizeof(struct page
*));
314 static void __exit
async_xor_exit(void)
319 module_init(async_xor_init
);
320 module_exit(async_xor_exit
);
322 MODULE_AUTHOR("Intel Corporation");
323 MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
324 MODULE_LICENSE("GPL");