2 * xor offload engine api
4 * Copyright © 2006, Intel Corporation.
6 * Dan Williams <dan.j.williams@intel.com>
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
26 #include <linux/kernel.h>
27 #include <linux/interrupt.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/raid/xor.h>
31 #include <linux/async_tx.h>
33 /* do_async_xor - dma map the pages and perform the xor with an engine.
34 * This routine is marked __always_inline so it can be compiled away
35 * when CONFIG_DMA_ENGINE=n
37 static __always_inline
struct dma_async_tx_descriptor
*
38 do_async_xor(struct dma_chan
*chan
, struct page
*dest
, struct page
**src_list
,
39 unsigned int offset
, int src_cnt
, size_t len
,
40 enum async_tx_flags flags
,
41 struct dma_async_tx_descriptor
*depend_tx
,
42 dma_async_tx_callback cb_fn
, void *cb_param
)
44 struct dma_device
*dma
= chan
->device
;
45 dma_addr_t
*dma_src
= (dma_addr_t
*) src_list
;
46 struct dma_async_tx_descriptor
*tx
= NULL
;
49 dma_async_tx_callback _cb_fn
;
51 enum async_tx_flags async_flags
;
52 enum dma_ctrl_flags dma_flags
;
56 dma_dest
= dma_map_page(dma
->dev
, dest
, offset
, len
, DMA_FROM_DEVICE
);
57 for (i
= 0; i
< src_cnt
; i
++)
58 dma_src
[i
] = dma_map_page(dma
->dev
, src_list
[i
], offset
,
64 xor_src_cnt
= min(src_cnt
, dma
->max_xor
);
65 /* if we are submitting additional xors, leave the chain open,
66 * clear the callback parameters, and leave the destination
69 if (src_cnt
> xor_src_cnt
) {
70 async_flags
&= ~ASYNC_TX_ACK
;
71 dma_flags
= DMA_COMPL_SKIP_DEST_UNMAP
;
79 dma_flags
|= DMA_PREP_INTERRUPT
;
81 /* Since we have clobbered the src_list we are committed
82 * to doing this asynchronously. Drivers force forward progress
83 * in case they can not provide a descriptor
85 tx
= dma
->device_prep_dma_xor(chan
, dma_dest
, &dma_src
[src_off
],
86 xor_src_cnt
, len
, dma_flags
);
88 if (unlikely(!tx
&& depend_tx
))
89 dma_wait_for_async_tx(depend_tx
);
91 /* spin wait for the preceeding transactions to complete */
93 tx
= dma
->device_prep_dma_xor(chan
, dma_dest
,
98 async_tx_submit(chan
, tx
, async_flags
, depend_tx
, _cb_fn
,
102 flags
|= ASYNC_TX_DEP_ACK
;
104 if (src_cnt
> xor_src_cnt
) {
105 /* drop completed sources */
106 src_cnt
-= xor_src_cnt
;
107 src_off
+= xor_src_cnt
;
109 /* use the intermediate result a source */
110 dma_src
[--src_off
] = dma_dest
;
120 do_sync_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
121 int src_cnt
, size_t len
, enum async_tx_flags flags
,
122 struct dma_async_tx_descriptor
*depend_tx
,
123 dma_async_tx_callback cb_fn
, void *cb_param
)
129 void **srcs
= (void **) src_list
;
131 /* reuse the 'src_list' array to convert to buffer pointers */
132 for (i
= 0; i
< src_cnt
; i
++)
133 srcs
[i
] = page_address(src_list
[i
]) + offset
;
135 /* set destination address */
136 dest_buf
= page_address(dest
) + offset
;
138 if (flags
& ASYNC_TX_XOR_ZERO_DST
)
139 memset(dest_buf
, 0, len
);
141 while (src_cnt
> 0) {
142 /* process up to 'MAX_XOR_BLOCKS' sources */
143 xor_src_cnt
= min(src_cnt
, MAX_XOR_BLOCKS
);
144 xor_blocks(xor_src_cnt
, len
, dest_buf
, &srcs
[src_off
]);
146 /* drop completed sources */
147 src_cnt
-= xor_src_cnt
;
148 src_off
+= xor_src_cnt
;
151 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
155 * async_xor - attempt to xor a set of blocks with a dma engine.
156 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
157 * flag must be set to not include dest data in the calculation. The
158 * assumption with dma eninges is that they only use the destination
159 * buffer as a source when it is explicity specified in the source list.
160 * @dest: destination page
161 * @src_list: array of source pages (if the dest is also a source it must be
162 * at index zero). The contents of this array may be overwritten.
163 * @offset: offset in pages to start transaction
164 * @src_cnt: number of source pages
165 * @len: length in bytes
166 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST,
167 * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
168 * @depend_tx: xor depends on the result of this transaction.
169 * @cb_fn: function to call when the xor completes
170 * @cb_param: parameter to pass to the callback routine
172 struct dma_async_tx_descriptor
*
173 async_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
174 int src_cnt
, size_t len
, enum async_tx_flags flags
,
175 struct dma_async_tx_descriptor
*depend_tx
,
176 dma_async_tx_callback cb_fn
, void *cb_param
)
178 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_XOR
,
181 BUG_ON(src_cnt
<= 1);
184 /* run the xor asynchronously */
185 pr_debug("%s (async): len: %zu\n", __func__
, len
);
187 return do_async_xor(chan
, dest
, src_list
, offset
, src_cnt
, len
,
188 flags
, depend_tx
, cb_fn
, cb_param
);
190 /* run the xor synchronously */
191 pr_debug("%s (sync): len: %zu\n", __func__
, len
);
193 /* in the sync case the dest is an implied source
194 * (assumes the dest is the first source)
196 if (flags
& ASYNC_TX_XOR_DROP_DST
) {
201 /* wait for any prerequisite operations */
203 /* if ack is already set then we cannot be sure
204 * we are referring to the correct operation
206 BUG_ON(async_tx_test_ack(depend_tx
));
207 if (dma_wait_for_async_tx(depend_tx
) == DMA_ERROR
)
208 panic("%s: DMA_ERROR waiting for depend_tx\n",
212 do_sync_xor(dest
, src_list
, offset
, src_cnt
, len
,
213 flags
, depend_tx
, cb_fn
, cb_param
);
218 EXPORT_SYMBOL_GPL(async_xor
);
220 static int page_is_zero(struct page
*p
, unsigned int offset
, size_t len
)
222 char *a
= page_address(p
) + offset
;
223 return ((*(u32
*) a
) == 0 &&
224 memcmp(a
, a
+ 4, len
- 4) == 0);
228 * async_xor_zero_sum - attempt a xor parity check with a dma engine.
229 * @dest: destination page used if the xor is performed synchronously
230 * @src_list: array of source pages. The dest page must be listed as a source
231 * at index zero. The contents of this array may be overwritten.
232 * @offset: offset in pages to start transaction
233 * @src_cnt: number of source pages
234 * @len: length in bytes
235 * @result: 0 if sum == 0 else non-zero
236 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
237 * @depend_tx: xor depends on the result of this transaction.
238 * @cb_fn: function to call when the xor completes
239 * @cb_param: parameter to pass to the callback routine
241 struct dma_async_tx_descriptor
*
242 async_xor_zero_sum(struct page
*dest
, struct page
**src_list
,
243 unsigned int offset
, int src_cnt
, size_t len
,
244 u32
*result
, enum async_tx_flags flags
,
245 struct dma_async_tx_descriptor
*depend_tx
,
246 dma_async_tx_callback cb_fn
, void *cb_param
)
248 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_ZERO_SUM
,
251 struct dma_device
*device
= chan
? chan
->device
: NULL
;
252 struct dma_async_tx_descriptor
*tx
= NULL
;
254 BUG_ON(src_cnt
<= 1);
256 if (device
&& src_cnt
<= device
->max_xor
) {
257 dma_addr_t
*dma_src
= (dma_addr_t
*) src_list
;
258 unsigned long dma_prep_flags
= cb_fn
? DMA_PREP_INTERRUPT
: 0;
261 pr_debug("%s: (async) len: %zu\n", __func__
, len
);
263 for (i
= 0; i
< src_cnt
; i
++)
264 dma_src
[i
] = dma_map_page(device
->dev
, src_list
[i
],
265 offset
, len
, DMA_TO_DEVICE
);
267 tx
= device
->device_prep_dma_zero_sum(chan
, dma_src
, src_cnt
,
272 dma_wait_for_async_tx(depend_tx
);
275 tx
= device
->device_prep_dma_zero_sum(chan
,
276 dma_src
, src_cnt
, len
, result
,
280 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
282 unsigned long xor_flags
= flags
;
284 pr_debug("%s: (sync) len: %zu\n", __func__
, len
);
286 xor_flags
|= ASYNC_TX_XOR_DROP_DST
;
287 xor_flags
&= ~ASYNC_TX_ACK
;
289 tx
= async_xor(dest
, src_list
, offset
, src_cnt
, len
, xor_flags
,
290 depend_tx
, NULL
, NULL
);
293 if (dma_wait_for_async_tx(tx
) == DMA_ERROR
)
294 panic("%s: DMA_ERROR waiting for tx\n",
299 *result
= page_is_zero(dest
, offset
, len
) ? 0 : 1;
303 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
308 EXPORT_SYMBOL_GPL(async_xor_zero_sum
);
310 static int __init
async_xor_init(void)
312 #ifdef CONFIG_DMA_ENGINE
313 /* To conserve stack space the input src_list (array of page pointers)
314 * is reused to hold the array of dma addresses passed to the driver.
315 * This conversion is only possible when dma_addr_t is less than the
316 * the size of a pointer. HIGHMEM64G is known to violate this
319 BUILD_BUG_ON(sizeof(dma_addr_t
) > sizeof(struct page
*));
325 static void __exit
async_xor_exit(void)
330 module_init(async_xor_init
);
331 module_exit(async_xor_exit
);
333 MODULE_AUTHOR("Intel Corporation");
334 MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
335 MODULE_LICENSE("GPL");