2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
24 #include <linux/device.h>
25 #include <linux/uio.h>
26 #include <linux/kref.h>
27 #include <linux/completion.h>
28 #include <linux/rcupdate.h>
29 #include <linux/dma-mapping.h>
32 * enum dma_state - resource PNP/power management state
33 * @DMA_RESOURCE_SUSPEND: DMA device going into low power state
34 * @DMA_RESOURCE_RESUME: DMA device returning to full power
35 * @DMA_RESOURCE_AVAILABLE: DMA device available to the system
36 * @DMA_RESOURCE_REMOVED: DMA device removed from the system
41 DMA_RESOURCE_AVAILABLE
,
46 * enum dma_state_client - state of the channel in the client
47 * @DMA_ACK: client would like to use, or was using this channel
48 * @DMA_DUP: client has already seen this channel, or is not using this channel
49 * @DMA_NAK: client does not want to see any more channels
51 enum dma_state_client
{
58 * typedef dma_cookie_t - an opaque DMA cookie
60 * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
62 typedef s32 dma_cookie_t
;
64 #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0)
67 * enum dma_status - DMA transaction status
68 * @DMA_SUCCESS: transaction completed successfully
69 * @DMA_IN_PROGRESS: transaction not yet processed
70 * @DMA_ERROR: transaction failed
79 * enum dma_transaction_type - DMA transaction types/indexes
81 enum dma_transaction_type
{
95 /* last transaction type for creation of the capabilities mask */
96 #define DMA_TX_TYPE_END (DMA_SLAVE + 1)
99 * enum dma_slave_width - DMA slave register access width.
100 * @DMA_SLAVE_WIDTH_8BIT: Do 8-bit slave register accesses
101 * @DMA_SLAVE_WIDTH_16BIT: Do 16-bit slave register accesses
102 * @DMA_SLAVE_WIDTH_32BIT: Do 32-bit slave register accesses
104 enum dma_slave_width
{
105 DMA_SLAVE_WIDTH_8BIT
,
106 DMA_SLAVE_WIDTH_16BIT
,
107 DMA_SLAVE_WIDTH_32BIT
,
111 * enum dma_ctrl_flags - DMA flags to augment operation preparation,
112 * control completion, and communicate status.
113 * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
115 * @DMA_CTRL_ACK - the descriptor cannot be reused until the client
116 * acknowledges receipt, i.e. has has a chance to establish any
118 * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s)
119 * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s)
121 enum dma_ctrl_flags
{
122 DMA_PREP_INTERRUPT
= (1 << 0),
123 DMA_CTRL_ACK
= (1 << 1),
124 DMA_COMPL_SKIP_SRC_UNMAP
= (1 << 2),
125 DMA_COMPL_SKIP_DEST_UNMAP
= (1 << 3),
129 * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
130 * See linux/cpumask.h
132 typedef struct { DECLARE_BITMAP(bits
, DMA_TX_TYPE_END
); } dma_cap_mask_t
;
135 * struct dma_slave - Information about a DMA slave
136 * @dev: device acting as DMA slave
137 * @dma_dev: required DMA master device. If non-NULL, the client can not be
138 * bound to other masters than this.
139 * @tx_reg: physical address of data register used for
140 * memory-to-peripheral transfers
141 * @rx_reg: physical address of data register used for
142 * peripheral-to-memory transfers
143 * @reg_width: peripheral register width
145 * If dma_dev is non-NULL, the client can not be bound to other DMA
146 * masters than the one corresponding to this device. The DMA master
147 * driver may use this to determine if there is controller-specific
148 * data wrapped around this struct. Drivers of platform code that sets
149 * the dma_dev field must therefore make sure to use an appropriate
150 * controller-specific dma slave structure wrapping this struct.
154 struct device
*dma_dev
;
157 enum dma_slave_width reg_width
;
161 * struct dma_chan_percpu - the per-CPU part of struct dma_chan
162 * @refcount: local_t used for open-coded "bigref" counting
163 * @memcpy_count: transaction counter
164 * @bytes_transferred: byte counter
167 struct dma_chan_percpu
{
169 unsigned long memcpy_count
;
170 unsigned long bytes_transferred
;
174 * struct dma_chan - devices supply DMA channels, clients use them
175 * @device: ptr to the dma device who supplies this channel, always !%NULL
176 * @cookie: last cookie value returned to client
177 * @chan_id: channel ID for sysfs
178 * @class_dev: class device for sysfs
179 * @refcount: kref, used in "bigref" slow-mode
180 * @slow_ref: indicates that the DMA channel is free
181 * @rcu: the DMA channel's RCU head
182 * @device_node: used to add this to the device chan list
183 * @local: per-cpu pointer to a struct dma_chan_percpu
184 * @client-count: how many clients are using this channel
185 * @table_count: number of appearances in the mem-to-mem allocation table
188 struct dma_device
*device
;
195 struct kref refcount
;
199 struct list_head device_node
;
200 struct dma_chan_percpu
*local
;
205 #define to_dma_chan(p) container_of(p, struct dma_chan, dev)
207 void dma_chan_cleanup(struct kref
*kref
);
210 * typedef dma_event_callback - function pointer to a DMA event callback
211 * For each channel added to the system this routine is called for each client.
212 * If the client would like to use the channel it returns '1' to signal (ack)
213 * the dmaengine core to take out a reference on the channel and its
214 * corresponding device. A client must not 'ack' an available channel more
215 * than once. When a channel is removed all clients are notified. If a client
216 * is using the channel it must 'ack' the removal. A client must not 'ack' a
217 * removed channel more than once.
218 * @client - 'this' pointer for the client context
219 * @chan - channel to be acted upon
220 * @state - available or removed
223 typedef enum dma_state_client (*dma_event_callback
) (struct dma_client
*client
,
224 struct dma_chan
*chan
, enum dma_state state
);
227 * struct dma_client - info on the entity making use of DMA services
228 * @event_callback: func ptr to call when something happens
229 * @cap_mask: only return channels that satisfy the requested capabilities
230 * a value of zero corresponds to any capability
231 * @slave: data for preparing slave transfer. Must be non-NULL iff the
232 * DMA_SLAVE capability is requested.
233 * @global_node: list_head for global dma_client_list
236 dma_event_callback event_callback
;
237 dma_cap_mask_t cap_mask
;
238 struct dma_slave
*slave
;
239 struct list_head global_node
;
242 typedef void (*dma_async_tx_callback
)(void *dma_async_param
);
244 * struct dma_async_tx_descriptor - async transaction descriptor
245 * ---dma generic offload fields---
246 * @cookie: tracking cookie for this transaction, set to -EBUSY if
247 * this tx is sitting on a dependency list
248 * @flags: flags to augment operation preparation, control completion, and
250 * @phys: physical address of the descriptor
251 * @tx_list: driver common field for operations that require multiple
253 * @chan: target channel for this operation
254 * @tx_submit: set the prepared descriptor(s) to be executed by the engine
255 * @callback: routine to call after this operation is complete
256 * @callback_param: general parameter to pass to the callback routine
257 * ---async_tx api specific fields---
258 * @next: at completion submit this descriptor
259 * @parent: pointer to the next level up in the dependency chain
260 * @lock: protect the parent and next pointers
262 struct dma_async_tx_descriptor
{
264 enum dma_ctrl_flags flags
; /* not a 'long' to pack with cookie */
266 struct list_head tx_list
;
267 struct dma_chan
*chan
;
268 dma_cookie_t (*tx_submit
)(struct dma_async_tx_descriptor
*tx
);
269 dma_async_tx_callback callback
;
270 void *callback_param
;
271 struct dma_async_tx_descriptor
*next
;
272 struct dma_async_tx_descriptor
*parent
;
277 * struct dma_device - info on the entity supplying DMA services
278 * @chancnt: how many DMA channels are supported
279 * @channels: the list of struct dma_chan
280 * @global_node: list_head for global dma_device_list
281 * @cap_mask: one or more dma_capability flags
282 * @max_xor: maximum number of xor sources, 0 if no capability
283 * @refcount: reference count
284 * @done: IO completion struct
285 * @dev_id: unique device ID
286 * @dev: struct device reference for dma mapping api
287 * @device_alloc_chan_resources: allocate resources and return the
288 * number of allocated descriptors
289 * @device_free_chan_resources: release DMA channel's resources
290 * @device_prep_dma_memcpy: prepares a memcpy operation
291 * @device_prep_dma_xor: prepares a xor operation
292 * @device_prep_dma_zero_sum: prepares a zero_sum operation
293 * @device_prep_dma_memset: prepares a memset operation
294 * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
295 * @device_prep_slave_sg: prepares a slave dma operation
296 * @device_terminate_all: terminate all pending operations
297 * @device_issue_pending: push pending transactions to hardware
301 unsigned int chancnt
;
302 struct list_head channels
;
303 struct list_head global_node
;
304 dma_cap_mask_t cap_mask
;
307 struct kref refcount
;
308 struct completion done
;
313 int (*device_alloc_chan_resources
)(struct dma_chan
*chan
,
314 struct dma_client
*client
);
315 void (*device_free_chan_resources
)(struct dma_chan
*chan
);
317 struct dma_async_tx_descriptor
*(*device_prep_dma_memcpy
)(
318 struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
319 size_t len
, unsigned long flags
);
320 struct dma_async_tx_descriptor
*(*device_prep_dma_xor
)(
321 struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t
*src
,
322 unsigned int src_cnt
, size_t len
, unsigned long flags
);
323 struct dma_async_tx_descriptor
*(*device_prep_dma_zero_sum
)(
324 struct dma_chan
*chan
, dma_addr_t
*src
, unsigned int src_cnt
,
325 size_t len
, u32
*result
, unsigned long flags
);
326 struct dma_async_tx_descriptor
*(*device_prep_dma_memset
)(
327 struct dma_chan
*chan
, dma_addr_t dest
, int value
, size_t len
,
328 unsigned long flags
);
329 struct dma_async_tx_descriptor
*(*device_prep_dma_interrupt
)(
330 struct dma_chan
*chan
, unsigned long flags
);
332 struct dma_async_tx_descriptor
*(*device_prep_slave_sg
)(
333 struct dma_chan
*chan
, struct scatterlist
*sgl
,
334 unsigned int sg_len
, enum dma_data_direction direction
,
335 unsigned long flags
);
336 void (*device_terminate_all
)(struct dma_chan
*chan
);
338 enum dma_status (*device_is_tx_complete
)(struct dma_chan
*chan
,
339 dma_cookie_t cookie
, dma_cookie_t
*last
,
341 void (*device_issue_pending
)(struct dma_chan
*chan
);
344 /* --- public DMA engine API --- */
346 void dma_async_client_register(struct dma_client
*client
);
347 void dma_async_client_unregister(struct dma_client
*client
);
348 void dma_async_client_chan_request(struct dma_client
*client
);
349 dma_cookie_t
dma_async_memcpy_buf_to_buf(struct dma_chan
*chan
,
350 void *dest
, void *src
, size_t len
);
351 dma_cookie_t
dma_async_memcpy_buf_to_pg(struct dma_chan
*chan
,
352 struct page
*page
, unsigned int offset
, void *kdata
, size_t len
);
353 dma_cookie_t
dma_async_memcpy_pg_to_pg(struct dma_chan
*chan
,
354 struct page
*dest_pg
, unsigned int dest_off
, struct page
*src_pg
,
355 unsigned int src_off
, size_t len
);
356 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor
*tx
,
357 struct dma_chan
*chan
);
359 static inline void async_tx_ack(struct dma_async_tx_descriptor
*tx
)
361 tx
->flags
|= DMA_CTRL_ACK
;
364 static inline bool async_tx_test_ack(struct dma_async_tx_descriptor
*tx
)
366 return (tx
->flags
& DMA_CTRL_ACK
) == DMA_CTRL_ACK
;
369 #define first_dma_cap(mask) __first_dma_cap(&(mask))
370 static inline int __first_dma_cap(const dma_cap_mask_t
*srcp
)
372 return min_t(int, DMA_TX_TYPE_END
,
373 find_first_bit(srcp
->bits
, DMA_TX_TYPE_END
));
376 #define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
377 static inline int __next_dma_cap(int n
, const dma_cap_mask_t
*srcp
)
379 return min_t(int, DMA_TX_TYPE_END
,
380 find_next_bit(srcp
->bits
, DMA_TX_TYPE_END
, n
+1));
383 #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
385 __dma_cap_set(enum dma_transaction_type tx_type
, dma_cap_mask_t
*dstp
)
387 set_bit(tx_type
, dstp
->bits
);
390 #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
392 __dma_has_cap(enum dma_transaction_type tx_type
, dma_cap_mask_t
*srcp
)
394 return test_bit(tx_type
, srcp
->bits
);
397 #define for_each_dma_cap_mask(cap, mask) \
398 for ((cap) = first_dma_cap(mask); \
399 (cap) < DMA_TX_TYPE_END; \
400 (cap) = next_dma_cap((cap), (mask)))
403 * dma_async_issue_pending - flush pending transactions to HW
404 * @chan: target DMA channel
406 * This allows drivers to push copies to HW in batches,
407 * reducing MMIO writes where possible.
409 static inline void dma_async_issue_pending(struct dma_chan
*chan
)
411 chan
->device
->device_issue_pending(chan
);
414 #define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
417 * dma_async_is_tx_complete - poll for transaction completion
419 * @cookie: transaction identifier to check status of
420 * @last: returns last completed cookie, can be NULL
421 * @used: returns last issued cookie, can be NULL
423 * If @last and @used are passed in, upon return they reflect the driver
424 * internal state and can be used with dma_async_is_complete() to check
425 * the status of multiple cookies without re-checking hardware state.
427 static inline enum dma_status
dma_async_is_tx_complete(struct dma_chan
*chan
,
428 dma_cookie_t cookie
, dma_cookie_t
*last
, dma_cookie_t
*used
)
430 return chan
->device
->device_is_tx_complete(chan
, cookie
, last
, used
);
433 #define dma_async_memcpy_complete(chan, cookie, last, used)\
434 dma_async_is_tx_complete(chan, cookie, last, used)
437 * dma_async_is_complete - test a cookie against chan state
438 * @cookie: transaction identifier to test status of
439 * @last_complete: last know completed transaction
440 * @last_used: last cookie value handed out
442 * dma_async_is_complete() is used in dma_async_memcpy_complete()
443 * the test logic is separated for lightweight testing of multiple cookies
445 static inline enum dma_status
dma_async_is_complete(dma_cookie_t cookie
,
446 dma_cookie_t last_complete
, dma_cookie_t last_used
)
448 if (last_complete
<= last_used
) {
449 if ((cookie
<= last_complete
) || (cookie
> last_used
))
452 if ((cookie
<= last_complete
) && (cookie
> last_used
))
455 return DMA_IN_PROGRESS
;
458 enum dma_status
dma_sync_wait(struct dma_chan
*chan
, dma_cookie_t cookie
);
459 #ifdef CONFIG_DMA_ENGINE
460 enum dma_status
dma_wait_for_async_tx(struct dma_async_tx_descriptor
*tx
);
462 static inline enum dma_status
dma_wait_for_async_tx(struct dma_async_tx_descriptor
*tx
)
468 /* --- DMA device --- */
470 int dma_async_device_register(struct dma_device
*device
);
471 void dma_async_device_unregister(struct dma_device
*device
);
472 void dma_run_dependencies(struct dma_async_tx_descriptor
*tx
);
473 struct dma_chan
*dma_find_channel(enum dma_transaction_type tx_type
);
475 /* --- Helper iov-locking functions --- */
477 struct dma_page_list
{
478 char __user
*base_address
;
483 struct dma_pinned_list
{
485 struct dma_page_list page_list
[0];
488 struct dma_pinned_list
*dma_pin_iovec_pages(struct iovec
*iov
, size_t len
);
489 void dma_unpin_iovec_pages(struct dma_pinned_list
* pinned_list
);
491 dma_cookie_t
dma_memcpy_to_iovec(struct dma_chan
*chan
, struct iovec
*iov
,
492 struct dma_pinned_list
*pinned_list
, unsigned char *kdata
, size_t len
);
493 dma_cookie_t
dma_memcpy_pg_to_iovec(struct dma_chan
*chan
, struct iovec
*iov
,
494 struct dma_pinned_list
*pinned_list
, struct page
*page
,
495 unsigned int offset
, size_t len
);
497 #endif /* DMAENGINE_H */