2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
23 #include <linux/config.h>
24 #ifdef CONFIG_DMA_ENGINE
26 #include <linux/device.h>
27 #include <linux/uio.h>
28 #include <linux/kref.h>
29 #include <linux/completion.h>
30 #include <linux/rcupdate.h>
33 * enum dma_event - resource PNP/power managment events
34 * @DMA_RESOURCE_SUSPEND: DMA device going into low power state
35 * @DMA_RESOURCE_RESUME: DMA device returning to full power
36 * @DMA_RESOURCE_ADDED: DMA device added to the system
37 * @DMA_RESOURCE_REMOVED: DMA device removed from the system
47 * typedef dma_cookie_t
49 * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
51 typedef s32 dma_cookie_t
;
53 #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0)
56 * enum dma_status - DMA transaction status
57 * @DMA_SUCCESS: transaction completed successfully
58 * @DMA_IN_PROGRESS: transaction not yet processed
59 * @DMA_ERROR: transaction failed
68 * struct dma_chan_percpu - the per-CPU part of struct dma_chan
69 * @refcount: local_t used for open-coded "bigref" counting
70 * @memcpy_count: transaction counter
71 * @bytes_transferred: byte counter
74 struct dma_chan_percpu
{
77 unsigned long memcpy_count
;
78 unsigned long bytes_transferred
;
82 * struct dma_chan - devices supply DMA channels, clients use them
83 * @client: ptr to the client user of this chan, will be NULL when unused
84 * @device: ptr to the dma device who supplies this channel, always !NULL
85 * @cookie: last cookie value returned to client
88 * @refcount: kref, used in "bigref" slow-mode
91 * @client_node: used to add this to the client chan list
92 * @device_node: used to add this to the device chan list
93 * @local: per-cpu pointer to a struct dma_chan_percpu
96 struct dma_client
*client
;
97 struct dma_device
*device
;
102 struct class_device class_dev
;
104 struct kref refcount
;
108 struct list_head client_node
;
109 struct list_head device_node
;
110 struct dma_chan_percpu
*local
;
113 void dma_chan_cleanup(struct kref
*kref
);
115 static inline void dma_chan_get(struct dma_chan
*chan
)
117 if (unlikely(chan
->slow_ref
))
118 kref_get(&chan
->refcount
);
120 local_inc(&(per_cpu_ptr(chan
->local
, get_cpu())->refcount
));
125 static inline void dma_chan_put(struct dma_chan
*chan
)
127 if (unlikely(chan
->slow_ref
))
128 kref_put(&chan
->refcount
, dma_chan_cleanup
);
130 local_dec(&(per_cpu_ptr(chan
->local
, get_cpu())->refcount
));
136 * typedef dma_event_callback - function pointer to a DMA event callback
138 typedef void (*dma_event_callback
) (struct dma_client
*client
,
139 struct dma_chan
*chan
, enum dma_event event
);
142 * struct dma_client - info on the entity making use of DMA services
143 * @event_callback: func ptr to call when something happens
144 * @chan_count: number of chans allocated
145 * @chans_desired: number of chans requested. Can be +/- chan_count
146 * @lock: protects access to the channels list
147 * @channels: the list of DMA channels allocated
148 * @global_node: list_head for global dma_client_list
151 dma_event_callback event_callback
;
152 unsigned int chan_count
;
153 unsigned int chans_desired
;
156 struct list_head channels
;
157 struct list_head global_node
;
161 * struct dma_device - info on the entity supplying DMA services
162 * @chancnt: how many DMA channels are supported
163 * @channels: the list of struct dma_chan
164 * @global_node: list_head for global dma_device_list
168 * Other func ptrs: used to make use of this device's capabilities
172 unsigned int chancnt
;
173 struct list_head channels
;
174 struct list_head global_node
;
176 struct kref refcount
;
177 struct completion done
;
181 int (*device_alloc_chan_resources
)(struct dma_chan
*chan
);
182 void (*device_free_chan_resources
)(struct dma_chan
*chan
);
183 dma_cookie_t (*device_memcpy_buf_to_buf
)(struct dma_chan
*chan
,
184 void *dest
, void *src
, size_t len
);
185 dma_cookie_t (*device_memcpy_buf_to_pg
)(struct dma_chan
*chan
,
186 struct page
*page
, unsigned int offset
, void *kdata
,
188 dma_cookie_t (*device_memcpy_pg_to_pg
)(struct dma_chan
*chan
,
189 struct page
*dest_pg
, unsigned int dest_off
,
190 struct page
*src_pg
, unsigned int src_off
, size_t len
);
191 enum dma_status (*device_memcpy_complete
)(struct dma_chan
*chan
,
192 dma_cookie_t cookie
, dma_cookie_t
*last
,
194 void (*device_memcpy_issue_pending
)(struct dma_chan
*chan
);
197 /* --- public DMA engine API --- */
199 struct dma_client
*dma_async_client_register(dma_event_callback event_callback
);
200 void dma_async_client_unregister(struct dma_client
*client
);
201 void dma_async_client_chan_request(struct dma_client
*client
,
202 unsigned int number
);
205 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
206 * @chan: DMA channel to offload copy to
207 * @dest: destination address (virtual)
208 * @src: source address (virtual)
211 * Both @dest and @src must be mappable to a bus address according to the
212 * DMA mapping API rules for streaming mappings.
213 * Both @dest and @src must stay memory resident (kernel memory or locked
216 static inline dma_cookie_t
dma_async_memcpy_buf_to_buf(struct dma_chan
*chan
,
217 void *dest
, void *src
, size_t len
)
220 per_cpu_ptr(chan
->local
, cpu
)->bytes_transferred
+= len
;
221 per_cpu_ptr(chan
->local
, cpu
)->memcpy_count
++;
224 return chan
->device
->device_memcpy_buf_to_buf(chan
, dest
, src
, len
);
228 * dma_async_memcpy_buf_to_pg - offloaded copy
229 * @chan: DMA channel to offload copy to
230 * @page: destination page
231 * @offset: offset in page to copy to
232 * @kdata: source address (virtual)
235 * Both @page/@offset and @kdata must be mappable to a bus address according
236 * to the DMA mapping API rules for streaming mappings.
237 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
238 * locked user space pages)
240 static inline dma_cookie_t
dma_async_memcpy_buf_to_pg(struct dma_chan
*chan
,
241 struct page
*page
, unsigned int offset
, void *kdata
, size_t len
)
244 per_cpu_ptr(chan
->local
, cpu
)->bytes_transferred
+= len
;
245 per_cpu_ptr(chan
->local
, cpu
)->memcpy_count
++;
248 return chan
->device
->device_memcpy_buf_to_pg(chan
, page
, offset
,
253 * dma_async_memcpy_buf_to_pg - offloaded copy
254 * @chan: DMA channel to offload copy to
255 * @dest_page: destination page
256 * @dest_off: offset in page to copy to
257 * @src_page: source page
258 * @src_off: offset in page to copy from
261 * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
262 * address according to the DMA mapping API rules for streaming mappings.
263 * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
264 * (kernel memory or locked user space pages)
266 static inline dma_cookie_t
dma_async_memcpy_pg_to_pg(struct dma_chan
*chan
,
267 struct page
*dest_pg
, unsigned int dest_off
, struct page
*src_pg
,
268 unsigned int src_off
, size_t len
)
271 per_cpu_ptr(chan
->local
, cpu
)->bytes_transferred
+= len
;
272 per_cpu_ptr(chan
->local
, cpu
)->memcpy_count
++;
275 return chan
->device
->device_memcpy_pg_to_pg(chan
, dest_pg
, dest_off
,
276 src_pg
, src_off
, len
);
280 * dma_async_memcpy_issue_pending - flush pending copies to HW
283 * This allows drivers to push copies to HW in batches,
284 * reducing MMIO writes where possible.
286 static inline void dma_async_memcpy_issue_pending(struct dma_chan
*chan
)
288 return chan
->device
->device_memcpy_issue_pending(chan
);
292 * dma_async_memcpy_complete - poll for transaction completion
294 * @cookie: transaction identifier to check status of
295 * @last: returns last completed cookie, can be NULL
296 * @used: returns last issued cookie, can be NULL
298 * If @last and @used are passed in, upon return they reflect the driver
299 * internal state and can be used with dma_async_is_complete() to check
300 * the status of multiple cookies without re-checking hardware state.
302 static inline enum dma_status
dma_async_memcpy_complete(struct dma_chan
*chan
,
303 dma_cookie_t cookie
, dma_cookie_t
*last
, dma_cookie_t
*used
)
305 return chan
->device
->device_memcpy_complete(chan
, cookie
, last
, used
);
309 * dma_async_is_complete - test a cookie against chan state
310 * @cookie: transaction identifier to test status of
311 * @last_complete: last know completed transaction
312 * @last_used: last cookie value handed out
314 * dma_async_is_complete() is used in dma_async_memcpy_complete()
315 * the test logic is seperated for lightweight testing of multiple cookies
317 static inline enum dma_status
dma_async_is_complete(dma_cookie_t cookie
,
318 dma_cookie_t last_complete
, dma_cookie_t last_used
)
320 if (last_complete
<= last_used
) {
321 if ((cookie
<= last_complete
) || (cookie
> last_used
))
324 if ((cookie
<= last_complete
) && (cookie
> last_used
))
327 return DMA_IN_PROGRESS
;
331 /* --- DMA device --- */
333 int dma_async_device_register(struct dma_device
*device
);
334 void dma_async_device_unregister(struct dma_device
*device
);
336 /* --- Helper iov-locking functions --- */
338 struct dma_page_list
{
344 struct dma_pinned_list
{
346 struct dma_page_list page_list
[0];
349 struct dma_pinned_list
*dma_pin_iovec_pages(struct iovec
*iov
, size_t len
);
350 void dma_unpin_iovec_pages(struct dma_pinned_list
* pinned_list
);
352 dma_cookie_t
dma_memcpy_to_iovec(struct dma_chan
*chan
, struct iovec
*iov
,
353 struct dma_pinned_list
*pinned_list
, unsigned char *kdata
, size_t len
);
354 dma_cookie_t
dma_memcpy_pg_to_iovec(struct dma_chan
*chan
, struct iovec
*iov
,
355 struct dma_pinned_list
*pinned_list
, struct page
*page
,
356 unsigned int offset
, size_t len
);
358 #endif /* CONFIG_DMA_ENGINE */
359 #endif /* DMAENGINE_H */