1 #include <linux/device.h>
2 #include <linux/dma-mapping.h>
3 #include <linux/dmaengine.h>
4 #include <linux/sizes.h>
5 #include <linux/platform_device.h>
10 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
12 #define EP_MODE_AUTOREG_NONE 0
13 #define EP_MODE_AUTOREG_ALL_NEOP 1
14 #define EP_MODE_AUTOREG_ALWAYS 3
16 #define EP_MODE_DMA_TRANSPARENT 0
17 #define EP_MODE_DMA_RNDIS 1
18 #define EP_MODE_DMA_GEN_RNDIS 3
20 #define USB_CTRL_TX_MODE 0x70
21 #define USB_CTRL_RX_MODE 0x74
22 #define USB_CTRL_AUTOREQ 0xd0
23 #define USB_TDOWN 0xd8
25 struct cppi41_dma_channel
{
26 struct dma_channel channel
;
27 struct cppi41_dma_controller
*controller
;
28 struct musb_hw_ep
*hw_ep
;
43 #define MUSB_DMA_NUM_CHANNELS 15
45 struct cppi41_dma_controller
{
46 struct dma_controller controller
;
47 struct cppi41_dma_channel rx_channel
[MUSB_DMA_NUM_CHANNELS
];
48 struct cppi41_dma_channel tx_channel
[MUSB_DMA_NUM_CHANNELS
];
55 static void save_rx_toggle(struct cppi41_dma_channel
*cppi41_channel
)
60 if (cppi41_channel
->is_tx
)
62 if (!is_host_active(cppi41_channel
->controller
->musb
))
65 csr
= musb_readw(cppi41_channel
->hw_ep
->regs
, MUSB_RXCSR
);
66 toggle
= csr
& MUSB_RXCSR_H_DATATOGGLE
? 1 : 0;
68 cppi41_channel
->usb_toggle
= toggle
;
71 static void update_rx_toggle(struct cppi41_dma_channel
*cppi41_channel
)
76 if (cppi41_channel
->is_tx
)
78 if (!is_host_active(cppi41_channel
->controller
->musb
))
81 csr
= musb_readw(cppi41_channel
->hw_ep
->regs
, MUSB_RXCSR
);
82 toggle
= csr
& MUSB_RXCSR_H_DATATOGGLE
? 1 : 0;
85 * AM335x Advisory 1.0.13: Due to internal synchronisation error the
86 * data toggle may reset from DATA1 to DATA0 during receiving data from
87 * more than one endpoint.
89 if (!toggle
&& toggle
== cppi41_channel
->usb_toggle
) {
90 csr
|= MUSB_RXCSR_H_DATATOGGLE
| MUSB_RXCSR_H_WR_DATATOGGLE
;
91 musb_writew(cppi41_channel
->hw_ep
->regs
, MUSB_RXCSR
, csr
);
92 dev_dbg(cppi41_channel
->controller
->musb
->controller
,
93 "Restoring DATA1 toggle.\n");
96 cppi41_channel
->usb_toggle
= toggle
;
99 static void cppi41_dma_callback(void *private_data
)
101 struct dma_channel
*channel
= private_data
;
102 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
103 struct musb_hw_ep
*hw_ep
= cppi41_channel
->hw_ep
;
104 struct musb
*musb
= hw_ep
->musb
;
106 struct dma_tx_state txstate
;
109 spin_lock_irqsave(&musb
->lock
, flags
);
111 dmaengine_tx_status(cppi41_channel
->dc
, cppi41_channel
->cookie
,
113 transferred
= cppi41_channel
->prog_len
- txstate
.residue
;
114 cppi41_channel
->transferred
+= transferred
;
116 dev_dbg(musb
->controller
, "DMA transfer done on hw_ep=%d bytes=%d/%d\n",
117 hw_ep
->epnum
, cppi41_channel
->transferred
,
118 cppi41_channel
->total_len
);
120 update_rx_toggle(cppi41_channel
);
122 if (cppi41_channel
->transferred
== cppi41_channel
->total_len
||
123 transferred
< cppi41_channel
->packet_sz
) {
126 cppi41_channel
->channel
.actual_len
=
127 cppi41_channel
->transferred
;
128 cppi41_channel
->channel
.status
= MUSB_DMA_STATUS_FREE
;
129 musb_dma_completion(musb
, hw_ep
->epnum
, cppi41_channel
->is_tx
);
131 /* next iteration, reload */
132 struct dma_chan
*dc
= cppi41_channel
->dc
;
133 struct dma_async_tx_descriptor
*dma_desc
;
134 enum dma_transfer_direction direction
;
137 void __iomem
*epio
= cppi41_channel
->hw_ep
->regs
;
139 cppi41_channel
->buf_addr
+= cppi41_channel
->packet_sz
;
141 remain_bytes
= cppi41_channel
->total_len
;
142 remain_bytes
-= cppi41_channel
->transferred
;
143 remain_bytes
= min(remain_bytes
, cppi41_channel
->packet_sz
);
144 cppi41_channel
->prog_len
= remain_bytes
;
146 direction
= cppi41_channel
->is_tx
? DMA_MEM_TO_DEV
148 dma_desc
= dmaengine_prep_slave_single(dc
,
149 cppi41_channel
->buf_addr
,
152 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
153 if (WARN_ON(!dma_desc
)) {
154 spin_unlock_irqrestore(&musb
->lock
, flags
);
158 dma_desc
->callback
= cppi41_dma_callback
;
159 dma_desc
->callback_param
= channel
;
160 cppi41_channel
->cookie
= dma_desc
->tx_submit(dma_desc
);
161 dma_async_issue_pending(dc
);
163 if (!cppi41_channel
->is_tx
) {
164 csr
= musb_readw(epio
, MUSB_RXCSR
);
165 csr
|= MUSB_RXCSR_H_REQPKT
;
166 musb_writew(epio
, MUSB_RXCSR
, csr
);
169 spin_unlock_irqrestore(&musb
->lock
, flags
);
172 static u32
update_ep_mode(unsigned ep
, unsigned mode
, u32 old
)
176 shift
= (ep
- 1) * 2;
177 old
&= ~(3 << shift
);
178 old
|= mode
<< shift
;
182 static void cppi41_set_dma_mode(struct cppi41_dma_channel
*cppi41_channel
,
185 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
190 if (cppi41_channel
->is_tx
)
191 old_mode
= controller
->tx_mode
;
193 old_mode
= controller
->rx_mode
;
194 port
= cppi41_channel
->port_num
;
195 new_mode
= update_ep_mode(port
, mode
, old_mode
);
197 if (new_mode
== old_mode
)
199 if (cppi41_channel
->is_tx
) {
200 controller
->tx_mode
= new_mode
;
201 musb_writel(controller
->musb
->ctrl_base
, USB_CTRL_TX_MODE
,
204 controller
->rx_mode
= new_mode
;
205 musb_writel(controller
->musb
->ctrl_base
, USB_CTRL_RX_MODE
,
210 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel
*cppi41_channel
,
213 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
218 old_mode
= controller
->auto_req
;
219 port
= cppi41_channel
->port_num
;
220 new_mode
= update_ep_mode(port
, mode
, old_mode
);
222 if (new_mode
== old_mode
)
224 controller
->auto_req
= new_mode
;
225 musb_writel(controller
->musb
->ctrl_base
, USB_CTRL_AUTOREQ
, new_mode
);
228 static bool cppi41_configure_channel(struct dma_channel
*channel
,
229 u16 packet_sz
, u8 mode
,
230 dma_addr_t dma_addr
, u32 len
)
232 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
233 struct dma_chan
*dc
= cppi41_channel
->dc
;
234 struct dma_async_tx_descriptor
*dma_desc
;
235 enum dma_transfer_direction direction
;
236 struct musb
*musb
= cppi41_channel
->controller
->musb
;
237 unsigned use_gen_rndis
= 0;
239 dev_dbg(musb
->controller
,
240 "configure ep%d/%x packet_sz=%d, mode=%d, dma_addr=0x%llx, len=%d is_tx=%d\n",
241 cppi41_channel
->port_num
, RNDIS_REG(cppi41_channel
->port_num
),
242 packet_sz
, mode
, (unsigned long long) dma_addr
,
243 len
, cppi41_channel
->is_tx
);
245 cppi41_channel
->buf_addr
= dma_addr
;
246 cppi41_channel
->total_len
= len
;
247 cppi41_channel
->transferred
= 0;
248 cppi41_channel
->packet_sz
= packet_sz
;
251 * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
252 * than max packet size at a time.
254 if (cppi41_channel
->is_tx
)
259 if (len
> packet_sz
) {
260 musb_writel(musb
->ctrl_base
,
261 RNDIS_REG(cppi41_channel
->port_num
), len
);
263 cppi41_set_dma_mode(cppi41_channel
,
264 EP_MODE_DMA_GEN_RNDIS
);
267 cppi41_set_autoreq_mode(cppi41_channel
,
268 EP_MODE_AUTOREG_ALL_NEOP
);
270 musb_writel(musb
->ctrl_base
,
271 RNDIS_REG(cppi41_channel
->port_num
), 0);
272 cppi41_set_dma_mode(cppi41_channel
,
273 EP_MODE_DMA_TRANSPARENT
);
274 cppi41_set_autoreq_mode(cppi41_channel
,
275 EP_MODE_AUTOREG_NONE
);
279 cppi41_set_dma_mode(cppi41_channel
, EP_MODE_DMA_TRANSPARENT
);
280 cppi41_set_autoreq_mode(cppi41_channel
, EP_MODE_AUTOREG_NONE
);
281 len
= min_t(u32
, packet_sz
, len
);
283 cppi41_channel
->prog_len
= len
;
284 direction
= cppi41_channel
->is_tx
? DMA_MEM_TO_DEV
: DMA_DEV_TO_MEM
;
285 dma_desc
= dmaengine_prep_slave_single(dc
, dma_addr
, len
, direction
,
286 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
290 dma_desc
->callback
= cppi41_dma_callback
;
291 dma_desc
->callback_param
= channel
;
292 cppi41_channel
->cookie
= dma_desc
->tx_submit(dma_desc
);
294 save_rx_toggle(cppi41_channel
);
295 dma_async_issue_pending(dc
);
299 static struct dma_channel
*cppi41_dma_channel_allocate(struct dma_controller
*c
,
300 struct musb_hw_ep
*hw_ep
, u8 is_tx
)
302 struct cppi41_dma_controller
*controller
= container_of(c
,
303 struct cppi41_dma_controller
, controller
);
304 struct cppi41_dma_channel
*cppi41_channel
= NULL
;
305 u8 ch_num
= hw_ep
->epnum
- 1;
307 if (ch_num
>= MUSB_DMA_NUM_CHANNELS
)
311 cppi41_channel
= &controller
->tx_channel
[ch_num
];
313 cppi41_channel
= &controller
->rx_channel
[ch_num
];
315 if (!cppi41_channel
->dc
)
318 if (cppi41_channel
->is_allocated
)
321 cppi41_channel
->hw_ep
= hw_ep
;
322 cppi41_channel
->is_allocated
= 1;
324 return &cppi41_channel
->channel
;
327 static void cppi41_dma_channel_release(struct dma_channel
*channel
)
329 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
331 if (cppi41_channel
->is_allocated
) {
332 cppi41_channel
->is_allocated
= 0;
333 channel
->status
= MUSB_DMA_STATUS_FREE
;
334 channel
->actual_len
= 0;
338 static int cppi41_dma_channel_program(struct dma_channel
*channel
,
339 u16 packet_sz
, u8 mode
,
340 dma_addr_t dma_addr
, u32 len
)
344 BUG_ON(channel
->status
== MUSB_DMA_STATUS_UNKNOWN
||
345 channel
->status
== MUSB_DMA_STATUS_BUSY
);
347 channel
->status
= MUSB_DMA_STATUS_BUSY
;
348 channel
->actual_len
= 0;
349 ret
= cppi41_configure_channel(channel
, packet_sz
, mode
, dma_addr
, len
);
351 channel
->status
= MUSB_DMA_STATUS_FREE
;
356 static int cppi41_is_compatible(struct dma_channel
*channel
, u16 maxpacket
,
357 void *buf
, u32 length
)
359 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
360 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
361 struct musb
*musb
= controller
->musb
;
363 if (is_host_active(musb
)) {
367 if (cppi41_channel
->is_tx
)
369 /* AM335x Advisory 1.0.13. No workaround for device RX mode */
373 static int cppi41_dma_channel_abort(struct dma_channel
*channel
)
375 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
376 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
377 struct musb
*musb
= controller
->musb
;
378 void __iomem
*epio
= cppi41_channel
->hw_ep
->regs
;
384 is_tx
= cppi41_channel
->is_tx
;
385 dev_dbg(musb
->controller
, "abort channel=%d, is_tx=%d\n",
386 cppi41_channel
->port_num
, is_tx
);
388 if (cppi41_channel
->channel
.status
== MUSB_DMA_STATUS_FREE
)
392 csr
= musb_readw(epio
, MUSB_TXCSR
);
393 csr
&= ~MUSB_TXCSR_DMAENAB
;
394 musb_writew(epio
, MUSB_TXCSR
, csr
);
396 csr
= musb_readw(epio
, MUSB_RXCSR
);
397 csr
&= ~(MUSB_RXCSR_H_REQPKT
| MUSB_RXCSR_DMAENAB
);
398 musb_writew(epio
, MUSB_RXCSR
, csr
);
400 csr
= musb_readw(epio
, MUSB_RXCSR
);
401 if (csr
& MUSB_RXCSR_RXPKTRDY
) {
402 csr
|= MUSB_RXCSR_FLUSHFIFO
;
403 musb_writew(epio
, MUSB_RXCSR
, csr
);
404 musb_writew(epio
, MUSB_RXCSR
, csr
);
408 tdbit
= 1 << cppi41_channel
->port_num
;
413 musb_writel(musb
->ctrl_base
, USB_TDOWN
, tdbit
);
414 ret
= dmaengine_terminate_all(cppi41_channel
->dc
);
415 } while (ret
== -EAGAIN
);
417 musb_writel(musb
->ctrl_base
, USB_TDOWN
, tdbit
);
420 csr
= musb_readw(epio
, MUSB_TXCSR
);
421 if (csr
& MUSB_TXCSR_TXPKTRDY
) {
422 csr
|= MUSB_TXCSR_FLUSHFIFO
;
423 musb_writew(epio
, MUSB_TXCSR
, csr
);
427 cppi41_channel
->channel
.status
= MUSB_DMA_STATUS_FREE
;
431 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller
*ctrl
)
436 for (i
= 0; i
< MUSB_DMA_NUM_CHANNELS
; i
++) {
437 dc
= ctrl
->tx_channel
[i
].dc
;
439 dma_release_channel(dc
);
440 dc
= ctrl
->rx_channel
[i
].dc
;
442 dma_release_channel(dc
);
446 static void cppi41_dma_controller_stop(struct cppi41_dma_controller
*controller
)
448 cppi41_release_all_dma_chans(controller
);
451 static int cppi41_dma_controller_start(struct cppi41_dma_controller
*controller
)
453 struct musb
*musb
= controller
->musb
;
454 struct device
*dev
= musb
->controller
;
455 struct device_node
*np
= dev
->of_node
;
456 struct cppi41_dma_channel
*cppi41_channel
;
461 count
= of_property_count_strings(np
, "dma-names");
465 for (i
= 0; i
< count
; i
++) {
467 struct dma_channel
*musb_dma
;
472 ret
= of_property_read_string_index(np
, "dma-names", i
, &str
);
475 if (!strncmp(str
, "tx", 2))
477 else if (!strncmp(str
, "rx", 2))
480 dev_err(dev
, "Wrong dmatype %s\n", str
);
483 ret
= kstrtouint(str
+ 2, 0, &port
);
488 if (port
> MUSB_DMA_NUM_CHANNELS
|| !port
)
491 cppi41_channel
= &controller
->tx_channel
[port
- 1];
493 cppi41_channel
= &controller
->rx_channel
[port
- 1];
495 cppi41_channel
->controller
= controller
;
496 cppi41_channel
->port_num
= port
;
497 cppi41_channel
->is_tx
= is_tx
;
499 musb_dma
= &cppi41_channel
->channel
;
500 musb_dma
->private_data
= cppi41_channel
;
501 musb_dma
->status
= MUSB_DMA_STATUS_FREE
;
502 musb_dma
->max_len
= SZ_4M
;
504 dc
= dma_request_slave_channel(dev
, str
);
506 dev_err(dev
, "Falied to request %s.\n", str
);
510 cppi41_channel
->dc
= dc
;
514 cppi41_release_all_dma_chans(controller
);
518 void dma_controller_destroy(struct dma_controller
*c
)
520 struct cppi41_dma_controller
*controller
= container_of(c
,
521 struct cppi41_dma_controller
, controller
);
523 cppi41_dma_controller_stop(controller
);
527 struct dma_controller
*dma_controller_create(struct musb
*musb
,
530 struct cppi41_dma_controller
*controller
;
533 if (!musb
->controller
->of_node
) {
534 dev_err(musb
->controller
, "Need DT for the DMA engine.\n");
538 controller
= kzalloc(sizeof(*controller
), GFP_KERNEL
);
542 controller
->musb
= musb
;
544 controller
->controller
.channel_alloc
= cppi41_dma_channel_allocate
;
545 controller
->controller
.channel_release
= cppi41_dma_channel_release
;
546 controller
->controller
.channel_program
= cppi41_dma_channel_program
;
547 controller
->controller
.channel_abort
= cppi41_dma_channel_abort
;
548 controller
->controller
.is_compatible
= cppi41_is_compatible
;
550 ret
= cppi41_dma_controller_start(controller
);
553 return &controller
->controller
;
558 if (ret
== -EPROBE_DEFER
)