?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge 4.18-rc7 into staging-next
[linux-2.6/btrfs-unstable.git] / drivers / dma / sprd-dma.c
blob55df0d41355b0622f4fb9ede60fd6bddbfe5a4e6
1 /*
2 * Copyright (C) 2017 Spreadtrum Communications Inc.
3 *
4 * SPDX-License-Identifier: GPL-2.0
5 */
6
7 #include <linux/clk.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/dma/sprd-dma.h>
10 #include <linux/errno.h>
11 #include <linux/init.h>
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/of_dma.h>
18 #include <linux/of_device.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/slab.h>
21
22 #include "virt-dma.h"
23
24 #define SPRD_DMA_CHN_REG_OFFSET 0x1000
25 #define SPRD_DMA_CHN_REG_LENGTH 0x40
26 #define SPRD_DMA_MEMCPY_MIN_SIZE 64
27
28 /* DMA global registers definition */
29 #define SPRD_DMA_GLB_PAUSE 0x0
30 #define SPRD_DMA_GLB_FRAG_WAIT 0x4
31 #define SPRD_DMA_GLB_REQ_PEND0_EN 0x8
32 #define SPRD_DMA_GLB_REQ_PEND1_EN 0xc
33 #define SPRD_DMA_GLB_INT_RAW_STS 0x10
34 #define SPRD_DMA_GLB_INT_MSK_STS 0x14
35 #define SPRD_DMA_GLB_REQ_STS 0x18
36 #define SPRD_DMA_GLB_CHN_EN_STS 0x1c
37 #define SPRD_DMA_GLB_DEBUG_STS 0x20
38 #define SPRD_DMA_GLB_ARB_SEL_STS 0x24
39 #define SPRD_DMA_GLB_REQ_UID(uid) (0x4 * ((uid) - 1))
40 #define SPRD_DMA_GLB_REQ_UID_OFFSET 0x2000
41
42 /* DMA channel registers definition */
43 #define SPRD_DMA_CHN_PAUSE 0x0
44 #define SPRD_DMA_CHN_REQ 0x4
45 #define SPRD_DMA_CHN_CFG 0x8
46 #define SPRD_DMA_CHN_INTC 0xc
47 #define SPRD_DMA_CHN_SRC_ADDR 0x10
48 #define SPRD_DMA_CHN_DES_ADDR 0x14
49 #define SPRD_DMA_CHN_FRG_LEN 0x18
50 #define SPRD_DMA_CHN_BLK_LEN 0x1c
51 #define SPRD_DMA_CHN_TRSC_LEN 0x20
52 #define SPRD_DMA_CHN_TRSF_STEP 0x24
53 #define SPRD_DMA_CHN_WARP_PTR 0x28
54 #define SPRD_DMA_CHN_WARP_TO 0x2c
55 #define SPRD_DMA_CHN_LLIST_PTR 0x30
56 #define SPRD_DMA_CHN_FRAG_STEP 0x34
57 #define SPRD_DMA_CHN_SRC_BLK_STEP 0x38
58 #define SPRD_DMA_CHN_DES_BLK_STEP 0x3c
59
60 /* SPRD_DMA_CHN_INTC register definition */
61 #define SPRD_DMA_INT_MASK GENMASK(4, 0)
62 #define SPRD_DMA_INT_CLR_OFFSET 24
63 #define SPRD_DMA_FRAG_INT_EN BIT(0)
64 #define SPRD_DMA_BLK_INT_EN BIT(1)
65 #define SPRD_DMA_TRANS_INT_EN BIT(2)
66 #define SPRD_DMA_LIST_INT_EN BIT(3)
67 #define SPRD_DMA_CFG_ERR_INT_EN BIT(4)
68
69 /* SPRD_DMA_CHN_CFG register definition */
70 #define SPRD_DMA_CHN_EN BIT(0)
71 #define SPRD_DMA_WAIT_BDONE_OFFSET 24
72 #define SPRD_DMA_DONOT_WAIT_BDONE 1
73
74 /* SPRD_DMA_CHN_REQ register definition */
75 #define SPRD_DMA_REQ_EN BIT(0)
76
77 /* SPRD_DMA_CHN_PAUSE register definition */
78 #define SPRD_DMA_PAUSE_EN BIT(0)
79 #define SPRD_DMA_PAUSE_STS BIT(2)
80 #define SPRD_DMA_PAUSE_CNT 0x2000
81
82 /* DMA_CHN_WARP_* register definition */
83 #define SPRD_DMA_HIGH_ADDR_MASK GENMASK(31, 28)
84 #define SPRD_DMA_LOW_ADDR_MASK GENMASK(31, 0)
85 #define SPRD_DMA_HIGH_ADDR_OFFSET 4
86
87 /* SPRD_DMA_CHN_INTC register definition */
88 #define SPRD_DMA_FRAG_INT_STS BIT(16)
89 #define SPRD_DMA_BLK_INT_STS BIT(17)
90 #define SPRD_DMA_TRSC_INT_STS BIT(18)
91 #define SPRD_DMA_LIST_INT_STS BIT(19)
92 #define SPRD_DMA_CFGERR_INT_STS BIT(20)
93 #define SPRD_DMA_CHN_INT_STS \
94 (SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS | \
95 SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS | \
96 SPRD_DMA_CFGERR_INT_STS)
97
98 /* SPRD_DMA_CHN_FRG_LEN register definition */
99 #define SPRD_DMA_SRC_DATAWIDTH_OFFSET 30
100 #define SPRD_DMA_DES_DATAWIDTH_OFFSET 28
101 #define SPRD_DMA_SWT_MODE_OFFSET 26
102 #define SPRD_DMA_REQ_MODE_OFFSET 24
103 #define SPRD_DMA_REQ_MODE_MASK GENMASK(1, 0)
104 #define SPRD_DMA_FIX_SEL_OFFSET 21
105 #define SPRD_DMA_FIX_EN_OFFSET 20
106 #define SPRD_DMA_LLIST_END_OFFSET 19
107 #define SPRD_DMA_FRG_LEN_MASK GENMASK(16, 0)
108
109 /* SPRD_DMA_CHN_BLK_LEN register definition */
110 #define SPRD_DMA_BLK_LEN_MASK GENMASK(16, 0)
111
112 /* SPRD_DMA_CHN_TRSC_LEN register definition */
113 #define SPRD_DMA_TRSC_LEN_MASK GENMASK(27, 0)
114
115 /* SPRD_DMA_CHN_TRSF_STEP register definition */
116 #define SPRD_DMA_DEST_TRSF_STEP_OFFSET 16
117 #define SPRD_DMA_SRC_TRSF_STEP_OFFSET 0
118 #define SPRD_DMA_TRSF_STEP_MASK GENMASK(15, 0)
119
120 /* define the DMA transfer step type */
121 #define SPRD_DMA_NONE_STEP 0
122 #define SPRD_DMA_BYTE_STEP 1
123 #define SPRD_DMA_SHORT_STEP 2
124 #define SPRD_DMA_WORD_STEP 4
125 #define SPRD_DMA_DWORD_STEP 8
126
127 #define SPRD_DMA_SOFTWARE_UID 0
128
129 /* dma data width values */
130 enum sprd_dma_datawidth {
131 SPRD_DMA_DATAWIDTH_1_BYTE,
132 SPRD_DMA_DATAWIDTH_2_BYTES,
133 SPRD_DMA_DATAWIDTH_4_BYTES,
134 SPRD_DMA_DATAWIDTH_8_BYTES,
135 };
136
137 /* dma channel hardware configuration */
138 struct sprd_dma_chn_hw {
139 u32 pause;
140 u32 req;
141 u32 cfg;
142 u32 intc;
143 u32 src_addr;
144 u32 des_addr;
145 u32 frg_len;
146 u32 blk_len;
147 u32 trsc_len;
148 u32 trsf_step;
149 u32 wrap_ptr;
150 u32 wrap_to;
151 u32 llist_ptr;
152 u32 frg_step;
153 u32 src_blk_step;
154 u32 des_blk_step;
155 };
156
157 /* dma request description */
158 struct sprd_dma_desc {
159 struct virt_dma_desc vd;
160 struct sprd_dma_chn_hw chn_hw;
161 };
162
163 /* dma channel description */
164 struct sprd_dma_chn {
165 struct virt_dma_chan vc;
166 void __iomem *chn_base;
167 struct dma_slave_config slave_cfg;
168 u32 chn_num;
169 u32 dev_id;
170 struct sprd_dma_desc *cur_desc;
171 };
172
173 /* SPRD dma device */
174 struct sprd_dma_dev {
175 struct dma_device dma_dev;
176 void __iomem *glb_base;
177 struct clk *clk;
178 struct clk *ashb_clk;
179 int irq;
180 u32 total_chns;
181 struct sprd_dma_chn channels[0];
182 };
183
184 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param);
185 static struct of_dma_filter_info sprd_dma_info = {
186 .filter_fn = sprd_dma_filter_fn,
187 };
188
189 static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c)
190 {
191 return container_of(c, struct sprd_dma_chn, vc.chan);
192 }
193
194 static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c)
195 {
196 struct sprd_dma_chn *schan = to_sprd_dma_chan(c);
197
198 return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]);
199 }
200
201 static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd)
202 {
203 return container_of(vd, struct sprd_dma_desc, vd);
204 }
205
206 static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,
207 u32 mask, u32 val)
208 {
209 u32 orig = readl(schan->chn_base + reg);
210 u32 tmp;
211
212 tmp = (orig & ~mask) | val;
213 writel(tmp, schan->chn_base + reg);
214 }
215
216 static int sprd_dma_enable(struct sprd_dma_dev *sdev)
217 {
218 int ret;
219
220 ret = clk_prepare_enable(sdev->clk);
221 if (ret)
222 return ret;
223
224 /*
225 * The ashb_clk is optional and only for AGCP DMA controller, so we
226 * need add one condition to check if the ashb_clk need enable.
227 */
228 if (!IS_ERR(sdev->ashb_clk))
229 ret = clk_prepare_enable(sdev->ashb_clk);
230
231 return ret;
232 }
233
234 static void sprd_dma_disable(struct sprd_dma_dev *sdev)
235 {
236 clk_disable_unprepare(sdev->clk);
237
238 /*
239 * Need to check if we need disable the optional ashb_clk for AGCP DMA.
240 */
241 if (!IS_ERR(sdev->ashb_clk))
242 clk_disable_unprepare(sdev->ashb_clk);
243 }
244
245 static void sprd_dma_set_uid(struct sprd_dma_chn *schan)
246 {
247 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
248 u32 dev_id = schan->dev_id;
249
250 if (dev_id != SPRD_DMA_SOFTWARE_UID) {
251 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
252 SPRD_DMA_GLB_REQ_UID(dev_id);
253
254 writel(schan->chn_num + 1, sdev->glb_base + uid_offset);
255 }
256 }
257
258 static void sprd_dma_unset_uid(struct sprd_dma_chn *schan)
259 {
260 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
261 u32 dev_id = schan->dev_id;
262
263 if (dev_id != SPRD_DMA_SOFTWARE_UID) {
264 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
265 SPRD_DMA_GLB_REQ_UID(dev_id);
266
267 writel(0, sdev->glb_base + uid_offset);
268 }
269 }
270
271 static void sprd_dma_clear_int(struct sprd_dma_chn *schan)
272 {
273 sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC,
274 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET,
275 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET);
276 }
277
278 static void sprd_dma_enable_chn(struct sprd_dma_chn *schan)
279 {
280 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN,
281 SPRD_DMA_CHN_EN);
282 }
283
284 static void sprd_dma_disable_chn(struct sprd_dma_chn *schan)
285 {
286 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0);
287 }
288
289 static void sprd_dma_soft_request(struct sprd_dma_chn *schan)
290 {
291 sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN,
292 SPRD_DMA_REQ_EN);
293 }
294
295 static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable)
296 {
297 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
298 u32 pause, timeout = SPRD_DMA_PAUSE_CNT;
299
300 if (enable) {
301 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
302 SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN);
303
304 do {
305 pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE);
306 if (pause & SPRD_DMA_PAUSE_STS)
307 break;
308
309 cpu_relax();
310 } while (--timeout > 0);
311
312 if (!timeout)
313 dev_warn(sdev->dma_dev.dev,
314 "pause dma controller timeout\n");
315 } else {
316 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
317 SPRD_DMA_PAUSE_EN, 0);
318 }
319 }
320
321 static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan)
322 {
323 u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG);
324
325 if (!(cfg & SPRD_DMA_CHN_EN))
326 return;
327
328 sprd_dma_pause_resume(schan, true);
329 sprd_dma_disable_chn(schan);
330 }
331
332 static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan)
333 {
334 unsigned long addr, addr_high;
335
336 addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
337 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) &
338 SPRD_DMA_HIGH_ADDR_MASK;
339
340 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET);
341 }
342
343 static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan)
344 {
345 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
346 u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) &
347 SPRD_DMA_CHN_INT_STS;
348
349 switch (intc_sts) {
350 case SPRD_DMA_CFGERR_INT_STS:
351 return SPRD_DMA_CFGERR_INT;
352
353 case SPRD_DMA_LIST_INT_STS:
354 return SPRD_DMA_LIST_INT;
355
356 case SPRD_DMA_TRSC_INT_STS:
357 return SPRD_DMA_TRANS_INT;
358
359 case SPRD_DMA_BLK_INT_STS:
360 return SPRD_DMA_BLK_INT;
361
362 case SPRD_DMA_FRAG_INT_STS:
363 return SPRD_DMA_FRAG_INT;
364
365 default:
366 dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n");
367 return SPRD_DMA_NO_INT;
368 }
369 }
370
371 static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan)
372 {
373 u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
374
375 return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK;
376 }
377
378 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan,
379 struct sprd_dma_desc *sdesc)
380 {
381 struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw;
382
383 writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE);
384 writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG);
385 writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC);
386 writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);
387 writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
388 writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
389 writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN);
390 writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN);
391 writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP);
392 writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR);
393 writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO);
394 writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR);
395 writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP);
396 writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP);
397 writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP);
398 writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ);
399 }
400
401 static void sprd_dma_start(struct sprd_dma_chn *schan)
402 {
403 struct virt_dma_desc *vd = vchan_next_desc(&schan->vc);
404
405 if (!vd)
406 return;
407
408 list_del(&vd->node);
409 schan->cur_desc = to_sprd_dma_desc(vd);
410
411 /*
412 * Copy the DMA configuration from DMA descriptor to this hardware
413 * channel.
414 */
415 sprd_dma_set_chn_config(schan, schan->cur_desc);
416 sprd_dma_set_uid(schan);
417 sprd_dma_enable_chn(schan);
418
419 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID)
420 sprd_dma_soft_request(schan);
421 }
422
423 static void sprd_dma_stop(struct sprd_dma_chn *schan)
424 {
425 sprd_dma_stop_and_disable(schan);
426 sprd_dma_unset_uid(schan);
427 sprd_dma_clear_int(schan);
428 }
429
430 static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc,
431 enum sprd_dma_int_type int_type,
432 enum sprd_dma_req_mode req_mode)
433 {
434 if (int_type == SPRD_DMA_NO_INT)
435 return false;
436
437 if (int_type >= req_mode + 1)
438 return true;
439 else
440 return false;
441 }
442
443 static irqreturn_t dma_irq_handle(int irq, void *dev_id)
444 {
445 struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id;
446 u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS);
447 struct sprd_dma_chn *schan;
448 struct sprd_dma_desc *sdesc;
449 enum sprd_dma_req_mode req_type;
450 enum sprd_dma_int_type int_type;
451 bool trans_done = false;
452 u32 i;
453
454 while (irq_status) {
455 i = __ffs(irq_status);
456 irq_status &= (irq_status - 1);
457 schan = &sdev->channels[i];
458
459 spin_lock(&schan->vc.lock);
460 int_type = sprd_dma_get_int_type(schan);
461 req_type = sprd_dma_get_req_type(schan);
462 sprd_dma_clear_int(schan);
463
464 sdesc = schan->cur_desc;
465
466 /* Check if the dma request descriptor is done. */
467 trans_done = sprd_dma_check_trans_done(sdesc, int_type,
468 req_type);
469 if (trans_done == true) {
470 vchan_cookie_complete(&sdesc->vd);
471 schan->cur_desc = NULL;
472 sprd_dma_start(schan);
473 }
474 spin_unlock(&schan->vc.lock);
475 }
476
477 return IRQ_HANDLED;
478 }
479
480 static int sprd_dma_alloc_chan_resources(struct dma_chan *chan)
481 {
482 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
483 int ret;
484
485 ret = pm_runtime_get_sync(chan->device->dev);
486 if (ret < 0)
487 return ret;
488
489 schan->dev_id = SPRD_DMA_SOFTWARE_UID;
490 return 0;
491 }
492
493 static void sprd_dma_free_chan_resources(struct dma_chan *chan)
494 {
495 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
496 unsigned long flags;
497
498 spin_lock_irqsave(&schan->vc.lock, flags);
499 sprd_dma_stop(schan);
500 spin_unlock_irqrestore(&schan->vc.lock, flags);
501
502 vchan_free_chan_resources(&schan->vc);
503 pm_runtime_put(chan->device->dev);
504 }
505
506 static enum dma_status sprd_dma_tx_status(struct dma_chan *chan,
507 dma_cookie_t cookie,
508 struct dma_tx_state *txstate)
509 {
510 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
511 struct virt_dma_desc *vd;
512 unsigned long flags;
513 enum dma_status ret;
514 u32 pos;
515
516 ret = dma_cookie_status(chan, cookie, txstate);
517 if (ret == DMA_COMPLETE || !txstate)
518 return ret;
519
520 spin_lock_irqsave(&schan->vc.lock, flags);
521 vd = vchan_find_desc(&schan->vc, cookie);
522 if (vd) {
523 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
524 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
525
526 if (hw->trsc_len > 0)
527 pos = hw->trsc_len;
528 else if (hw->blk_len > 0)
529 pos = hw->blk_len;
530 else if (hw->frg_len > 0)
531 pos = hw->frg_len;
532 else
533 pos = 0;
534 } else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) {
535 pos = sprd_dma_get_dst_addr(schan);
536 } else {
537 pos = 0;
538 }
539 spin_unlock_irqrestore(&schan->vc.lock, flags);
540
541 dma_set_residue(txstate, pos);
542 return ret;
543 }
544
545 static void sprd_dma_issue_pending(struct dma_chan *chan)
546 {
547 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
548 unsigned long flags;
549
550 spin_lock_irqsave(&schan->vc.lock, flags);
551 if (vchan_issue_pending(&schan->vc) && !schan->cur_desc)
552 sprd_dma_start(schan);
553 spin_unlock_irqrestore(&schan->vc.lock, flags);
554 }
555
556 static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth)
557 {
558 switch (buswidth) {
559 case DMA_SLAVE_BUSWIDTH_1_BYTE:
560 case DMA_SLAVE_BUSWIDTH_2_BYTES:
561 case DMA_SLAVE_BUSWIDTH_4_BYTES:
562 case DMA_SLAVE_BUSWIDTH_8_BYTES:
563 return ffs(buswidth) - 1;
564
565 default:
566 return -EINVAL;
567 }
568 }
569
570 static int sprd_dma_get_step(enum dma_slave_buswidth buswidth)
571 {
572 switch (buswidth) {
573 case DMA_SLAVE_BUSWIDTH_1_BYTE:
574 case DMA_SLAVE_BUSWIDTH_2_BYTES:
575 case DMA_SLAVE_BUSWIDTH_4_BYTES:
576 case DMA_SLAVE_BUSWIDTH_8_BYTES:
577 return buswidth;
578
579 default:
580 return -EINVAL;
581 }
582 }
583
584 static int sprd_dma_fill_desc(struct dma_chan *chan,
585 struct sprd_dma_desc *sdesc,
586 dma_addr_t src, dma_addr_t dst, u32 len,
587 enum dma_transfer_direction dir,
588 unsigned long flags,
589 struct dma_slave_config *slave_cfg)
590 {
591 struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
592 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
593 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
594 u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK;
595 u32 int_mode = flags & SPRD_DMA_INT_MASK;
596 int src_datawidth, dst_datawidth, src_step, dst_step;
597 u32 temp, fix_mode = 0, fix_en = 0;
598
599 if (dir == DMA_MEM_TO_DEV) {
600 src_step = sprd_dma_get_step(slave_cfg->src_addr_width);
601 if (src_step < 0) {
602 dev_err(sdev->dma_dev.dev, "invalid source step\n");
603 return src_step;
604 }
605 dst_step = SPRD_DMA_NONE_STEP;
606 } else {
607 dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width);
608 if (dst_step < 0) {
609 dev_err(sdev->dma_dev.dev, "invalid destination step\n");
610 return dst_step;
611 }
612 src_step = SPRD_DMA_NONE_STEP;
613 }
614
615 src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width);
616 if (src_datawidth < 0) {
617 dev_err(sdev->dma_dev.dev, "invalid source datawidth\n");
618 return src_datawidth;
619 }
620
621 dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width);
622 if (dst_datawidth < 0) {
623 dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n");
624 return dst_datawidth;
625 }
626
627 if (slave_cfg->slave_id)
628 schan->dev_id = slave_cfg->slave_id;
629
630 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
631
632 /*
633 * wrap_ptr and wrap_to will save the high 4 bits source address and
634 * destination address.
635 */
636 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
637 hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
638 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
639 hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK;
640
641 /*
642 * If the src step and dst step both are 0 or both are not 0, that means
643 * we can not enable the fix mode. If one is 0 and another one is not,
644 * we can enable the fix mode.
645 */
646 if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) {
647 fix_en = 0;
648 } else {
649 fix_en = 1;
650 if (src_step)
651 fix_mode = 1;
652 else
653 fix_mode = 0;
654 }
655
656 hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN;
657
658 temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
659 temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
660 temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET;
661 temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET;
662 temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET;
663 temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK;
664 hw->frg_len = temp;
665
666 hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK;
667 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
668
669 temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
670 temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
671 hw->trsf_step = temp;
672
673 hw->frg_step = 0;
674 hw->src_blk_step = 0;
675 hw->des_blk_step = 0;
676 return 0;
677 }
678
679 static struct dma_async_tx_descriptor *
680 sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
681 size_t len, unsigned long flags)
682 {
683 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
684 struct sprd_dma_desc *sdesc;
685 struct sprd_dma_chn_hw *hw;
686 enum sprd_dma_datawidth datawidth;
687 u32 step, temp;
688
689 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
690 if (!sdesc)
691 return NULL;
692
693 hw = &sdesc->chn_hw;
694
695 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
696 hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN;
697 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
698 hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK;
699 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) &
700 SPRD_DMA_HIGH_ADDR_MASK;
701 hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) &
702 SPRD_DMA_HIGH_ADDR_MASK;
703
704 if (IS_ALIGNED(len, 8)) {
705 datawidth = SPRD_DMA_DATAWIDTH_8_BYTES;
706 step = SPRD_DMA_DWORD_STEP;
707 } else if (IS_ALIGNED(len, 4)) {
708 datawidth = SPRD_DMA_DATAWIDTH_4_BYTES;
709 step = SPRD_DMA_WORD_STEP;
710 } else if (IS_ALIGNED(len, 2)) {
711 datawidth = SPRD_DMA_DATAWIDTH_2_BYTES;
712 step = SPRD_DMA_SHORT_STEP;
713 } else {
714 datawidth = SPRD_DMA_DATAWIDTH_1_BYTE;
715 step = SPRD_DMA_BYTE_STEP;
716 }
717
718 temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
719 temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
720 temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET;
721 temp |= len & SPRD_DMA_FRG_LEN_MASK;
722 hw->frg_len = temp;
723
724 hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK;
725 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
726
727 temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
728 temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
729 hw->trsf_step = temp;
730
731 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
732 }
733
734 static struct dma_async_tx_descriptor *
735 sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
736 unsigned int sglen, enum dma_transfer_direction dir,
737 unsigned long flags, void *context)
738 {
739 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
740 struct dma_slave_config *slave_cfg = &schan->slave_cfg;
741 dma_addr_t src = 0, dst = 0;
742 struct sprd_dma_desc *sdesc;
743 struct scatterlist *sg;
744 u32 len = 0;
745 int ret, i;
746
747 /* TODO: now we only support one sg for each DMA configuration. */
748 if (!is_slave_direction(dir) || sglen > 1)
749 return NULL;
750
751 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
752 if (!sdesc)
753 return NULL;
754
755 for_each_sg(sgl, sg, sglen, i) {
756 len = sg_dma_len(sg);
757
758 if (dir == DMA_MEM_TO_DEV) {
759 src = sg_dma_address(sg);
760 dst = slave_cfg->dst_addr;
761 } else {
762 src = slave_cfg->src_addr;
763 dst = sg_dma_address(sg);
764 }
765 }
766
767 ret = sprd_dma_fill_desc(chan, sdesc, src, dst, len, dir, flags,
768 slave_cfg);
769 if (ret) {
770 kfree(sdesc);
771 return NULL;
772 }
773
774 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
775 }
776
777 static int sprd_dma_slave_config(struct dma_chan *chan,
778 struct dma_slave_config *config)
779 {
780 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
781 struct dma_slave_config *slave_cfg = &schan->slave_cfg;
782
783 if (!is_slave_direction(config->direction))
784 return -EINVAL;
785
786 memcpy(slave_cfg, config, sizeof(*config));
787 return 0;
788 }
789
790 static int sprd_dma_pause(struct dma_chan *chan)
791 {
792 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
793 unsigned long flags;
794
795 spin_lock_irqsave(&schan->vc.lock, flags);
796 sprd_dma_pause_resume(schan, true);
797 spin_unlock_irqrestore(&schan->vc.lock, flags);
798
799 return 0;
800 }
801
802 static int sprd_dma_resume(struct dma_chan *chan)
803 {
804 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
805 unsigned long flags;
806
807 spin_lock_irqsave(&schan->vc.lock, flags);
808 sprd_dma_pause_resume(schan, false);
809 spin_unlock_irqrestore(&schan->vc.lock, flags);
810
811 return 0;
812 }
813
814 static int sprd_dma_terminate_all(struct dma_chan *chan)
815 {
816 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
817 unsigned long flags;
818 LIST_HEAD(head);
819
820 spin_lock_irqsave(&schan->vc.lock, flags);
821 sprd_dma_stop(schan);
822
823 vchan_get_all_descriptors(&schan->vc, &head);
824 spin_unlock_irqrestore(&schan->vc.lock, flags);
825
826 vchan_dma_desc_free_list(&schan->vc, &head);
827 return 0;
828 }
829
830 static void sprd_dma_free_desc(struct virt_dma_desc *vd)
831 {
832 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
833
834 kfree(sdesc);
835 }
836
837 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param)
838 {
839 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
840 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
841 u32 req = *(u32 *)param;
842
843 if (req < sdev->total_chns)
844 return req == schan->chn_num + 1;
845 else
846 return false;
847 }
848
849 static int sprd_dma_probe(struct platform_device *pdev)
850 {
851 struct device_node *np = pdev->dev.of_node;
852 struct sprd_dma_dev *sdev;
853 struct sprd_dma_chn *dma_chn;
854 struct resource *res;
855 u32 chn_count;
856 int ret, i;
857
858 ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count);
859 if (ret) {
860 dev_err(&pdev->dev, "get dma channels count failed\n");
861 return ret;
862 }
863
864 sdev = devm_kzalloc(&pdev->dev,
865 struct_size(sdev, channels, chn_count),
866 GFP_KERNEL);
867 if (!sdev)
868 return -ENOMEM;
869
870 sdev->clk = devm_clk_get(&pdev->dev, "enable");
871 if (IS_ERR(sdev->clk)) {
872 dev_err(&pdev->dev, "get enable clock failed\n");
873 return PTR_ERR(sdev->clk);
874 }
875
876 /* ashb clock is optional for AGCP DMA */
877 sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb");
878 if (IS_ERR(sdev->ashb_clk))
879 dev_warn(&pdev->dev, "no optional ashb eb clock\n");
880
881 /*
882 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP
883 * DMA controller, it can or do not request the irq, which will save
884 * system power without resuming system by DMA interrupts if AGCP DMA
885 * does not request the irq. Thus the DMA interrupts property should
886 * be optional.
887 */
888 sdev->irq = platform_get_irq(pdev, 0);
889 if (sdev->irq > 0) {
890 ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle,
891 0, "sprd_dma", (void *)sdev);
892 if (ret < 0) {
893 dev_err(&pdev->dev, "request dma irq failed\n");
894 return ret;
895 }
896 } else {
897 dev_warn(&pdev->dev, "no interrupts for the dma controller\n");
898 }
899
900 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
901 sdev->glb_base = devm_ioremap_resource(&pdev->dev, res);
902 if (IS_ERR(sdev->glb_base))
903 return PTR_ERR(sdev->glb_base);
904
905 dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask);
906 sdev->total_chns = chn_count;
907 sdev->dma_dev.chancnt = chn_count;
908 INIT_LIST_HEAD(&sdev->dma_dev.channels);
909 INIT_LIST_HEAD(&sdev->dma_dev.global_node);
910 sdev->dma_dev.dev = &pdev->dev;
911 sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources;
912 sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources;
913 sdev->dma_dev.device_tx_status = sprd_dma_tx_status;
914 sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending;
915 sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy;
916 sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg;
917 sdev->dma_dev.device_config = sprd_dma_slave_config;
918 sdev->dma_dev.device_pause = sprd_dma_pause;
919 sdev->dma_dev.device_resume = sprd_dma_resume;
920 sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all;
921
922 for (i = 0; i < chn_count; i++) {
923 dma_chn = &sdev->channels[i];
924 dma_chn->chn_num = i;
925 dma_chn->cur_desc = NULL;
926 /* get each channel's registers base address. */
927 dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET +
928 SPRD_DMA_CHN_REG_LENGTH * i;
929
930 dma_chn->vc.desc_free = sprd_dma_free_desc;
931 vchan_init(&dma_chn->vc, &sdev->dma_dev);
932 }
933
934 platform_set_drvdata(pdev, sdev);
935 ret = sprd_dma_enable(sdev);
936 if (ret)
937 return ret;
938
939 pm_runtime_set_active(&pdev->dev);
940 pm_runtime_enable(&pdev->dev);
941
942 ret = pm_runtime_get_sync(&pdev->dev);
943 if (ret < 0)
944 goto err_rpm;
945
946 ret = dma_async_device_register(&sdev->dma_dev);
947 if (ret < 0) {
948 dev_err(&pdev->dev, "register dma device failed:%d\n", ret);
949 goto err_register;
950 }
951
952 sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask;
953 ret = of_dma_controller_register(np, of_dma_simple_xlate,
954 &sprd_dma_info);
955 if (ret)
956 goto err_of_register;
957
958 pm_runtime_put(&pdev->dev);
959 return 0;
960
961 err_of_register:
962 dma_async_device_unregister(&sdev->dma_dev);
963 err_register:
964 pm_runtime_put_noidle(&pdev->dev);
965 pm_runtime_disable(&pdev->dev);
966 err_rpm:
967 sprd_dma_disable(sdev);
968 return ret;
969 }
970
971 static int sprd_dma_remove(struct platform_device *pdev)
972 {
973 struct sprd_dma_dev *sdev = platform_get_drvdata(pdev);
974 struct sprd_dma_chn *c, *cn;
975 int ret;
976
977 ret = pm_runtime_get_sync(&pdev->dev);
978 if (ret < 0)
979 return ret;
980
981 /* explicitly free the irq */
982 if (sdev->irq > 0)
983 devm_free_irq(&pdev->dev, sdev->irq, sdev);
984
985 list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels,
986 vc.chan.device_node) {
987 list_del(&c->vc.chan.device_node);
988 tasklet_kill(&c->vc.task);
989 }
990
991 of_dma_controller_free(pdev->dev.of_node);
992 dma_async_device_unregister(&sdev->dma_dev);
993 sprd_dma_disable(sdev);
994
995 pm_runtime_put_noidle(&pdev->dev);
996 pm_runtime_disable(&pdev->dev);
997 return 0;
998 }
999
1000 static const struct of_device_id sprd_dma_match[] = {
1001 { .compatible = "sprd,sc9860-dma", },
1002 {},
1003 };
1004
1005 static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev)
1006 {
1007 struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
1008
1009 sprd_dma_disable(sdev);
1010 return 0;
1011 }
1012
1013 static int __maybe_unused sprd_dma_runtime_resume(struct device *dev)
1014 {
1015 struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
1016 int ret;
1017
1018 ret = sprd_dma_enable(sdev);
1019 if (ret)
1020 dev_err(sdev->dma_dev.dev, "enable dma failed\n");
1021
1022 return ret;
1023 }
1024
1025 static const struct dev_pm_ops sprd_dma_pm_ops = {
1026 SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend,
1027 sprd_dma_runtime_resume,
1028 NULL)
1029 };
1030
1031 static struct platform_driver sprd_dma_driver = {
1032 .probe = sprd_dma_probe,
1033 .remove = sprd_dma_remove,
1034 .driver = {
1035 .name = "sprd-dma",
1036 .of_match_table = sprd_dma_match,
1037 .pm = &sprd_dma_pm_ops,
1038 },
1039 };
1040 module_platform_driver(sprd_dma_driver);
1041
1042 MODULE_LICENSE("GPL v2");
1043 MODULE_DESCRIPTION("DMA driver for Spreadtrum");
1044 MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>");
1045 MODULE_ALIAS("platform:sprd-dma");
(deprecated) Btrfs devel tree