?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/i915: Fix use of invalid array size for ring->sync_seqno
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / dma / ste_dma40.c
blob6e1d46a65d0eec5cbcd1e3d5dee89a57931effd5
1 /*
2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
6 * License terms: GNU General Public License (GPL) version 2
7 */
8
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
11 #include <linux/dmaengine.h>
12 #include <linux/platform_device.h>
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/err.h>
16
17 #include <plat/ste_dma40.h>
18
19 #include "ste_dma40_ll.h"
20
21 #define D40_NAME "dma40"
22
23 #define D40_PHY_CHAN -1
24
25 /* For masking out/in 2 bit channel positions */
26 #define D40_CHAN_POS(chan) (2 * (chan / 2))
27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
28
29 /* Maximum iterations taken before giving up suspending a channel */
30 #define D40_SUSPEND_MAX_IT 500
31
32 /* Hardware requirement on LCLA alignment */
33 #define LCLA_ALIGNMENT 0x40000
34
35 /* Max number of links per event group */
36 #define D40_LCLA_LINK_PER_EVENT_GRP 128
37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
38
39 /* Attempts before giving up to trying to get pages that are aligned */
40 #define MAX_LCLA_ALLOC_ATTEMPTS 256
41
42 /* Bit markings for allocation map */
43 #define D40_ALLOC_FREE (1 << 31)
44 #define D40_ALLOC_PHY (1 << 30)
45 #define D40_ALLOC_LOG_FREE 0
46
47 /* Hardware designer of the block */
48 #define D40_HW_DESIGNER 0x8
49
50 /**
51 * enum 40_command - The different commands and/or statuses.
52 *
53 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
54 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
55 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
56 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
57 */
58 enum d40_command {
59 D40_DMA_STOP = 0,
60 D40_DMA_RUN = 1,
61 D40_DMA_SUSPEND_REQ = 2,
62 D40_DMA_SUSPENDED = 3
63 };
64
65 /**
66 * struct d40_lli_pool - Structure for keeping LLIs in memory
67 *
68 * @base: Pointer to memory area when the pre_alloc_lli's are not large
69 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
70 * pre_alloc_lli is used.
71 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
72 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
73 * one buffer to one buffer.
74 */
75 struct d40_lli_pool {
76 void *base;
77 int size;
78 /* Space for dst and src, plus an extra for padding */
79 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
80 };
81
82 /**
83 * struct d40_desc - A descriptor is one DMA job.
84 *
85 * @lli_phy: LLI settings for physical channel. Both src and dst=
86 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
87 * lli_len equals one.
88 * @lli_log: Same as above but for logical channels.
89 * @lli_pool: The pool with two entries pre-allocated.
90 * @lli_len: Number of llis of current descriptor.
91 * @lli_current: Number of transfered llis.
92 * @lcla_alloc: Number of LCLA entries allocated.
93 * @txd: DMA engine struct. Used for among other things for communication
94 * during a transfer.
95 * @node: List entry.
96 * @is_in_client_list: true if the client owns this descriptor.
97 * @is_hw_linked: true if this job will automatically be continued for
98 * the previous one.
99 *
100 * This descriptor is used for both logical and physical transfers.
101 */
102 struct d40_desc {
103 /* LLI physical */
104 struct d40_phy_lli_bidir lli_phy;
105 /* LLI logical */
106 struct d40_log_lli_bidir lli_log;
107
108 struct d40_lli_pool lli_pool;
109 int lli_len;
110 int lli_current;
111 int lcla_alloc;
112
113 struct dma_async_tx_descriptor txd;
114 struct list_head node;
115
116 bool is_in_client_list;
117 bool is_hw_linked;
118 };
119
120 /**
121 * struct d40_lcla_pool - LCLA pool settings and data.
122 *
123 * @base: The virtual address of LCLA. 18 bit aligned.
124 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
125 * This pointer is only there for clean-up on error.
126 * @pages: The number of pages needed for all physical channels.
127 * Only used later for clean-up on error
128 * @lock: Lock to protect the content in this struct.
129 * @alloc_map: big map over which LCLA entry is own by which job.
130 */
131 struct d40_lcla_pool {
132 void *base;
133 void *base_unaligned;
134 int pages;
135 spinlock_t lock;
136 struct d40_desc **alloc_map;
137 };
138
139 /**
140 * struct d40_phy_res - struct for handling eventlines mapped to physical
141 * channels.
142 *
143 * @lock: A lock protection this entity.
144 * @num: The physical channel number of this entity.
145 * @allocated_src: Bit mapped to show which src event line's are mapped to
146 * this physical channel. Can also be free or physically allocated.
147 * @allocated_dst: Same as for src but is dst.
148 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
149 * event line number.
150 */
151 struct d40_phy_res {
152 spinlock_t lock;
153 int num;
154 u32 allocated_src;
155 u32 allocated_dst;
156 };
157
158 struct d40_base;
159
160 /**
161 * struct d40_chan - Struct that describes a channel.
162 *
163 * @lock: A spinlock to protect this struct.
164 * @log_num: The logical number, if any of this channel.
165 * @completed: Starts with 1, after first interrupt it is set to dma engine's
166 * current cookie.
167 * @pending_tx: The number of pending transfers. Used between interrupt handler
168 * and tasklet.
169 * @busy: Set to true when transfer is ongoing on this channel.
170 * @phy_chan: Pointer to physical channel which this instance runs on. If this
171 * point is NULL, then the channel is not allocated.
172 * @chan: DMA engine handle.
173 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
174 * transfer and call client callback.
175 * @client: Cliented owned descriptor list.
176 * @active: Active descriptor.
177 * @queue: Queued jobs.
178 * @dma_cfg: The client configuration of this dma channel.
179 * @configured: whether the dma_cfg configuration is valid
180 * @base: Pointer to the device instance struct.
181 * @src_def_cfg: Default cfg register setting for src.
182 * @dst_def_cfg: Default cfg register setting for dst.
183 * @log_def: Default logical channel settings.
184 * @lcla: Space for one dst src pair for logical channel transfers.
185 * @lcpa: Pointer to dst and src lcpa settings.
186 *
187 * This struct can either "be" a logical or a physical channel.
188 */
189 struct d40_chan {
190 spinlock_t lock;
191 int log_num;
192 /* ID of the most recent completed transfer */
193 int completed;
194 int pending_tx;
195 bool busy;
196 struct d40_phy_res *phy_chan;
197 struct dma_chan chan;
198 struct tasklet_struct tasklet;
199 struct list_head client;
200 struct list_head active;
201 struct list_head queue;
202 struct stedma40_chan_cfg dma_cfg;
203 bool configured;
204 struct d40_base *base;
205 /* Default register configurations */
206 u32 src_def_cfg;
207 u32 dst_def_cfg;
208 struct d40_def_lcsp log_def;
209 struct d40_log_lli_full *lcpa;
210 /* Runtime reconfiguration */
211 dma_addr_t runtime_addr;
212 enum dma_data_direction runtime_direction;
213 };
214
215 /**
216 * struct d40_base - The big global struct, one for each probe'd instance.
217 *
218 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
219 * @execmd_lock: Lock for execute command usage since several channels share
220 * the same physical register.
221 * @dev: The device structure.
222 * @virtbase: The virtual base address of the DMA's register.
223 * @rev: silicon revision detected.
224 * @clk: Pointer to the DMA clock structure.
225 * @phy_start: Physical memory start of the DMA registers.
226 * @phy_size: Size of the DMA register map.
227 * @irq: The IRQ number.
228 * @num_phy_chans: The number of physical channels. Read from HW. This
229 * is the number of available channels for this driver, not counting "Secure
230 * mode" allocated physical channels.
231 * @num_log_chans: The number of logical channels. Calculated from
232 * num_phy_chans.
233 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
234 * @dma_slave: dma_device channels that can do only do slave transfers.
235 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
236 * @log_chans: Room for all possible logical channels in system.
237 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
238 * to log_chans entries.
239 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
240 * to phy_chans entries.
241 * @plat_data: Pointer to provided platform_data which is the driver
242 * configuration.
243 * @phy_res: Vector containing all physical channels.
244 * @lcla_pool: lcla pool settings and data.
245 * @lcpa_base: The virtual mapped address of LCPA.
246 * @phy_lcpa: The physical address of the LCPA.
247 * @lcpa_size: The size of the LCPA area.
248 * @desc_slab: cache for descriptors.
249 */
250 struct d40_base {
251 spinlock_t interrupt_lock;
252 spinlock_t execmd_lock;
253 struct device *dev;
254 void __iomem *virtbase;
255 u8 rev:4;
256 struct clk *clk;
257 phys_addr_t phy_start;
258 resource_size_t phy_size;
259 int irq;
260 int num_phy_chans;
261 int num_log_chans;
262 struct dma_device dma_both;
263 struct dma_device dma_slave;
264 struct dma_device dma_memcpy;
265 struct d40_chan *phy_chans;
266 struct d40_chan *log_chans;
267 struct d40_chan **lookup_log_chans;
268 struct d40_chan **lookup_phy_chans;
269 struct stedma40_platform_data *plat_data;
270 /* Physical half channels */
271 struct d40_phy_res *phy_res;
272 struct d40_lcla_pool lcla_pool;
273 void *lcpa_base;
274 dma_addr_t phy_lcpa;
275 resource_size_t lcpa_size;
276 struct kmem_cache *desc_slab;
277 };
278
279 /**
280 * struct d40_interrupt_lookup - lookup table for interrupt handler
281 *
282 * @src: Interrupt mask register.
283 * @clr: Interrupt clear register.
284 * @is_error: true if this is an error interrupt.
285 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
286 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
287 */
288 struct d40_interrupt_lookup {
289 u32 src;
290 u32 clr;
291 bool is_error;
292 int offset;
293 };
294
295 /**
296 * struct d40_reg_val - simple lookup struct
297 *
298 * @reg: The register.
299 * @val: The value that belongs to the register in reg.
300 */
301 struct d40_reg_val {
302 unsigned int reg;
303 unsigned int val;
304 };
305
306 static int d40_pool_lli_alloc(struct d40_desc *d40d,
307 int lli_len, bool is_log)
308 {
309 u32 align;
310 void *base;
311
312 if (is_log)
313 align = sizeof(struct d40_log_lli);
314 else
315 align = sizeof(struct d40_phy_lli);
316
317 if (lli_len == 1) {
318 base = d40d->lli_pool.pre_alloc_lli;
319 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
320 d40d->lli_pool.base = NULL;
321 } else {
322 d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
323
324 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
325 d40d->lli_pool.base = base;
326
327 if (d40d->lli_pool.base == NULL)
328 return -ENOMEM;
329 }
330
331 if (is_log) {
332 d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
333 align);
334 d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
335 align);
336 } else {
337 d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
338 align);
339 d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
340 align);
341 }
342
343 return 0;
344 }
345
346 static void d40_pool_lli_free(struct d40_desc *d40d)
347 {
348 kfree(d40d->lli_pool.base);
349 d40d->lli_pool.base = NULL;
350 d40d->lli_pool.size = 0;
351 d40d->lli_log.src = NULL;
352 d40d->lli_log.dst = NULL;
353 d40d->lli_phy.src = NULL;
354 d40d->lli_phy.dst = NULL;
355 }
356
357 static int d40_lcla_alloc_one(struct d40_chan *d40c,
358 struct d40_desc *d40d)
359 {
360 unsigned long flags;
361 int i;
362 int ret = -EINVAL;
363 int p;
364
365 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
366
367 p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
368
369 /*
370 * Allocate both src and dst at the same time, therefore the half
371 * start on 1 since 0 can't be used since zero is used as end marker.
372 */
373 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
374 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
375 d40c->base->lcla_pool.alloc_map[p + i] = d40d;
376 d40d->lcla_alloc++;
377 ret = i;
378 break;
379 }
380 }
381
382 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
383
384 return ret;
385 }
386
387 static int d40_lcla_free_all(struct d40_chan *d40c,
388 struct d40_desc *d40d)
389 {
390 unsigned long flags;
391 int i;
392 int ret = -EINVAL;
393
394 if (d40c->log_num == D40_PHY_CHAN)
395 return 0;
396
397 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
398
399 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
400 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
401 D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
402 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
403 D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
404 d40d->lcla_alloc--;
405 if (d40d->lcla_alloc == 0) {
406 ret = 0;
407 break;
408 }
409 }
410 }
411
412 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
413
414 return ret;
415
416 }
417
418 static void d40_desc_remove(struct d40_desc *d40d)
419 {
420 list_del(&d40d->node);
421 }
422
423 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
424 {
425 struct d40_desc *desc = NULL;
426
427 if (!list_empty(&d40c->client)) {
428 struct d40_desc *d;
429 struct d40_desc *_d;
430
431 list_for_each_entry_safe(d, _d, &d40c->client, node)
432 if (async_tx_test_ack(&d->txd)) {
433 d40_pool_lli_free(d);
434 d40_desc_remove(d);
435 desc = d;
436 memset(desc, 0, sizeof(*desc));
437 break;
438 }
439 }
440
441 if (!desc)
442 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
443
444 if (desc)
445 INIT_LIST_HEAD(&desc->node);
446
447 return desc;
448 }
449
450 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
451 {
452
453 d40_lcla_free_all(d40c, d40d);
454 kmem_cache_free(d40c->base->desc_slab, d40d);
455 }
456
457 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
458 {
459 list_add_tail(&desc->node, &d40c->active);
460 }
461
462 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
463 {
464 int curr_lcla = -EINVAL, next_lcla;
465
466 if (d40c->log_num == D40_PHY_CHAN) {
467 d40_phy_lli_write(d40c->base->virtbase,
468 d40c->phy_chan->num,
469 d40d->lli_phy.dst,
470 d40d->lli_phy.src);
471 d40d->lli_current = d40d->lli_len;
472 } else {
473
474 if ((d40d->lli_len - d40d->lli_current) > 1)
475 curr_lcla = d40_lcla_alloc_one(d40c, d40d);
476
477 d40_log_lli_lcpa_write(d40c->lcpa,
478 &d40d->lli_log.dst[d40d->lli_current],
479 &d40d->lli_log.src[d40d->lli_current],
480 curr_lcla);
481
482 d40d->lli_current++;
483 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
484 struct d40_log_lli *lcla;
485
486 if (d40d->lli_current + 1 < d40d->lli_len)
487 next_lcla = d40_lcla_alloc_one(d40c, d40d);
488 else
489 next_lcla = -EINVAL;
490
491 lcla = d40c->base->lcla_pool.base +
492 d40c->phy_chan->num * 1024 +
493 8 * curr_lcla * 2;
494
495 d40_log_lli_lcla_write(lcla,
496 &d40d->lli_log.dst[d40d->lli_current],
497 &d40d->lli_log.src[d40d->lli_current],
498 next_lcla);
499
500 (void) dma_map_single(d40c->base->dev, lcla,
501 2 * sizeof(struct d40_log_lli),
502 DMA_TO_DEVICE);
503
504 curr_lcla = next_lcla;
505
506 if (curr_lcla == -EINVAL) {
507 d40d->lli_current++;
508 break;
509 }
510
511 }
512 }
513 }
514
515 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
516 {
517 struct d40_desc *d;
518
519 if (list_empty(&d40c->active))
520 return NULL;
521
522 d = list_first_entry(&d40c->active,
523 struct d40_desc,
524 node);
525 return d;
526 }
527
528 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
529 {
530 list_add_tail(&desc->node, &d40c->queue);
531 }
532
533 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
534 {
535 struct d40_desc *d;
536
537 if (list_empty(&d40c->queue))
538 return NULL;
539
540 d = list_first_entry(&d40c->queue,
541 struct d40_desc,
542 node);
543 return d;
544 }
545
546 static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
547 {
548 struct d40_desc *d;
549
550 if (list_empty(&d40c->queue))
551 return NULL;
552 list_for_each_entry(d, &d40c->queue, node)
553 if (list_is_last(&d->node, &d40c->queue))
554 break;
555 return d;
556 }
557
558 static int d40_psize_2_burst_size(bool is_log, int psize)
559 {
560 if (is_log) {
561 if (psize == STEDMA40_PSIZE_LOG_1)
562 return 1;
563 } else {
564 if (psize == STEDMA40_PSIZE_PHY_1)
565 return 1;
566 }
567
568 return 2 << psize;
569 }
570
571 /*
572 * The dma only supports transmitting packages up to
573 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
574 * dma elements required to send the entire sg list
575 */
576 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
577 {
578 int dmalen;
579 u32 max_w = max(data_width1, data_width2);
580 u32 min_w = min(data_width1, data_width2);
581 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
582
583 if (seg_max > STEDMA40_MAX_SEG_SIZE)
584 seg_max -= (1 << max_w);
585
586 if (!IS_ALIGNED(size, 1 << max_w))
587 return -EINVAL;
588
589 if (size <= seg_max)
590 dmalen = 1;
591 else {
592 dmalen = size / seg_max;
593 if (dmalen * seg_max < size)
594 dmalen++;
595 }
596 return dmalen;
597 }
598
599 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
600 u32 data_width1, u32 data_width2)
601 {
602 struct scatterlist *sg;
603 int i;
604 int len = 0;
605 int ret;
606
607 for_each_sg(sgl, sg, sg_len, i) {
608 ret = d40_size_2_dmalen(sg_dma_len(sg),
609 data_width1, data_width2);
610 if (ret < 0)
611 return ret;
612 len += ret;
613 }
614 return len;
615 }
616
617 /* Support functions for logical channels */
618
619 static int d40_channel_execute_command(struct d40_chan *d40c,
620 enum d40_command command)
621 {
622 u32 status;
623 int i;
624 void __iomem *active_reg;
625 int ret = 0;
626 unsigned long flags;
627 u32 wmask;
628
629 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
630
631 if (d40c->phy_chan->num % 2 == 0)
632 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
633 else
634 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
635
636 if (command == D40_DMA_SUSPEND_REQ) {
637 status = (readl(active_reg) &
638 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
639 D40_CHAN_POS(d40c->phy_chan->num);
640
641 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
642 goto done;
643 }
644
645 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
646 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
647 active_reg);
648
649 if (command == D40_DMA_SUSPEND_REQ) {
650
651 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
652 status = (readl(active_reg) &
653 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
654 D40_CHAN_POS(d40c->phy_chan->num);
655
656 cpu_relax();
657 /*
658 * Reduce the number of bus accesses while
659 * waiting for the DMA to suspend.
660 */
661 udelay(3);
662
663 if (status == D40_DMA_STOP ||
664 status == D40_DMA_SUSPENDED)
665 break;
666 }
667
668 if (i == D40_SUSPEND_MAX_IT) {
669 dev_err(&d40c->chan.dev->device,
670 "[%s]: unable to suspend the chl %d (log: %d) status %x\n",
671 __func__, d40c->phy_chan->num, d40c->log_num,
672 status);
673 dump_stack();
674 ret = -EBUSY;
675 }
676
677 }
678 done:
679 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
680 return ret;
681 }
682
683 static void d40_term_all(struct d40_chan *d40c)
684 {
685 struct d40_desc *d40d;
686
687 /* Release active descriptors */
688 while ((d40d = d40_first_active_get(d40c))) {
689 d40_desc_remove(d40d);
690 d40_desc_free(d40c, d40d);
691 }
692
693 /* Release queued descriptors waiting for transfer */
694 while ((d40d = d40_first_queued(d40c))) {
695 d40_desc_remove(d40d);
696 d40_desc_free(d40c, d40d);
697 }
698
699
700 d40c->pending_tx = 0;
701 d40c->busy = false;
702 }
703
704 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
705 {
706 u32 val;
707 unsigned long flags;
708
709 /* Notice, that disable requires the physical channel to be stopped */
710 if (do_enable)
711 val = D40_ACTIVATE_EVENTLINE;
712 else
713 val = D40_DEACTIVATE_EVENTLINE;
714
715 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
716
717 /* Enable event line connected to device (or memcpy) */
718 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
719 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
720 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
721
722 writel((val << D40_EVENTLINE_POS(event)) |
723 ~D40_EVENTLINE_MASK(event),
724 d40c->base->virtbase + D40_DREG_PCBASE +
725 d40c->phy_chan->num * D40_DREG_PCDELTA +
726 D40_CHAN_REG_SSLNK);
727 }
728 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
729 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
730
731 writel((val << D40_EVENTLINE_POS(event)) |
732 ~D40_EVENTLINE_MASK(event),
733 d40c->base->virtbase + D40_DREG_PCBASE +
734 d40c->phy_chan->num * D40_DREG_PCDELTA +
735 D40_CHAN_REG_SDLNK);
736 }
737
738 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
739 }
740
741 static u32 d40_chan_has_events(struct d40_chan *d40c)
742 {
743 u32 val;
744
745 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
746 d40c->phy_chan->num * D40_DREG_PCDELTA +
747 D40_CHAN_REG_SSLNK);
748
749 val |= readl(d40c->base->virtbase + D40_DREG_PCBASE +
750 d40c->phy_chan->num * D40_DREG_PCDELTA +
751 D40_CHAN_REG_SDLNK);
752 return val;
753 }
754
755 static u32 d40_get_prmo(struct d40_chan *d40c)
756 {
757 static const unsigned int phy_map[] = {
758 [STEDMA40_PCHAN_BASIC_MODE]
759 = D40_DREG_PRMO_PCHAN_BASIC,
760 [STEDMA40_PCHAN_MODULO_MODE]
761 = D40_DREG_PRMO_PCHAN_MODULO,
762 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
763 = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
764 };
765 static const unsigned int log_map[] = {
766 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
767 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
768 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
769 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
770 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
771 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
772 };
773
774 if (d40c->log_num == D40_PHY_CHAN)
775 return phy_map[d40c->dma_cfg.mode_opt];
776 else
777 return log_map[d40c->dma_cfg.mode_opt];
778 }
779
780 static void d40_config_write(struct d40_chan *d40c)
781 {
782 u32 addr_base;
783 u32 var;
784
785 /* Odd addresses are even addresses + 4 */
786 addr_base = (d40c->phy_chan->num % 2) * 4;
787 /* Setup channel mode to logical or physical */
788 var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
789 D40_CHAN_POS(d40c->phy_chan->num);
790 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
791
792 /* Setup operational mode option register */
793 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
794
795 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
796
797 if (d40c->log_num != D40_PHY_CHAN) {
798 /* Set default config for CFG reg */
799 writel(d40c->src_def_cfg,
800 d40c->base->virtbase + D40_DREG_PCBASE +
801 d40c->phy_chan->num * D40_DREG_PCDELTA +
802 D40_CHAN_REG_SSCFG);
803 writel(d40c->dst_def_cfg,
804 d40c->base->virtbase + D40_DREG_PCBASE +
805 d40c->phy_chan->num * D40_DREG_PCDELTA +
806 D40_CHAN_REG_SDCFG);
807
808 /* Set LIDX for lcla */
809 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
810 D40_SREG_ELEM_LOG_LIDX_MASK,
811 d40c->base->virtbase + D40_DREG_PCBASE +
812 d40c->phy_chan->num * D40_DREG_PCDELTA +
813 D40_CHAN_REG_SDELT);
814
815 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
816 D40_SREG_ELEM_LOG_LIDX_MASK,
817 d40c->base->virtbase + D40_DREG_PCBASE +
818 d40c->phy_chan->num * D40_DREG_PCDELTA +
819 D40_CHAN_REG_SSELT);
820
821 }
822 }
823
824 static u32 d40_residue(struct d40_chan *d40c)
825 {
826 u32 num_elt;
827
828 if (d40c->log_num != D40_PHY_CHAN)
829 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
830 >> D40_MEM_LCSP2_ECNT_POS;
831 else
832 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
833 d40c->phy_chan->num * D40_DREG_PCDELTA +
834 D40_CHAN_REG_SDELT) &
835 D40_SREG_ELEM_PHY_ECNT_MASK) >>
836 D40_SREG_ELEM_PHY_ECNT_POS;
837 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
838 }
839
840 static bool d40_tx_is_linked(struct d40_chan *d40c)
841 {
842 bool is_link;
843
844 if (d40c->log_num != D40_PHY_CHAN)
845 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
846 else
847 is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
848 d40c->phy_chan->num * D40_DREG_PCDELTA +
849 D40_CHAN_REG_SDLNK) &
850 D40_SREG_LNK_PHYS_LNK_MASK;
851 return is_link;
852 }
853
854 static int d40_pause(struct dma_chan *chan)
855 {
856 struct d40_chan *d40c =
857 container_of(chan, struct d40_chan, chan);
858 int res = 0;
859 unsigned long flags;
860
861 if (!d40c->busy)
862 return 0;
863
864 spin_lock_irqsave(&d40c->lock, flags);
865
866 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
867 if (res == 0) {
868 if (d40c->log_num != D40_PHY_CHAN) {
869 d40_config_set_event(d40c, false);
870 /* Resume the other logical channels if any */
871 if (d40_chan_has_events(d40c))
872 res = d40_channel_execute_command(d40c,
873 D40_DMA_RUN);
874 }
875 }
876
877 spin_unlock_irqrestore(&d40c->lock, flags);
878 return res;
879 }
880
881 static int d40_resume(struct dma_chan *chan)
882 {
883 struct d40_chan *d40c =
884 container_of(chan, struct d40_chan, chan);
885 int res = 0;
886 unsigned long flags;
887
888 if (!d40c->busy)
889 return 0;
890
891 spin_lock_irqsave(&d40c->lock, flags);
892
893 if (d40c->base->rev == 0)
894 if (d40c->log_num != D40_PHY_CHAN) {
895 res = d40_channel_execute_command(d40c,
896 D40_DMA_SUSPEND_REQ);
897 goto no_suspend;
898 }
899
900 /* If bytes left to transfer or linked tx resume job */
901 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
902
903 if (d40c->log_num != D40_PHY_CHAN)
904 d40_config_set_event(d40c, true);
905
906 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
907 }
908
909 no_suspend:
910 spin_unlock_irqrestore(&d40c->lock, flags);
911 return res;
912 }
913
914 static void d40_tx_submit_log(struct d40_chan *d40c, struct d40_desc *d40d)
915 {
916 /* TODO: Write */
917 }
918
919 static void d40_tx_submit_phy(struct d40_chan *d40c, struct d40_desc *d40d)
920 {
921 struct d40_desc *d40d_prev = NULL;
922 int i;
923 u32 val;
924
925 if (!list_empty(&d40c->queue))
926 d40d_prev = d40_last_queued(d40c);
927 else if (!list_empty(&d40c->active))
928 d40d_prev = d40_first_active_get(d40c);
929
930 if (!d40d_prev)
931 return;
932
933 /* Here we try to join this job with previous jobs */
934 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
935 d40c->phy_chan->num * D40_DREG_PCDELTA +
936 D40_CHAN_REG_SSLNK);
937
938 /* Figure out which link we're currently transmitting */
939 for (i = 0; i < d40d_prev->lli_len; i++)
940 if (val == d40d_prev->lli_phy.src[i].reg_lnk)
941 break;
942
943 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
944 d40c->phy_chan->num * D40_DREG_PCDELTA +
945 D40_CHAN_REG_SSELT) >> D40_SREG_ELEM_LOG_ECNT_POS;
946
947 if (i == (d40d_prev->lli_len - 1) && val > 0) {
948 /* Change the current one */
949 writel(virt_to_phys(d40d->lli_phy.src),
950 d40c->base->virtbase + D40_DREG_PCBASE +
951 d40c->phy_chan->num * D40_DREG_PCDELTA +
952 D40_CHAN_REG_SSLNK);
953 writel(virt_to_phys(d40d->lli_phy.dst),
954 d40c->base->virtbase + D40_DREG_PCBASE +
955 d40c->phy_chan->num * D40_DREG_PCDELTA +
956 D40_CHAN_REG_SDLNK);
957
958 d40d->is_hw_linked = true;
959
960 } else if (i < d40d_prev->lli_len) {
961 (void) dma_unmap_single(d40c->base->dev,
962 virt_to_phys(d40d_prev->lli_phy.src),
963 d40d_prev->lli_pool.size,
964 DMA_TO_DEVICE);
965
966 /* Keep the settings */
967 val = d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk &
968 ~D40_SREG_LNK_PHYS_LNK_MASK;
969 d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk =
970 val | virt_to_phys(d40d->lli_phy.src);
971
972 val = d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk &
973 ~D40_SREG_LNK_PHYS_LNK_MASK;
974 d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk =
975 val | virt_to_phys(d40d->lli_phy.dst);
976
977 (void) dma_map_single(d40c->base->dev,
978 d40d_prev->lli_phy.src,
979 d40d_prev->lli_pool.size,
980 DMA_TO_DEVICE);
981 d40d->is_hw_linked = true;
982 }
983 }
984
985 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
986 {
987 struct d40_chan *d40c = container_of(tx->chan,
988 struct d40_chan,
989 chan);
990 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
991 unsigned long flags;
992
993 (void) d40_pause(&d40c->chan);
994
995 spin_lock_irqsave(&d40c->lock, flags);
996
997 d40c->chan.cookie++;
998
999 if (d40c->chan.cookie < 0)
1000 d40c->chan.cookie = 1;
1001
1002 d40d->txd.cookie = d40c->chan.cookie;
1003
1004 if (d40c->log_num == D40_PHY_CHAN)
1005 d40_tx_submit_phy(d40c, d40d);
1006 else
1007 d40_tx_submit_log(d40c, d40d);
1008
1009 d40_desc_queue(d40c, d40d);
1010
1011 spin_unlock_irqrestore(&d40c->lock, flags);
1012
1013 (void) d40_resume(&d40c->chan);
1014
1015 return tx->cookie;
1016 }
1017
1018 static int d40_start(struct d40_chan *d40c)
1019 {
1020 if (d40c->base->rev == 0) {
1021 int err;
1022
1023 if (d40c->log_num != D40_PHY_CHAN) {
1024 err = d40_channel_execute_command(d40c,
1025 D40_DMA_SUSPEND_REQ);
1026 if (err)
1027 return err;
1028 }
1029 }
1030
1031 if (d40c->log_num != D40_PHY_CHAN)
1032 d40_config_set_event(d40c, true);
1033
1034 return d40_channel_execute_command(d40c, D40_DMA_RUN);
1035 }
1036
1037 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1038 {
1039 struct d40_desc *d40d;
1040 int err;
1041
1042 /* Start queued jobs, if any */
1043 d40d = d40_first_queued(d40c);
1044
1045 if (d40d != NULL) {
1046 d40c->busy = true;
1047
1048 /* Remove from queue */
1049 d40_desc_remove(d40d);
1050
1051 /* Add to active queue */
1052 d40_desc_submit(d40c, d40d);
1053
1054 /*
1055 * If this job is already linked in hw,
1056 * do not submit it.
1057 */
1058
1059 if (!d40d->is_hw_linked) {
1060 /* Initiate DMA job */
1061 d40_desc_load(d40c, d40d);
1062
1063 /* Start dma job */
1064 err = d40_start(d40c);
1065
1066 if (err)
1067 return NULL;
1068 }
1069 }
1070
1071 return d40d;
1072 }
1073
1074 /* called from interrupt context */
1075 static void dma_tc_handle(struct d40_chan *d40c)
1076 {
1077 struct d40_desc *d40d;
1078
1079 /* Get first active entry from list */
1080 d40d = d40_first_active_get(d40c);
1081
1082 if (d40d == NULL)
1083 return;
1084
1085 d40_lcla_free_all(d40c, d40d);
1086
1087 if (d40d->lli_current < d40d->lli_len) {
1088 d40_desc_load(d40c, d40d);
1089 /* Start dma job */
1090 (void) d40_start(d40c);
1091 return;
1092 }
1093
1094 if (d40_queue_start(d40c) == NULL)
1095 d40c->busy = false;
1096
1097 d40c->pending_tx++;
1098 tasklet_schedule(&d40c->tasklet);
1099
1100 }
1101
1102 static void dma_tasklet(unsigned long data)
1103 {
1104 struct d40_chan *d40c = (struct d40_chan *) data;
1105 struct d40_desc *d40d;
1106 unsigned long flags;
1107 dma_async_tx_callback callback;
1108 void *callback_param;
1109
1110 spin_lock_irqsave(&d40c->lock, flags);
1111
1112 /* Get first active entry from list */
1113 d40d = d40_first_active_get(d40c);
1114
1115 if (d40d == NULL)
1116 goto err;
1117
1118 d40c->completed = d40d->txd.cookie;
1119
1120 /*
1121 * If terminating a channel pending_tx is set to zero.
1122 * This prevents any finished active jobs to return to the client.
1123 */
1124 if (d40c->pending_tx == 0) {
1125 spin_unlock_irqrestore(&d40c->lock, flags);
1126 return;
1127 }
1128
1129 /* Callback to client */
1130 callback = d40d->txd.callback;
1131 callback_param = d40d->txd.callback_param;
1132
1133 if (async_tx_test_ack(&d40d->txd)) {
1134 d40_pool_lli_free(d40d);
1135 d40_desc_remove(d40d);
1136 d40_desc_free(d40c, d40d);
1137 } else {
1138 if (!d40d->is_in_client_list) {
1139 d40_desc_remove(d40d);
1140 d40_lcla_free_all(d40c, d40d);
1141 list_add_tail(&d40d->node, &d40c->client);
1142 d40d->is_in_client_list = true;
1143 }
1144 }
1145
1146 d40c->pending_tx--;
1147
1148 if (d40c->pending_tx)
1149 tasklet_schedule(&d40c->tasklet);
1150
1151 spin_unlock_irqrestore(&d40c->lock, flags);
1152
1153 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1154 callback(callback_param);
1155
1156 return;
1157
1158 err:
1159 /* Rescue manouver if receiving double interrupts */
1160 if (d40c->pending_tx > 0)
1161 d40c->pending_tx--;
1162 spin_unlock_irqrestore(&d40c->lock, flags);
1163 }
1164
1165 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1166 {
1167 static const struct d40_interrupt_lookup il[] = {
1168 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
1169 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1170 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1171 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1172 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
1173 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
1174 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
1175 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
1176 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
1177 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
1178 };
1179
1180 int i;
1181 u32 regs[ARRAY_SIZE(il)];
1182 u32 idx;
1183 u32 row;
1184 long chan = -1;
1185 struct d40_chan *d40c;
1186 unsigned long flags;
1187 struct d40_base *base = data;
1188
1189 spin_lock_irqsave(&base->interrupt_lock, flags);
1190
1191 /* Read interrupt status of both logical and physical channels */
1192 for (i = 0; i < ARRAY_SIZE(il); i++)
1193 regs[i] = readl(base->virtbase + il[i].src);
1194
1195 for (;;) {
1196
1197 chan = find_next_bit((unsigned long *)regs,
1198 BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1199
1200 /* No more set bits found? */
1201 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1202 break;
1203
1204 row = chan / BITS_PER_LONG;
1205 idx = chan & (BITS_PER_LONG - 1);
1206
1207 /* ACK interrupt */
1208 writel(1 << idx, base->virtbase + il[row].clr);
1209
1210 if (il[row].offset == D40_PHY_CHAN)
1211 d40c = base->lookup_phy_chans[idx];
1212 else
1213 d40c = base->lookup_log_chans[il[row].offset + idx];
1214 spin_lock(&d40c->lock);
1215
1216 if (!il[row].is_error)
1217 dma_tc_handle(d40c);
1218 else
1219 dev_err(base->dev,
1220 "[%s] IRQ chan: %ld offset %d idx %d\n",
1221 __func__, chan, il[row].offset, idx);
1222
1223 spin_unlock(&d40c->lock);
1224 }
1225
1226 spin_unlock_irqrestore(&base->interrupt_lock, flags);
1227
1228 return IRQ_HANDLED;
1229 }
1230
1231 static int d40_validate_conf(struct d40_chan *d40c,
1232 struct stedma40_chan_cfg *conf)
1233 {
1234 int res = 0;
1235 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1236 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1237 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1238
1239 if (!conf->dir) {
1240 dev_err(&d40c->chan.dev->device, "[%s] Invalid direction.\n",
1241 __func__);
1242 res = -EINVAL;
1243 }
1244
1245 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1246 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1247 d40c->runtime_addr == 0) {
1248
1249 dev_err(&d40c->chan.dev->device,
1250 "[%s] Invalid TX channel address (%d)\n",
1251 __func__, conf->dst_dev_type);
1252 res = -EINVAL;
1253 }
1254
1255 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1256 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1257 d40c->runtime_addr == 0) {
1258 dev_err(&d40c->chan.dev->device,
1259 "[%s] Invalid RX channel address (%d)\n",
1260 __func__, conf->src_dev_type);
1261 res = -EINVAL;
1262 }
1263
1264 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1265 dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1266 dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
1267 __func__);
1268 res = -EINVAL;
1269 }
1270
1271 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1272 src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1273 dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
1274 __func__);
1275 res = -EINVAL;
1276 }
1277
1278 if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1279 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1280 dev_err(&d40c->chan.dev->device,
1281 "[%s] No event line\n", __func__);
1282 res = -EINVAL;
1283 }
1284
1285 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1286 (src_event_group != dst_event_group)) {
1287 dev_err(&d40c->chan.dev->device,
1288 "[%s] Invalid event group\n", __func__);
1289 res = -EINVAL;
1290 }
1291
1292 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1293 /*
1294 * DMAC HW supports it. Will be added to this driver,
1295 * in case any dma client requires it.
1296 */
1297 dev_err(&d40c->chan.dev->device,
1298 "[%s] periph to periph not supported\n",
1299 __func__);
1300 res = -EINVAL;
1301 }
1302
1303 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1304 (1 << conf->src_info.data_width) !=
1305 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1306 (1 << conf->dst_info.data_width)) {
1307 /*
1308 * The DMAC hardware only supports
1309 * src (burst x width) == dst (burst x width)
1310 */
1311
1312 dev_err(&d40c->chan.dev->device,
1313 "[%s] src (burst x width) != dst (burst x width)\n",
1314 __func__);
1315 res = -EINVAL;
1316 }
1317
1318 return res;
1319 }
1320
1321 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1322 int log_event_line, bool is_log)
1323 {
1324 unsigned long flags;
1325 spin_lock_irqsave(&phy->lock, flags);
1326 if (!is_log) {
1327 /* Physical interrupts are masked per physical full channel */
1328 if (phy->allocated_src == D40_ALLOC_FREE &&
1329 phy->allocated_dst == D40_ALLOC_FREE) {
1330 phy->allocated_dst = D40_ALLOC_PHY;
1331 phy->allocated_src = D40_ALLOC_PHY;
1332 goto found;
1333 } else
1334 goto not_found;
1335 }
1336
1337 /* Logical channel */
1338 if (is_src) {
1339 if (phy->allocated_src == D40_ALLOC_PHY)
1340 goto not_found;
1341
1342 if (phy->allocated_src == D40_ALLOC_FREE)
1343 phy->allocated_src = D40_ALLOC_LOG_FREE;
1344
1345 if (!(phy->allocated_src & (1 << log_event_line))) {
1346 phy->allocated_src |= 1 << log_event_line;
1347 goto found;
1348 } else
1349 goto not_found;
1350 } else {
1351 if (phy->allocated_dst == D40_ALLOC_PHY)
1352 goto not_found;
1353
1354 if (phy->allocated_dst == D40_ALLOC_FREE)
1355 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1356
1357 if (!(phy->allocated_dst & (1 << log_event_line))) {
1358 phy->allocated_dst |= 1 << log_event_line;
1359 goto found;
1360 } else
1361 goto not_found;
1362 }
1363
1364 not_found:
1365 spin_unlock_irqrestore(&phy->lock, flags);
1366 return false;
1367 found:
1368 spin_unlock_irqrestore(&phy->lock, flags);
1369 return true;
1370 }
1371
1372 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1373 int log_event_line)
1374 {
1375 unsigned long flags;
1376 bool is_free = false;
1377
1378 spin_lock_irqsave(&phy->lock, flags);
1379 if (!log_event_line) {
1380 phy->allocated_dst = D40_ALLOC_FREE;
1381 phy->allocated_src = D40_ALLOC_FREE;
1382 is_free = true;
1383 goto out;
1384 }
1385
1386 /* Logical channel */
1387 if (is_src) {
1388 phy->allocated_src &= ~(1 << log_event_line);
1389 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1390 phy->allocated_src = D40_ALLOC_FREE;
1391 } else {
1392 phy->allocated_dst &= ~(1 << log_event_line);
1393 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1394 phy->allocated_dst = D40_ALLOC_FREE;
1395 }
1396
1397 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1398 D40_ALLOC_FREE);
1399
1400 out:
1401 spin_unlock_irqrestore(&phy->lock, flags);
1402
1403 return is_free;
1404 }
1405
1406 static int d40_allocate_channel(struct d40_chan *d40c)
1407 {
1408 int dev_type;
1409 int event_group;
1410 int event_line;
1411 struct d40_phy_res *phys;
1412 int i;
1413 int j;
1414 int log_num;
1415 bool is_src;
1416 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1417
1418 phys = d40c->base->phy_res;
1419
1420 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1421 dev_type = d40c->dma_cfg.src_dev_type;
1422 log_num = 2 * dev_type;
1423 is_src = true;
1424 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1425 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1426 /* dst event lines are used for logical memcpy */
1427 dev_type = d40c->dma_cfg.dst_dev_type;
1428 log_num = 2 * dev_type + 1;
1429 is_src = false;
1430 } else
1431 return -EINVAL;
1432
1433 event_group = D40_TYPE_TO_GROUP(dev_type);
1434 event_line = D40_TYPE_TO_EVENT(dev_type);
1435
1436 if (!is_log) {
1437 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1438 /* Find physical half channel */
1439 for (i = 0; i < d40c->base->num_phy_chans; i++) {
1440
1441 if (d40_alloc_mask_set(&phys[i], is_src,
1442 0, is_log))
1443 goto found_phy;
1444 }
1445 } else
1446 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1447 int phy_num = j + event_group * 2;
1448 for (i = phy_num; i < phy_num + 2; i++) {
1449 if (d40_alloc_mask_set(&phys[i],
1450 is_src,
1451 0,
1452 is_log))
1453 goto found_phy;
1454 }
1455 }
1456 return -EINVAL;
1457 found_phy:
1458 d40c->phy_chan = &phys[i];
1459 d40c->log_num = D40_PHY_CHAN;
1460 goto out;
1461 }
1462 if (dev_type == -1)
1463 return -EINVAL;
1464
1465 /* Find logical channel */
1466 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1467 int phy_num = j + event_group * 2;
1468 /*
1469 * Spread logical channels across all available physical rather
1470 * than pack every logical channel at the first available phy
1471 * channels.
1472 */
1473 if (is_src) {
1474 for (i = phy_num; i < phy_num + 2; i++) {
1475 if (d40_alloc_mask_set(&phys[i], is_src,
1476 event_line, is_log))
1477 goto found_log;
1478 }
1479 } else {
1480 for (i = phy_num + 1; i >= phy_num; i--) {
1481 if (d40_alloc_mask_set(&phys[i], is_src,
1482 event_line, is_log))
1483 goto found_log;
1484 }
1485 }
1486 }
1487 return -EINVAL;
1488
1489 found_log:
1490 d40c->phy_chan = &phys[i];
1491 d40c->log_num = log_num;
1492 out:
1493
1494 if (is_log)
1495 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1496 else
1497 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1498
1499 return 0;
1500
1501 }
1502
1503 static int d40_config_memcpy(struct d40_chan *d40c)
1504 {
1505 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1506
1507 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1508 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1509 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1510 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1511 memcpy[d40c->chan.chan_id];
1512
1513 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1514 dma_has_cap(DMA_SLAVE, cap)) {
1515 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1516 } else {
1517 dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
1518 __func__);
1519 return -EINVAL;
1520 }
1521
1522 return 0;
1523 }
1524
1525
1526 static int d40_free_dma(struct d40_chan *d40c)
1527 {
1528
1529 int res = 0;
1530 u32 event;
1531 struct d40_phy_res *phy = d40c->phy_chan;
1532 bool is_src;
1533 struct d40_desc *d;
1534 struct d40_desc *_d;
1535
1536
1537 /* Terminate all queued and active transfers */
1538 d40_term_all(d40c);
1539
1540 /* Release client owned descriptors */
1541 if (!list_empty(&d40c->client))
1542 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1543 d40_pool_lli_free(d);
1544 d40_desc_remove(d);
1545 d40_desc_free(d40c, d);
1546 }
1547
1548 if (phy == NULL) {
1549 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
1550 __func__);
1551 return -EINVAL;
1552 }
1553
1554 if (phy->allocated_src == D40_ALLOC_FREE &&
1555 phy->allocated_dst == D40_ALLOC_FREE) {
1556 dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
1557 __func__);
1558 return -EINVAL;
1559 }
1560
1561 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1562 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1563 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1564 is_src = false;
1565 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1566 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1567 is_src = true;
1568 } else {
1569 dev_err(&d40c->chan.dev->device,
1570 "[%s] Unknown direction\n", __func__);
1571 return -EINVAL;
1572 }
1573
1574 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1575 if (res) {
1576 dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n",
1577 __func__);
1578 return res;
1579 }
1580
1581 if (d40c->log_num != D40_PHY_CHAN) {
1582 /* Release logical channel, deactivate the event line */
1583
1584 d40_config_set_event(d40c, false);
1585 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1586
1587 /*
1588 * Check if there are more logical allocation
1589 * on this phy channel.
1590 */
1591 if (!d40_alloc_mask_free(phy, is_src, event)) {
1592 /* Resume the other logical channels if any */
1593 if (d40_chan_has_events(d40c)) {
1594 res = d40_channel_execute_command(d40c,
1595 D40_DMA_RUN);
1596 if (res) {
1597 dev_err(&d40c->chan.dev->device,
1598 "[%s] Executing RUN command\n",
1599 __func__);
1600 return res;
1601 }
1602 }
1603 return 0;
1604 }
1605 } else {
1606 (void) d40_alloc_mask_free(phy, is_src, 0);
1607 }
1608
1609 /* Release physical channel */
1610 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1611 if (res) {
1612 dev_err(&d40c->chan.dev->device,
1613 "[%s] Failed to stop channel\n", __func__);
1614 return res;
1615 }
1616 d40c->phy_chan = NULL;
1617 d40c->configured = false;
1618 d40c->base->lookup_phy_chans[phy->num] = NULL;
1619
1620 return 0;
1621 }
1622
1623 static bool d40_is_paused(struct d40_chan *d40c)
1624 {
1625 bool is_paused = false;
1626 unsigned long flags;
1627 void __iomem *active_reg;
1628 u32 status;
1629 u32 event;
1630
1631 spin_lock_irqsave(&d40c->lock, flags);
1632
1633 if (d40c->log_num == D40_PHY_CHAN) {
1634 if (d40c->phy_chan->num % 2 == 0)
1635 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1636 else
1637 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1638
1639 status = (readl(active_reg) &
1640 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1641 D40_CHAN_POS(d40c->phy_chan->num);
1642 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1643 is_paused = true;
1644
1645 goto _exit;
1646 }
1647
1648 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1649 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1650 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1651 status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1652 d40c->phy_chan->num * D40_DREG_PCDELTA +
1653 D40_CHAN_REG_SDLNK);
1654 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1655 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1656 status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1657 d40c->phy_chan->num * D40_DREG_PCDELTA +
1658 D40_CHAN_REG_SSLNK);
1659 } else {
1660 dev_err(&d40c->chan.dev->device,
1661 "[%s] Unknown direction\n", __func__);
1662 goto _exit;
1663 }
1664
1665 status = (status & D40_EVENTLINE_MASK(event)) >>
1666 D40_EVENTLINE_POS(event);
1667
1668 if (status != D40_DMA_RUN)
1669 is_paused = true;
1670 _exit:
1671 spin_unlock_irqrestore(&d40c->lock, flags);
1672 return is_paused;
1673
1674 }
1675
1676
1677 static u32 stedma40_residue(struct dma_chan *chan)
1678 {
1679 struct d40_chan *d40c =
1680 container_of(chan, struct d40_chan, chan);
1681 u32 bytes_left;
1682 unsigned long flags;
1683
1684 spin_lock_irqsave(&d40c->lock, flags);
1685 bytes_left = d40_residue(d40c);
1686 spin_unlock_irqrestore(&d40c->lock, flags);
1687
1688 return bytes_left;
1689 }
1690
1691 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1692 struct scatterlist *sgl_dst,
1693 struct scatterlist *sgl_src,
1694 unsigned int sgl_len,
1695 unsigned long dma_flags)
1696 {
1697 int res;
1698 struct d40_desc *d40d;
1699 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1700 chan);
1701 unsigned long flags;
1702
1703 if (d40c->phy_chan == NULL) {
1704 dev_err(&d40c->chan.dev->device,
1705 "[%s] Unallocated channel.\n", __func__);
1706 return ERR_PTR(-EINVAL);
1707 }
1708
1709 spin_lock_irqsave(&d40c->lock, flags);
1710 d40d = d40_desc_get(d40c);
1711
1712 if (d40d == NULL)
1713 goto err;
1714
1715 d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len,
1716 d40c->dma_cfg.src_info.data_width,
1717 d40c->dma_cfg.dst_info.data_width);
1718 if (d40d->lli_len < 0) {
1719 dev_err(&d40c->chan.dev->device,
1720 "[%s] Unaligned size\n", __func__);
1721 goto err;
1722 }
1723
1724 d40d->lli_current = 0;
1725 d40d->txd.flags = dma_flags;
1726
1727 if (d40c->log_num != D40_PHY_CHAN) {
1728
1729 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1730 dev_err(&d40c->chan.dev->device,
1731 "[%s] Out of memory\n", __func__);
1732 goto err;
1733 }
1734
1735 (void) d40_log_sg_to_lli(sgl_src,
1736 sgl_len,
1737 d40d->lli_log.src,
1738 d40c->log_def.lcsp1,
1739 d40c->dma_cfg.src_info.data_width,
1740 d40c->dma_cfg.dst_info.data_width);
1741
1742 (void) d40_log_sg_to_lli(sgl_dst,
1743 sgl_len,
1744 d40d->lli_log.dst,
1745 d40c->log_def.lcsp3,
1746 d40c->dma_cfg.dst_info.data_width,
1747 d40c->dma_cfg.src_info.data_width);
1748 } else {
1749 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1750 dev_err(&d40c->chan.dev->device,
1751 "[%s] Out of memory\n", __func__);
1752 goto err;
1753 }
1754
1755 res = d40_phy_sg_to_lli(sgl_src,
1756 sgl_len,
1757 0,
1758 d40d->lli_phy.src,
1759 virt_to_phys(d40d->lli_phy.src),
1760 d40c->src_def_cfg,
1761 d40c->dma_cfg.src_info.data_width,
1762 d40c->dma_cfg.dst_info.data_width,
1763 d40c->dma_cfg.src_info.psize);
1764
1765 if (res < 0)
1766 goto err;
1767
1768 res = d40_phy_sg_to_lli(sgl_dst,
1769 sgl_len,
1770 0,
1771 d40d->lli_phy.dst,
1772 virt_to_phys(d40d->lli_phy.dst),
1773 d40c->dst_def_cfg,
1774 d40c->dma_cfg.dst_info.data_width,
1775 d40c->dma_cfg.src_info.data_width,
1776 d40c->dma_cfg.dst_info.psize);
1777
1778 if (res < 0)
1779 goto err;
1780
1781 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1782 d40d->lli_pool.size, DMA_TO_DEVICE);
1783 }
1784
1785 dma_async_tx_descriptor_init(&d40d->txd, chan);
1786
1787 d40d->txd.tx_submit = d40_tx_submit;
1788
1789 spin_unlock_irqrestore(&d40c->lock, flags);
1790
1791 return &d40d->txd;
1792 err:
1793 if (d40d)
1794 d40_desc_free(d40c, d40d);
1795 spin_unlock_irqrestore(&d40c->lock, flags);
1796 return NULL;
1797 }
1798 EXPORT_SYMBOL(stedma40_memcpy_sg);
1799
1800 bool stedma40_filter(struct dma_chan *chan, void *data)
1801 {
1802 struct stedma40_chan_cfg *info = data;
1803 struct d40_chan *d40c =
1804 container_of(chan, struct d40_chan, chan);
1805 int err;
1806
1807 if (data) {
1808 err = d40_validate_conf(d40c, info);
1809 if (!err)
1810 d40c->dma_cfg = *info;
1811 } else
1812 err = d40_config_memcpy(d40c);
1813
1814 if (!err)
1815 d40c->configured = true;
1816
1817 return err == 0;
1818 }
1819 EXPORT_SYMBOL(stedma40_filter);
1820
1821 /* DMA ENGINE functions */
1822 static int d40_alloc_chan_resources(struct dma_chan *chan)
1823 {
1824 int err;
1825 unsigned long flags;
1826 struct d40_chan *d40c =
1827 container_of(chan, struct d40_chan, chan);
1828 bool is_free_phy;
1829 spin_lock_irqsave(&d40c->lock, flags);
1830
1831 d40c->completed = chan->cookie = 1;
1832
1833 /* If no dma configuration is set use default configuration (memcpy) */
1834 if (!d40c->configured) {
1835 err = d40_config_memcpy(d40c);
1836 if (err) {
1837 dev_err(&d40c->chan.dev->device,
1838 "[%s] Failed to configure memcpy channel\n",
1839 __func__);
1840 goto fail;
1841 }
1842 }
1843 is_free_phy = (d40c->phy_chan == NULL);
1844
1845 err = d40_allocate_channel(d40c);
1846 if (err) {
1847 dev_err(&d40c->chan.dev->device,
1848 "[%s] Failed to allocate channel\n", __func__);
1849 goto fail;
1850 }
1851
1852 /* Fill in basic CFG register values */
1853 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1854 &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1855
1856 if (d40c->log_num != D40_PHY_CHAN) {
1857 d40_log_cfg(&d40c->dma_cfg,
1858 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1859
1860 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1861 d40c->lcpa = d40c->base->lcpa_base +
1862 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1863 else
1864 d40c->lcpa = d40c->base->lcpa_base +
1865 d40c->dma_cfg.dst_dev_type *
1866 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1867 }
1868
1869 /*
1870 * Only write channel configuration to the DMA if the physical
1871 * resource is free. In case of multiple logical channels
1872 * on the same physical resource, only the first write is necessary.
1873 */
1874 if (is_free_phy)
1875 d40_config_write(d40c);
1876 fail:
1877 spin_unlock_irqrestore(&d40c->lock, flags);
1878 return err;
1879 }
1880
1881 static void d40_free_chan_resources(struct dma_chan *chan)
1882 {
1883 struct d40_chan *d40c =
1884 container_of(chan, struct d40_chan, chan);
1885 int err;
1886 unsigned long flags;
1887
1888 if (d40c->phy_chan == NULL) {
1889 dev_err(&d40c->chan.dev->device,
1890 "[%s] Cannot free unallocated channel\n", __func__);
1891 return;
1892 }
1893
1894
1895 spin_lock_irqsave(&d40c->lock, flags);
1896
1897 err = d40_free_dma(d40c);
1898
1899 if (err)
1900 dev_err(&d40c->chan.dev->device,
1901 "[%s] Failed to free channel\n", __func__);
1902 spin_unlock_irqrestore(&d40c->lock, flags);
1903 }
1904
1905 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1906 dma_addr_t dst,
1907 dma_addr_t src,
1908 size_t size,
1909 unsigned long dma_flags)
1910 {
1911 struct d40_desc *d40d;
1912 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1913 chan);
1914 unsigned long flags;
1915
1916 if (d40c->phy_chan == NULL) {
1917 dev_err(&d40c->chan.dev->device,
1918 "[%s] Channel is not allocated.\n", __func__);
1919 return ERR_PTR(-EINVAL);
1920 }
1921
1922 spin_lock_irqsave(&d40c->lock, flags);
1923 d40d = d40_desc_get(d40c);
1924
1925 if (d40d == NULL) {
1926 dev_err(&d40c->chan.dev->device,
1927 "[%s] Descriptor is NULL\n", __func__);
1928 goto err;
1929 }
1930
1931 d40d->txd.flags = dma_flags;
1932 d40d->lli_len = d40_size_2_dmalen(size,
1933 d40c->dma_cfg.src_info.data_width,
1934 d40c->dma_cfg.dst_info.data_width);
1935 if (d40d->lli_len < 0) {
1936 dev_err(&d40c->chan.dev->device,
1937 "[%s] Unaligned size\n", __func__);
1938 goto err;
1939 }
1940
1941
1942 dma_async_tx_descriptor_init(&d40d->txd, chan);
1943
1944 d40d->txd.tx_submit = d40_tx_submit;
1945
1946 if (d40c->log_num != D40_PHY_CHAN) {
1947
1948 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1949 dev_err(&d40c->chan.dev->device,
1950 "[%s] Out of memory\n", __func__);
1951 goto err;
1952 }
1953 d40d->lli_current = 0;
1954
1955 if (d40_log_buf_to_lli(d40d->lli_log.src,
1956 src,
1957 size,
1958 d40c->log_def.lcsp1,
1959 d40c->dma_cfg.src_info.data_width,
1960 d40c->dma_cfg.dst_info.data_width,
1961 true) == NULL)
1962 goto err;
1963
1964 if (d40_log_buf_to_lli(d40d->lli_log.dst,
1965 dst,
1966 size,
1967 d40c->log_def.lcsp3,
1968 d40c->dma_cfg.dst_info.data_width,
1969 d40c->dma_cfg.src_info.data_width,
1970 true) == NULL)
1971 goto err;
1972
1973 } else {
1974
1975 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1976 dev_err(&d40c->chan.dev->device,
1977 "[%s] Out of memory\n", __func__);
1978 goto err;
1979 }
1980
1981 if (d40_phy_buf_to_lli(d40d->lli_phy.src,
1982 src,
1983 size,
1984 d40c->dma_cfg.src_info.psize,
1985 0,
1986 d40c->src_def_cfg,
1987 true,
1988 d40c->dma_cfg.src_info.data_width,
1989 d40c->dma_cfg.dst_info.data_width,
1990 false) == NULL)
1991 goto err;
1992
1993 if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
1994 dst,
1995 size,
1996 d40c->dma_cfg.dst_info.psize,
1997 0,
1998 d40c->dst_def_cfg,
1999 true,
2000 d40c->dma_cfg.dst_info.data_width,
2001 d40c->dma_cfg.src_info.data_width,
2002 false) == NULL)
2003 goto err;
2004
2005 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2006 d40d->lli_pool.size, DMA_TO_DEVICE);
2007 }
2008
2009 spin_unlock_irqrestore(&d40c->lock, flags);
2010 return &d40d->txd;
2011
2012 err:
2013 if (d40d)
2014 d40_desc_free(d40c, d40d);
2015 spin_unlock_irqrestore(&d40c->lock, flags);
2016 return NULL;
2017 }
2018
2019 static struct dma_async_tx_descriptor *
2020 d40_prep_sg(struct dma_chan *chan,
2021 struct scatterlist *dst_sg, unsigned int dst_nents,
2022 struct scatterlist *src_sg, unsigned int src_nents,
2023 unsigned long dma_flags)
2024 {
2025 if (dst_nents != src_nents)
2026 return NULL;
2027
2028 return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags);
2029 }
2030
2031 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
2032 struct d40_chan *d40c,
2033 struct scatterlist *sgl,
2034 unsigned int sg_len,
2035 enum dma_data_direction direction,
2036 unsigned long dma_flags)
2037 {
2038 dma_addr_t dev_addr = 0;
2039 int total_size;
2040
2041 d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
2042 d40c->dma_cfg.src_info.data_width,
2043 d40c->dma_cfg.dst_info.data_width);
2044 if (d40d->lli_len < 0) {
2045 dev_err(&d40c->chan.dev->device,
2046 "[%s] Unaligned size\n", __func__);
2047 return -EINVAL;
2048 }
2049
2050 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
2051 dev_err(&d40c->chan.dev->device,
2052 "[%s] Out of memory\n", __func__);
2053 return -ENOMEM;
2054 }
2055
2056 d40d->lli_current = 0;
2057
2058 if (direction == DMA_FROM_DEVICE)
2059 if (d40c->runtime_addr)
2060 dev_addr = d40c->runtime_addr;
2061 else
2062 dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2063 else if (direction == DMA_TO_DEVICE)
2064 if (d40c->runtime_addr)
2065 dev_addr = d40c->runtime_addr;
2066 else
2067 dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
2068
2069 else
2070 return -EINVAL;
2071
2072 total_size = d40_log_sg_to_dev(sgl, sg_len,
2073 &d40d->lli_log,
2074 &d40c->log_def,
2075 d40c->dma_cfg.src_info.data_width,
2076 d40c->dma_cfg.dst_info.data_width,
2077 direction,
2078 dev_addr);
2079
2080 if (total_size < 0)
2081 return -EINVAL;
2082
2083 return 0;
2084 }
2085
2086 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
2087 struct d40_chan *d40c,
2088 struct scatterlist *sgl,
2089 unsigned int sgl_len,
2090 enum dma_data_direction direction,
2091 unsigned long dma_flags)
2092 {
2093 dma_addr_t src_dev_addr;
2094 dma_addr_t dst_dev_addr;
2095 int res;
2096
2097 d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
2098 d40c->dma_cfg.src_info.data_width,
2099 d40c->dma_cfg.dst_info.data_width);
2100 if (d40d->lli_len < 0) {
2101 dev_err(&d40c->chan.dev->device,
2102 "[%s] Unaligned size\n", __func__);
2103 return -EINVAL;
2104 }
2105
2106 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
2107 dev_err(&d40c->chan.dev->device,
2108 "[%s] Out of memory\n", __func__);
2109 return -ENOMEM;
2110 }
2111
2112 d40d->lli_current = 0;
2113
2114 if (direction == DMA_FROM_DEVICE) {
2115 dst_dev_addr = 0;
2116 if (d40c->runtime_addr)
2117 src_dev_addr = d40c->runtime_addr;
2118 else
2119 src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2120 } else if (direction == DMA_TO_DEVICE) {
2121 if (d40c->runtime_addr)
2122 dst_dev_addr = d40c->runtime_addr;
2123 else
2124 dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
2125 src_dev_addr = 0;
2126 } else
2127 return -EINVAL;
2128
2129 res = d40_phy_sg_to_lli(sgl,
2130 sgl_len,
2131 src_dev_addr,
2132 d40d->lli_phy.src,
2133 virt_to_phys(d40d->lli_phy.src),
2134 d40c->src_def_cfg,
2135 d40c->dma_cfg.src_info.data_width,
2136 d40c->dma_cfg.dst_info.data_width,
2137 d40c->dma_cfg.src_info.psize);
2138 if (res < 0)
2139 return res;
2140
2141 res = d40_phy_sg_to_lli(sgl,
2142 sgl_len,
2143 dst_dev_addr,
2144 d40d->lli_phy.dst,
2145 virt_to_phys(d40d->lli_phy.dst),
2146 d40c->dst_def_cfg,
2147 d40c->dma_cfg.dst_info.data_width,
2148 d40c->dma_cfg.src_info.data_width,
2149 d40c->dma_cfg.dst_info.psize);
2150 if (res < 0)
2151 return res;
2152
2153 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2154 d40d->lli_pool.size, DMA_TO_DEVICE);
2155 return 0;
2156 }
2157
2158 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2159 struct scatterlist *sgl,
2160 unsigned int sg_len,
2161 enum dma_data_direction direction,
2162 unsigned long dma_flags)
2163 {
2164 struct d40_desc *d40d;
2165 struct d40_chan *d40c = container_of(chan, struct d40_chan,
2166 chan);
2167 unsigned long flags;
2168 int err;
2169
2170 if (d40c->phy_chan == NULL) {
2171 dev_err(&d40c->chan.dev->device,
2172 "[%s] Cannot prepare unallocated channel\n", __func__);
2173 return ERR_PTR(-EINVAL);
2174 }
2175
2176 spin_lock_irqsave(&d40c->lock, flags);
2177 d40d = d40_desc_get(d40c);
2178
2179 if (d40d == NULL)
2180 goto err;
2181
2182 if (d40c->log_num != D40_PHY_CHAN)
2183 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2184 direction, dma_flags);
2185 else
2186 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2187 direction, dma_flags);
2188 if (err) {
2189 dev_err(&d40c->chan.dev->device,
2190 "[%s] Failed to prepare %s slave sg job: %d\n",
2191 __func__,
2192 d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
2193 goto err;
2194 }
2195
2196 d40d->txd.flags = dma_flags;
2197
2198 dma_async_tx_descriptor_init(&d40d->txd, chan);
2199
2200 d40d->txd.tx_submit = d40_tx_submit;
2201
2202 spin_unlock_irqrestore(&d40c->lock, flags);
2203 return &d40d->txd;
2204
2205 err:
2206 if (d40d)
2207 d40_desc_free(d40c, d40d);
2208 spin_unlock_irqrestore(&d40c->lock, flags);
2209 return NULL;
2210 }
2211
2212 static enum dma_status d40_tx_status(struct dma_chan *chan,
2213 dma_cookie_t cookie,
2214 struct dma_tx_state *txstate)
2215 {
2216 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2217 dma_cookie_t last_used;
2218 dma_cookie_t last_complete;
2219 int ret;
2220
2221 if (d40c->phy_chan == NULL) {
2222 dev_err(&d40c->chan.dev->device,
2223 "[%s] Cannot read status of unallocated channel\n",
2224 __func__);
2225 return -EINVAL;
2226 }
2227
2228 last_complete = d40c->completed;
2229 last_used = chan->cookie;
2230
2231 if (d40_is_paused(d40c))
2232 ret = DMA_PAUSED;
2233 else
2234 ret = dma_async_is_complete(cookie, last_complete, last_used);
2235
2236 dma_set_tx_state(txstate, last_complete, last_used,
2237 stedma40_residue(chan));
2238
2239 return ret;
2240 }
2241
2242 static void d40_issue_pending(struct dma_chan *chan)
2243 {
2244 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2245 unsigned long flags;
2246
2247 if (d40c->phy_chan == NULL) {
2248 dev_err(&d40c->chan.dev->device,
2249 "[%s] Channel is not allocated!\n", __func__);
2250 return;
2251 }
2252
2253 spin_lock_irqsave(&d40c->lock, flags);
2254
2255 /* Busy means that pending jobs are already being processed */
2256 if (!d40c->busy)
2257 (void) d40_queue_start(d40c);
2258
2259 spin_unlock_irqrestore(&d40c->lock, flags);
2260 }
2261
2262 /* Runtime reconfiguration extension */
2263 static void d40_set_runtime_config(struct dma_chan *chan,
2264 struct dma_slave_config *config)
2265 {
2266 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2267 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2268 enum dma_slave_buswidth config_addr_width;
2269 dma_addr_t config_addr;
2270 u32 config_maxburst;
2271 enum stedma40_periph_data_width addr_width;
2272 int psize;
2273
2274 if (config->direction == DMA_FROM_DEVICE) {
2275 dma_addr_t dev_addr_rx =
2276 d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2277
2278 config_addr = config->src_addr;
2279 if (dev_addr_rx)
2280 dev_dbg(d40c->base->dev,
2281 "channel has a pre-wired RX address %08x "
2282 "overriding with %08x\n",
2283 dev_addr_rx, config_addr);
2284 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2285 dev_dbg(d40c->base->dev,
2286 "channel was not configured for peripheral "
2287 "to memory transfer (%d) overriding\n",
2288 cfg->dir);
2289 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2290
2291 config_addr_width = config->src_addr_width;
2292 config_maxburst = config->src_maxburst;
2293
2294 } else if (config->direction == DMA_TO_DEVICE) {
2295 dma_addr_t dev_addr_tx =
2296 d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2297
2298 config_addr = config->dst_addr;
2299 if (dev_addr_tx)
2300 dev_dbg(d40c->base->dev,
2301 "channel has a pre-wired TX address %08x "
2302 "overriding with %08x\n",
2303 dev_addr_tx, config_addr);
2304 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2305 dev_dbg(d40c->base->dev,
2306 "channel was not configured for memory "
2307 "to peripheral transfer (%d) overriding\n",
2308 cfg->dir);
2309 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2310
2311 config_addr_width = config->dst_addr_width;
2312 config_maxburst = config->dst_maxburst;
2313
2314 } else {
2315 dev_err(d40c->base->dev,
2316 "unrecognized channel direction %d\n",
2317 config->direction);
2318 return;
2319 }
2320
2321 switch (config_addr_width) {
2322 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2323 addr_width = STEDMA40_BYTE_WIDTH;
2324 break;
2325 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2326 addr_width = STEDMA40_HALFWORD_WIDTH;
2327 break;
2328 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2329 addr_width = STEDMA40_WORD_WIDTH;
2330 break;
2331 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2332 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2333 break;
2334 default:
2335 dev_err(d40c->base->dev,
2336 "illegal peripheral address width "
2337 "requested (%d)\n",
2338 config->src_addr_width);
2339 return;
2340 }
2341
2342 if (d40c->log_num != D40_PHY_CHAN) {
2343 if (config_maxburst >= 16)
2344 psize = STEDMA40_PSIZE_LOG_16;
2345 else if (config_maxburst >= 8)
2346 psize = STEDMA40_PSIZE_LOG_8;
2347 else if (config_maxburst >= 4)
2348 psize = STEDMA40_PSIZE_LOG_4;
2349 else
2350 psize = STEDMA40_PSIZE_LOG_1;
2351 } else {
2352 if (config_maxburst >= 16)
2353 psize = STEDMA40_PSIZE_PHY_16;
2354 else if (config_maxburst >= 8)
2355 psize = STEDMA40_PSIZE_PHY_8;
2356 else if (config_maxburst >= 4)
2357 psize = STEDMA40_PSIZE_PHY_4;
2358 else if (config_maxburst >= 2)
2359 psize = STEDMA40_PSIZE_PHY_2;
2360 else
2361 psize = STEDMA40_PSIZE_PHY_1;
2362 }
2363
2364 /* Set up all the endpoint configs */
2365 cfg->src_info.data_width = addr_width;
2366 cfg->src_info.psize = psize;
2367 cfg->src_info.big_endian = false;
2368 cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2369 cfg->dst_info.data_width = addr_width;
2370 cfg->dst_info.psize = psize;
2371 cfg->dst_info.big_endian = false;
2372 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2373
2374 /* Fill in register values */
2375 if (d40c->log_num != D40_PHY_CHAN)
2376 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2377 else
2378 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2379 &d40c->dst_def_cfg, false);
2380
2381 /* These settings will take precedence later */
2382 d40c->runtime_addr = config_addr;
2383 d40c->runtime_direction = config->direction;
2384 dev_dbg(d40c->base->dev,
2385 "configured channel %s for %s, data width %d, "
2386 "maxburst %d bytes, LE, no flow control\n",
2387 dma_chan_name(chan),
2388 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2389 config_addr_width,
2390 config_maxburst);
2391 }
2392
2393 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2394 unsigned long arg)
2395 {
2396 unsigned long flags;
2397 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2398
2399 if (d40c->phy_chan == NULL) {
2400 dev_err(&d40c->chan.dev->device,
2401 "[%s] Channel is not allocated!\n", __func__);
2402 return -EINVAL;
2403 }
2404
2405 switch (cmd) {
2406 case DMA_TERMINATE_ALL:
2407 spin_lock_irqsave(&d40c->lock, flags);
2408 d40_term_all(d40c);
2409 spin_unlock_irqrestore(&d40c->lock, flags);
2410 return 0;
2411 case DMA_PAUSE:
2412 return d40_pause(chan);
2413 case DMA_RESUME:
2414 return d40_resume(chan);
2415 case DMA_SLAVE_CONFIG:
2416 d40_set_runtime_config(chan,
2417 (struct dma_slave_config *) arg);
2418 return 0;
2419 default:
2420 break;
2421 }
2422
2423 /* Other commands are unimplemented */
2424 return -ENXIO;
2425 }
2426
2427 /* Initialization functions */
2428
2429 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2430 struct d40_chan *chans, int offset,
2431 int num_chans)
2432 {
2433 int i = 0;
2434 struct d40_chan *d40c;
2435
2436 INIT_LIST_HEAD(&dma->channels);
2437
2438 for (i = offset; i < offset + num_chans; i++) {
2439 d40c = &chans[i];
2440 d40c->base = base;
2441 d40c->chan.device = dma;
2442
2443 spin_lock_init(&d40c->lock);
2444
2445 d40c->log_num = D40_PHY_CHAN;
2446
2447 INIT_LIST_HEAD(&d40c->active);
2448 INIT_LIST_HEAD(&d40c->queue);
2449 INIT_LIST_HEAD(&d40c->client);
2450
2451 tasklet_init(&d40c->tasklet, dma_tasklet,
2452 (unsigned long) d40c);
2453
2454 list_add_tail(&d40c->chan.device_node,
2455 &dma->channels);
2456 }
2457 }
2458
2459 static int __init d40_dmaengine_init(struct d40_base *base,
2460 int num_reserved_chans)
2461 {
2462 int err ;
2463
2464 d40_chan_init(base, &base->dma_slave, base->log_chans,
2465 0, base->num_log_chans);
2466
2467 dma_cap_zero(base->dma_slave.cap_mask);
2468 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2469
2470 base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2471 base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2472 base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2473 base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2474 base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2475 base->dma_slave.device_tx_status = d40_tx_status;
2476 base->dma_slave.device_issue_pending = d40_issue_pending;
2477 base->dma_slave.device_control = d40_control;
2478 base->dma_slave.dev = base->dev;
2479
2480 err = dma_async_device_register(&base->dma_slave);
2481
2482 if (err) {
2483 dev_err(base->dev,
2484 "[%s] Failed to register slave channels\n",
2485 __func__);
2486 goto failure1;
2487 }
2488
2489 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2490 base->num_log_chans, base->plat_data->memcpy_len);
2491
2492 dma_cap_zero(base->dma_memcpy.cap_mask);
2493 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2494 dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2495
2496 base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2497 base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2498 base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2499 base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2500 base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2501 base->dma_memcpy.device_tx_status = d40_tx_status;
2502 base->dma_memcpy.device_issue_pending = d40_issue_pending;
2503 base->dma_memcpy.device_control = d40_control;
2504 base->dma_memcpy.dev = base->dev;
2505 /*
2506 * This controller can only access address at even
2507 * 32bit boundaries, i.e. 2^2
2508 */
2509 base->dma_memcpy.copy_align = 2;
2510
2511 err = dma_async_device_register(&base->dma_memcpy);
2512
2513 if (err) {
2514 dev_err(base->dev,
2515 "[%s] Failed to regsiter memcpy only channels\n",
2516 __func__);
2517 goto failure2;
2518 }
2519
2520 d40_chan_init(base, &base->dma_both, base->phy_chans,
2521 0, num_reserved_chans);
2522
2523 dma_cap_zero(base->dma_both.cap_mask);
2524 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2525 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2526 dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2527
2528 base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2529 base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2530 base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2531 base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2532 base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2533 base->dma_both.device_tx_status = d40_tx_status;
2534 base->dma_both.device_issue_pending = d40_issue_pending;
2535 base->dma_both.device_control = d40_control;
2536 base->dma_both.dev = base->dev;
2537 base->dma_both.copy_align = 2;
2538 err = dma_async_device_register(&base->dma_both);
2539
2540 if (err) {
2541 dev_err(base->dev,
2542 "[%s] Failed to register logical and physical capable channels\n",
2543 __func__);
2544 goto failure3;
2545 }
2546 return 0;
2547 failure3:
2548 dma_async_device_unregister(&base->dma_memcpy);
2549 failure2:
2550 dma_async_device_unregister(&base->dma_slave);
2551 failure1:
2552 return err;
2553 }
2554
2555 /* Initialization functions. */
2556
2557 static int __init d40_phy_res_init(struct d40_base *base)
2558 {
2559 int i;
2560 int num_phy_chans_avail = 0;
2561 u32 val[2];
2562 int odd_even_bit = -2;
2563
2564 val[0] = readl(base->virtbase + D40_DREG_PRSME);
2565 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2566
2567 for (i = 0; i < base->num_phy_chans; i++) {
2568 base->phy_res[i].num = i;
2569 odd_even_bit += 2 * ((i % 2) == 0);
2570 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2571 /* Mark security only channels as occupied */
2572 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2573 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2574 } else {
2575 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2576 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2577 num_phy_chans_avail++;
2578 }
2579 spin_lock_init(&base->phy_res[i].lock);
2580 }
2581
2582 /* Mark disabled channels as occupied */
2583 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2584 int chan = base->plat_data->disabled_channels[i];
2585
2586 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2587 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2588 num_phy_chans_avail--;
2589 }
2590
2591 dev_info(base->dev, "%d of %d physical DMA channels available\n",
2592 num_phy_chans_avail, base->num_phy_chans);
2593
2594 /* Verify settings extended vs standard */
2595 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2596
2597 for (i = 0; i < base->num_phy_chans; i++) {
2598
2599 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2600 (val[0] & 0x3) != 1)
2601 dev_info(base->dev,
2602 "[%s] INFO: channel %d is misconfigured (%d)\n",
2603 __func__, i, val[0] & 0x3);
2604
2605 val[0] = val[0] >> 2;
2606 }
2607
2608 return num_phy_chans_avail;
2609 }
2610
2611 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2612 {
2613 static const struct d40_reg_val dma_id_regs[] = {
2614 /* Peripheral Id */
2615 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2616 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2617 /*
2618 * D40_DREG_PERIPHID2 Depends on HW revision:
2619 * MOP500/HREF ED has 0x0008,
2620 * ? has 0x0018,
2621 * HREF V1 has 0x0028
2622 */
2623 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2624
2625 /* PCell Id */
2626 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2627 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2628 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2629 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2630 };
2631 struct stedma40_platform_data *plat_data;
2632 struct clk *clk = NULL;
2633 void __iomem *virtbase = NULL;
2634 struct resource *res = NULL;
2635 struct d40_base *base = NULL;
2636 int num_log_chans = 0;
2637 int num_phy_chans;
2638 int i;
2639 u32 val;
2640 u32 rev;
2641
2642 clk = clk_get(&pdev->dev, NULL);
2643
2644 if (IS_ERR(clk)) {
2645 dev_err(&pdev->dev, "[%s] No matching clock found\n",
2646 __func__);
2647 goto failure;
2648 }
2649
2650 clk_enable(clk);
2651
2652 /* Get IO for DMAC base address */
2653 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2654 if (!res)
2655 goto failure;
2656
2657 if (request_mem_region(res->start, resource_size(res),
2658 D40_NAME " I/O base") == NULL)
2659 goto failure;
2660
2661 virtbase = ioremap(res->start, resource_size(res));
2662 if (!virtbase)
2663 goto failure;
2664
2665 /* HW version check */
2666 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2667 if (dma_id_regs[i].val !=
2668 readl(virtbase + dma_id_regs[i].reg)) {
2669 dev_err(&pdev->dev,
2670 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2671 __func__,
2672 dma_id_regs[i].val,
2673 dma_id_regs[i].reg,
2674 readl(virtbase + dma_id_regs[i].reg));
2675 goto failure;
2676 }
2677 }
2678
2679 /* Get silicon revision and designer */
2680 val = readl(virtbase + D40_DREG_PERIPHID2);
2681
2682 if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2683 D40_HW_DESIGNER) {
2684 dev_err(&pdev->dev,
2685 "[%s] Unknown designer! Got %x wanted %x\n",
2686 __func__, val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2687 D40_HW_DESIGNER);
2688 goto failure;
2689 }
2690
2691 rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2692 D40_DREG_PERIPHID2_REV_POS;
2693
2694 /* The number of physical channels on this HW */
2695 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2696
2697 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2698 rev, res->start);
2699
2700 plat_data = pdev->dev.platform_data;
2701
2702 /* Count the number of logical channels in use */
2703 for (i = 0; i < plat_data->dev_len; i++)
2704 if (plat_data->dev_rx[i] != 0)
2705 num_log_chans++;
2706
2707 for (i = 0; i < plat_data->dev_len; i++)
2708 if (plat_data->dev_tx[i] != 0)
2709 num_log_chans++;
2710
2711 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2712 (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2713 sizeof(struct d40_chan), GFP_KERNEL);
2714
2715 if (base == NULL) {
2716 dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
2717 goto failure;
2718 }
2719
2720 base->rev = rev;
2721 base->clk = clk;
2722 base->num_phy_chans = num_phy_chans;
2723 base->num_log_chans = num_log_chans;
2724 base->phy_start = res->start;
2725 base->phy_size = resource_size(res);
2726 base->virtbase = virtbase;
2727 base->plat_data = plat_data;
2728 base->dev = &pdev->dev;
2729 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2730 base->log_chans = &base->phy_chans[num_phy_chans];
2731
2732 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2733 GFP_KERNEL);
2734 if (!base->phy_res)
2735 goto failure;
2736
2737 base->lookup_phy_chans = kzalloc(num_phy_chans *
2738 sizeof(struct d40_chan *),
2739 GFP_KERNEL);
2740 if (!base->lookup_phy_chans)
2741 goto failure;
2742
2743 if (num_log_chans + plat_data->memcpy_len) {
2744 /*
2745 * The max number of logical channels are event lines for all
2746 * src devices and dst devices
2747 */
2748 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2749 sizeof(struct d40_chan *),
2750 GFP_KERNEL);
2751 if (!base->lookup_log_chans)
2752 goto failure;
2753 }
2754
2755 base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2756 sizeof(struct d40_desc *) *
2757 D40_LCLA_LINK_PER_EVENT_GRP,
2758 GFP_KERNEL);
2759 if (!base->lcla_pool.alloc_map)
2760 goto failure;
2761
2762 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2763 0, SLAB_HWCACHE_ALIGN,
2764 NULL);
2765 if (base->desc_slab == NULL)
2766 goto failure;
2767
2768 return base;
2769
2770 failure:
2771 if (!IS_ERR(clk)) {
2772 clk_disable(clk);
2773 clk_put(clk);
2774 }
2775 if (virtbase)
2776 iounmap(virtbase);
2777 if (res)
2778 release_mem_region(res->start,
2779 resource_size(res));
2780 if (virtbase)
2781 iounmap(virtbase);
2782
2783 if (base) {
2784 kfree(base->lcla_pool.alloc_map);
2785 kfree(base->lookup_log_chans);
2786 kfree(base->lookup_phy_chans);
2787 kfree(base->phy_res);
2788 kfree(base);
2789 }
2790
2791 return NULL;
2792 }
2793
2794 static void __init d40_hw_init(struct d40_base *base)
2795 {
2796
2797 static const struct d40_reg_val dma_init_reg[] = {
2798 /* Clock every part of the DMA block from start */
2799 { .reg = D40_DREG_GCC, .val = 0x0000ff01},
2800
2801 /* Interrupts on all logical channels */
2802 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2803 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2804 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2805 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2806 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2807 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2808 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2809 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2810 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2811 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2812 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2813 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2814 };
2815 int i;
2816 u32 prmseo[2] = {0, 0};
2817 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2818 u32 pcmis = 0;
2819 u32 pcicr = 0;
2820
2821 for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2822 writel(dma_init_reg[i].val,
2823 base->virtbase + dma_init_reg[i].reg);
2824
2825 /* Configure all our dma channels to default settings */
2826 for (i = 0; i < base->num_phy_chans; i++) {
2827
2828 activeo[i % 2] = activeo[i % 2] << 2;
2829
2830 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2831 == D40_ALLOC_PHY) {
2832 activeo[i % 2] |= 3;
2833 continue;
2834 }
2835
2836 /* Enable interrupt # */
2837 pcmis = (pcmis << 1) | 1;
2838
2839 /* Clear interrupt # */
2840 pcicr = (pcicr << 1) | 1;
2841
2842 /* Set channel to physical mode */
2843 prmseo[i % 2] = prmseo[i % 2] << 2;
2844 prmseo[i % 2] |= 1;
2845
2846 }
2847
2848 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2849 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2850 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2851 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2852
2853 /* Write which interrupt to enable */
2854 writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2855
2856 /* Write which interrupt to clear */
2857 writel(pcicr, base->virtbase + D40_DREG_PCICR);
2858
2859 }
2860
2861 static int __init d40_lcla_allocate(struct d40_base *base)
2862 {
2863 unsigned long *page_list;
2864 int i, j;
2865 int ret = 0;
2866
2867 /*
2868 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2869 * To full fill this hardware requirement without wasting 256 kb
2870 * we allocate pages until we get an aligned one.
2871 */
2872 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2873 GFP_KERNEL);
2874
2875 if (!page_list) {
2876 ret = -ENOMEM;
2877 goto failure;
2878 }
2879
2880 /* Calculating how many pages that are required */
2881 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2882
2883 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2884 page_list[i] = __get_free_pages(GFP_KERNEL,
2885 base->lcla_pool.pages);
2886 if (!page_list[i]) {
2887
2888 dev_err(base->dev,
2889 "[%s] Failed to allocate %d pages.\n",
2890 __func__, base->lcla_pool.pages);
2891
2892 for (j = 0; j < i; j++)
2893 free_pages(page_list[j], base->lcla_pool.pages);
2894 goto failure;
2895 }
2896
2897 if ((virt_to_phys((void *)page_list[i]) &
2898 (LCLA_ALIGNMENT - 1)) == 0)
2899 break;
2900 }
2901
2902 for (j = 0; j < i; j++)
2903 free_pages(page_list[j], base->lcla_pool.pages);
2904
2905 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2906 base->lcla_pool.base = (void *)page_list[i];
2907 } else {
2908 /*
2909 * After many attempts and no succees with finding the correct
2910 * alignment, try with allocating a big buffer.
2911 */
2912 dev_warn(base->dev,
2913 "[%s] Failed to get %d pages @ 18 bit align.\n",
2914 __func__, base->lcla_pool.pages);
2915 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2916 base->num_phy_chans +
2917 LCLA_ALIGNMENT,
2918 GFP_KERNEL);
2919 if (!base->lcla_pool.base_unaligned) {
2920 ret = -ENOMEM;
2921 goto failure;
2922 }
2923
2924 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2925 LCLA_ALIGNMENT);
2926 }
2927
2928 writel(virt_to_phys(base->lcla_pool.base),
2929 base->virtbase + D40_DREG_LCLA);
2930 failure:
2931 kfree(page_list);
2932 return ret;
2933 }
2934
2935 static int __init d40_probe(struct platform_device *pdev)
2936 {
2937 int err;
2938 int ret = -ENOENT;
2939 struct d40_base *base;
2940 struct resource *res = NULL;
2941 int num_reserved_chans;
2942 u32 val;
2943
2944 base = d40_hw_detect_init(pdev);
2945
2946 if (!base)
2947 goto failure;
2948
2949 num_reserved_chans = d40_phy_res_init(base);
2950
2951 platform_set_drvdata(pdev, base);
2952
2953 spin_lock_init(&base->interrupt_lock);
2954 spin_lock_init(&base->execmd_lock);
2955
2956 /* Get IO for logical channel parameter address */
2957 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2958 if (!res) {
2959 ret = -ENOENT;
2960 dev_err(&pdev->dev,
2961 "[%s] No \"lcpa\" memory resource\n",
2962 __func__);
2963 goto failure;
2964 }
2965 base->lcpa_size = resource_size(res);
2966 base->phy_lcpa = res->start;
2967
2968 if (request_mem_region(res->start, resource_size(res),
2969 D40_NAME " I/O lcpa") == NULL) {
2970 ret = -EBUSY;
2971 dev_err(&pdev->dev,
2972 "[%s] Failed to request LCPA region 0x%x-0x%x\n",
2973 __func__, res->start, res->end);
2974 goto failure;
2975 }
2976
2977 /* We make use of ESRAM memory for this. */
2978 val = readl(base->virtbase + D40_DREG_LCPA);
2979 if (res->start != val && val != 0) {
2980 dev_warn(&pdev->dev,
2981 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2982 __func__, val, res->start);
2983 } else
2984 writel(res->start, base->virtbase + D40_DREG_LCPA);
2985
2986 base->lcpa_base = ioremap(res->start, resource_size(res));
2987 if (!base->lcpa_base) {
2988 ret = -ENOMEM;
2989 dev_err(&pdev->dev,
2990 "[%s] Failed to ioremap LCPA region\n",
2991 __func__);
2992 goto failure;
2993 }
2994
2995 ret = d40_lcla_allocate(base);
2996 if (ret) {
2997 dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n",
2998 __func__);
2999 goto failure;
3000 }
3001
3002 spin_lock_init(&base->lcla_pool.lock);
3003
3004 base->irq = platform_get_irq(pdev, 0);
3005
3006 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
3007
3008 if (ret) {
3009 dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
3010 goto failure;
3011 }
3012
3013 err = d40_dmaengine_init(base, num_reserved_chans);
3014 if (err)
3015 goto failure;
3016
3017 d40_hw_init(base);
3018
3019 dev_info(base->dev, "initialized\n");
3020 return 0;
3021
3022 failure:
3023 if (base) {
3024 if (base->desc_slab)
3025 kmem_cache_destroy(base->desc_slab);
3026 if (base->virtbase)
3027 iounmap(base->virtbase);
3028 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3029 free_pages((unsigned long)base->lcla_pool.base,
3030 base->lcla_pool.pages);
3031
3032 kfree(base->lcla_pool.base_unaligned);
3033
3034 if (base->phy_lcpa)
3035 release_mem_region(base->phy_lcpa,
3036 base->lcpa_size);
3037 if (base->phy_start)
3038 release_mem_region(base->phy_start,
3039 base->phy_size);
3040 if (base->clk) {
3041 clk_disable(base->clk);
3042 clk_put(base->clk);
3043 }
3044
3045 kfree(base->lcla_pool.alloc_map);
3046 kfree(base->lookup_log_chans);
3047 kfree(base->lookup_phy_chans);
3048 kfree(base->phy_res);
3049 kfree(base);
3050 }
3051
3052 dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
3053 return ret;
3054 }
3055
3056 static struct platform_driver d40_driver = {
3057 .driver = {
3058 .owner = THIS_MODULE,
3059 .name = D40_NAME,
3060 },
3061 };
3062
3063 int __init stedma40_init(void)
3064 {
3065 return platform_driver_probe(&d40_driver, d40_probe);
3066 }
3067 arch_initcall(stedma40_init);
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree