?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf: Fix multi-event parsing bug
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / dma / ste_dma40.c
blob94ee15dd3aed7ca0c0dc55c0a06f4e12e7834097
1 /*
2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
6 * License terms: GNU General Public License (GPL) version 2
7 */
8
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
11 #include <linux/dmaengine.h>
12 #include <linux/platform_device.h>
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/err.h>
16
17 #include <plat/ste_dma40.h>
18
19 #include "ste_dma40_ll.h"
20
21 #define D40_NAME "dma40"
22
23 #define D40_PHY_CHAN -1
24
25 /* For masking out/in 2 bit channel positions */
26 #define D40_CHAN_POS(chan) (2 * (chan / 2))
27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
28
29 /* Maximum iterations taken before giving up suspending a channel */
30 #define D40_SUSPEND_MAX_IT 500
31
32 /* Hardware requirement on LCLA alignment */
33 #define LCLA_ALIGNMENT 0x40000
34
35 /* Max number of links per event group */
36 #define D40_LCLA_LINK_PER_EVENT_GRP 128
37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
38
39 /* Attempts before giving up to trying to get pages that are aligned */
40 #define MAX_LCLA_ALLOC_ATTEMPTS 256
41
42 /* Bit markings for allocation map */
43 #define D40_ALLOC_FREE (1 << 31)
44 #define D40_ALLOC_PHY (1 << 30)
45 #define D40_ALLOC_LOG_FREE 0
46
47 /* Hardware designer of the block */
48 #define D40_HW_DESIGNER 0x8
49
50 /**
51 * enum 40_command - The different commands and/or statuses.
52 *
53 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
54 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
55 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
56 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
57 */
58 enum d40_command {
59 D40_DMA_STOP = 0,
60 D40_DMA_RUN = 1,
61 D40_DMA_SUSPEND_REQ = 2,
62 D40_DMA_SUSPENDED = 3
63 };
64
65 /**
66 * struct d40_lli_pool - Structure for keeping LLIs in memory
67 *
68 * @base: Pointer to memory area when the pre_alloc_lli's are not large
69 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
70 * pre_alloc_lli is used.
71 * @dma_addr: DMA address, if mapped
72 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
73 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
74 * one buffer to one buffer.
75 */
76 struct d40_lli_pool {
77 void *base;
78 int size;
79 dma_addr_t dma_addr;
80 /* Space for dst and src, plus an extra for padding */
81 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
82 };
83
84 /**
85 * struct d40_desc - A descriptor is one DMA job.
86 *
87 * @lli_phy: LLI settings for physical channel. Both src and dst=
88 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
89 * lli_len equals one.
90 * @lli_log: Same as above but for logical channels.
91 * @lli_pool: The pool with two entries pre-allocated.
92 * @lli_len: Number of llis of current descriptor.
93 * @lli_current: Number of transferred llis.
94 * @lcla_alloc: Number of LCLA entries allocated.
95 * @txd: DMA engine struct. Used for among other things for communication
96 * during a transfer.
97 * @node: List entry.
98 * @is_in_client_list: true if the client owns this descriptor.
99 * the previous one.
100 *
101 * This descriptor is used for both logical and physical transfers.
102 */
103 struct d40_desc {
104 /* LLI physical */
105 struct d40_phy_lli_bidir lli_phy;
106 /* LLI logical */
107 struct d40_log_lli_bidir lli_log;
108
109 struct d40_lli_pool lli_pool;
110 int lli_len;
111 int lli_current;
112 int lcla_alloc;
113
114 struct dma_async_tx_descriptor txd;
115 struct list_head node;
116
117 bool is_in_client_list;
118 bool cyclic;
119 };
120
121 /**
122 * struct d40_lcla_pool - LCLA pool settings and data.
123 *
124 * @base: The virtual address of LCLA. 18 bit aligned.
125 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
126 * This pointer is only there for clean-up on error.
127 * @pages: The number of pages needed for all physical channels.
128 * Only used later for clean-up on error
129 * @lock: Lock to protect the content in this struct.
130 * @alloc_map: big map over which LCLA entry is own by which job.
131 */
132 struct d40_lcla_pool {
133 void *base;
134 dma_addr_t dma_addr;
135 void *base_unaligned;
136 int pages;
137 spinlock_t lock;
138 struct d40_desc **alloc_map;
139 };
140
141 /**
142 * struct d40_phy_res - struct for handling eventlines mapped to physical
143 * channels.
144 *
145 * @lock: A lock protection this entity.
146 * @num: The physical channel number of this entity.
147 * @allocated_src: Bit mapped to show which src event line's are mapped to
148 * this physical channel. Can also be free or physically allocated.
149 * @allocated_dst: Same as for src but is dst.
150 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
151 * event line number.
152 */
153 struct d40_phy_res {
154 spinlock_t lock;
155 int num;
156 u32 allocated_src;
157 u32 allocated_dst;
158 };
159
160 struct d40_base;
161
162 /**
163 * struct d40_chan - Struct that describes a channel.
164 *
165 * @lock: A spinlock to protect this struct.
166 * @log_num: The logical number, if any of this channel.
167 * @completed: Starts with 1, after first interrupt it is set to dma engine's
168 * current cookie.
169 * @pending_tx: The number of pending transfers. Used between interrupt handler
170 * and tasklet.
171 * @busy: Set to true when transfer is ongoing on this channel.
172 * @phy_chan: Pointer to physical channel which this instance runs on. If this
173 * point is NULL, then the channel is not allocated.
174 * @chan: DMA engine handle.
175 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
176 * transfer and call client callback.
177 * @client: Cliented owned descriptor list.
178 * @active: Active descriptor.
179 * @queue: Queued jobs.
180 * @dma_cfg: The client configuration of this dma channel.
181 * @configured: whether the dma_cfg configuration is valid
182 * @base: Pointer to the device instance struct.
183 * @src_def_cfg: Default cfg register setting for src.
184 * @dst_def_cfg: Default cfg register setting for dst.
185 * @log_def: Default logical channel settings.
186 * @lcla: Space for one dst src pair for logical channel transfers.
187 * @lcpa: Pointer to dst and src lcpa settings.
188 *
189 * This struct can either "be" a logical or a physical channel.
190 */
191 struct d40_chan {
192 spinlock_t lock;
193 int log_num;
194 /* ID of the most recent completed transfer */
195 int completed;
196 int pending_tx;
197 bool busy;
198 struct d40_phy_res *phy_chan;
199 struct dma_chan chan;
200 struct tasklet_struct tasklet;
201 struct list_head client;
202 struct list_head active;
203 struct list_head queue;
204 struct stedma40_chan_cfg dma_cfg;
205 bool configured;
206 struct d40_base *base;
207 /* Default register configurations */
208 u32 src_def_cfg;
209 u32 dst_def_cfg;
210 struct d40_def_lcsp log_def;
211 struct d40_log_lli_full *lcpa;
212 /* Runtime reconfiguration */
213 dma_addr_t runtime_addr;
214 enum dma_data_direction runtime_direction;
215 };
216
217 /**
218 * struct d40_base - The big global struct, one for each probe'd instance.
219 *
220 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
221 * @execmd_lock: Lock for execute command usage since several channels share
222 * the same physical register.
223 * @dev: The device structure.
224 * @virtbase: The virtual base address of the DMA's register.
225 * @rev: silicon revision detected.
226 * @clk: Pointer to the DMA clock structure.
227 * @phy_start: Physical memory start of the DMA registers.
228 * @phy_size: Size of the DMA register map.
229 * @irq: The IRQ number.
230 * @num_phy_chans: The number of physical channels. Read from HW. This
231 * is the number of available channels for this driver, not counting "Secure
232 * mode" allocated physical channels.
233 * @num_log_chans: The number of logical channels. Calculated from
234 * num_phy_chans.
235 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
236 * @dma_slave: dma_device channels that can do only do slave transfers.
237 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
238 * @log_chans: Room for all possible logical channels in system.
239 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
240 * to log_chans entries.
241 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
242 * to phy_chans entries.
243 * @plat_data: Pointer to provided platform_data which is the driver
244 * configuration.
245 * @phy_res: Vector containing all physical channels.
246 * @lcla_pool: lcla pool settings and data.
247 * @lcpa_base: The virtual mapped address of LCPA.
248 * @phy_lcpa: The physical address of the LCPA.
249 * @lcpa_size: The size of the LCPA area.
250 * @desc_slab: cache for descriptors.
251 */
252 struct d40_base {
253 spinlock_t interrupt_lock;
254 spinlock_t execmd_lock;
255 struct device *dev;
256 void __iomem *virtbase;
257 u8 rev:4;
258 struct clk *clk;
259 phys_addr_t phy_start;
260 resource_size_t phy_size;
261 int irq;
262 int num_phy_chans;
263 int num_log_chans;
264 struct dma_device dma_both;
265 struct dma_device dma_slave;
266 struct dma_device dma_memcpy;
267 struct d40_chan *phy_chans;
268 struct d40_chan *log_chans;
269 struct d40_chan **lookup_log_chans;
270 struct d40_chan **lookup_phy_chans;
271 struct stedma40_platform_data *plat_data;
272 /* Physical half channels */
273 struct d40_phy_res *phy_res;
274 struct d40_lcla_pool lcla_pool;
275 void *lcpa_base;
276 dma_addr_t phy_lcpa;
277 resource_size_t lcpa_size;
278 struct kmem_cache *desc_slab;
279 };
280
281 /**
282 * struct d40_interrupt_lookup - lookup table for interrupt handler
283 *
284 * @src: Interrupt mask register.
285 * @clr: Interrupt clear register.
286 * @is_error: true if this is an error interrupt.
287 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
288 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
289 */
290 struct d40_interrupt_lookup {
291 u32 src;
292 u32 clr;
293 bool is_error;
294 int offset;
295 };
296
297 /**
298 * struct d40_reg_val - simple lookup struct
299 *
300 * @reg: The register.
301 * @val: The value that belongs to the register in reg.
302 */
303 struct d40_reg_val {
304 unsigned int reg;
305 unsigned int val;
306 };
307
308 static struct device *chan2dev(struct d40_chan *d40c)
309 {
310 return &d40c->chan.dev->device;
311 }
312
313 static bool chan_is_physical(struct d40_chan *chan)
314 {
315 return chan->log_num == D40_PHY_CHAN;
316 }
317
318 static bool chan_is_logical(struct d40_chan *chan)
319 {
320 return !chan_is_physical(chan);
321 }
322
323 static void __iomem *chan_base(struct d40_chan *chan)
324 {
325 return chan->base->virtbase + D40_DREG_PCBASE +
326 chan->phy_chan->num * D40_DREG_PCDELTA;
327 }
328
329 #define d40_err(dev, format, arg...) \
330 dev_err(dev, "[%s] " format, __func__, ## arg)
331
332 #define chan_err(d40c, format, arg...) \
333 d40_err(chan2dev(d40c), format, ## arg)
334
335 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
336 int lli_len)
337 {
338 bool is_log = chan_is_logical(d40c);
339 u32 align;
340 void *base;
341
342 if (is_log)
343 align = sizeof(struct d40_log_lli);
344 else
345 align = sizeof(struct d40_phy_lli);
346
347 if (lli_len == 1) {
348 base = d40d->lli_pool.pre_alloc_lli;
349 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
350 d40d->lli_pool.base = NULL;
351 } else {
352 d40d->lli_pool.size = lli_len * 2 * align;
353
354 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
355 d40d->lli_pool.base = base;
356
357 if (d40d->lli_pool.base == NULL)
358 return -ENOMEM;
359 }
360
361 if (is_log) {
362 d40d->lli_log.src = PTR_ALIGN(base, align);
363 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
364
365 d40d->lli_pool.dma_addr = 0;
366 } else {
367 d40d->lli_phy.src = PTR_ALIGN(base, align);
368 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
369
370 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
371 d40d->lli_phy.src,
372 d40d->lli_pool.size,
373 DMA_TO_DEVICE);
374
375 if (dma_mapping_error(d40c->base->dev,
376 d40d->lli_pool.dma_addr)) {
377 kfree(d40d->lli_pool.base);
378 d40d->lli_pool.base = NULL;
379 d40d->lli_pool.dma_addr = 0;
380 return -ENOMEM;
381 }
382 }
383
384 return 0;
385 }
386
387 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
388 {
389 if (d40d->lli_pool.dma_addr)
390 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
391 d40d->lli_pool.size, DMA_TO_DEVICE);
392
393 kfree(d40d->lli_pool.base);
394 d40d->lli_pool.base = NULL;
395 d40d->lli_pool.size = 0;
396 d40d->lli_log.src = NULL;
397 d40d->lli_log.dst = NULL;
398 d40d->lli_phy.src = NULL;
399 d40d->lli_phy.dst = NULL;
400 }
401
402 static int d40_lcla_alloc_one(struct d40_chan *d40c,
403 struct d40_desc *d40d)
404 {
405 unsigned long flags;
406 int i;
407 int ret = -EINVAL;
408 int p;
409
410 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
411
412 p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
413
414 /*
415 * Allocate both src and dst at the same time, therefore the half
416 * start on 1 since 0 can't be used since zero is used as end marker.
417 */
418 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
419 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
420 d40c->base->lcla_pool.alloc_map[p + i] = d40d;
421 d40d->lcla_alloc++;
422 ret = i;
423 break;
424 }
425 }
426
427 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
428
429 return ret;
430 }
431
432 static int d40_lcla_free_all(struct d40_chan *d40c,
433 struct d40_desc *d40d)
434 {
435 unsigned long flags;
436 int i;
437 int ret = -EINVAL;
438
439 if (chan_is_physical(d40c))
440 return 0;
441
442 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
443
444 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
445 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
446 D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
447 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
448 D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
449 d40d->lcla_alloc--;
450 if (d40d->lcla_alloc == 0) {
451 ret = 0;
452 break;
453 }
454 }
455 }
456
457 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
458
459 return ret;
460
461 }
462
463 static void d40_desc_remove(struct d40_desc *d40d)
464 {
465 list_del(&d40d->node);
466 }
467
468 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
469 {
470 struct d40_desc *desc = NULL;
471
472 if (!list_empty(&d40c->client)) {
473 struct d40_desc *d;
474 struct d40_desc *_d;
475
476 list_for_each_entry_safe(d, _d, &d40c->client, node)
477 if (async_tx_test_ack(&d->txd)) {
478 d40_pool_lli_free(d40c, d);
479 d40_desc_remove(d);
480 desc = d;
481 memset(desc, 0, sizeof(*desc));
482 break;
483 }
484 }
485
486 if (!desc)
487 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
488
489 if (desc)
490 INIT_LIST_HEAD(&desc->node);
491
492 return desc;
493 }
494
495 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
496 {
497
498 d40_pool_lli_free(d40c, d40d);
499 d40_lcla_free_all(d40c, d40d);
500 kmem_cache_free(d40c->base->desc_slab, d40d);
501 }
502
503 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
504 {
505 list_add_tail(&desc->node, &d40c->active);
506 }
507
508 static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
509 {
510 struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
511 struct d40_phy_lli *lli_src = desc->lli_phy.src;
512 void __iomem *base = chan_base(chan);
513
514 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
515 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
516 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
517 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
518
519 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
520 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
521 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
522 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
523 }
524
525 static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
526 {
527 struct d40_lcla_pool *pool = &chan->base->lcla_pool;
528 struct d40_log_lli_bidir *lli = &desc->lli_log;
529 int lli_current = desc->lli_current;
530 int lli_len = desc->lli_len;
531 bool cyclic = desc->cyclic;
532 int curr_lcla = -EINVAL;
533 int first_lcla = 0;
534 bool linkback;
535
536 /*
537 * We may have partially running cyclic transfers, in case we did't get
538 * enough LCLA entries.
539 */
540 linkback = cyclic && lli_current == 0;
541
542 /*
543 * For linkback, we need one LCLA even with only one link, because we
544 * can't link back to the one in LCPA space
545 */
546 if (linkback || (lli_len - lli_current > 1)) {
547 curr_lcla = d40_lcla_alloc_one(chan, desc);
548 first_lcla = curr_lcla;
549 }
550
551 /*
552 * For linkback, we normally load the LCPA in the loop since we need to
553 * link it to the second LCLA and not the first. However, if we
554 * couldn't even get a first LCLA, then we have to run in LCPA and
555 * reload manually.
556 */
557 if (!linkback || curr_lcla == -EINVAL) {
558 unsigned int flags = 0;
559
560 if (curr_lcla == -EINVAL)
561 flags |= LLI_TERM_INT;
562
563 d40_log_lli_lcpa_write(chan->lcpa,
564 &lli->dst[lli_current],
565 &lli->src[lli_current],
566 curr_lcla,
567 flags);
568 lli_current++;
569 }
570
571 if (curr_lcla < 0)
572 goto out;
573
574 for (; lli_current < lli_len; lli_current++) {
575 unsigned int lcla_offset = chan->phy_chan->num * 1024 +
576 8 * curr_lcla * 2;
577 struct d40_log_lli *lcla = pool->base + lcla_offset;
578 unsigned int flags = 0;
579 int next_lcla;
580
581 if (lli_current + 1 < lli_len)
582 next_lcla = d40_lcla_alloc_one(chan, desc);
583 else
584 next_lcla = linkback ? first_lcla : -EINVAL;
585
586 if (cyclic || next_lcla == -EINVAL)
587 flags |= LLI_TERM_INT;
588
589 if (linkback && curr_lcla == first_lcla) {
590 /* First link goes in both LCPA and LCLA */
591 d40_log_lli_lcpa_write(chan->lcpa,
592 &lli->dst[lli_current],
593 &lli->src[lli_current],
594 next_lcla, flags);
595 }
596
597 /*
598 * One unused LCLA in the cyclic case if the very first
599 * next_lcla fails...
600 */
601 d40_log_lli_lcla_write(lcla,
602 &lli->dst[lli_current],
603 &lli->src[lli_current],
604 next_lcla, flags);
605
606 dma_sync_single_range_for_device(chan->base->dev,
607 pool->dma_addr, lcla_offset,
608 2 * sizeof(struct d40_log_lli),
609 DMA_TO_DEVICE);
610
611 curr_lcla = next_lcla;
612
613 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
614 lli_current++;
615 break;
616 }
617 }
618
619 out:
620 desc->lli_current = lli_current;
621 }
622
623 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
624 {
625 if (chan_is_physical(d40c)) {
626 d40_phy_lli_load(d40c, d40d);
627 d40d->lli_current = d40d->lli_len;
628 } else
629 d40_log_lli_to_lcxa(d40c, d40d);
630 }
631
632 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
633 {
634 struct d40_desc *d;
635
636 if (list_empty(&d40c->active))
637 return NULL;
638
639 d = list_first_entry(&d40c->active,
640 struct d40_desc,
641 node);
642 return d;
643 }
644
645 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
646 {
647 list_add_tail(&desc->node, &d40c->queue);
648 }
649
650 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
651 {
652 struct d40_desc *d;
653
654 if (list_empty(&d40c->queue))
655 return NULL;
656
657 d = list_first_entry(&d40c->queue,
658 struct d40_desc,
659 node);
660 return d;
661 }
662
663 static int d40_psize_2_burst_size(bool is_log, int psize)
664 {
665 if (is_log) {
666 if (psize == STEDMA40_PSIZE_LOG_1)
667 return 1;
668 } else {
669 if (psize == STEDMA40_PSIZE_PHY_1)
670 return 1;
671 }
672
673 return 2 << psize;
674 }
675
676 /*
677 * The dma only supports transmitting packages up to
678 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
679 * dma elements required to send the entire sg list
680 */
681 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
682 {
683 int dmalen;
684 u32 max_w = max(data_width1, data_width2);
685 u32 min_w = min(data_width1, data_width2);
686 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
687
688 if (seg_max > STEDMA40_MAX_SEG_SIZE)
689 seg_max -= (1 << max_w);
690
691 if (!IS_ALIGNED(size, 1 << max_w))
692 return -EINVAL;
693
694 if (size <= seg_max)
695 dmalen = 1;
696 else {
697 dmalen = size / seg_max;
698 if (dmalen * seg_max < size)
699 dmalen++;
700 }
701 return dmalen;
702 }
703
704 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
705 u32 data_width1, u32 data_width2)
706 {
707 struct scatterlist *sg;
708 int i;
709 int len = 0;
710 int ret;
711
712 for_each_sg(sgl, sg, sg_len, i) {
713 ret = d40_size_2_dmalen(sg_dma_len(sg),
714 data_width1, data_width2);
715 if (ret < 0)
716 return ret;
717 len += ret;
718 }
719 return len;
720 }
721
722 /* Support functions for logical channels */
723
724 static int d40_channel_execute_command(struct d40_chan *d40c,
725 enum d40_command command)
726 {
727 u32 status;
728 int i;
729 void __iomem *active_reg;
730 int ret = 0;
731 unsigned long flags;
732 u32 wmask;
733
734 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
735
736 if (d40c->phy_chan->num % 2 == 0)
737 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
738 else
739 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
740
741 if (command == D40_DMA_SUSPEND_REQ) {
742 status = (readl(active_reg) &
743 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
744 D40_CHAN_POS(d40c->phy_chan->num);
745
746 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
747 goto done;
748 }
749
750 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
751 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
752 active_reg);
753
754 if (command == D40_DMA_SUSPEND_REQ) {
755
756 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
757 status = (readl(active_reg) &
758 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
759 D40_CHAN_POS(d40c->phy_chan->num);
760
761 cpu_relax();
762 /*
763 * Reduce the number of bus accesses while
764 * waiting for the DMA to suspend.
765 */
766 udelay(3);
767
768 if (status == D40_DMA_STOP ||
769 status == D40_DMA_SUSPENDED)
770 break;
771 }
772
773 if (i == D40_SUSPEND_MAX_IT) {
774 chan_err(d40c,
775 "unable to suspend the chl %d (log: %d) status %x\n",
776 d40c->phy_chan->num, d40c->log_num,
777 status);
778 dump_stack();
779 ret = -EBUSY;
780 }
781
782 }
783 done:
784 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
785 return ret;
786 }
787
788 static void d40_term_all(struct d40_chan *d40c)
789 {
790 struct d40_desc *d40d;
791
792 /* Release active descriptors */
793 while ((d40d = d40_first_active_get(d40c))) {
794 d40_desc_remove(d40d);
795 d40_desc_free(d40c, d40d);
796 }
797
798 /* Release queued descriptors waiting for transfer */
799 while ((d40d = d40_first_queued(d40c))) {
800 d40_desc_remove(d40d);
801 d40_desc_free(d40c, d40d);
802 }
803
804
805 d40c->pending_tx = 0;
806 d40c->busy = false;
807 }
808
809 static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
810 u32 event, int reg)
811 {
812 void __iomem *addr = chan_base(d40c) + reg;
813 int tries;
814
815 if (!enable) {
816 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
817 | ~D40_EVENTLINE_MASK(event), addr);
818 return;
819 }
820
821 /*
822 * The hardware sometimes doesn't register the enable when src and dst
823 * event lines are active on the same logical channel. Retry to ensure
824 * it does. Usually only one retry is sufficient.
825 */
826 tries = 100;
827 while (--tries) {
828 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
829 | ~D40_EVENTLINE_MASK(event), addr);
830
831 if (readl(addr) & D40_EVENTLINE_MASK(event))
832 break;
833 }
834
835 if (tries != 99)
836 dev_dbg(chan2dev(d40c),
837 "[%s] workaround enable S%cLNK (%d tries)\n",
838 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
839 100 - tries);
840
841 WARN_ON(!tries);
842 }
843
844 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
845 {
846 unsigned long flags;
847
848 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
849
850 /* Enable event line connected to device (or memcpy) */
851 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
852 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
853 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
854
855 __d40_config_set_event(d40c, do_enable, event,
856 D40_CHAN_REG_SSLNK);
857 }
858
859 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
860 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
861
862 __d40_config_set_event(d40c, do_enable, event,
863 D40_CHAN_REG_SDLNK);
864 }
865
866 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
867 }
868
869 static u32 d40_chan_has_events(struct d40_chan *d40c)
870 {
871 void __iomem *chanbase = chan_base(d40c);
872 u32 val;
873
874 val = readl(chanbase + D40_CHAN_REG_SSLNK);
875 val |= readl(chanbase + D40_CHAN_REG_SDLNK);
876
877 return val;
878 }
879
880 static u32 d40_get_prmo(struct d40_chan *d40c)
881 {
882 static const unsigned int phy_map[] = {
883 [STEDMA40_PCHAN_BASIC_MODE]
884 = D40_DREG_PRMO_PCHAN_BASIC,
885 [STEDMA40_PCHAN_MODULO_MODE]
886 = D40_DREG_PRMO_PCHAN_MODULO,
887 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
888 = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
889 };
890 static const unsigned int log_map[] = {
891 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
892 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
893 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
894 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
895 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
896 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
897 };
898
899 if (chan_is_physical(d40c))
900 return phy_map[d40c->dma_cfg.mode_opt];
901 else
902 return log_map[d40c->dma_cfg.mode_opt];
903 }
904
905 static void d40_config_write(struct d40_chan *d40c)
906 {
907 u32 addr_base;
908 u32 var;
909
910 /* Odd addresses are even addresses + 4 */
911 addr_base = (d40c->phy_chan->num % 2) * 4;
912 /* Setup channel mode to logical or physical */
913 var = ((u32)(chan_is_logical(d40c)) + 1) <<
914 D40_CHAN_POS(d40c->phy_chan->num);
915 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
916
917 /* Setup operational mode option register */
918 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
919
920 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
921
922 if (chan_is_logical(d40c)) {
923 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
924 & D40_SREG_ELEM_LOG_LIDX_MASK;
925 void __iomem *chanbase = chan_base(d40c);
926
927 /* Set default config for CFG reg */
928 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
929 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
930
931 /* Set LIDX for lcla */
932 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
933 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
934 }
935 }
936
937 static u32 d40_residue(struct d40_chan *d40c)
938 {
939 u32 num_elt;
940
941 if (chan_is_logical(d40c))
942 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
943 >> D40_MEM_LCSP2_ECNT_POS;
944 else {
945 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
946 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
947 >> D40_SREG_ELEM_PHY_ECNT_POS;
948 }
949
950 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
951 }
952
953 static bool d40_tx_is_linked(struct d40_chan *d40c)
954 {
955 bool is_link;
956
957 if (chan_is_logical(d40c))
958 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
959 else
960 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
961 & D40_SREG_LNK_PHYS_LNK_MASK;
962
963 return is_link;
964 }
965
966 static int d40_pause(struct d40_chan *d40c)
967 {
968 int res = 0;
969 unsigned long flags;
970
971 if (!d40c->busy)
972 return 0;
973
974 spin_lock_irqsave(&d40c->lock, flags);
975
976 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
977 if (res == 0) {
978 if (chan_is_logical(d40c)) {
979 d40_config_set_event(d40c, false);
980 /* Resume the other logical channels if any */
981 if (d40_chan_has_events(d40c))
982 res = d40_channel_execute_command(d40c,
983 D40_DMA_RUN);
984 }
985 }
986
987 spin_unlock_irqrestore(&d40c->lock, flags);
988 return res;
989 }
990
991 static int d40_resume(struct d40_chan *d40c)
992 {
993 int res = 0;
994 unsigned long flags;
995
996 if (!d40c->busy)
997 return 0;
998
999 spin_lock_irqsave(&d40c->lock, flags);
1000
1001 if (d40c->base->rev == 0)
1002 if (chan_is_logical(d40c)) {
1003 res = d40_channel_execute_command(d40c,
1004 D40_DMA_SUSPEND_REQ);
1005 goto no_suspend;
1006 }
1007
1008 /* If bytes left to transfer or linked tx resume job */
1009 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
1010
1011 if (chan_is_logical(d40c))
1012 d40_config_set_event(d40c, true);
1013
1014 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1015 }
1016
1017 no_suspend:
1018 spin_unlock_irqrestore(&d40c->lock, flags);
1019 return res;
1020 }
1021
1022 static int d40_terminate_all(struct d40_chan *chan)
1023 {
1024 unsigned long flags;
1025 int ret = 0;
1026
1027 ret = d40_pause(chan);
1028 if (!ret && chan_is_physical(chan))
1029 ret = d40_channel_execute_command(chan, D40_DMA_STOP);
1030
1031 spin_lock_irqsave(&chan->lock, flags);
1032 d40_term_all(chan);
1033 spin_unlock_irqrestore(&chan->lock, flags);
1034
1035 return ret;
1036 }
1037
1038 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1039 {
1040 struct d40_chan *d40c = container_of(tx->chan,
1041 struct d40_chan,
1042 chan);
1043 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1044 unsigned long flags;
1045
1046 spin_lock_irqsave(&d40c->lock, flags);
1047
1048 d40c->chan.cookie++;
1049
1050 if (d40c->chan.cookie < 0)
1051 d40c->chan.cookie = 1;
1052
1053 d40d->txd.cookie = d40c->chan.cookie;
1054
1055 d40_desc_queue(d40c, d40d);
1056
1057 spin_unlock_irqrestore(&d40c->lock, flags);
1058
1059 return tx->cookie;
1060 }
1061
1062 static int d40_start(struct d40_chan *d40c)
1063 {
1064 if (d40c->base->rev == 0) {
1065 int err;
1066
1067 if (chan_is_logical(d40c)) {
1068 err = d40_channel_execute_command(d40c,
1069 D40_DMA_SUSPEND_REQ);
1070 if (err)
1071 return err;
1072 }
1073 }
1074
1075 if (chan_is_logical(d40c))
1076 d40_config_set_event(d40c, true);
1077
1078 return d40_channel_execute_command(d40c, D40_DMA_RUN);
1079 }
1080
1081 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1082 {
1083 struct d40_desc *d40d;
1084 int err;
1085
1086 /* Start queued jobs, if any */
1087 d40d = d40_first_queued(d40c);
1088
1089 if (d40d != NULL) {
1090 d40c->busy = true;
1091
1092 /* Remove from queue */
1093 d40_desc_remove(d40d);
1094
1095 /* Add to active queue */
1096 d40_desc_submit(d40c, d40d);
1097
1098 /* Initiate DMA job */
1099 d40_desc_load(d40c, d40d);
1100
1101 /* Start dma job */
1102 err = d40_start(d40c);
1103
1104 if (err)
1105 return NULL;
1106 }
1107
1108 return d40d;
1109 }
1110
1111 /* called from interrupt context */
1112 static void dma_tc_handle(struct d40_chan *d40c)
1113 {
1114 struct d40_desc *d40d;
1115
1116 /* Get first active entry from list */
1117 d40d = d40_first_active_get(d40c);
1118
1119 if (d40d == NULL)
1120 return;
1121
1122 if (d40d->cyclic) {
1123 /*
1124 * If this was a paritially loaded list, we need to reloaded
1125 * it, and only when the list is completed. We need to check
1126 * for done because the interrupt will hit for every link, and
1127 * not just the last one.
1128 */
1129 if (d40d->lli_current < d40d->lli_len
1130 && !d40_tx_is_linked(d40c)
1131 && !d40_residue(d40c)) {
1132 d40_lcla_free_all(d40c, d40d);
1133 d40_desc_load(d40c, d40d);
1134 (void) d40_start(d40c);
1135
1136 if (d40d->lli_current == d40d->lli_len)
1137 d40d->lli_current = 0;
1138 }
1139 } else {
1140 d40_lcla_free_all(d40c, d40d);
1141
1142 if (d40d->lli_current < d40d->lli_len) {
1143 d40_desc_load(d40c, d40d);
1144 /* Start dma job */
1145 (void) d40_start(d40c);
1146 return;
1147 }
1148
1149 if (d40_queue_start(d40c) == NULL)
1150 d40c->busy = false;
1151 }
1152
1153 d40c->pending_tx++;
1154 tasklet_schedule(&d40c->tasklet);
1155
1156 }
1157
1158 static void dma_tasklet(unsigned long data)
1159 {
1160 struct d40_chan *d40c = (struct d40_chan *) data;
1161 struct d40_desc *d40d;
1162 unsigned long flags;
1163 dma_async_tx_callback callback;
1164 void *callback_param;
1165
1166 spin_lock_irqsave(&d40c->lock, flags);
1167
1168 /* Get first active entry from list */
1169 d40d = d40_first_active_get(d40c);
1170 if (d40d == NULL)
1171 goto err;
1172
1173 if (!d40d->cyclic)
1174 d40c->completed = d40d->txd.cookie;
1175
1176 /*
1177 * If terminating a channel pending_tx is set to zero.
1178 * This prevents any finished active jobs to return to the client.
1179 */
1180 if (d40c->pending_tx == 0) {
1181 spin_unlock_irqrestore(&d40c->lock, flags);
1182 return;
1183 }
1184
1185 /* Callback to client */
1186 callback = d40d->txd.callback;
1187 callback_param = d40d->txd.callback_param;
1188
1189 if (!d40d->cyclic) {
1190 if (async_tx_test_ack(&d40d->txd)) {
1191 d40_pool_lli_free(d40c, d40d);
1192 d40_desc_remove(d40d);
1193 d40_desc_free(d40c, d40d);
1194 } else {
1195 if (!d40d->is_in_client_list) {
1196 d40_desc_remove(d40d);
1197 d40_lcla_free_all(d40c, d40d);
1198 list_add_tail(&d40d->node, &d40c->client);
1199 d40d->is_in_client_list = true;
1200 }
1201 }
1202 }
1203
1204 d40c->pending_tx--;
1205
1206 if (d40c->pending_tx)
1207 tasklet_schedule(&d40c->tasklet);
1208
1209 spin_unlock_irqrestore(&d40c->lock, flags);
1210
1211 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1212 callback(callback_param);
1213
1214 return;
1215
1216 err:
1217 /* Rescue manoeuvre if receiving double interrupts */
1218 if (d40c->pending_tx > 0)
1219 d40c->pending_tx--;
1220 spin_unlock_irqrestore(&d40c->lock, flags);
1221 }
1222
1223 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1224 {
1225 static const struct d40_interrupt_lookup il[] = {
1226 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
1227 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1228 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1229 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1230 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
1231 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
1232 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
1233 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
1234 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
1235 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
1236 };
1237
1238 int i;
1239 u32 regs[ARRAY_SIZE(il)];
1240 u32 idx;
1241 u32 row;
1242 long chan = -1;
1243 struct d40_chan *d40c;
1244 unsigned long flags;
1245 struct d40_base *base = data;
1246
1247 spin_lock_irqsave(&base->interrupt_lock, flags);
1248
1249 /* Read interrupt status of both logical and physical channels */
1250 for (i = 0; i < ARRAY_SIZE(il); i++)
1251 regs[i] = readl(base->virtbase + il[i].src);
1252
1253 for (;;) {
1254
1255 chan = find_next_bit((unsigned long *)regs,
1256 BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1257
1258 /* No more set bits found? */
1259 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1260 break;
1261
1262 row = chan / BITS_PER_LONG;
1263 idx = chan & (BITS_PER_LONG - 1);
1264
1265 /* ACK interrupt */
1266 writel(1 << idx, base->virtbase + il[row].clr);
1267
1268 if (il[row].offset == D40_PHY_CHAN)
1269 d40c = base->lookup_phy_chans[idx];
1270 else
1271 d40c = base->lookup_log_chans[il[row].offset + idx];
1272 spin_lock(&d40c->lock);
1273
1274 if (!il[row].is_error)
1275 dma_tc_handle(d40c);
1276 else
1277 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1278 chan, il[row].offset, idx);
1279
1280 spin_unlock(&d40c->lock);
1281 }
1282
1283 spin_unlock_irqrestore(&base->interrupt_lock, flags);
1284
1285 return IRQ_HANDLED;
1286 }
1287
1288 static int d40_validate_conf(struct d40_chan *d40c,
1289 struct stedma40_chan_cfg *conf)
1290 {
1291 int res = 0;
1292 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1293 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1294 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1295
1296 if (!conf->dir) {
1297 chan_err(d40c, "Invalid direction.\n");
1298 res = -EINVAL;
1299 }
1300
1301 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1302 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1303 d40c->runtime_addr == 0) {
1304
1305 chan_err(d40c, "Invalid TX channel address (%d)\n",
1306 conf->dst_dev_type);
1307 res = -EINVAL;
1308 }
1309
1310 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1311 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1312 d40c->runtime_addr == 0) {
1313 chan_err(d40c, "Invalid RX channel address (%d)\n",
1314 conf->src_dev_type);
1315 res = -EINVAL;
1316 }
1317
1318 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1319 dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1320 chan_err(d40c, "Invalid dst\n");
1321 res = -EINVAL;
1322 }
1323
1324 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1325 src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1326 chan_err(d40c, "Invalid src\n");
1327 res = -EINVAL;
1328 }
1329
1330 if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1331 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1332 chan_err(d40c, "No event line\n");
1333 res = -EINVAL;
1334 }
1335
1336 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1337 (src_event_group != dst_event_group)) {
1338 chan_err(d40c, "Invalid event group\n");
1339 res = -EINVAL;
1340 }
1341
1342 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1343 /*
1344 * DMAC HW supports it. Will be added to this driver,
1345 * in case any dma client requires it.
1346 */
1347 chan_err(d40c, "periph to periph not supported\n");
1348 res = -EINVAL;
1349 }
1350
1351 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1352 (1 << conf->src_info.data_width) !=
1353 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1354 (1 << conf->dst_info.data_width)) {
1355 /*
1356 * The DMAC hardware only supports
1357 * src (burst x width) == dst (burst x width)
1358 */
1359
1360 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1361 res = -EINVAL;
1362 }
1363
1364 return res;
1365 }
1366
1367 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1368 int log_event_line, bool is_log)
1369 {
1370 unsigned long flags;
1371 spin_lock_irqsave(&phy->lock, flags);
1372 if (!is_log) {
1373 /* Physical interrupts are masked per physical full channel */
1374 if (phy->allocated_src == D40_ALLOC_FREE &&
1375 phy->allocated_dst == D40_ALLOC_FREE) {
1376 phy->allocated_dst = D40_ALLOC_PHY;
1377 phy->allocated_src = D40_ALLOC_PHY;
1378 goto found;
1379 } else
1380 goto not_found;
1381 }
1382
1383 /* Logical channel */
1384 if (is_src) {
1385 if (phy->allocated_src == D40_ALLOC_PHY)
1386 goto not_found;
1387
1388 if (phy->allocated_src == D40_ALLOC_FREE)
1389 phy->allocated_src = D40_ALLOC_LOG_FREE;
1390
1391 if (!(phy->allocated_src & (1 << log_event_line))) {
1392 phy->allocated_src |= 1 << log_event_line;
1393 goto found;
1394 } else
1395 goto not_found;
1396 } else {
1397 if (phy->allocated_dst == D40_ALLOC_PHY)
1398 goto not_found;
1399
1400 if (phy->allocated_dst == D40_ALLOC_FREE)
1401 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1402
1403 if (!(phy->allocated_dst & (1 << log_event_line))) {
1404 phy->allocated_dst |= 1 << log_event_line;
1405 goto found;
1406 } else
1407 goto not_found;
1408 }
1409
1410 not_found:
1411 spin_unlock_irqrestore(&phy->lock, flags);
1412 return false;
1413 found:
1414 spin_unlock_irqrestore(&phy->lock, flags);
1415 return true;
1416 }
1417
1418 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1419 int log_event_line)
1420 {
1421 unsigned long flags;
1422 bool is_free = false;
1423
1424 spin_lock_irqsave(&phy->lock, flags);
1425 if (!log_event_line) {
1426 phy->allocated_dst = D40_ALLOC_FREE;
1427 phy->allocated_src = D40_ALLOC_FREE;
1428 is_free = true;
1429 goto out;
1430 }
1431
1432 /* Logical channel */
1433 if (is_src) {
1434 phy->allocated_src &= ~(1 << log_event_line);
1435 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1436 phy->allocated_src = D40_ALLOC_FREE;
1437 } else {
1438 phy->allocated_dst &= ~(1 << log_event_line);
1439 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1440 phy->allocated_dst = D40_ALLOC_FREE;
1441 }
1442
1443 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1444 D40_ALLOC_FREE);
1445
1446 out:
1447 spin_unlock_irqrestore(&phy->lock, flags);
1448
1449 return is_free;
1450 }
1451
1452 static int d40_allocate_channel(struct d40_chan *d40c)
1453 {
1454 int dev_type;
1455 int event_group;
1456 int event_line;
1457 struct d40_phy_res *phys;
1458 int i;
1459 int j;
1460 int log_num;
1461 bool is_src;
1462 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1463
1464 phys = d40c->base->phy_res;
1465
1466 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1467 dev_type = d40c->dma_cfg.src_dev_type;
1468 log_num = 2 * dev_type;
1469 is_src = true;
1470 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1471 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1472 /* dst event lines are used for logical memcpy */
1473 dev_type = d40c->dma_cfg.dst_dev_type;
1474 log_num = 2 * dev_type + 1;
1475 is_src = false;
1476 } else
1477 return -EINVAL;
1478
1479 event_group = D40_TYPE_TO_GROUP(dev_type);
1480 event_line = D40_TYPE_TO_EVENT(dev_type);
1481
1482 if (!is_log) {
1483 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1484 /* Find physical half channel */
1485 for (i = 0; i < d40c->base->num_phy_chans; i++) {
1486
1487 if (d40_alloc_mask_set(&phys[i], is_src,
1488 0, is_log))
1489 goto found_phy;
1490 }
1491 } else
1492 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1493 int phy_num = j + event_group * 2;
1494 for (i = phy_num; i < phy_num + 2; i++) {
1495 if (d40_alloc_mask_set(&phys[i],
1496 is_src,
1497 0,
1498 is_log))
1499 goto found_phy;
1500 }
1501 }
1502 return -EINVAL;
1503 found_phy:
1504 d40c->phy_chan = &phys[i];
1505 d40c->log_num = D40_PHY_CHAN;
1506 goto out;
1507 }
1508 if (dev_type == -1)
1509 return -EINVAL;
1510
1511 /* Find logical channel */
1512 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1513 int phy_num = j + event_group * 2;
1514 /*
1515 * Spread logical channels across all available physical rather
1516 * than pack every logical channel at the first available phy
1517 * channels.
1518 */
1519 if (is_src) {
1520 for (i = phy_num; i < phy_num + 2; i++) {
1521 if (d40_alloc_mask_set(&phys[i], is_src,
1522 event_line, is_log))
1523 goto found_log;
1524 }
1525 } else {
1526 for (i = phy_num + 1; i >= phy_num; i--) {
1527 if (d40_alloc_mask_set(&phys[i], is_src,
1528 event_line, is_log))
1529 goto found_log;
1530 }
1531 }
1532 }
1533 return -EINVAL;
1534
1535 found_log:
1536 d40c->phy_chan = &phys[i];
1537 d40c->log_num = log_num;
1538 out:
1539
1540 if (is_log)
1541 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1542 else
1543 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1544
1545 return 0;
1546
1547 }
1548
1549 static int d40_config_memcpy(struct d40_chan *d40c)
1550 {
1551 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1552
1553 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1554 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1555 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1556 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1557 memcpy[d40c->chan.chan_id];
1558
1559 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1560 dma_has_cap(DMA_SLAVE, cap)) {
1561 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1562 } else {
1563 chan_err(d40c, "No memcpy\n");
1564 return -EINVAL;
1565 }
1566
1567 return 0;
1568 }
1569
1570
1571 static int d40_free_dma(struct d40_chan *d40c)
1572 {
1573
1574 int res = 0;
1575 u32 event;
1576 struct d40_phy_res *phy = d40c->phy_chan;
1577 bool is_src;
1578 struct d40_desc *d;
1579 struct d40_desc *_d;
1580
1581
1582 /* Terminate all queued and active transfers */
1583 d40_term_all(d40c);
1584
1585 /* Release client owned descriptors */
1586 if (!list_empty(&d40c->client))
1587 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1588 d40_pool_lli_free(d40c, d);
1589 d40_desc_remove(d);
1590 d40_desc_free(d40c, d);
1591 }
1592
1593 if (phy == NULL) {
1594 chan_err(d40c, "phy == null\n");
1595 return -EINVAL;
1596 }
1597
1598 if (phy->allocated_src == D40_ALLOC_FREE &&
1599 phy->allocated_dst == D40_ALLOC_FREE) {
1600 chan_err(d40c, "channel already free\n");
1601 return -EINVAL;
1602 }
1603
1604 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1605 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1606 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1607 is_src = false;
1608 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1609 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1610 is_src = true;
1611 } else {
1612 chan_err(d40c, "Unknown direction\n");
1613 return -EINVAL;
1614 }
1615
1616 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1617 if (res) {
1618 chan_err(d40c, "suspend failed\n");
1619 return res;
1620 }
1621
1622 if (chan_is_logical(d40c)) {
1623 /* Release logical channel, deactivate the event line */
1624
1625 d40_config_set_event(d40c, false);
1626 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1627
1628 /*
1629 * Check if there are more logical allocation
1630 * on this phy channel.
1631 */
1632 if (!d40_alloc_mask_free(phy, is_src, event)) {
1633 /* Resume the other logical channels if any */
1634 if (d40_chan_has_events(d40c)) {
1635 res = d40_channel_execute_command(d40c,
1636 D40_DMA_RUN);
1637 if (res) {
1638 chan_err(d40c,
1639 "Executing RUN command\n");
1640 return res;
1641 }
1642 }
1643 return 0;
1644 }
1645 } else {
1646 (void) d40_alloc_mask_free(phy, is_src, 0);
1647 }
1648
1649 /* Release physical channel */
1650 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1651 if (res) {
1652 chan_err(d40c, "Failed to stop channel\n");
1653 return res;
1654 }
1655 d40c->phy_chan = NULL;
1656 d40c->configured = false;
1657 d40c->base->lookup_phy_chans[phy->num] = NULL;
1658
1659 return 0;
1660 }
1661
1662 static bool d40_is_paused(struct d40_chan *d40c)
1663 {
1664 void __iomem *chanbase = chan_base(d40c);
1665 bool is_paused = false;
1666 unsigned long flags;
1667 void __iomem *active_reg;
1668 u32 status;
1669 u32 event;
1670
1671 spin_lock_irqsave(&d40c->lock, flags);
1672
1673 if (chan_is_physical(d40c)) {
1674 if (d40c->phy_chan->num % 2 == 0)
1675 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1676 else
1677 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1678
1679 status = (readl(active_reg) &
1680 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1681 D40_CHAN_POS(d40c->phy_chan->num);
1682 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1683 is_paused = true;
1684
1685 goto _exit;
1686 }
1687
1688 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1689 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1690 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1691 status = readl(chanbase + D40_CHAN_REG_SDLNK);
1692 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1693 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1694 status = readl(chanbase + D40_CHAN_REG_SSLNK);
1695 } else {
1696 chan_err(d40c, "Unknown direction\n");
1697 goto _exit;
1698 }
1699
1700 status = (status & D40_EVENTLINE_MASK(event)) >>
1701 D40_EVENTLINE_POS(event);
1702
1703 if (status != D40_DMA_RUN)
1704 is_paused = true;
1705 _exit:
1706 spin_unlock_irqrestore(&d40c->lock, flags);
1707 return is_paused;
1708
1709 }
1710
1711
1712 static u32 stedma40_residue(struct dma_chan *chan)
1713 {
1714 struct d40_chan *d40c =
1715 container_of(chan, struct d40_chan, chan);
1716 u32 bytes_left;
1717 unsigned long flags;
1718
1719 spin_lock_irqsave(&d40c->lock, flags);
1720 bytes_left = d40_residue(d40c);
1721 spin_unlock_irqrestore(&d40c->lock, flags);
1722
1723 return bytes_left;
1724 }
1725
1726 static int
1727 d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
1728 struct scatterlist *sg_src, struct scatterlist *sg_dst,
1729 unsigned int sg_len, dma_addr_t src_dev_addr,
1730 dma_addr_t dst_dev_addr)
1731 {
1732 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1733 struct stedma40_half_channel_info *src_info = &cfg->src_info;
1734 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1735 int ret;
1736
1737 ret = d40_log_sg_to_lli(sg_src, sg_len,
1738 src_dev_addr,
1739 desc->lli_log.src,
1740 chan->log_def.lcsp1,
1741 src_info->data_width,
1742 dst_info->data_width);
1743
1744 ret = d40_log_sg_to_lli(sg_dst, sg_len,
1745 dst_dev_addr,
1746 desc->lli_log.dst,
1747 chan->log_def.lcsp3,
1748 dst_info->data_width,
1749 src_info->data_width);
1750
1751 return ret < 0 ? ret : 0;
1752 }
1753
1754 static int
1755 d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
1756 struct scatterlist *sg_src, struct scatterlist *sg_dst,
1757 unsigned int sg_len, dma_addr_t src_dev_addr,
1758 dma_addr_t dst_dev_addr)
1759 {
1760 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1761 struct stedma40_half_channel_info *src_info = &cfg->src_info;
1762 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1763 unsigned long flags = 0;
1764 int ret;
1765
1766 if (desc->cyclic)
1767 flags |= LLI_CYCLIC | LLI_TERM_INT;
1768
1769 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
1770 desc->lli_phy.src,
1771 virt_to_phys(desc->lli_phy.src),
1772 chan->src_def_cfg,
1773 src_info, dst_info, flags);
1774
1775 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
1776 desc->lli_phy.dst,
1777 virt_to_phys(desc->lli_phy.dst),
1778 chan->dst_def_cfg,
1779 dst_info, src_info, flags);
1780
1781 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
1782 desc->lli_pool.size, DMA_TO_DEVICE);
1783
1784 return ret < 0 ? ret : 0;
1785 }
1786
1787
1788 static struct d40_desc *
1789 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
1790 unsigned int sg_len, unsigned long dma_flags)
1791 {
1792 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1793 struct d40_desc *desc;
1794 int ret;
1795
1796 desc = d40_desc_get(chan);
1797 if (!desc)
1798 return NULL;
1799
1800 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
1801 cfg->dst_info.data_width);
1802 if (desc->lli_len < 0) {
1803 chan_err(chan, "Unaligned size\n");
1804 goto err;
1805 }
1806
1807 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
1808 if (ret < 0) {
1809 chan_err(chan, "Could not allocate lli\n");
1810 goto err;
1811 }
1812
1813
1814 desc->lli_current = 0;
1815 desc->txd.flags = dma_flags;
1816 desc->txd.tx_submit = d40_tx_submit;
1817
1818 dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
1819
1820 return desc;
1821
1822 err:
1823 d40_desc_free(chan, desc);
1824 return NULL;
1825 }
1826
1827 static dma_addr_t
1828 d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
1829 {
1830 struct stedma40_platform_data *plat = chan->base->plat_data;
1831 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1832 dma_addr_t addr;
1833
1834 if (chan->runtime_addr)
1835 return chan->runtime_addr;
1836
1837 if (direction == DMA_FROM_DEVICE)
1838 addr = plat->dev_rx[cfg->src_dev_type];
1839 else if (direction == DMA_TO_DEVICE)
1840 addr = plat->dev_tx[cfg->dst_dev_type];
1841
1842 return addr;
1843 }
1844
1845 static struct dma_async_tx_descriptor *
1846 d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
1847 struct scatterlist *sg_dst, unsigned int sg_len,
1848 enum dma_data_direction direction, unsigned long dma_flags)
1849 {
1850 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
1851 dma_addr_t src_dev_addr = 0;
1852 dma_addr_t dst_dev_addr = 0;
1853 struct d40_desc *desc;
1854 unsigned long flags;
1855 int ret;
1856
1857 if (!chan->phy_chan) {
1858 chan_err(chan, "Cannot prepare unallocated channel\n");
1859 return NULL;
1860 }
1861
1862
1863 spin_lock_irqsave(&chan->lock, flags);
1864
1865 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
1866 if (desc == NULL)
1867 goto err;
1868
1869 if (sg_next(&sg_src[sg_len - 1]) == sg_src)
1870 desc->cyclic = true;
1871
1872 if (direction != DMA_NONE) {
1873 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
1874
1875 if (direction == DMA_FROM_DEVICE)
1876 src_dev_addr = dev_addr;
1877 else if (direction == DMA_TO_DEVICE)
1878 dst_dev_addr = dev_addr;
1879 }
1880
1881 if (chan_is_logical(chan))
1882 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
1883 sg_len, src_dev_addr, dst_dev_addr);
1884 else
1885 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
1886 sg_len, src_dev_addr, dst_dev_addr);
1887
1888 if (ret) {
1889 chan_err(chan, "Failed to prepare %s sg job: %d\n",
1890 chan_is_logical(chan) ? "log" : "phy", ret);
1891 goto err;
1892 }
1893
1894 spin_unlock_irqrestore(&chan->lock, flags);
1895
1896 return &desc->txd;
1897
1898 err:
1899 if (desc)
1900 d40_desc_free(chan, desc);
1901 spin_unlock_irqrestore(&chan->lock, flags);
1902 return NULL;
1903 }
1904
1905 bool stedma40_filter(struct dma_chan *chan, void *data)
1906 {
1907 struct stedma40_chan_cfg *info = data;
1908 struct d40_chan *d40c =
1909 container_of(chan, struct d40_chan, chan);
1910 int err;
1911
1912 if (data) {
1913 err = d40_validate_conf(d40c, info);
1914 if (!err)
1915 d40c->dma_cfg = *info;
1916 } else
1917 err = d40_config_memcpy(d40c);
1918
1919 if (!err)
1920 d40c->configured = true;
1921
1922 return err == 0;
1923 }
1924 EXPORT_SYMBOL(stedma40_filter);
1925
1926 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
1927 {
1928 bool realtime = d40c->dma_cfg.realtime;
1929 bool highprio = d40c->dma_cfg.high_priority;
1930 u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
1931 u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
1932 u32 event = D40_TYPE_TO_EVENT(dev_type);
1933 u32 group = D40_TYPE_TO_GROUP(dev_type);
1934 u32 bit = 1 << event;
1935
1936 /* Destination event lines are stored in the upper halfword */
1937 if (!src)
1938 bit <<= 16;
1939
1940 writel(bit, d40c->base->virtbase + prioreg + group * 4);
1941 writel(bit, d40c->base->virtbase + rtreg + group * 4);
1942 }
1943
1944 static void d40_set_prio_realtime(struct d40_chan *d40c)
1945 {
1946 if (d40c->base->rev < 3)
1947 return;
1948
1949 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
1950 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1951 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
1952
1953 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
1954 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1955 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
1956 }
1957
1958 /* DMA ENGINE functions */
1959 static int d40_alloc_chan_resources(struct dma_chan *chan)
1960 {
1961 int err;
1962 unsigned long flags;
1963 struct d40_chan *d40c =
1964 container_of(chan, struct d40_chan, chan);
1965 bool is_free_phy;
1966 spin_lock_irqsave(&d40c->lock, flags);
1967
1968 d40c->completed = chan->cookie = 1;
1969
1970 /* If no dma configuration is set use default configuration (memcpy) */
1971 if (!d40c->configured) {
1972 err = d40_config_memcpy(d40c);
1973 if (err) {
1974 chan_err(d40c, "Failed to configure memcpy channel\n");
1975 goto fail;
1976 }
1977 }
1978 is_free_phy = (d40c->phy_chan == NULL);
1979
1980 err = d40_allocate_channel(d40c);
1981 if (err) {
1982 chan_err(d40c, "Failed to allocate channel\n");
1983 goto fail;
1984 }
1985
1986 /* Fill in basic CFG register values */
1987 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1988 &d40c->dst_def_cfg, chan_is_logical(d40c));
1989
1990 d40_set_prio_realtime(d40c);
1991
1992 if (chan_is_logical(d40c)) {
1993 d40_log_cfg(&d40c->dma_cfg,
1994 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1995
1996 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1997 d40c->lcpa = d40c->base->lcpa_base +
1998 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1999 else
2000 d40c->lcpa = d40c->base->lcpa_base +
2001 d40c->dma_cfg.dst_dev_type *
2002 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
2003 }
2004
2005 /*
2006 * Only write channel configuration to the DMA if the physical
2007 * resource is free. In case of multiple logical channels
2008 * on the same physical resource, only the first write is necessary.
2009 */
2010 if (is_free_phy)
2011 d40_config_write(d40c);
2012 fail:
2013 spin_unlock_irqrestore(&d40c->lock, flags);
2014 return err;
2015 }
2016
2017 static void d40_free_chan_resources(struct dma_chan *chan)
2018 {
2019 struct d40_chan *d40c =
2020 container_of(chan, struct d40_chan, chan);
2021 int err;
2022 unsigned long flags;
2023
2024 if (d40c->phy_chan == NULL) {
2025 chan_err(d40c, "Cannot free unallocated channel\n");
2026 return;
2027 }
2028
2029
2030 spin_lock_irqsave(&d40c->lock, flags);
2031
2032 err = d40_free_dma(d40c);
2033
2034 if (err)
2035 chan_err(d40c, "Failed to free channel\n");
2036 spin_unlock_irqrestore(&d40c->lock, flags);
2037 }
2038
2039 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2040 dma_addr_t dst,
2041 dma_addr_t src,
2042 size_t size,
2043 unsigned long dma_flags)
2044 {
2045 struct scatterlist dst_sg;
2046 struct scatterlist src_sg;
2047
2048 sg_init_table(&dst_sg, 1);
2049 sg_init_table(&src_sg, 1);
2050
2051 sg_dma_address(&dst_sg) = dst;
2052 sg_dma_address(&src_sg) = src;
2053
2054 sg_dma_len(&dst_sg) = size;
2055 sg_dma_len(&src_sg) = size;
2056
2057 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
2058 }
2059
2060 static struct dma_async_tx_descriptor *
2061 d40_prep_memcpy_sg(struct dma_chan *chan,
2062 struct scatterlist *dst_sg, unsigned int dst_nents,
2063 struct scatterlist *src_sg, unsigned int src_nents,
2064 unsigned long dma_flags)
2065 {
2066 if (dst_nents != src_nents)
2067 return NULL;
2068
2069 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
2070 }
2071
2072 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2073 struct scatterlist *sgl,
2074 unsigned int sg_len,
2075 enum dma_data_direction direction,
2076 unsigned long dma_flags)
2077 {
2078 if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
2079 return NULL;
2080
2081 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
2082 }
2083
2084 static struct dma_async_tx_descriptor *
2085 dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2086 size_t buf_len, size_t period_len,
2087 enum dma_data_direction direction)
2088 {
2089 unsigned int periods = buf_len / period_len;
2090 struct dma_async_tx_descriptor *txd;
2091 struct scatterlist *sg;
2092 int i;
2093
2094 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_KERNEL);
2095 for (i = 0; i < periods; i++) {
2096 sg_dma_address(&sg[i]) = dma_addr;
2097 sg_dma_len(&sg[i]) = period_len;
2098 dma_addr += period_len;
2099 }
2100
2101 sg[periods].offset = 0;
2102 sg[periods].length = 0;
2103 sg[periods].page_link =
2104 ((unsigned long)sg | 0x01) & ~0x02;
2105
2106 txd = d40_prep_sg(chan, sg, sg, periods, direction,
2107 DMA_PREP_INTERRUPT);
2108
2109 kfree(sg);
2110
2111 return txd;
2112 }
2113
2114 static enum dma_status d40_tx_status(struct dma_chan *chan,
2115 dma_cookie_t cookie,
2116 struct dma_tx_state *txstate)
2117 {
2118 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2119 dma_cookie_t last_used;
2120 dma_cookie_t last_complete;
2121 int ret;
2122
2123 if (d40c->phy_chan == NULL) {
2124 chan_err(d40c, "Cannot read status of unallocated channel\n");
2125 return -EINVAL;
2126 }
2127
2128 last_complete = d40c->completed;
2129 last_used = chan->cookie;
2130
2131 if (d40_is_paused(d40c))
2132 ret = DMA_PAUSED;
2133 else
2134 ret = dma_async_is_complete(cookie, last_complete, last_used);
2135
2136 dma_set_tx_state(txstate, last_complete, last_used,
2137 stedma40_residue(chan));
2138
2139 return ret;
2140 }
2141
2142 static void d40_issue_pending(struct dma_chan *chan)
2143 {
2144 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2145 unsigned long flags;
2146
2147 if (d40c->phy_chan == NULL) {
2148 chan_err(d40c, "Channel is not allocated!\n");
2149 return;
2150 }
2151
2152 spin_lock_irqsave(&d40c->lock, flags);
2153
2154 /* Busy means that pending jobs are already being processed */
2155 if (!d40c->busy)
2156 (void) d40_queue_start(d40c);
2157
2158 spin_unlock_irqrestore(&d40c->lock, flags);
2159 }
2160
2161 /* Runtime reconfiguration extension */
2162 static void d40_set_runtime_config(struct dma_chan *chan,
2163 struct dma_slave_config *config)
2164 {
2165 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2166 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2167 enum dma_slave_buswidth config_addr_width;
2168 dma_addr_t config_addr;
2169 u32 config_maxburst;
2170 enum stedma40_periph_data_width addr_width;
2171 int psize;
2172
2173 if (config->direction == DMA_FROM_DEVICE) {
2174 dma_addr_t dev_addr_rx =
2175 d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2176
2177 config_addr = config->src_addr;
2178 if (dev_addr_rx)
2179 dev_dbg(d40c->base->dev,
2180 "channel has a pre-wired RX address %08x "
2181 "overriding with %08x\n",
2182 dev_addr_rx, config_addr);
2183 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2184 dev_dbg(d40c->base->dev,
2185 "channel was not configured for peripheral "
2186 "to memory transfer (%d) overriding\n",
2187 cfg->dir);
2188 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2189
2190 config_addr_width = config->src_addr_width;
2191 config_maxburst = config->src_maxburst;
2192
2193 } else if (config->direction == DMA_TO_DEVICE) {
2194 dma_addr_t dev_addr_tx =
2195 d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2196
2197 config_addr = config->dst_addr;
2198 if (dev_addr_tx)
2199 dev_dbg(d40c->base->dev,
2200 "channel has a pre-wired TX address %08x "
2201 "overriding with %08x\n",
2202 dev_addr_tx, config_addr);
2203 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2204 dev_dbg(d40c->base->dev,
2205 "channel was not configured for memory "
2206 "to peripheral transfer (%d) overriding\n",
2207 cfg->dir);
2208 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2209
2210 config_addr_width = config->dst_addr_width;
2211 config_maxburst = config->dst_maxburst;
2212
2213 } else {
2214 dev_err(d40c->base->dev,
2215 "unrecognized channel direction %d\n",
2216 config->direction);
2217 return;
2218 }
2219
2220 switch (config_addr_width) {
2221 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2222 addr_width = STEDMA40_BYTE_WIDTH;
2223 break;
2224 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2225 addr_width = STEDMA40_HALFWORD_WIDTH;
2226 break;
2227 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2228 addr_width = STEDMA40_WORD_WIDTH;
2229 break;
2230 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2231 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2232 break;
2233 default:
2234 dev_err(d40c->base->dev,
2235 "illegal peripheral address width "
2236 "requested (%d)\n",
2237 config->src_addr_width);
2238 return;
2239 }
2240
2241 if (chan_is_logical(d40c)) {
2242 if (config_maxburst >= 16)
2243 psize = STEDMA40_PSIZE_LOG_16;
2244 else if (config_maxburst >= 8)
2245 psize = STEDMA40_PSIZE_LOG_8;
2246 else if (config_maxburst >= 4)
2247 psize = STEDMA40_PSIZE_LOG_4;
2248 else
2249 psize = STEDMA40_PSIZE_LOG_1;
2250 } else {
2251 if (config_maxburst >= 16)
2252 psize = STEDMA40_PSIZE_PHY_16;
2253 else if (config_maxburst >= 8)
2254 psize = STEDMA40_PSIZE_PHY_8;
2255 else if (config_maxburst >= 4)
2256 psize = STEDMA40_PSIZE_PHY_4;
2257 else if (config_maxburst >= 2)
2258 psize = STEDMA40_PSIZE_PHY_2;
2259 else
2260 psize = STEDMA40_PSIZE_PHY_1;
2261 }
2262
2263 /* Set up all the endpoint configs */
2264 cfg->src_info.data_width = addr_width;
2265 cfg->src_info.psize = psize;
2266 cfg->src_info.big_endian = false;
2267 cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2268 cfg->dst_info.data_width = addr_width;
2269 cfg->dst_info.psize = psize;
2270 cfg->dst_info.big_endian = false;
2271 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2272
2273 /* Fill in register values */
2274 if (chan_is_logical(d40c))
2275 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2276 else
2277 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2278 &d40c->dst_def_cfg, false);
2279
2280 /* These settings will take precedence later */
2281 d40c->runtime_addr = config_addr;
2282 d40c->runtime_direction = config->direction;
2283 dev_dbg(d40c->base->dev,
2284 "configured channel %s for %s, data width %d, "
2285 "maxburst %d bytes, LE, no flow control\n",
2286 dma_chan_name(chan),
2287 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2288 config_addr_width,
2289 config_maxburst);
2290 }
2291
2292 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2293 unsigned long arg)
2294 {
2295 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2296
2297 if (d40c->phy_chan == NULL) {
2298 chan_err(d40c, "Channel is not allocated!\n");
2299 return -EINVAL;
2300 }
2301
2302 switch (cmd) {
2303 case DMA_TERMINATE_ALL:
2304 return d40_terminate_all(d40c);
2305 case DMA_PAUSE:
2306 return d40_pause(d40c);
2307 case DMA_RESUME:
2308 return d40_resume(d40c);
2309 case DMA_SLAVE_CONFIG:
2310 d40_set_runtime_config(chan,
2311 (struct dma_slave_config *) arg);
2312 return 0;
2313 default:
2314 break;
2315 }
2316
2317 /* Other commands are unimplemented */
2318 return -ENXIO;
2319 }
2320
2321 /* Initialization functions */
2322
2323 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2324 struct d40_chan *chans, int offset,
2325 int num_chans)
2326 {
2327 int i = 0;
2328 struct d40_chan *d40c;
2329
2330 INIT_LIST_HEAD(&dma->channels);
2331
2332 for (i = offset; i < offset + num_chans; i++) {
2333 d40c = &chans[i];
2334 d40c->base = base;
2335 d40c->chan.device = dma;
2336
2337 spin_lock_init(&d40c->lock);
2338
2339 d40c->log_num = D40_PHY_CHAN;
2340
2341 INIT_LIST_HEAD(&d40c->active);
2342 INIT_LIST_HEAD(&d40c->queue);
2343 INIT_LIST_HEAD(&d40c->client);
2344
2345 tasklet_init(&d40c->tasklet, dma_tasklet,
2346 (unsigned long) d40c);
2347
2348 list_add_tail(&d40c->chan.device_node,
2349 &dma->channels);
2350 }
2351 }
2352
2353 static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2354 {
2355 if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2356 dev->device_prep_slave_sg = d40_prep_slave_sg;
2357
2358 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2359 dev->device_prep_dma_memcpy = d40_prep_memcpy;
2360
2361 /*
2362 * This controller can only access address at even
2363 * 32bit boundaries, i.e. 2^2
2364 */
2365 dev->copy_align = 2;
2366 }
2367
2368 if (dma_has_cap(DMA_SG, dev->cap_mask))
2369 dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2370
2371 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2372 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2373
2374 dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2375 dev->device_free_chan_resources = d40_free_chan_resources;
2376 dev->device_issue_pending = d40_issue_pending;
2377 dev->device_tx_status = d40_tx_status;
2378 dev->device_control = d40_control;
2379 dev->dev = base->dev;
2380 }
2381
2382 static int __init d40_dmaengine_init(struct d40_base *base,
2383 int num_reserved_chans)
2384 {
2385 int err ;
2386
2387 d40_chan_init(base, &base->dma_slave, base->log_chans,
2388 0, base->num_log_chans);
2389
2390 dma_cap_zero(base->dma_slave.cap_mask);
2391 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2392 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2393
2394 d40_ops_init(base, &base->dma_slave);
2395
2396 err = dma_async_device_register(&base->dma_slave);
2397
2398 if (err) {
2399 d40_err(base->dev, "Failed to register slave channels\n");
2400 goto failure1;
2401 }
2402
2403 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2404 base->num_log_chans, base->plat_data->memcpy_len);
2405
2406 dma_cap_zero(base->dma_memcpy.cap_mask);
2407 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2408 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
2409
2410 d40_ops_init(base, &base->dma_memcpy);
2411
2412 err = dma_async_device_register(&base->dma_memcpy);
2413
2414 if (err) {
2415 d40_err(base->dev,
2416 "Failed to regsiter memcpy only channels\n");
2417 goto failure2;
2418 }
2419
2420 d40_chan_init(base, &base->dma_both, base->phy_chans,
2421 0, num_reserved_chans);
2422
2423 dma_cap_zero(base->dma_both.cap_mask);
2424 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2425 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2426 dma_cap_set(DMA_SG, base->dma_both.cap_mask);
2427 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2428
2429 d40_ops_init(base, &base->dma_both);
2430 err = dma_async_device_register(&base->dma_both);
2431
2432 if (err) {
2433 d40_err(base->dev,
2434 "Failed to register logical and physical capable channels\n");
2435 goto failure3;
2436 }
2437 return 0;
2438 failure3:
2439 dma_async_device_unregister(&base->dma_memcpy);
2440 failure2:
2441 dma_async_device_unregister(&base->dma_slave);
2442 failure1:
2443 return err;
2444 }
2445
2446 /* Initialization functions. */
2447
2448 static int __init d40_phy_res_init(struct d40_base *base)
2449 {
2450 int i;
2451 int num_phy_chans_avail = 0;
2452 u32 val[2];
2453 int odd_even_bit = -2;
2454
2455 val[0] = readl(base->virtbase + D40_DREG_PRSME);
2456 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2457
2458 for (i = 0; i < base->num_phy_chans; i++) {
2459 base->phy_res[i].num = i;
2460 odd_even_bit += 2 * ((i % 2) == 0);
2461 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2462 /* Mark security only channels as occupied */
2463 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2464 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2465 } else {
2466 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2467 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2468 num_phy_chans_avail++;
2469 }
2470 spin_lock_init(&base->phy_res[i].lock);
2471 }
2472
2473 /* Mark disabled channels as occupied */
2474 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2475 int chan = base->plat_data->disabled_channels[i];
2476
2477 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2478 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2479 num_phy_chans_avail--;
2480 }
2481
2482 dev_info(base->dev, "%d of %d physical DMA channels available\n",
2483 num_phy_chans_avail, base->num_phy_chans);
2484
2485 /* Verify settings extended vs standard */
2486 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2487
2488 for (i = 0; i < base->num_phy_chans; i++) {
2489
2490 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2491 (val[0] & 0x3) != 1)
2492 dev_info(base->dev,
2493 "[%s] INFO: channel %d is misconfigured (%d)\n",
2494 __func__, i, val[0] & 0x3);
2495
2496 val[0] = val[0] >> 2;
2497 }
2498
2499 return num_phy_chans_avail;
2500 }
2501
2502 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2503 {
2504 static const struct d40_reg_val dma_id_regs[] = {
2505 /* Peripheral Id */
2506 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2507 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2508 /*
2509 * D40_DREG_PERIPHID2 Depends on HW revision:
2510 * DB8500ed has 0x0008,
2511 * ? has 0x0018,
2512 * DB8500v1 has 0x0028
2513 * DB8500v2 has 0x0038
2514 */
2515 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2516
2517 /* PCell Id */
2518 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2519 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2520 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2521 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2522 };
2523 struct stedma40_platform_data *plat_data;
2524 struct clk *clk = NULL;
2525 void __iomem *virtbase = NULL;
2526 struct resource *res = NULL;
2527 struct d40_base *base = NULL;
2528 int num_log_chans = 0;
2529 int num_phy_chans;
2530 int i;
2531 u32 val;
2532 u32 rev;
2533
2534 clk = clk_get(&pdev->dev, NULL);
2535
2536 if (IS_ERR(clk)) {
2537 d40_err(&pdev->dev, "No matching clock found\n");
2538 goto failure;
2539 }
2540
2541 clk_enable(clk);
2542
2543 /* Get IO for DMAC base address */
2544 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2545 if (!res)
2546 goto failure;
2547
2548 if (request_mem_region(res->start, resource_size(res),
2549 D40_NAME " I/O base") == NULL)
2550 goto failure;
2551
2552 virtbase = ioremap(res->start, resource_size(res));
2553 if (!virtbase)
2554 goto failure;
2555
2556 /* HW version check */
2557 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2558 if (dma_id_regs[i].val !=
2559 readl(virtbase + dma_id_regs[i].reg)) {
2560 d40_err(&pdev->dev,
2561 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2562 dma_id_regs[i].val,
2563 dma_id_regs[i].reg,
2564 readl(virtbase + dma_id_regs[i].reg));
2565 goto failure;
2566 }
2567 }
2568
2569 /* Get silicon revision and designer */
2570 val = readl(virtbase + D40_DREG_PERIPHID2);
2571
2572 if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2573 D40_HW_DESIGNER) {
2574 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2575 val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2576 D40_HW_DESIGNER);
2577 goto failure;
2578 }
2579
2580 rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2581 D40_DREG_PERIPHID2_REV_POS;
2582
2583 /* The number of physical channels on this HW */
2584 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2585
2586 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2587 rev, res->start);
2588
2589 plat_data = pdev->dev.platform_data;
2590
2591 /* Count the number of logical channels in use */
2592 for (i = 0; i < plat_data->dev_len; i++)
2593 if (plat_data->dev_rx[i] != 0)
2594 num_log_chans++;
2595
2596 for (i = 0; i < plat_data->dev_len; i++)
2597 if (plat_data->dev_tx[i] != 0)
2598 num_log_chans++;
2599
2600 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2601 (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2602 sizeof(struct d40_chan), GFP_KERNEL);
2603
2604 if (base == NULL) {
2605 d40_err(&pdev->dev, "Out of memory\n");
2606 goto failure;
2607 }
2608
2609 base->rev = rev;
2610 base->clk = clk;
2611 base->num_phy_chans = num_phy_chans;
2612 base->num_log_chans = num_log_chans;
2613 base->phy_start = res->start;
2614 base->phy_size = resource_size(res);
2615 base->virtbase = virtbase;
2616 base->plat_data = plat_data;
2617 base->dev = &pdev->dev;
2618 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2619 base->log_chans = &base->phy_chans[num_phy_chans];
2620
2621 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2622 GFP_KERNEL);
2623 if (!base->phy_res)
2624 goto failure;
2625
2626 base->lookup_phy_chans = kzalloc(num_phy_chans *
2627 sizeof(struct d40_chan *),
2628 GFP_KERNEL);
2629 if (!base->lookup_phy_chans)
2630 goto failure;
2631
2632 if (num_log_chans + plat_data->memcpy_len) {
2633 /*
2634 * The max number of logical channels are event lines for all
2635 * src devices and dst devices
2636 */
2637 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2638 sizeof(struct d40_chan *),
2639 GFP_KERNEL);
2640 if (!base->lookup_log_chans)
2641 goto failure;
2642 }
2643
2644 base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2645 sizeof(struct d40_desc *) *
2646 D40_LCLA_LINK_PER_EVENT_GRP,
2647 GFP_KERNEL);
2648 if (!base->lcla_pool.alloc_map)
2649 goto failure;
2650
2651 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2652 0, SLAB_HWCACHE_ALIGN,
2653 NULL);
2654 if (base->desc_slab == NULL)
2655 goto failure;
2656
2657 return base;
2658
2659 failure:
2660 if (!IS_ERR(clk)) {
2661 clk_disable(clk);
2662 clk_put(clk);
2663 }
2664 if (virtbase)
2665 iounmap(virtbase);
2666 if (res)
2667 release_mem_region(res->start,
2668 resource_size(res));
2669 if (virtbase)
2670 iounmap(virtbase);
2671
2672 if (base) {
2673 kfree(base->lcla_pool.alloc_map);
2674 kfree(base->lookup_log_chans);
2675 kfree(base->lookup_phy_chans);
2676 kfree(base->phy_res);
2677 kfree(base);
2678 }
2679
2680 return NULL;
2681 }
2682
2683 static void __init d40_hw_init(struct d40_base *base)
2684 {
2685
2686 static const struct d40_reg_val dma_init_reg[] = {
2687 /* Clock every part of the DMA block from start */
2688 { .reg = D40_DREG_GCC, .val = 0x0000ff01},
2689
2690 /* Interrupts on all logical channels */
2691 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2692 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2693 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2694 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2695 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2696 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2697 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2698 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2699 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2700 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2701 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2702 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2703 };
2704 int i;
2705 u32 prmseo[2] = {0, 0};
2706 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2707 u32 pcmis = 0;
2708 u32 pcicr = 0;
2709
2710 for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2711 writel(dma_init_reg[i].val,
2712 base->virtbase + dma_init_reg[i].reg);
2713
2714 /* Configure all our dma channels to default settings */
2715 for (i = 0; i < base->num_phy_chans; i++) {
2716
2717 activeo[i % 2] = activeo[i % 2] << 2;
2718
2719 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2720 == D40_ALLOC_PHY) {
2721 activeo[i % 2] |= 3;
2722 continue;
2723 }
2724
2725 /* Enable interrupt # */
2726 pcmis = (pcmis << 1) | 1;
2727
2728 /* Clear interrupt # */
2729 pcicr = (pcicr << 1) | 1;
2730
2731 /* Set channel to physical mode */
2732 prmseo[i % 2] = prmseo[i % 2] << 2;
2733 prmseo[i % 2] |= 1;
2734
2735 }
2736
2737 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2738 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2739 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2740 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2741
2742 /* Write which interrupt to enable */
2743 writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2744
2745 /* Write which interrupt to clear */
2746 writel(pcicr, base->virtbase + D40_DREG_PCICR);
2747
2748 }
2749
2750 static int __init d40_lcla_allocate(struct d40_base *base)
2751 {
2752 struct d40_lcla_pool *pool = &base->lcla_pool;
2753 unsigned long *page_list;
2754 int i, j;
2755 int ret = 0;
2756
2757 /*
2758 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2759 * To full fill this hardware requirement without wasting 256 kb
2760 * we allocate pages until we get an aligned one.
2761 */
2762 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2763 GFP_KERNEL);
2764
2765 if (!page_list) {
2766 ret = -ENOMEM;
2767 goto failure;
2768 }
2769
2770 /* Calculating how many pages that are required */
2771 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2772
2773 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2774 page_list[i] = __get_free_pages(GFP_KERNEL,
2775 base->lcla_pool.pages);
2776 if (!page_list[i]) {
2777
2778 d40_err(base->dev, "Failed to allocate %d pages.\n",
2779 base->lcla_pool.pages);
2780
2781 for (j = 0; j < i; j++)
2782 free_pages(page_list[j], base->lcla_pool.pages);
2783 goto failure;
2784 }
2785
2786 if ((virt_to_phys((void *)page_list[i]) &
2787 (LCLA_ALIGNMENT - 1)) == 0)
2788 break;
2789 }
2790
2791 for (j = 0; j < i; j++)
2792 free_pages(page_list[j], base->lcla_pool.pages);
2793
2794 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2795 base->lcla_pool.base = (void *)page_list[i];
2796 } else {
2797 /*
2798 * After many attempts and no succees with finding the correct
2799 * alignment, try with allocating a big buffer.
2800 */
2801 dev_warn(base->dev,
2802 "[%s] Failed to get %d pages @ 18 bit align.\n",
2803 __func__, base->lcla_pool.pages);
2804 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2805 base->num_phy_chans +
2806 LCLA_ALIGNMENT,
2807 GFP_KERNEL);
2808 if (!base->lcla_pool.base_unaligned) {
2809 ret = -ENOMEM;
2810 goto failure;
2811 }
2812
2813 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2814 LCLA_ALIGNMENT);
2815 }
2816
2817 pool->dma_addr = dma_map_single(base->dev, pool->base,
2818 SZ_1K * base->num_phy_chans,
2819 DMA_TO_DEVICE);
2820 if (dma_mapping_error(base->dev, pool->dma_addr)) {
2821 pool->dma_addr = 0;
2822 ret = -ENOMEM;
2823 goto failure;
2824 }
2825
2826 writel(virt_to_phys(base->lcla_pool.base),
2827 base->virtbase + D40_DREG_LCLA);
2828 failure:
2829 kfree(page_list);
2830 return ret;
2831 }
2832
2833 static int __init d40_probe(struct platform_device *pdev)
2834 {
2835 int err;
2836 int ret = -ENOENT;
2837 struct d40_base *base;
2838 struct resource *res = NULL;
2839 int num_reserved_chans;
2840 u32 val;
2841
2842 base = d40_hw_detect_init(pdev);
2843
2844 if (!base)
2845 goto failure;
2846
2847 num_reserved_chans = d40_phy_res_init(base);
2848
2849 platform_set_drvdata(pdev, base);
2850
2851 spin_lock_init(&base->interrupt_lock);
2852 spin_lock_init(&base->execmd_lock);
2853
2854 /* Get IO for logical channel parameter address */
2855 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2856 if (!res) {
2857 ret = -ENOENT;
2858 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
2859 goto failure;
2860 }
2861 base->lcpa_size = resource_size(res);
2862 base->phy_lcpa = res->start;
2863
2864 if (request_mem_region(res->start, resource_size(res),
2865 D40_NAME " I/O lcpa") == NULL) {
2866 ret = -EBUSY;
2867 d40_err(&pdev->dev,
2868 "Failed to request LCPA region 0x%x-0x%x\n",
2869 res->start, res->end);
2870 goto failure;
2871 }
2872
2873 /* We make use of ESRAM memory for this. */
2874 val = readl(base->virtbase + D40_DREG_LCPA);
2875 if (res->start != val && val != 0) {
2876 dev_warn(&pdev->dev,
2877 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2878 __func__, val, res->start);
2879 } else
2880 writel(res->start, base->virtbase + D40_DREG_LCPA);
2881
2882 base->lcpa_base = ioremap(res->start, resource_size(res));
2883 if (!base->lcpa_base) {
2884 ret = -ENOMEM;
2885 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
2886 goto failure;
2887 }
2888
2889 ret = d40_lcla_allocate(base);
2890 if (ret) {
2891 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
2892 goto failure;
2893 }
2894
2895 spin_lock_init(&base->lcla_pool.lock);
2896
2897 base->irq = platform_get_irq(pdev, 0);
2898
2899 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2900 if (ret) {
2901 d40_err(&pdev->dev, "No IRQ defined\n");
2902 goto failure;
2903 }
2904
2905 err = d40_dmaengine_init(base, num_reserved_chans);
2906 if (err)
2907 goto failure;
2908
2909 d40_hw_init(base);
2910
2911 dev_info(base->dev, "initialized\n");
2912 return 0;
2913
2914 failure:
2915 if (base) {
2916 if (base->desc_slab)
2917 kmem_cache_destroy(base->desc_slab);
2918 if (base->virtbase)
2919 iounmap(base->virtbase);
2920
2921 if (base->lcla_pool.dma_addr)
2922 dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
2923 SZ_1K * base->num_phy_chans,
2924 DMA_TO_DEVICE);
2925
2926 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2927 free_pages((unsigned long)base->lcla_pool.base,
2928 base->lcla_pool.pages);
2929
2930 kfree(base->lcla_pool.base_unaligned);
2931
2932 if (base->phy_lcpa)
2933 release_mem_region(base->phy_lcpa,
2934 base->lcpa_size);
2935 if (base->phy_start)
2936 release_mem_region(base->phy_start,
2937 base->phy_size);
2938 if (base->clk) {
2939 clk_disable(base->clk);
2940 clk_put(base->clk);
2941 }
2942
2943 kfree(base->lcla_pool.alloc_map);
2944 kfree(base->lookup_log_chans);
2945 kfree(base->lookup_phy_chans);
2946 kfree(base->phy_res);
2947 kfree(base);
2948 }
2949
2950 d40_err(&pdev->dev, "probe failed\n");
2951 return ret;
2952 }
2953
2954 static struct platform_driver d40_driver = {
2955 .driver = {
2956 .owner = THIS_MODULE,
2957 .name = D40_NAME,
2958 },
2959 };
2960
2961 static int __init stedma40_init(void)
2962 {
2963 return platform_driver_probe(&d40_driver, d40_probe);
2964 }
2965 arch_initcall(stedma40_init);
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree