3 * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
5 * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/init.h>
13 #include <linux/platform_device.h>
14 #include <linux/err.h>
15 #include <linux/spinlock.h>
16 #include <linux/delay.h>
17 #include <linux/list.h>
18 #include <linux/clk.h>
19 #include <linux/vmalloc.h>
20 #include <linux/string.h>
21 #include <linux/interrupt.h>
26 #include "ipu_intern.h"
28 #define FS_VF_IN_VALID 0x00000002
29 #define FS_ENC_IN_VALID 0x00000001
31 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
35 * There can be only one, we could allocate it dynamically, but then we'd have
36 * to add an extra parameter to some functions, and use something as ugly as
37 * struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
40 static struct ipu ipu_data
;
42 #define to_ipu(id) container_of(id, struct ipu, idmac)
44 static u32
__idmac_read_icreg(struct ipu
*ipu
, unsigned long reg
)
46 return __raw_readl(ipu
->reg_ic
+ reg
);
49 #define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
51 static void __idmac_write_icreg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
53 __raw_writel(value
, ipu
->reg_ic
+ reg
);
56 #define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
58 static u32
idmac_read_ipureg(struct ipu
*ipu
, unsigned long reg
)
60 return __raw_readl(ipu
->reg_ipu
+ reg
);
63 static void idmac_write_ipureg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
65 __raw_writel(value
, ipu
->reg_ipu
+ reg
);
68 /*****************************************************************************
69 * IPU / IC common functions
71 static void dump_idmac_reg(struct ipu
*ipu
)
73 dev_dbg(ipu
->dev
, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
74 "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
75 idmac_read_icreg(ipu
, IDMAC_CONF
),
76 idmac_read_icreg(ipu
, IC_CONF
),
77 idmac_read_icreg(ipu
, IDMAC_CHA_EN
),
78 idmac_read_icreg(ipu
, IDMAC_CHA_PRI
),
79 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
));
80 dev_dbg(ipu
->dev
, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
81 "DB_MODE 0x%x, TASKS_STAT 0x%x\n",
82 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
83 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
84 idmac_read_ipureg(ipu
, IPU_CHA_CUR_BUF
),
85 idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
),
86 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
89 static uint32_t bytes_per_pixel(enum pixel_fmt fmt
)
92 case IPU_PIX_FMT_GENERIC
: /* generic data */
93 case IPU_PIX_FMT_RGB332
:
94 case IPU_PIX_FMT_YUV420P
:
95 case IPU_PIX_FMT_YUV422P
:
98 case IPU_PIX_FMT_RGB565
:
99 case IPU_PIX_FMT_YUYV
:
100 case IPU_PIX_FMT_UYVY
:
102 case IPU_PIX_FMT_BGR24
:
103 case IPU_PIX_FMT_RGB24
:
105 case IPU_PIX_FMT_GENERIC_32
: /* generic data */
106 case IPU_PIX_FMT_BGR32
:
107 case IPU_PIX_FMT_RGB32
:
108 case IPU_PIX_FMT_ABGR32
:
113 /* Enable direct write to memory by the Camera Sensor Interface */
114 static void ipu_ic_enable_task(struct ipu
*ipu
, enum ipu_channel channel
)
116 uint32_t ic_conf
, mask
;
120 mask
= IC_CONF_PRPENC_EN
;
123 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
128 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) | mask
;
129 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
132 /* Called under spin_lock_irqsave(&ipu_data.lock) */
133 static void ipu_ic_disable_task(struct ipu
*ipu
, enum ipu_channel channel
)
135 uint32_t ic_conf
, mask
;
139 mask
= IC_CONF_PRPENC_EN
;
142 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
147 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) & ~mask
;
148 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
151 static uint32_t ipu_channel_status(struct ipu
*ipu
, enum ipu_channel channel
)
153 uint32_t stat
= TASK_STAT_IDLE
;
154 uint32_t task_stat_reg
= idmac_read_ipureg(ipu
, IPU_TASKS_STAT
);
158 stat
= (task_stat_reg
& TSTAT_CSI2MEM_MASK
) >>
159 TSTAT_CSI2MEM_OFFSET
;
170 struct chan_param_mem_planar
{
208 } __attribute__ ((packed
));
210 struct chan_param_mem_interleaved
{
267 } __attribute__ ((packed
));
269 union chan_param_mem
{
270 struct chan_param_mem_planar pp
;
271 struct chan_param_mem_interleaved ip
;
274 static void ipu_ch_param_set_plane_offset(union chan_param_mem
*params
,
275 u32 u_offset
, u32 v_offset
)
277 params
->pp
.ubo_l
= u_offset
& 0x7ff;
278 params
->pp
.ubo_h
= u_offset
>> 11;
279 params
->pp
.vbo_l
= v_offset
& 0x1ffff;
280 params
->pp
.vbo_h
= v_offset
>> 17;
283 static void ipu_ch_param_set_size(union chan_param_mem
*params
,
284 uint32_t pixel_fmt
, uint16_t width
,
285 uint16_t height
, uint16_t stride
)
290 params
->pp
.fw
= width
- 1;
291 params
->pp
.fh_l
= height
- 1;
292 params
->pp
.fh_h
= (height
- 1) >> 8;
293 params
->pp
.sl
= stride
- 1;
296 case IPU_PIX_FMT_GENERIC
:
297 /*Represents 8-bit Generic data */
301 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
303 case IPU_PIX_FMT_GENERIC_32
:
304 /*Represents 32-bit Generic data */
308 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
310 case IPU_PIX_FMT_RGB565
:
314 params
->ip
.sat
= 2; /* SAT = 32-bit access */
315 params
->ip
.ofs0
= 0; /* Red bit offset */
316 params
->ip
.ofs1
= 5; /* Green bit offset */
317 params
->ip
.ofs2
= 11; /* Blue bit offset */
318 params
->ip
.ofs3
= 16; /* Alpha bit offset */
319 params
->ip
.wid0
= 4; /* Red bit width - 1 */
320 params
->ip
.wid1
= 5; /* Green bit width - 1 */
321 params
->ip
.wid2
= 4; /* Blue bit width - 1 */
323 case IPU_PIX_FMT_BGR24
:
324 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
327 params
->ip
.sat
= 2; /* SAT = 32-bit access */
328 params
->ip
.ofs0
= 0; /* Red bit offset */
329 params
->ip
.ofs1
= 8; /* Green bit offset */
330 params
->ip
.ofs2
= 16; /* Blue bit offset */
331 params
->ip
.ofs3
= 24; /* Alpha bit offset */
332 params
->ip
.wid0
= 7; /* Red bit width - 1 */
333 params
->ip
.wid1
= 7; /* Green bit width - 1 */
334 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
336 case IPU_PIX_FMT_RGB24
:
337 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
340 params
->ip
.sat
= 2; /* SAT = 32-bit access */
341 params
->ip
.ofs0
= 16; /* Red bit offset */
342 params
->ip
.ofs1
= 8; /* Green bit offset */
343 params
->ip
.ofs2
= 0; /* Blue bit offset */
344 params
->ip
.ofs3
= 24; /* Alpha bit offset */
345 params
->ip
.wid0
= 7; /* Red bit width - 1 */
346 params
->ip
.wid1
= 7; /* Green bit width - 1 */
347 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
349 case IPU_PIX_FMT_BGRA32
:
350 case IPU_PIX_FMT_BGR32
:
354 params
->ip
.sat
= 2; /* SAT = 32-bit access */
355 params
->ip
.ofs0
= 8; /* Red bit offset */
356 params
->ip
.ofs1
= 16; /* Green bit offset */
357 params
->ip
.ofs2
= 24; /* Blue bit offset */
358 params
->ip
.ofs3
= 0; /* Alpha bit offset */
359 params
->ip
.wid0
= 7; /* Red bit width - 1 */
360 params
->ip
.wid1
= 7; /* Green bit width - 1 */
361 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
362 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
364 case IPU_PIX_FMT_RGBA32
:
365 case IPU_PIX_FMT_RGB32
:
369 params
->ip
.sat
= 2; /* SAT = 32-bit access */
370 params
->ip
.ofs0
= 24; /* Red bit offset */
371 params
->ip
.ofs1
= 16; /* Green bit offset */
372 params
->ip
.ofs2
= 8; /* Blue bit offset */
373 params
->ip
.ofs3
= 0; /* Alpha bit offset */
374 params
->ip
.wid0
= 7; /* Red bit width - 1 */
375 params
->ip
.wid1
= 7; /* Green bit width - 1 */
376 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
377 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
379 case IPU_PIX_FMT_ABGR32
:
383 params
->ip
.sat
= 2; /* SAT = 32-bit access */
384 params
->ip
.ofs0
= 8; /* Red bit offset */
385 params
->ip
.ofs1
= 16; /* Green bit offset */
386 params
->ip
.ofs2
= 24; /* Blue bit offset */
387 params
->ip
.ofs3
= 0; /* Alpha bit offset */
388 params
->ip
.wid0
= 7; /* Red bit width - 1 */
389 params
->ip
.wid1
= 7; /* Green bit width - 1 */
390 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
391 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
393 case IPU_PIX_FMT_UYVY
:
397 params
->ip
.sat
= 2; /* SAT = 32-bit access */
399 case IPU_PIX_FMT_YUV420P2
:
400 case IPU_PIX_FMT_YUV420P
:
404 params
->ip
.sat
= 2; /* SAT = 32-bit access */
405 u_offset
= stride
* height
;
406 v_offset
= u_offset
+ u_offset
/ 4;
407 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
409 case IPU_PIX_FMT_YVU422P
:
413 params
->ip
.sat
= 2; /* SAT = 32-bit access */
414 v_offset
= stride
* height
;
415 u_offset
= v_offset
+ v_offset
/ 2;
416 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
418 case IPU_PIX_FMT_YUV422P
:
422 params
->ip
.sat
= 2; /* SAT = 32-bit access */
423 u_offset
= stride
* height
;
424 v_offset
= u_offset
+ u_offset
/ 2;
425 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
428 dev_err(ipu_data
.dev
,
429 "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt
);
436 static void ipu_ch_param_set_burst_size(union chan_param_mem
*params
,
437 uint16_t burst_pixels
)
439 params
->pp
.npb
= burst_pixels
- 1;
442 static void ipu_ch_param_set_buffer(union chan_param_mem
*params
,
443 dma_addr_t buf0
, dma_addr_t buf1
)
445 params
->pp
.eba0
= buf0
;
446 params
->pp
.eba1
= buf1
;
449 static void ipu_ch_param_set_rotation(union chan_param_mem
*params
,
450 enum ipu_rotate_mode rotate
)
452 params
->pp
.bam
= rotate
;
455 static void ipu_write_param_mem(uint32_t addr
, uint32_t *data
,
458 for (; num_words
> 0; num_words
--) {
459 dev_dbg(ipu_data
.dev
,
460 "write param mem - addr = 0x%08X, data = 0x%08X\n",
462 idmac_write_ipureg(&ipu_data
, addr
, IPU_IMA_ADDR
);
463 idmac_write_ipureg(&ipu_data
, *data
++, IPU_IMA_DATA
);
465 if ((addr
& 0x7) == 5) {
466 addr
&= ~0x7; /* set to word 0 */
467 addr
+= 8; /* increment to next row */
472 static int calc_resize_coeffs(uint32_t in_size
, uint32_t out_size
,
473 uint32_t *resize_coeff
,
474 uint32_t *downsize_coeff
)
477 uint32_t temp_downsize
;
479 *resize_coeff
= 1 << 13;
480 *downsize_coeff
= 1 << 13;
482 /* Cannot downsize more than 8:1 */
483 if (out_size
<< 3 < in_size
)
486 /* compute downsizing coefficient */
489 while (temp_size
>= out_size
* 2 && temp_downsize
< 2) {
493 *downsize_coeff
= temp_downsize
;
496 * compute resizing coefficient using the following formula:
497 * resize_coeff = M*(SI -1)/(SO - 1)
498 * where M = 2^13, SI - input size, SO - output size
500 *resize_coeff
= (8192L * (temp_size
- 1)) / (out_size
- 1);
501 if (*resize_coeff
>= 16384L) {
502 dev_err(ipu_data
.dev
, "Warning! Overflow on resize coeff.\n");
503 *resize_coeff
= 0x3FFF;
506 dev_dbg(ipu_data
.dev
, "resizing from %u -> %u pixels, "
507 "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size
, out_size
,
508 *downsize_coeff
, *resize_coeff
>= 8192L ? 1 : 0,
509 ((*resize_coeff
& 0x1FFF) * 10000L) / 8192L, *resize_coeff
);
514 static enum ipu_color_space
format_to_colorspace(enum pixel_fmt fmt
)
517 case IPU_PIX_FMT_RGB565
:
518 case IPU_PIX_FMT_BGR24
:
519 case IPU_PIX_FMT_RGB24
:
520 case IPU_PIX_FMT_BGR32
:
521 case IPU_PIX_FMT_RGB32
:
522 return IPU_COLORSPACE_RGB
;
524 return IPU_COLORSPACE_YCBCR
;
528 static int ipu_ic_init_prpenc(struct ipu
*ipu
,
529 union ipu_channel_param
*params
, bool src_is_csi
)
531 uint32_t reg
, ic_conf
;
532 uint32_t downsize_coeff
, resize_coeff
;
533 enum ipu_color_space in_fmt
, out_fmt
;
535 /* Setup vertical resizing */
536 calc_resize_coeffs(params
->video
.in_height
,
537 params
->video
.out_height
,
538 &resize_coeff
, &downsize_coeff
);
539 reg
= (downsize_coeff
<< 30) | (resize_coeff
<< 16);
541 /* Setup horizontal resizing */
542 calc_resize_coeffs(params
->video
.in_width
,
543 params
->video
.out_width
,
544 &resize_coeff
, &downsize_coeff
);
545 reg
|= (downsize_coeff
<< 14) | resize_coeff
;
547 /* Setup color space conversion */
548 in_fmt
= format_to_colorspace(params
->video
.in_pixel_fmt
);
549 out_fmt
= format_to_colorspace(params
->video
.out_pixel_fmt
);
552 * Colourspace conversion unsupported yet - see _init_csc() in
555 if (in_fmt
!= out_fmt
) {
556 dev_err(ipu
->dev
, "Colourspace conversion unsupported!\n");
560 idmac_write_icreg(ipu
, reg
, IC_PRP_ENC_RSC
);
562 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
);
565 ic_conf
&= ~IC_CONF_RWS_EN
;
567 ic_conf
|= IC_CONF_RWS_EN
;
569 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
574 static uint32_t dma_param_addr(uint32_t dma_ch
)
576 /* Channel Parameter Memory */
577 return 0x10000 | (dma_ch
<< 4);
580 static void ipu_channel_set_priority(struct ipu
*ipu
, enum ipu_channel channel
,
583 u32 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_PRI
);
586 reg
|= 1UL << channel
;
588 reg
&= ~(1UL << channel
);
590 idmac_write_icreg(ipu
, reg
, IDMAC_CHA_PRI
);
595 static uint32_t ipu_channel_conf_mask(enum ipu_channel channel
)
602 mask
= IPU_CONF_CSI_EN
| IPU_CONF_IC_EN
;
606 mask
= IPU_CONF_SDC_EN
| IPU_CONF_DI_EN
;
617 * ipu_enable_channel() - enable an IPU channel.
618 * @idmac: IPU DMAC context.
619 * @ichan: IDMAC channel.
620 * @return: 0 on success or negative error code on failure.
622 static int ipu_enable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
624 struct ipu
*ipu
= to_ipu(idmac
);
625 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
629 spin_lock_irqsave(&ipu
->lock
, flags
);
631 /* Reset to buffer 0 */
632 idmac_write_ipureg(ipu
, 1UL << channel
, IPU_CHA_CUR_BUF
);
633 ichan
->active_buffer
= 0;
634 ichan
->status
= IPU_CHANNEL_ENABLED
;
640 ipu_channel_set_priority(ipu
, channel
, true);
645 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
647 idmac_write_icreg(ipu
, reg
| (1UL << channel
), IDMAC_CHA_EN
);
649 ipu_ic_enable_task(ipu
, channel
);
651 spin_unlock_irqrestore(&ipu
->lock
, flags
);
656 * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
657 * @ichan: IDMAC channel.
658 * @pixel_fmt: pixel format of buffer. Pixel format is a FOURCC ASCII code.
659 * @width: width of buffer in pixels.
660 * @height: height of buffer in pixels.
661 * @stride: stride length of buffer in pixels.
662 * @rot_mode: rotation mode of buffer. A rotation setting other than
663 * IPU_ROTATE_VERT_FLIP should only be used for input buffers of
665 * @phyaddr_0: buffer 0 physical address.
666 * @phyaddr_1: buffer 1 physical address. Setting this to a value other than
667 * NULL enables double buffering mode.
668 * @return: 0 on success or negative error code on failure.
670 static int ipu_init_channel_buffer(struct idmac_channel
*ichan
,
671 enum pixel_fmt pixel_fmt
,
672 uint16_t width
, uint16_t height
,
674 enum ipu_rotate_mode rot_mode
,
675 dma_addr_t phyaddr_0
, dma_addr_t phyaddr_1
)
677 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
678 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
679 struct ipu
*ipu
= to_ipu(idmac
);
680 union chan_param_mem params
= {};
683 uint32_t stride_bytes
;
685 stride_bytes
= stride
* bytes_per_pixel(pixel_fmt
);
687 if (stride_bytes
% 4) {
689 "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
690 stride
, stride_bytes
);
694 /* IC channel's stride must be a multiple of 8 pixels */
695 if ((channel
<= IDMAC_IC_13
) && (stride
% 8)) {
696 dev_err(ipu
->dev
, "Stride must be 8 pixel multiple\n");
700 /* Build parameter memory data for DMA channel */
701 ipu_ch_param_set_size(¶ms
, pixel_fmt
, width
, height
, stride_bytes
);
702 ipu_ch_param_set_buffer(¶ms
, phyaddr_0
, phyaddr_1
);
703 ipu_ch_param_set_rotation(¶ms
, rot_mode
);
704 /* Some channels (rotation) have restriction on burst length */
706 case IDMAC_IC_7
: /* Hangs with burst 8, 16, other values
707 invalid - Table 44-30 */
709 ipu_ch_param_set_burst_size(¶ms, 8);
714 /* In original code only IPU_PIX_FMT_RGB565 was setting burst */
715 ipu_ch_param_set_burst_size(¶ms
, 16);
722 spin_lock_irqsave(&ipu
->lock
, flags
);
724 ipu_write_param_mem(dma_param_addr(channel
), (uint32_t *)¶ms
, 10);
726 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
729 reg
|= 1UL << channel
;
731 reg
&= ~(1UL << channel
);
733 idmac_write_ipureg(ipu
, reg
, IPU_CHA_DB_MODE_SEL
);
735 ichan
->status
= IPU_CHANNEL_READY
;
737 spin_unlock_irqrestore(&ipu
->lock
, flags
);
743 * ipu_select_buffer() - mark a channel's buffer as ready.
744 * @channel: channel ID.
745 * @buffer_n: buffer number to mark ready.
747 static void ipu_select_buffer(enum ipu_channel channel
, int buffer_n
)
749 /* No locking - this is a write-one-to-set register, cleared by IPU */
751 /* Mark buffer 0 as ready. */
752 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF0_RDY
);
754 /* Mark buffer 1 as ready. */
755 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF1_RDY
);
759 * ipu_update_channel_buffer() - update physical address of a channel buffer.
760 * @ichan: IDMAC channel.
761 * @buffer_n: buffer number to update.
762 * 0 or 1 are the only valid values.
763 * @phyaddr: buffer physical address.
764 * @return: Returns 0 on success or negative error code on failure. This
765 * function will fail if the buffer is set to ready.
767 /* Called under spin_lock(_irqsave)(&ichan->lock) */
768 static int ipu_update_channel_buffer(struct idmac_channel
*ichan
,
769 int buffer_n
, dma_addr_t phyaddr
)
771 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
775 spin_lock_irqsave(&ipu_data
.lock
, flags
);
778 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
779 if (reg
& (1UL << channel
)) {
780 ipu_ic_disable_task(&ipu_data
, channel
);
781 ichan
->status
= IPU_CHANNEL_READY
;
784 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
785 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
786 0x0008UL
, IPU_IMA_ADDR
);
787 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
789 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
790 if (reg
& (1UL << channel
)) {
791 ipu_ic_disable_task(&ipu_data
, channel
);
792 ichan
->status
= IPU_CHANNEL_READY
;
795 /* Check if double-buffering is already enabled */
796 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_DB_MODE_SEL
);
798 if (!(reg
& (1UL << channel
)))
799 idmac_write_ipureg(&ipu_data
, reg
| (1UL << channel
),
800 IPU_CHA_DB_MODE_SEL
);
802 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
803 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
804 0x0009UL
, IPU_IMA_ADDR
);
805 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
808 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
813 /* Called under spin_lock_irqsave(&ichan->lock) */
814 static int ipu_submit_buffer(struct idmac_channel
*ichan
,
815 struct idmac_tx_desc
*desc
, struct scatterlist
*sg
, int buf_idx
)
817 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
818 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
821 if (async_tx_test_ack(&desc
->txd
))
825 * On first invocation this shouldn't be necessary, the call to
826 * ipu_init_channel_buffer() above will set addresses for us, so we
827 * could make it conditional on status >= IPU_CHANNEL_ENABLED, but
828 * doing it again shouldn't hurt either.
830 ret
= ipu_update_channel_buffer(ichan
, buf_idx
,
834 dev_err(dev
, "Updating sg %p on channel 0x%x buffer %d failed!\n",
835 sg
, chan_id
, buf_idx
);
839 ipu_select_buffer(chan_id
, buf_idx
);
840 dev_dbg(dev
, "Updated sg %p on channel 0x%x buffer %d\n",
841 sg
, chan_id
, buf_idx
);
846 /* Called under spin_lock_irqsave(&ichan->lock) */
847 static int ipu_submit_channel_buffers(struct idmac_channel
*ichan
,
848 struct idmac_tx_desc
*desc
)
850 struct scatterlist
*sg
;
853 for (i
= 0, sg
= desc
->sg
; i
< 2 && sg
; i
++) {
857 ret
= ipu_submit_buffer(ichan
, desc
, sg
, i
);
868 static dma_cookie_t
idmac_tx_submit(struct dma_async_tx_descriptor
*tx
)
870 struct idmac_tx_desc
*desc
= to_tx_desc(tx
);
871 struct idmac_channel
*ichan
= to_idmac_chan(tx
->chan
);
872 struct idmac
*idmac
= to_idmac(tx
->chan
->device
);
873 struct ipu
*ipu
= to_ipu(idmac
);
874 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
880 if (!list_empty(&desc
->list
)) {
881 /* The descriptor doesn't belong to client */
882 dev_err(dev
, "Descriptor %p not prepared!\n", tx
);
886 mutex_lock(&ichan
->chan_mutex
);
888 async_tx_clear_ack(tx
);
890 if (ichan
->status
< IPU_CHANNEL_READY
) {
891 struct idmac_video_param
*video
= &ichan
->params
.video
;
893 * Initial buffer assignment - the first two sg-entries from
894 * the descriptor will end up in the IDMAC buffers
896 dma_addr_t dma_1
= sg_is_last(desc
->sg
) ? 0 :
897 sg_dma_address(&desc
->sg
[1]);
899 WARN_ON(ichan
->sg
[0] || ichan
->sg
[1]);
901 cookie
= ipu_init_channel_buffer(ichan
,
902 video
->out_pixel_fmt
,
907 sg_dma_address(&desc
->sg
[0]),
913 dev_dbg(dev
, "Submitting sg %p\n", &desc
->sg
[0]);
915 cookie
= ichan
->dma_chan
.cookie
;
920 /* from dmaengine.h: "last cookie value returned to client" */
921 ichan
->dma_chan
.cookie
= cookie
;
924 /* ipu->lock can be taken under ichan->lock, but not v.v. */
925 spin_lock_irqsave(&ichan
->lock
, flags
);
927 list_add_tail(&desc
->list
, &ichan
->queue
);
928 /* submit_buffers() atomically verifies and fills empty sg slots */
929 ret
= ipu_submit_channel_buffers(ichan
, desc
);
931 spin_unlock_irqrestore(&ichan
->lock
, flags
);
938 if (ichan
->status
< IPU_CHANNEL_ENABLED
) {
939 ret
= ipu_enable_channel(idmac
, ichan
);
950 spin_lock_irqsave(&ichan
->lock
, flags
);
951 list_del_init(&desc
->list
);
952 spin_unlock_irqrestore(&ichan
->lock
, flags
);
954 ichan
->dma_chan
.cookie
= cookie
;
958 mutex_unlock(&ichan
->chan_mutex
);
963 /* Called with ichan->chan_mutex held */
964 static int idmac_desc_alloc(struct idmac_channel
*ichan
, int n
)
966 struct idmac_tx_desc
*desc
= vmalloc(n
* sizeof(struct idmac_tx_desc
));
967 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
972 /* No interrupts, just disable the tasklet for a moment */
973 tasklet_disable(&to_ipu(idmac
)->tasklet
);
975 ichan
->n_tx_desc
= n
;
977 INIT_LIST_HEAD(&ichan
->queue
);
978 INIT_LIST_HEAD(&ichan
->free_list
);
981 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
983 memset(txd
, 0, sizeof(*txd
));
984 dma_async_tx_descriptor_init(txd
, &ichan
->dma_chan
);
985 txd
->tx_submit
= idmac_tx_submit
;
987 list_add(&desc
->list
, &ichan
->free_list
);
992 tasklet_enable(&to_ipu(idmac
)->tasklet
);
998 * ipu_init_channel() - initialize an IPU channel.
999 * @idmac: IPU DMAC context.
1000 * @ichan: pointer to the channel object.
1001 * @return 0 on success or negative error code on failure.
1003 static int ipu_init_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
1005 union ipu_channel_param
*params
= &ichan
->params
;
1007 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1008 unsigned long flags
;
1010 struct ipu
*ipu
= to_ipu(idmac
);
1011 int ret
= 0, n_desc
= 0;
1013 dev_dbg(ipu
->dev
, "init channel = %d\n", channel
);
1015 if (channel
!= IDMAC_SDC_0
&& channel
!= IDMAC_SDC_1
&&
1016 channel
!= IDMAC_IC_7
)
1019 spin_lock_irqsave(&ipu
->lock
, flags
);
1024 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1025 idmac_write_icreg(ipu
, reg
& ~IC_CONF_CSI_MEM_WR_EN
, IC_CONF
);
1029 reg
= idmac_read_ipureg(ipu
, IPU_FS_PROC_FLOW
);
1030 idmac_write_ipureg(ipu
, reg
& ~FS_ENC_IN_VALID
, IPU_FS_PROC_FLOW
);
1031 ret
= ipu_ic_init_prpenc(ipu
, params
, true);
1040 ipu
->channel_init_mask
|= 1L << channel
;
1042 /* Enable IPU sub module */
1043 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) |
1044 ipu_channel_conf_mask(channel
);
1045 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1047 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1049 if (n_desc
&& !ichan
->desc
)
1050 ret
= idmac_desc_alloc(ichan
, n_desc
);
1052 dump_idmac_reg(ipu
);
1058 * ipu_uninit_channel() - uninitialize an IPU channel.
1059 * @idmac: IPU DMAC context.
1060 * @ichan: pointer to the channel object.
1062 static void ipu_uninit_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
1064 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1065 unsigned long flags
;
1067 unsigned long chan_mask
= 1UL << channel
;
1069 struct ipu
*ipu
= to_ipu(idmac
);
1071 spin_lock_irqsave(&ipu
->lock
, flags
);
1073 if (!(ipu
->channel_init_mask
& chan_mask
)) {
1074 dev_err(ipu
->dev
, "Channel already uninitialized %d\n",
1076 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1080 /* Reset the double buffer */
1081 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
1082 idmac_write_ipureg(ipu
, reg
& ~chan_mask
, IPU_CHA_DB_MODE_SEL
);
1084 ichan
->sec_chan_en
= false;
1088 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1089 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
),
1093 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1094 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_PRPENC_EN
| IC_CONF_PRPENC_CSC1
),
1103 ipu
->channel_init_mask
&= ~(1L << channel
);
1105 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) &
1106 ~ipu_channel_conf_mask(channel
);
1107 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1109 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1111 ichan
->n_tx_desc
= 0;
1117 * ipu_disable_channel() - disable an IPU channel.
1118 * @idmac: IPU DMAC context.
1119 * @ichan: channel object pointer.
1120 * @wait_for_stop: flag to set whether to wait for channel end of frame or
1121 * return immediately.
1122 * @return: 0 on success or negative error code on failure.
1124 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
1127 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1128 struct ipu
*ipu
= to_ipu(idmac
);
1130 unsigned long flags
;
1131 unsigned long chan_mask
= 1UL << channel
;
1132 unsigned int timeout
;
1134 if (wait_for_stop
&& channel
!= IDMAC_SDC_1
&& channel
!= IDMAC_SDC_0
) {
1136 /* This waiting always fails. Related to spurious irq problem */
1137 while ((idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
) & chan_mask
) ||
1138 (ipu_channel_status(ipu
, channel
) == TASK_STAT_ACTIVE
)) {
1144 "Warning: timeout waiting for channel %u to "
1145 "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
1146 "busy = 0x%08X, tstat = 0x%08X\n", channel
,
1147 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
1148 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
1149 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
),
1150 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
1154 dev_dbg(ipu
->dev
, "timeout = %d * 10ms\n", 40 - timeout
);
1156 /* SDC BG and FG must be disabled before DMA is disabled */
1157 if (wait_for_stop
&& (channel
== IDMAC_SDC_0
||
1158 channel
== IDMAC_SDC_1
)) {
1160 timeout
&& !ipu_irq_status(ichan
->eof_irq
); timeout
--)
1164 spin_lock_irqsave(&ipu
->lock
, flags
);
1166 /* Disable IC task */
1167 ipu_ic_disable_task(ipu
, channel
);
1169 /* Disable DMA channel(s) */
1170 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
1171 idmac_write_icreg(ipu
, reg
& ~chan_mask
, IDMAC_CHA_EN
);
1174 * Problem (observed with channel DMAIC_7): after enabling the channel
1175 * and initialising buffers, there comes an interrupt with current still
1176 * pointing at buffer 0, whereas it should use buffer 0 first and only
1177 * generate an interrupt when it is done, then current should already
1178 * point to buffer 1. This spurious interrupt also comes on channel
1179 * DMASDC_0. With DMAIC_7 normally, is we just leave the ISR after the
1180 * first interrupt, there comes the second with current correctly
1181 * pointing to buffer 1 this time. But sometimes this second interrupt
1182 * doesn't come and the channel hangs. Clearing BUFx_RDY when disabling
1183 * the channel seems to prevent the channel from hanging, but it doesn't
1184 * prevent the spurious interrupt. This might also be unsafe. Think
1185 * about the IDMAC controller trying to switch to a buffer, when we
1186 * clear the ready bit, and re-enable it a moment later.
1188 reg
= idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
);
1189 idmac_write_ipureg(ipu
, 0, IPU_CHA_BUF0_RDY
);
1190 idmac_write_ipureg(ipu
, reg
& ~(1UL << channel
), IPU_CHA_BUF0_RDY
);
1192 reg
= idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
);
1193 idmac_write_ipureg(ipu
, 0, IPU_CHA_BUF1_RDY
);
1194 idmac_write_ipureg(ipu
, reg
& ~(1UL << channel
), IPU_CHA_BUF1_RDY
);
1196 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1201 static struct scatterlist
*idmac_sg_next(struct idmac_channel
*ichan
,
1202 struct idmac_tx_desc
**desc
, struct scatterlist
*sg
)
1204 struct scatterlist
*sgnew
= sg
? sg_next(sg
) : NULL
;
1207 /* next sg-element in this list */
1210 if ((*desc
)->list
.next
== &ichan
->queue
)
1211 /* No more descriptors on the queue */
1214 /* Fetch next descriptor */
1215 *desc
= list_entry((*desc
)->list
.next
, struct idmac_tx_desc
, list
);
1220 * We have several possibilities here:
1221 * current BUF next BUF
1223 * not last sg next not last sg
1224 * not last sg next last sg
1225 * last sg first sg from next descriptor
1228 * Besides, the descriptor queue might be empty or not. We process all these
1231 static irqreturn_t
idmac_interrupt(int irq
, void *dev_id
)
1233 struct idmac_channel
*ichan
= dev_id
;
1234 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
1235 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
1236 struct scatterlist
**sg
, *sgnext
, *sgnew
= NULL
;
1237 /* Next transfer descriptor */
1238 struct idmac_tx_desc
*desc
, *descnew
;
1239 dma_async_tx_callback callback
;
1240 void *callback_param
;
1242 u32 ready0
, ready1
, curbuf
, err
;
1243 unsigned long flags
;
1245 /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
1247 dev_dbg(dev
, "IDMAC irq %d, buf %d\n", irq
, ichan
->active_buffer
);
1249 spin_lock_irqsave(&ipu_data
.lock
, flags
);
1251 ready0
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
1252 ready1
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
1253 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1254 err
= idmac_read_ipureg(&ipu_data
, IPU_INT_STAT_4
);
1256 if (err
& (1 << chan_id
)) {
1257 idmac_write_ipureg(&ipu_data
, 1 << chan_id
, IPU_INT_STAT_4
);
1258 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1261 * ichan->sg[0] = ichan->sg[1] = NULL;
1262 * you can force channel re-enable on the next tx_submit(), but
1263 * this is dirty - think about descriptors with multiple
1266 dev_warn(dev
, "NFB4EOF on channel %d, ready %x, %x, cur %x\n",
1267 chan_id
, ready0
, ready1
, curbuf
);
1270 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1272 /* Other interrupts do not interfere with this channel */
1273 spin_lock(&ichan
->lock
);
1274 if (unlikely(chan_id
!= IDMAC_SDC_0
&& chan_id
!= IDMAC_SDC_1
&&
1275 ((curbuf
>> chan_id
) & 1) == ichan
->active_buffer
)) {
1278 /* This doesn't help. See comment in ipu_disable_channel() */
1280 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1281 if (((curbuf
>> chan_id
) & 1) != ichan
->active_buffer
)
1287 spin_unlock(&ichan
->lock
);
1289 "IRQ on active buffer on channel %x, active "
1290 "%d, ready %x, %x, current %x!\n", chan_id
,
1291 ichan
->active_buffer
, ready0
, ready1
, curbuf
);
1295 "Buffer deactivated on channel %x, active "
1296 "%d, ready %x, %x, current %x, rest %d!\n", chan_id
,
1297 ichan
->active_buffer
, ready0
, ready1
, curbuf
, i
);
1300 if (unlikely((ichan
->active_buffer
&& (ready1
>> chan_id
) & 1) ||
1301 (!ichan
->active_buffer
&& (ready0
>> chan_id
) & 1)
1303 spin_unlock(&ichan
->lock
);
1305 "IRQ with active buffer still ready on channel %x, "
1306 "active %d, ready %x, %x!\n", chan_id
,
1307 ichan
->active_buffer
, ready0
, ready1
);
1311 if (unlikely(list_empty(&ichan
->queue
))) {
1312 ichan
->sg
[ichan
->active_buffer
] = NULL
;
1313 spin_unlock(&ichan
->lock
);
1315 "IRQ without queued buffers on channel %x, active %d, "
1316 "ready %x, %x!\n", chan_id
,
1317 ichan
->active_buffer
, ready0
, ready1
);
1322 * active_buffer is a software flag, it shows which buffer we are
1323 * currently expecting back from the hardware, IDMAC should be
1324 * processing the other buffer already
1326 sg
= &ichan
->sg
[ichan
->active_buffer
];
1327 sgnext
= ichan
->sg
[!ichan
->active_buffer
];
1330 spin_unlock(&ichan
->lock
);
1334 desc
= list_entry(ichan
->queue
.next
, struct idmac_tx_desc
, list
);
1337 dev_dbg(dev
, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n",
1338 irq
, sg_dma_address(*sg
), sgnext
? sg_dma_address(sgnext
) : 0, ichan
->active_buffer
, curbuf
);
1340 /* Find the descriptor of sgnext */
1341 sgnew
= idmac_sg_next(ichan
, &descnew
, *sg
);
1342 if (sgnext
!= sgnew
)
1343 dev_err(dev
, "Submitted buffer %p, next buffer %p\n", sgnext
, sgnew
);
1346 * if sgnext == NULL sg must be the last element in a scatterlist and
1347 * queue must be empty
1349 if (unlikely(!sgnext
)) {
1350 if (!WARN_ON(sg_next(*sg
)))
1351 dev_dbg(dev
, "Underrun on channel %x\n", chan_id
);
1352 ichan
->sg
[!ichan
->active_buffer
] = sgnew
;
1354 if (unlikely(sgnew
)) {
1355 ipu_submit_buffer(ichan
, descnew
, sgnew
, !ichan
->active_buffer
);
1357 spin_lock_irqsave(&ipu_data
.lock
, flags
);
1358 ipu_ic_disable_task(&ipu_data
, chan_id
);
1359 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1360 ichan
->status
= IPU_CHANNEL_READY
;
1361 /* Continue to check for complete descriptor */
1365 /* Calculate and submit the next sg element */
1366 sgnew
= idmac_sg_next(ichan
, &descnew
, sgnew
);
1368 if (unlikely(!sg_next(*sg
)) || !sgnext
) {
1370 * Last element in scatterlist done, remove from the queue,
1371 * _init for debugging
1373 list_del_init(&desc
->list
);
1379 if (likely(sgnew
) &&
1380 ipu_submit_buffer(ichan
, descnew
, sgnew
, ichan
->active_buffer
) < 0) {
1381 callback
= desc
->txd
.callback
;
1382 callback_param
= desc
->txd
.callback_param
;
1383 spin_unlock(&ichan
->lock
);
1384 callback(callback_param
);
1385 spin_lock(&ichan
->lock
);
1388 /* Flip the active buffer - even if update above failed */
1389 ichan
->active_buffer
= !ichan
->active_buffer
;
1391 ichan
->completed
= desc
->txd
.cookie
;
1393 callback
= desc
->txd
.callback
;
1394 callback_param
= desc
->txd
.callback_param
;
1396 spin_unlock(&ichan
->lock
);
1398 if (done
&& (desc
->txd
.flags
& DMA_PREP_INTERRUPT
) && callback
)
1399 callback(callback_param
);
1404 static void ipu_gc_tasklet(unsigned long arg
)
1406 struct ipu
*ipu
= (struct ipu
*)arg
;
1409 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1410 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1411 struct idmac_tx_desc
*desc
;
1412 unsigned long flags
;
1413 struct scatterlist
*sg
;
1416 for (j
= 0; j
< ichan
->n_tx_desc
; j
++) {
1417 desc
= ichan
->desc
+ j
;
1418 spin_lock_irqsave(&ichan
->lock
, flags
);
1419 if (async_tx_test_ack(&desc
->txd
)) {
1420 list_move(&desc
->list
, &ichan
->free_list
);
1421 for_each_sg(desc
->sg
, sg
, desc
->sg_len
, k
) {
1422 if (ichan
->sg
[0] == sg
)
1423 ichan
->sg
[0] = NULL
;
1424 else if (ichan
->sg
[1] == sg
)
1425 ichan
->sg
[1] = NULL
;
1427 async_tx_clear_ack(&desc
->txd
);
1429 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1434 /* Allocate and initialise a transfer descriptor. */
1435 static struct dma_async_tx_descriptor
*idmac_prep_slave_sg(struct dma_chan
*chan
,
1436 struct scatterlist
*sgl
, unsigned int sg_len
,
1437 enum dma_data_direction direction
, unsigned long tx_flags
)
1439 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1440 struct idmac_tx_desc
*desc
= NULL
;
1441 struct dma_async_tx_descriptor
*txd
= NULL
;
1442 unsigned long flags
;
1444 /* We only can handle these three channels so far */
1445 if (chan
->chan_id
!= IDMAC_SDC_0
&& chan
->chan_id
!= IDMAC_SDC_1
&&
1446 chan
->chan_id
!= IDMAC_IC_7
)
1449 if (direction
!= DMA_FROM_DEVICE
&& direction
!= DMA_TO_DEVICE
) {
1450 dev_err(chan
->device
->dev
, "Invalid DMA direction %d!\n", direction
);
1454 mutex_lock(&ichan
->chan_mutex
);
1456 spin_lock_irqsave(&ichan
->lock
, flags
);
1457 if (!list_empty(&ichan
->free_list
)) {
1458 desc
= list_entry(ichan
->free_list
.next
,
1459 struct idmac_tx_desc
, list
);
1461 list_del_init(&desc
->list
);
1463 desc
->sg_len
= sg_len
;
1466 txd
->flags
= tx_flags
;
1468 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1470 mutex_unlock(&ichan
->chan_mutex
);
1472 tasklet_schedule(&to_ipu(to_idmac(chan
->device
))->tasklet
);
1477 /* Re-select the current buffer and re-activate the channel */
1478 static void idmac_issue_pending(struct dma_chan
*chan
)
1480 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1481 struct idmac
*idmac
= to_idmac(chan
->device
);
1482 struct ipu
*ipu
= to_ipu(idmac
);
1483 unsigned long flags
;
1485 /* This is not always needed, but doesn't hurt either */
1486 spin_lock_irqsave(&ipu
->lock
, flags
);
1487 ipu_select_buffer(chan
->chan_id
, ichan
->active_buffer
);
1488 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1491 * Might need to perform some parts of initialisation from
1492 * ipu_enable_channel(), but not all, we do not want to reset to buffer
1493 * 0, don't need to set priority again either, but re-enabling the task
1494 * and the channel might be a good idea.
1498 static void __idmac_terminate_all(struct dma_chan
*chan
)
1500 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1501 struct idmac
*idmac
= to_idmac(chan
->device
);
1502 unsigned long flags
;
1505 ipu_disable_channel(idmac
, ichan
,
1506 ichan
->status
>= IPU_CHANNEL_ENABLED
);
1508 tasklet_disable(&to_ipu(idmac
)->tasklet
);
1510 /* ichan->queue is modified in ISR, have to spinlock */
1511 spin_lock_irqsave(&ichan
->lock
, flags
);
1512 list_splice_init(&ichan
->queue
, &ichan
->free_list
);
1515 for (i
= 0; i
< ichan
->n_tx_desc
; i
++) {
1516 struct idmac_tx_desc
*desc
= ichan
->desc
+ i
;
1517 if (list_empty(&desc
->list
))
1518 /* Descriptor was prepared, but not submitted */
1519 list_add(&desc
->list
, &ichan
->free_list
);
1521 async_tx_clear_ack(&desc
->txd
);
1524 ichan
->sg
[0] = NULL
;
1525 ichan
->sg
[1] = NULL
;
1526 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1528 tasklet_enable(&to_ipu(idmac
)->tasklet
);
1530 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1533 static void idmac_terminate_all(struct dma_chan
*chan
)
1535 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1537 mutex_lock(&ichan
->chan_mutex
);
1539 __idmac_terminate_all(chan
);
1541 mutex_unlock(&ichan
->chan_mutex
);
1545 static irqreturn_t
ic_sof_irq(int irq
, void *dev_id
)
1547 struct idmac_channel
*ichan
= dev_id
;
1548 printk(KERN_DEBUG
"Got SOF IRQ %d on Channel %d\n",
1549 irq
, ichan
->dma_chan
.chan_id
);
1554 static irqreturn_t
ic_eof_irq(int irq
, void *dev_id
)
1556 struct idmac_channel
*ichan
= dev_id
;
1557 printk(KERN_DEBUG
"Got EOF IRQ %d on Channel %d\n",
1558 irq
, ichan
->dma_chan
.chan_id
);
1563 static int ic_sof
= -EINVAL
, ic_eof
= -EINVAL
;
1566 static int idmac_alloc_chan_resources(struct dma_chan
*chan
)
1568 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1569 struct idmac
*idmac
= to_idmac(chan
->device
);
1572 /* dmaengine.c now guarantees to only offer free channels */
1573 BUG_ON(chan
->client_count
> 1);
1574 WARN_ON(ichan
->status
!= IPU_CHANNEL_FREE
);
1577 ichan
->completed
= -ENXIO
;
1579 ret
= ipu_irq_map(chan
->chan_id
);
1583 ichan
->eof_irq
= ret
;
1586 * Important to first disable the channel, because maybe someone
1587 * used it before us, e.g., the bootloader
1589 ipu_disable_channel(idmac
, ichan
, true);
1591 ret
= ipu_init_channel(idmac
, ichan
);
1595 ret
= request_irq(ichan
->eof_irq
, idmac_interrupt
, 0,
1596 ichan
->eof_name
, ichan
);
1601 if (chan
->chan_id
== IDMAC_IC_7
) {
1602 ic_sof
= ipu_irq_map(69);
1604 request_irq(ic_sof
, ic_sof_irq
, 0, "IC SOF", ichan
);
1605 ic_eof
= ipu_irq_map(70);
1607 request_irq(ic_eof
, ic_eof_irq
, 0, "IC EOF", ichan
);
1611 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1613 dev_dbg(&chan
->dev
->device
, "Found channel 0x%x, irq %d\n",
1614 chan
->chan_id
, ichan
->eof_irq
);
1619 ipu_uninit_channel(idmac
, ichan
);
1621 ipu_irq_unmap(chan
->chan_id
);
1626 static void idmac_free_chan_resources(struct dma_chan
*chan
)
1628 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1629 struct idmac
*idmac
= to_idmac(chan
->device
);
1631 mutex_lock(&ichan
->chan_mutex
);
1633 __idmac_terminate_all(chan
);
1635 if (ichan
->status
> IPU_CHANNEL_FREE
) {
1637 if (chan
->chan_id
== IDMAC_IC_7
) {
1639 free_irq(ic_sof
, ichan
);
1644 free_irq(ic_eof
, ichan
);
1650 free_irq(ichan
->eof_irq
, ichan
);
1651 ipu_irq_unmap(chan
->chan_id
);
1654 ichan
->status
= IPU_CHANNEL_FREE
;
1656 ipu_uninit_channel(idmac
, ichan
);
1658 mutex_unlock(&ichan
->chan_mutex
);
1660 tasklet_schedule(&to_ipu(idmac
)->tasklet
);
1663 static enum dma_status
idmac_is_tx_complete(struct dma_chan
*chan
,
1664 dma_cookie_t cookie
, dma_cookie_t
*done
, dma_cookie_t
*used
)
1666 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1669 *done
= ichan
->completed
;
1671 *used
= chan
->cookie
;
1672 if (cookie
!= chan
->cookie
)
1677 static int __init
ipu_idmac_init(struct ipu
*ipu
)
1679 struct idmac
*idmac
= &ipu
->idmac
;
1680 struct dma_device
*dma
= &idmac
->dma
;
1683 dma_cap_set(DMA_SLAVE
, dma
->cap_mask
);
1684 dma_cap_set(DMA_PRIVATE
, dma
->cap_mask
);
1686 /* Compulsory common fields */
1687 dma
->dev
= ipu
->dev
;
1688 dma
->device_alloc_chan_resources
= idmac_alloc_chan_resources
;
1689 dma
->device_free_chan_resources
= idmac_free_chan_resources
;
1690 dma
->device_is_tx_complete
= idmac_is_tx_complete
;
1691 dma
->device_issue_pending
= idmac_issue_pending
;
1693 /* Compulsory for DMA_SLAVE fields */
1694 dma
->device_prep_slave_sg
= idmac_prep_slave_sg
;
1695 dma
->device_terminate_all
= idmac_terminate_all
;
1697 INIT_LIST_HEAD(&dma
->channels
);
1698 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1699 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1700 struct dma_chan
*dma_chan
= &ichan
->dma_chan
;
1702 spin_lock_init(&ichan
->lock
);
1703 mutex_init(&ichan
->chan_mutex
);
1705 ichan
->status
= IPU_CHANNEL_FREE
;
1706 ichan
->sec_chan_en
= false;
1707 ichan
->completed
= -ENXIO
;
1708 snprintf(ichan
->eof_name
, sizeof(ichan
->eof_name
), "IDMAC EOF %d", i
);
1710 dma_chan
->device
= &idmac
->dma
;
1711 dma_chan
->cookie
= 1;
1712 dma_chan
->chan_id
= i
;
1713 list_add_tail(&dma_chan
->device_node
, &dma
->channels
);
1716 idmac_write_icreg(ipu
, 0x00000070, IDMAC_CONF
);
1718 return dma_async_device_register(&idmac
->dma
);
1721 static void __exit
ipu_idmac_exit(struct ipu
*ipu
)
1724 struct idmac
*idmac
= &ipu
->idmac
;
1726 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1727 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1729 idmac_terminate_all(&ichan
->dma_chan
);
1730 idmac_prep_slave_sg(&ichan
->dma_chan
, NULL
, 0, DMA_NONE
, 0);
1733 dma_async_device_unregister(&idmac
->dma
);
1736 /*****************************************************************************
1737 * IPU common probe / remove
1740 static int __init
ipu_probe(struct platform_device
*pdev
)
1742 struct ipu_platform_data
*pdata
= pdev
->dev
.platform_data
;
1743 struct resource
*mem_ipu
, *mem_ic
;
1746 spin_lock_init(&ipu_data
.lock
);
1748 mem_ipu
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1749 mem_ic
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
1750 if (!pdata
|| !mem_ipu
|| !mem_ic
)
1753 ipu_data
.dev
= &pdev
->dev
;
1755 platform_set_drvdata(pdev
, &ipu_data
);
1757 ret
= platform_get_irq(pdev
, 0);
1761 ipu_data
.irq_fn
= ret
;
1762 ret
= platform_get_irq(pdev
, 1);
1766 ipu_data
.irq_err
= ret
;
1767 ipu_data
.irq_base
= pdata
->irq_base
;
1769 dev_dbg(&pdev
->dev
, "fn irq %u, err irq %u, irq-base %u\n",
1770 ipu_data
.irq_fn
, ipu_data
.irq_err
, ipu_data
.irq_base
);
1772 /* Remap IPU common registers */
1773 ipu_data
.reg_ipu
= ioremap(mem_ipu
->start
,
1774 mem_ipu
->end
- mem_ipu
->start
+ 1);
1775 if (!ipu_data
.reg_ipu
) {
1777 goto err_ioremap_ipu
;
1780 /* Remap Image Converter and Image DMA Controller registers */
1781 ipu_data
.reg_ic
= ioremap(mem_ic
->start
,
1782 mem_ic
->end
- mem_ic
->start
+ 1);
1783 if (!ipu_data
.reg_ic
) {
1785 goto err_ioremap_ic
;
1789 ipu_data
.ipu_clk
= clk_get(&pdev
->dev
, NULL
);
1790 if (IS_ERR(ipu_data
.ipu_clk
)) {
1791 ret
= PTR_ERR(ipu_data
.ipu_clk
);
1795 /* Make sure IPU HSP clock is running */
1796 clk_enable(ipu_data
.ipu_clk
);
1798 /* Disable all interrupts */
1799 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_1
);
1800 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_2
);
1801 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_3
);
1802 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_4
);
1803 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_5
);
1805 dev_dbg(&pdev
->dev
, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev
->name
,
1806 (unsigned long)mem_ipu
->start
, ipu_data
.irq_fn
, ipu_data
.irq_err
);
1808 ret
= ipu_irq_attach_irq(&ipu_data
, pdev
);
1810 goto err_attach_irq
;
1812 /* Initialize DMA engine */
1813 ret
= ipu_idmac_init(&ipu_data
);
1815 goto err_idmac_init
;
1817 tasklet_init(&ipu_data
.tasklet
, ipu_gc_tasklet
, (unsigned long)&ipu_data
);
1819 ipu_data
.dev
= &pdev
->dev
;
1821 dev_dbg(ipu_data
.dev
, "IPU initialized\n");
1827 ipu_irq_detach_irq(&ipu_data
, pdev
);
1828 clk_disable(ipu_data
.ipu_clk
);
1829 clk_put(ipu_data
.ipu_clk
);
1831 iounmap(ipu_data
.reg_ic
);
1833 iounmap(ipu_data
.reg_ipu
);
1836 dev_err(&pdev
->dev
, "Failed to probe IPU: %d\n", ret
);
1840 static int __exit
ipu_remove(struct platform_device
*pdev
)
1842 struct ipu
*ipu
= platform_get_drvdata(pdev
);
1844 ipu_idmac_exit(ipu
);
1845 ipu_irq_detach_irq(ipu
, pdev
);
1846 clk_disable(ipu
->ipu_clk
);
1847 clk_put(ipu
->ipu_clk
);
1848 iounmap(ipu
->reg_ic
);
1849 iounmap(ipu
->reg_ipu
);
1850 tasklet_kill(&ipu
->tasklet
);
1851 platform_set_drvdata(pdev
, NULL
);
1857 * We need two MEM resources - with IPU-common and Image Converter registers,
1858 * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
1860 static struct platform_driver ipu_platform_driver
= {
1863 .owner
= THIS_MODULE
,
1865 .remove
= __exit_p(ipu_remove
),
1868 static int __init
ipu_init(void)
1870 return platform_driver_probe(&ipu_platform_driver
, ipu_probe
);
1872 subsys_initcall(ipu_init
);
1874 MODULE_DESCRIPTION("IPU core driver");
1875 MODULE_LICENSE("GPL v2");
1876 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
1877 MODULE_ALIAS("platform:ipu-core");