2 * videobuf2-dma-contig.c - DMA contig memory allocator for videobuf2
4 * Copyright (C) 2010 Samsung Electronics
6 * Author: Pawel Osciak <pawel@osciak.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation.
13 #include <linux/dma-buf.h>
14 #include <linux/module.h>
15 #include <linux/scatterlist.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/dma-mapping.h>
20 #include <media/videobuf2-core.h>
21 #include <media/videobuf2-dma-contig.h>
22 #include <media/videobuf2-memops.h>
33 enum dma_data_direction dma_dir
;
34 struct sg_table
*dma_sgt
;
35 struct frame_vector
*vec
;
38 struct vb2_vmarea_handler handler
;
40 struct sg_table
*sgt_base
;
43 struct dma_buf_attachment
*db_attach
;
46 /*********************************************/
47 /* scatterlist table functions */
48 /*********************************************/
50 static unsigned long vb2_dc_get_contiguous_size(struct sg_table
*sgt
)
52 struct scatterlist
*s
;
53 dma_addr_t expected
= sg_dma_address(sgt
->sgl
);
55 unsigned long size
= 0;
57 for_each_sg(sgt
->sgl
, s
, sgt
->nents
, i
) {
58 if (sg_dma_address(s
) != expected
)
60 expected
= sg_dma_address(s
) + sg_dma_len(s
);
61 size
+= sg_dma_len(s
);
66 /*********************************************/
67 /* callbacks for all buffers */
68 /*********************************************/
70 static void *vb2_dc_cookie(void *buf_priv
)
72 struct vb2_dc_buf
*buf
= buf_priv
;
74 return &buf
->dma_addr
;
77 static void *vb2_dc_vaddr(void *buf_priv
)
79 struct vb2_dc_buf
*buf
= buf_priv
;
81 if (!buf
->vaddr
&& buf
->db_attach
)
82 buf
->vaddr
= dma_buf_vmap(buf
->db_attach
->dmabuf
);
87 static unsigned int vb2_dc_num_users(void *buf_priv
)
89 struct vb2_dc_buf
*buf
= buf_priv
;
91 return atomic_read(&buf
->refcount
);
94 static void vb2_dc_prepare(void *buf_priv
)
96 struct vb2_dc_buf
*buf
= buf_priv
;
97 struct sg_table
*sgt
= buf
->dma_sgt
;
99 /* DMABUF exporter will flush the cache for us */
100 if (!sgt
|| buf
->db_attach
)
103 dma_sync_sg_for_device(buf
->dev
, sgt
->sgl
, sgt
->nents
, buf
->dma_dir
);
106 static void vb2_dc_finish(void *buf_priv
)
108 struct vb2_dc_buf
*buf
= buf_priv
;
109 struct sg_table
*sgt
= buf
->dma_sgt
;
111 /* DMABUF exporter will flush the cache for us */
112 if (!sgt
|| buf
->db_attach
)
115 dma_sync_sg_for_cpu(buf
->dev
, sgt
->sgl
, sgt
->nents
, buf
->dma_dir
);
118 /*********************************************/
119 /* callbacks for MMAP buffers */
120 /*********************************************/
122 static void vb2_dc_put(void *buf_priv
)
124 struct vb2_dc_buf
*buf
= buf_priv
;
126 if (!atomic_dec_and_test(&buf
->refcount
))
130 sg_free_table(buf
->sgt_base
);
131 kfree(buf
->sgt_base
);
133 dma_free_coherent(buf
->dev
, buf
->size
, buf
->vaddr
, buf
->dma_addr
);
134 put_device(buf
->dev
);
138 static void *vb2_dc_alloc(void *alloc_ctx
, unsigned long size
,
139 enum dma_data_direction dma_dir
, gfp_t gfp_flags
)
141 struct vb2_dc_conf
*conf
= alloc_ctx
;
142 struct device
*dev
= conf
->dev
;
143 struct vb2_dc_buf
*buf
;
145 buf
= kzalloc(sizeof *buf
, GFP_KERNEL
);
147 return ERR_PTR(-ENOMEM
);
149 buf
->vaddr
= dma_alloc_coherent(dev
, size
, &buf
->dma_addr
,
150 GFP_KERNEL
| gfp_flags
);
152 dev_err(dev
, "dma_alloc_coherent of size %ld failed\n", size
);
154 return ERR_PTR(-ENOMEM
);
157 /* Prevent the device from being released while the buffer is used */
158 buf
->dev
= get_device(dev
);
160 buf
->dma_dir
= dma_dir
;
162 buf
->handler
.refcount
= &buf
->refcount
;
163 buf
->handler
.put
= vb2_dc_put
;
164 buf
->handler
.arg
= buf
;
166 atomic_inc(&buf
->refcount
);
171 static int vb2_dc_mmap(void *buf_priv
, struct vm_area_struct
*vma
)
173 struct vb2_dc_buf
*buf
= buf_priv
;
177 printk(KERN_ERR
"No buffer to map\n");
182 * dma_mmap_* uses vm_pgoff as in-buffer offset, but we want to
187 ret
= dma_mmap_coherent(buf
->dev
, vma
, buf
->vaddr
,
188 buf
->dma_addr
, buf
->size
);
191 pr_err("Remapping memory failed, error: %d\n", ret
);
195 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
;
196 vma
->vm_private_data
= &buf
->handler
;
197 vma
->vm_ops
= &vb2_common_vm_ops
;
199 vma
->vm_ops
->open(vma
);
201 pr_debug("%s: mapped dma addr 0x%08lx at 0x%08lx, size %ld\n",
202 __func__
, (unsigned long)buf
->dma_addr
, vma
->vm_start
,
208 /*********************************************/
209 /* DMABUF ops for exporters */
210 /*********************************************/
212 struct vb2_dc_attachment
{
214 enum dma_data_direction dma_dir
;
217 static int vb2_dc_dmabuf_ops_attach(struct dma_buf
*dbuf
, struct device
*dev
,
218 struct dma_buf_attachment
*dbuf_attach
)
220 struct vb2_dc_attachment
*attach
;
222 struct scatterlist
*rd
, *wr
;
223 struct sg_table
*sgt
;
224 struct vb2_dc_buf
*buf
= dbuf
->priv
;
227 attach
= kzalloc(sizeof(*attach
), GFP_KERNEL
);
232 /* Copy the buf->base_sgt scatter list to the attachment, as we can't
233 * map the same scatter list to multiple attachments at the same time.
235 ret
= sg_alloc_table(sgt
, buf
->sgt_base
->orig_nents
, GFP_KERNEL
);
241 rd
= buf
->sgt_base
->sgl
;
243 for (i
= 0; i
< sgt
->orig_nents
; ++i
) {
244 sg_set_page(wr
, sg_page(rd
), rd
->length
, rd
->offset
);
249 attach
->dma_dir
= DMA_NONE
;
250 dbuf_attach
->priv
= attach
;
255 static void vb2_dc_dmabuf_ops_detach(struct dma_buf
*dbuf
,
256 struct dma_buf_attachment
*db_attach
)
258 struct vb2_dc_attachment
*attach
= db_attach
->priv
;
259 struct sg_table
*sgt
;
266 /* release the scatterlist cache */
267 if (attach
->dma_dir
!= DMA_NONE
)
268 dma_unmap_sg(db_attach
->dev
, sgt
->sgl
, sgt
->orig_nents
,
272 db_attach
->priv
= NULL
;
275 static struct sg_table
*vb2_dc_dmabuf_ops_map(
276 struct dma_buf_attachment
*db_attach
, enum dma_data_direction dma_dir
)
278 struct vb2_dc_attachment
*attach
= db_attach
->priv
;
279 /* stealing dmabuf mutex to serialize map/unmap operations */
280 struct mutex
*lock
= &db_attach
->dmabuf
->lock
;
281 struct sg_table
*sgt
;
286 /* return previously mapped sg table */
287 if (attach
->dma_dir
== dma_dir
) {
292 /* release any previous cache */
293 if (attach
->dma_dir
!= DMA_NONE
) {
294 dma_unmap_sg(db_attach
->dev
, sgt
->sgl
, sgt
->orig_nents
,
296 attach
->dma_dir
= DMA_NONE
;
299 /* mapping to the client with new direction */
300 sgt
->nents
= dma_map_sg(db_attach
->dev
, sgt
->sgl
, sgt
->orig_nents
,
303 pr_err("failed to map scatterlist\n");
305 return ERR_PTR(-EIO
);
308 attach
->dma_dir
= dma_dir
;
315 static void vb2_dc_dmabuf_ops_unmap(struct dma_buf_attachment
*db_attach
,
316 struct sg_table
*sgt
, enum dma_data_direction dma_dir
)
318 /* nothing to be done here */
321 static void vb2_dc_dmabuf_ops_release(struct dma_buf
*dbuf
)
323 /* drop reference obtained in vb2_dc_get_dmabuf */
324 vb2_dc_put(dbuf
->priv
);
327 static void *vb2_dc_dmabuf_ops_kmap(struct dma_buf
*dbuf
, unsigned long pgnum
)
329 struct vb2_dc_buf
*buf
= dbuf
->priv
;
331 return buf
->vaddr
+ pgnum
* PAGE_SIZE
;
334 static void *vb2_dc_dmabuf_ops_vmap(struct dma_buf
*dbuf
)
336 struct vb2_dc_buf
*buf
= dbuf
->priv
;
341 static int vb2_dc_dmabuf_ops_mmap(struct dma_buf
*dbuf
,
342 struct vm_area_struct
*vma
)
344 return vb2_dc_mmap(dbuf
->priv
, vma
);
347 static struct dma_buf_ops vb2_dc_dmabuf_ops
= {
348 .attach
= vb2_dc_dmabuf_ops_attach
,
349 .detach
= vb2_dc_dmabuf_ops_detach
,
350 .map_dma_buf
= vb2_dc_dmabuf_ops_map
,
351 .unmap_dma_buf
= vb2_dc_dmabuf_ops_unmap
,
352 .kmap
= vb2_dc_dmabuf_ops_kmap
,
353 .kmap_atomic
= vb2_dc_dmabuf_ops_kmap
,
354 .vmap
= vb2_dc_dmabuf_ops_vmap
,
355 .mmap
= vb2_dc_dmabuf_ops_mmap
,
356 .release
= vb2_dc_dmabuf_ops_release
,
359 static struct sg_table
*vb2_dc_get_base_sgt(struct vb2_dc_buf
*buf
)
362 struct sg_table
*sgt
;
364 sgt
= kmalloc(sizeof(*sgt
), GFP_KERNEL
);
366 dev_err(buf
->dev
, "failed to alloc sg table\n");
370 ret
= dma_get_sgtable(buf
->dev
, sgt
, buf
->vaddr
, buf
->dma_addr
,
373 dev_err(buf
->dev
, "failed to get scatterlist from DMA API\n");
381 static struct dma_buf
*vb2_dc_get_dmabuf(void *buf_priv
, unsigned long flags
)
383 struct vb2_dc_buf
*buf
= buf_priv
;
384 struct dma_buf
*dbuf
;
385 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
387 exp_info
.ops
= &vb2_dc_dmabuf_ops
;
388 exp_info
.size
= buf
->size
;
389 exp_info
.flags
= flags
;
393 buf
->sgt_base
= vb2_dc_get_base_sgt(buf
);
395 if (WARN_ON(!buf
->sgt_base
))
398 dbuf
= dma_buf_export(&exp_info
);
402 /* dmabuf keeps reference to vb2 buffer */
403 atomic_inc(&buf
->refcount
);
408 /*********************************************/
409 /* callbacks for USERPTR buffers */
410 /*********************************************/
412 static void vb2_dc_put_userptr(void *buf_priv
)
414 struct vb2_dc_buf
*buf
= buf_priv
;
415 struct sg_table
*sgt
= buf
->dma_sgt
;
420 DEFINE_DMA_ATTRS(attrs
);
422 dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC
, &attrs
);
424 * No need to sync to CPU, it's already synced to the CPU
425 * since the finish() memop will have been called before this.
427 dma_unmap_sg_attrs(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
428 buf
->dma_dir
, &attrs
);
429 pages
= frame_vector_pages(buf
->vec
);
430 /* sgt should exist only if vector contains pages... */
431 BUG_ON(IS_ERR(pages
));
432 for (i
= 0; i
< frame_vector_count(buf
->vec
); i
++)
433 set_page_dirty_lock(pages
[i
]);
437 vb2_destroy_framevec(buf
->vec
);
442 * For some kind of reserved memory there might be no struct page available,
443 * so all that can be done to support such 'pages' is to try to convert
444 * pfn to dma address or at the last resort just assume that
445 * dma address == physical address (like it has been assumed in earlier version
446 * of videobuf2-dma-contig
449 #ifdef __arch_pfn_to_dma
450 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
452 return (dma_addr_t
)__arch_pfn_to_dma(dev
, pfn
);
454 #elif defined(__pfn_to_bus)
455 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
457 return (dma_addr_t
)__pfn_to_bus(pfn
);
459 #elif defined(__pfn_to_phys)
460 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
462 return (dma_addr_t
)__pfn_to_phys(pfn
);
465 static inline dma_addr_t
vb2_dc_pfn_to_dma(struct device
*dev
, unsigned long pfn
)
467 /* really, we cannot do anything better at this point */
468 return (dma_addr_t
)(pfn
) << PAGE_SHIFT
;
472 static void *vb2_dc_get_userptr(void *alloc_ctx
, unsigned long vaddr
,
473 unsigned long size
, enum dma_data_direction dma_dir
)
475 struct vb2_dc_conf
*conf
= alloc_ctx
;
476 struct vb2_dc_buf
*buf
;
477 struct frame_vector
*vec
;
478 unsigned long offset
;
481 struct sg_table
*sgt
;
482 unsigned long contig_size
;
483 unsigned long dma_align
= dma_get_cache_alignment();
484 DEFINE_DMA_ATTRS(attrs
);
486 dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC
, &attrs
);
488 /* Only cache aligned DMA transfers are reliable */
489 if (!IS_ALIGNED(vaddr
| size
, dma_align
)) {
490 pr_debug("user data must be aligned to %lu bytes\n", dma_align
);
491 return ERR_PTR(-EINVAL
);
495 pr_debug("size is zero\n");
496 return ERR_PTR(-EINVAL
);
499 buf
= kzalloc(sizeof *buf
, GFP_KERNEL
);
501 return ERR_PTR(-ENOMEM
);
503 buf
->dev
= conf
->dev
;
504 buf
->dma_dir
= dma_dir
;
506 offset
= vaddr
& ~PAGE_MASK
;
507 vec
= vb2_create_framevec(vaddr
, size
, dma_dir
== DMA_FROM_DEVICE
);
513 n_pages
= frame_vector_count(vec
);
514 ret
= frame_vector_to_pages(vec
);
516 unsigned long *nums
= frame_vector_pfns(vec
);
519 * Failed to convert to pages... Check the memory is physically
520 * contiguous and use direct mapping
522 for (i
= 1; i
< n_pages
; i
++)
523 if (nums
[i
-1] + 1 != nums
[i
])
525 buf
->dma_addr
= vb2_dc_pfn_to_dma(buf
->dev
, nums
[0]);
529 sgt
= kzalloc(sizeof(*sgt
), GFP_KERNEL
);
531 pr_err("failed to allocate sg table\n");
536 ret
= sg_alloc_table_from_pages(sgt
, frame_vector_pages(vec
), n_pages
,
537 offset
, size
, GFP_KERNEL
);
539 pr_err("failed to initialize sg table\n");
544 * No need to sync to the device, this will happen later when the
545 * prepare() memop is called.
547 sgt
->nents
= dma_map_sg_attrs(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
548 buf
->dma_dir
, &attrs
);
549 if (sgt
->nents
<= 0) {
550 pr_err("failed to map scatterlist\n");
555 contig_size
= vb2_dc_get_contiguous_size(sgt
);
556 if (contig_size
< size
) {
557 pr_err("contiguous mapping is too small %lu/%lu\n",
563 buf
->dma_addr
= sg_dma_address(sgt
->sgl
);
571 dma_unmap_sg_attrs(buf
->dev
, sgt
->sgl
, sgt
->orig_nents
,
572 buf
->dma_dir
, &attrs
);
581 vb2_destroy_framevec(vec
);
589 /*********************************************/
590 /* callbacks for DMABUF buffers */
591 /*********************************************/
593 static int vb2_dc_map_dmabuf(void *mem_priv
)
595 struct vb2_dc_buf
*buf
= mem_priv
;
596 struct sg_table
*sgt
;
597 unsigned long contig_size
;
599 if (WARN_ON(!buf
->db_attach
)) {
600 pr_err("trying to pin a non attached buffer\n");
604 if (WARN_ON(buf
->dma_sgt
)) {
605 pr_err("dmabuf buffer is already pinned\n");
609 /* get the associated scatterlist for this buffer */
610 sgt
= dma_buf_map_attachment(buf
->db_attach
, buf
->dma_dir
);
612 pr_err("Error getting dmabuf scatterlist\n");
616 /* checking if dmabuf is big enough to store contiguous chunk */
617 contig_size
= vb2_dc_get_contiguous_size(sgt
);
618 if (contig_size
< buf
->size
) {
619 pr_err("contiguous chunk is too small %lu/%lu b\n",
620 contig_size
, buf
->size
);
621 dma_buf_unmap_attachment(buf
->db_attach
, sgt
, buf
->dma_dir
);
625 buf
->dma_addr
= sg_dma_address(sgt
->sgl
);
632 static void vb2_dc_unmap_dmabuf(void *mem_priv
)
634 struct vb2_dc_buf
*buf
= mem_priv
;
635 struct sg_table
*sgt
= buf
->dma_sgt
;
637 if (WARN_ON(!buf
->db_attach
)) {
638 pr_err("trying to unpin a not attached buffer\n");
643 pr_err("dmabuf buffer is already unpinned\n");
648 dma_buf_vunmap(buf
->db_attach
->dmabuf
, buf
->vaddr
);
651 dma_buf_unmap_attachment(buf
->db_attach
, sgt
, buf
->dma_dir
);
657 static void vb2_dc_detach_dmabuf(void *mem_priv
)
659 struct vb2_dc_buf
*buf
= mem_priv
;
661 /* if vb2 works correctly you should never detach mapped buffer */
662 if (WARN_ON(buf
->dma_addr
))
663 vb2_dc_unmap_dmabuf(buf
);
665 /* detach this attachment */
666 dma_buf_detach(buf
->db_attach
->dmabuf
, buf
->db_attach
);
670 static void *vb2_dc_attach_dmabuf(void *alloc_ctx
, struct dma_buf
*dbuf
,
671 unsigned long size
, enum dma_data_direction dma_dir
)
673 struct vb2_dc_conf
*conf
= alloc_ctx
;
674 struct vb2_dc_buf
*buf
;
675 struct dma_buf_attachment
*dba
;
677 if (dbuf
->size
< size
)
678 return ERR_PTR(-EFAULT
);
680 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
682 return ERR_PTR(-ENOMEM
);
684 buf
->dev
= conf
->dev
;
685 /* create attachment for the dmabuf with the user device */
686 dba
= dma_buf_attach(dbuf
, buf
->dev
);
688 pr_err("failed to attach dmabuf\n");
693 buf
->dma_dir
= dma_dir
;
695 buf
->db_attach
= dba
;
700 /*********************************************/
701 /* DMA CONTIG exported functions */
702 /*********************************************/
704 const struct vb2_mem_ops vb2_dma_contig_memops
= {
705 .alloc
= vb2_dc_alloc
,
707 .get_dmabuf
= vb2_dc_get_dmabuf
,
708 .cookie
= vb2_dc_cookie
,
709 .vaddr
= vb2_dc_vaddr
,
711 .get_userptr
= vb2_dc_get_userptr
,
712 .put_userptr
= vb2_dc_put_userptr
,
713 .prepare
= vb2_dc_prepare
,
714 .finish
= vb2_dc_finish
,
715 .map_dmabuf
= vb2_dc_map_dmabuf
,
716 .unmap_dmabuf
= vb2_dc_unmap_dmabuf
,
717 .attach_dmabuf
= vb2_dc_attach_dmabuf
,
718 .detach_dmabuf
= vb2_dc_detach_dmabuf
,
719 .num_users
= vb2_dc_num_users
,
721 EXPORT_SYMBOL_GPL(vb2_dma_contig_memops
);
723 void *vb2_dma_contig_init_ctx(struct device
*dev
)
725 struct vb2_dc_conf
*conf
;
727 conf
= kzalloc(sizeof *conf
, GFP_KERNEL
);
729 return ERR_PTR(-ENOMEM
);
735 EXPORT_SYMBOL_GPL(vb2_dma_contig_init_ctx
);
737 void vb2_dma_contig_cleanup_ctx(void *alloc_ctx
)
739 if (!IS_ERR_OR_NULL(alloc_ctx
))
742 EXPORT_SYMBOL_GPL(vb2_dma_contig_cleanup_ctx
);
744 MODULE_DESCRIPTION("DMA-contig memory handling routines for videobuf2");
745 MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>");
746 MODULE_LICENSE("GPL");