?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/exynos: add userptr feature for g2d module
[linux-2.6/btrfs-unstable.git] / drivers / gpu / drm / exynos / exynos_drm_g2d.c
bloba9002adc4694c0ca7b702a3d863ef8638e02bfc7
1 /*
2 * Copyright (C) 2012 Samsung Electronics Co.Ltd
3 * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundationr
8 */
9
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/clk.h>
13 #include <linux/err.h>
14 #include <linux/interrupt.h>
15 #include <linux/io.h>
16 #include <linux/platform_device.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/slab.h>
19 #include <linux/workqueue.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/dma-attrs.h>
22
23 #include <drm/drmP.h>
24 #include <drm/exynos_drm.h>
25 #include "exynos_drm_drv.h"
26 #include "exynos_drm_gem.h"
27 #include "exynos_drm_iommu.h"
28
29 #define G2D_HW_MAJOR_VER 4
30 #define G2D_HW_MINOR_VER 1
31
32 /* vaild register range set from user: 0x0104 ~ 0x0880 */
33 #define G2D_VALID_START 0x0104
34 #define G2D_VALID_END 0x0880
35
36 /* general registers */
37 #define G2D_SOFT_RESET 0x0000
38 #define G2D_INTEN 0x0004
39 #define G2D_INTC_PEND 0x000C
40 #define G2D_DMA_SFR_BASE_ADDR 0x0080
41 #define G2D_DMA_COMMAND 0x0084
42 #define G2D_DMA_STATUS 0x008C
43 #define G2D_DMA_HOLD_CMD 0x0090
44
45 /* command registers */
46 #define G2D_BITBLT_START 0x0100
47
48 /* registers for base address */
49 #define G2D_SRC_BASE_ADDR 0x0304
50 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318
51 #define G2D_DST_BASE_ADDR 0x0404
52 #define G2D_DST_PLANE2_BASE_ADDR 0x0418
53 #define G2D_PAT_BASE_ADDR 0x0500
54 #define G2D_MSK_BASE_ADDR 0x0520
55
56 /* G2D_SOFT_RESET */
57 #define G2D_SFRCLEAR (1 << 1)
58 #define G2D_R (1 << 0)
59
60 /* G2D_INTEN */
61 #define G2D_INTEN_ACF (1 << 3)
62 #define G2D_INTEN_UCF (1 << 2)
63 #define G2D_INTEN_GCF (1 << 1)
64 #define G2D_INTEN_SCF (1 << 0)
65
66 /* G2D_INTC_PEND */
67 #define G2D_INTP_ACMD_FIN (1 << 3)
68 #define G2D_INTP_UCMD_FIN (1 << 2)
69 #define G2D_INTP_GCMD_FIN (1 << 1)
70 #define G2D_INTP_SCMD_FIN (1 << 0)
71
72 /* G2D_DMA_COMMAND */
73 #define G2D_DMA_HALT (1 << 2)
74 #define G2D_DMA_CONTINUE (1 << 1)
75 #define G2D_DMA_START (1 << 0)
76
77 /* G2D_DMA_STATUS */
78 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17)
79 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1)
80 #define G2D_DMA_DONE (1 << 0)
81 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17
82
83 /* G2D_DMA_HOLD_CMD */
84 #define G2D_USET_HOLD (1 << 2)
85 #define G2D_LIST_HOLD (1 << 1)
86 #define G2D_BITBLT_HOLD (1 << 0)
87
88 /* G2D_BITBLT_START */
89 #define G2D_START_CASESEL (1 << 2)
90 #define G2D_START_NHOLT (1 << 1)
91 #define G2D_START_BITBLT (1 << 0)
92
93 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4)
94 #define G2D_CMDLIST_NUM 64
95 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
96 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
97
98 #define MAX_BUF_ADDR_NR 6
99
100 /* maximum buffer pool size of userptr is 64MB as default */
101 #define MAX_POOL (64 * 1024 * 1024)
102
103 enum {
104 BUF_TYPE_GEM = 1,
105 BUF_TYPE_USERPTR,
106 };
107
108 /* cmdlist data structure */
109 struct g2d_cmdlist {
110 u32 head;
111 unsigned long data[G2D_CMDLIST_DATA_NUM];
112 u32 last; /* last data offset */
113 };
114
115 struct drm_exynos_pending_g2d_event {
116 struct drm_pending_event base;
117 struct drm_exynos_g2d_event event;
118 };
119
120 struct g2d_cmdlist_userptr {
121 struct list_head list;
122 dma_addr_t dma_addr;
123 unsigned long userptr;
124 unsigned long size;
125 struct page **pages;
126 unsigned int npages;
127 struct sg_table *sgt;
128 struct vm_area_struct *vma;
129 atomic_t refcount;
130 bool in_pool;
131 bool out_of_list;
132 };
133
134 struct g2d_cmdlist_node {
135 struct list_head list;
136 struct g2d_cmdlist *cmdlist;
137 unsigned int map_nr;
138 unsigned long handles[MAX_BUF_ADDR_NR];
139 unsigned int obj_type[MAX_BUF_ADDR_NR];
140 dma_addr_t dma_addr;
141
142 struct drm_exynos_pending_g2d_event *event;
143 };
144
145 struct g2d_runqueue_node {
146 struct list_head list;
147 struct list_head run_cmdlist;
148 struct list_head event_list;
149 struct drm_file *filp;
150 pid_t pid;
151 struct completion complete;
152 int async;
153 };
154
155 struct g2d_data {
156 struct device *dev;
157 struct clk *gate_clk;
158 void __iomem *regs;
159 int irq;
160 struct workqueue_struct *g2d_workq;
161 struct work_struct runqueue_work;
162 struct exynos_drm_subdrv subdrv;
163 bool suspended;
164
165 /* cmdlist */
166 struct g2d_cmdlist_node *cmdlist_node;
167 struct list_head free_cmdlist;
168 struct mutex cmdlist_mutex;
169 dma_addr_t cmdlist_pool;
170 void *cmdlist_pool_virt;
171 struct dma_attrs cmdlist_dma_attrs;
172
173 /* runqueue*/
174 struct g2d_runqueue_node *runqueue_node;
175 struct list_head runqueue;
176 struct mutex runqueue_mutex;
177 struct kmem_cache *runqueue_slab;
178
179 unsigned long current_pool;
180 unsigned long max_pool;
181 };
182
183 static int g2d_init_cmdlist(struct g2d_data *g2d)
184 {
185 struct device *dev = g2d->dev;
186 struct g2d_cmdlist_node *node = g2d->cmdlist_node;
187 struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
188 int nr;
189 int ret;
190
191 init_dma_attrs(&g2d->cmdlist_dma_attrs);
192 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &g2d->cmdlist_dma_attrs);
193
194 g2d->cmdlist_pool_virt = dma_alloc_attrs(subdrv->drm_dev->dev,
195 G2D_CMDLIST_POOL_SIZE,
196 &g2d->cmdlist_pool, GFP_KERNEL,
197 &g2d->cmdlist_dma_attrs);
198 if (!g2d->cmdlist_pool_virt) {
199 dev_err(dev, "failed to allocate dma memory\n");
200 return -ENOMEM;
201 }
202
203 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
204 if (!node) {
205 dev_err(dev, "failed to allocate memory\n");
206 ret = -ENOMEM;
207 goto err;
208 }
209
210 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
211 node[nr].cmdlist =
212 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
213 node[nr].dma_addr =
214 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
215
216 list_add_tail(&node[nr].list, &g2d->free_cmdlist);
217 }
218
219 return 0;
220
221 err:
222 dma_free_attrs(subdrv->drm_dev->dev, G2D_CMDLIST_POOL_SIZE,
223 g2d->cmdlist_pool_virt,
224 g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs);
225 return ret;
226 }
227
228 static void g2d_fini_cmdlist(struct g2d_data *g2d)
229 {
230 struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
231
232 kfree(g2d->cmdlist_node);
233 dma_free_attrs(subdrv->drm_dev->dev, G2D_CMDLIST_POOL_SIZE,
234 g2d->cmdlist_pool_virt,
235 g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs);
236 }
237
238 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
239 {
240 struct device *dev = g2d->dev;
241 struct g2d_cmdlist_node *node;
242
243 mutex_lock(&g2d->cmdlist_mutex);
244 if (list_empty(&g2d->free_cmdlist)) {
245 dev_err(dev, "there is no free cmdlist\n");
246 mutex_unlock(&g2d->cmdlist_mutex);
247 return NULL;
248 }
249
250 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
251 list);
252 list_del_init(&node->list);
253 mutex_unlock(&g2d->cmdlist_mutex);
254
255 return node;
256 }
257
258 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
259 {
260 mutex_lock(&g2d->cmdlist_mutex);
261 list_move_tail(&node->list, &g2d->free_cmdlist);
262 mutex_unlock(&g2d->cmdlist_mutex);
263 }
264
265 static void g2d_add_cmdlist_to_inuse(struct exynos_drm_g2d_private *g2d_priv,
266 struct g2d_cmdlist_node *node)
267 {
268 struct g2d_cmdlist_node *lnode;
269
270 if (list_empty(&g2d_priv->inuse_cmdlist))
271 goto add_to_list;
272
273 /* this links to base address of new cmdlist */
274 lnode = list_entry(g2d_priv->inuse_cmdlist.prev,
275 struct g2d_cmdlist_node, list);
276 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
277
278 add_to_list:
279 list_add_tail(&node->list, &g2d_priv->inuse_cmdlist);
280
281 if (node->event)
282 list_add_tail(&node->event->base.link, &g2d_priv->event_list);
283 }
284
285 static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev,
286 unsigned long obj,
287 bool force)
288 {
289 struct g2d_cmdlist_userptr *g2d_userptr =
290 (struct g2d_cmdlist_userptr *)obj;
291
292 if (!obj)
293 return;
294
295 if (force)
296 goto out;
297
298 atomic_dec(&g2d_userptr->refcount);
299
300 if (atomic_read(&g2d_userptr->refcount) > 0)
301 return;
302
303 if (g2d_userptr->in_pool)
304 return;
305
306 out:
307 exynos_gem_unmap_sgt_from_dma(drm_dev, g2d_userptr->sgt,
308 DMA_BIDIRECTIONAL);
309
310 exynos_gem_put_pages_to_userptr(g2d_userptr->pages,
311 g2d_userptr->npages,
312 g2d_userptr->vma);
313
314 if (!g2d_userptr->out_of_list)
315 list_del_init(&g2d_userptr->list);
316
317 sg_free_table(g2d_userptr->sgt);
318 kfree(g2d_userptr->sgt);
319 g2d_userptr->sgt = NULL;
320
321 kfree(g2d_userptr->pages);
322 kfree(g2d_userptr);
323 g2d_userptr->pages = NULL;
324 g2d_userptr = NULL;
325 }
326
327 dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
328 unsigned long userptr,
329 unsigned long size,
330 struct drm_file *filp,
331 unsigned long *obj)
332 {
333 struct drm_exynos_file_private *file_priv = filp->driver_priv;
334 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
335 struct g2d_cmdlist_userptr *g2d_userptr;
336 struct g2d_data *g2d;
337 struct page **pages;
338 struct sg_table *sgt;
339 struct vm_area_struct *vma;
340 unsigned long start, end;
341 unsigned int npages, offset;
342 int ret;
343
344 if (!size) {
345 DRM_ERROR("invalid userptr size.\n");
346 return ERR_PTR(-EINVAL);
347 }
348
349 g2d = dev_get_drvdata(g2d_priv->dev);
350
351 /* check if userptr already exists in userptr_list. */
352 list_for_each_entry(g2d_userptr, &g2d_priv->userptr_list, list) {
353 if (g2d_userptr->userptr == userptr) {
354 /*
355 * also check size because there could be same address
356 * and different size.
357 */
358 if (g2d_userptr->size == size) {
359 atomic_inc(&g2d_userptr->refcount);
360 *obj = (unsigned long)g2d_userptr;
361
362 return &g2d_userptr->dma_addr;
363 }
364
365 /*
366 * at this moment, maybe g2d dma is accessing this
367 * g2d_userptr memory region so just remove this
368 * g2d_userptr object from userptr_list not to be
369 * referred again and also except it the userptr
370 * pool to be released after the dma access completion.
371 */
372 g2d_userptr->out_of_list = true;
373 g2d_userptr->in_pool = false;
374 list_del_init(&g2d_userptr->list);
375
376 break;
377 }
378 }
379
380 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
381 if (!g2d_userptr) {
382 DRM_ERROR("failed to allocate g2d_userptr.\n");
383 return ERR_PTR(-ENOMEM);
384 }
385
386 atomic_set(&g2d_userptr->refcount, 1);
387
388 start = userptr & PAGE_MASK;
389 offset = userptr & ~PAGE_MASK;
390 end = PAGE_ALIGN(userptr + size);
391 npages = (end - start) >> PAGE_SHIFT;
392 g2d_userptr->npages = npages;
393
394 pages = kzalloc(npages * sizeof(struct page *), GFP_KERNEL);
395 if (!pages) {
396 DRM_ERROR("failed to allocate pages.\n");
397 kfree(g2d_userptr);
398 return ERR_PTR(-ENOMEM);
399 }
400
401 vma = find_vma(current->mm, userptr);
402 if (!vma) {
403 DRM_ERROR("failed to get vm region.\n");
404 ret = -EFAULT;
405 goto err_free_pages;
406 }
407
408 if (vma->vm_end < userptr + size) {
409 DRM_ERROR("vma is too small.\n");
410 ret = -EFAULT;
411 goto err_free_pages;
412 }
413
414 g2d_userptr->vma = exynos_gem_get_vma(vma);
415 if (!g2d_userptr->vma) {
416 DRM_ERROR("failed to copy vma.\n");
417 ret = -ENOMEM;
418 goto err_free_pages;
419 }
420
421 g2d_userptr->size = size;
422
423 ret = exynos_gem_get_pages_from_userptr(start & PAGE_MASK,
424 npages, pages, vma);
425 if (ret < 0) {
426 DRM_ERROR("failed to get user pages from userptr.\n");
427 goto err_put_vma;
428 }
429
430 g2d_userptr->pages = pages;
431
432 sgt = kzalloc(sizeof *sgt, GFP_KERNEL);
433 if (!sgt) {
434 DRM_ERROR("failed to allocate sg table.\n");
435 ret = -ENOMEM;
436 goto err_free_userptr;
437 }
438
439 ret = sg_alloc_table_from_pages(sgt, pages, npages, offset,
440 size, GFP_KERNEL);
441 if (ret < 0) {
442 DRM_ERROR("failed to get sgt from pages.\n");
443 goto err_free_sgt;
444 }
445
446 g2d_userptr->sgt = sgt;
447
448 ret = exynos_gem_map_sgt_with_dma(drm_dev, g2d_userptr->sgt,
449 DMA_BIDIRECTIONAL);
450 if (ret < 0) {
451 DRM_ERROR("failed to map sgt with dma region.\n");
452 goto err_free_sgt;
453 }
454
455 g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
456 g2d_userptr->userptr = userptr;
457
458 list_add_tail(&g2d_userptr->list, &g2d_priv->userptr_list);
459
460 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
461 g2d->current_pool += npages << PAGE_SHIFT;
462 g2d_userptr->in_pool = true;
463 }
464
465 *obj = (unsigned long)g2d_userptr;
466
467 return &g2d_userptr->dma_addr;
468
469 err_free_sgt:
470 sg_free_table(sgt);
471 kfree(sgt);
472 sgt = NULL;
473
474 err_free_userptr:
475 exynos_gem_put_pages_to_userptr(g2d_userptr->pages,
476 g2d_userptr->npages,
477 g2d_userptr->vma);
478
479 err_put_vma:
480 exynos_gem_put_vma(g2d_userptr->vma);
481
482 err_free_pages:
483 kfree(pages);
484 kfree(g2d_userptr);
485 pages = NULL;
486 g2d_userptr = NULL;
487
488 return ERR_PTR(ret);
489 }
490
491 static void g2d_userptr_free_all(struct drm_device *drm_dev,
492 struct g2d_data *g2d,
493 struct drm_file *filp)
494 {
495 struct drm_exynos_file_private *file_priv = filp->driver_priv;
496 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
497 struct g2d_cmdlist_userptr *g2d_userptr, *n;
498
499 list_for_each_entry_safe(g2d_userptr, n, &g2d_priv->userptr_list, list)
500 if (g2d_userptr->in_pool)
501 g2d_userptr_put_dma_addr(drm_dev,
502 (unsigned long)g2d_userptr,
503 true);
504
505 g2d->current_pool = 0;
506 }
507
508 static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
509 struct g2d_cmdlist_node *node,
510 struct drm_device *drm_dev,
511 struct drm_file *file)
512 {
513 struct g2d_cmdlist *cmdlist = node->cmdlist;
514 int offset;
515 int i;
516
517 for (i = 0; i < node->map_nr; i++) {
518 unsigned long handle;
519 dma_addr_t *addr;
520
521 offset = cmdlist->last - (i * 2 + 1);
522 handle = cmdlist->data[offset];
523
524 if (node->obj_type[i] == BUF_TYPE_GEM) {
525 addr = exynos_drm_gem_get_dma_addr(drm_dev, handle,
526 file);
527 if (IS_ERR(addr)) {
528 node->map_nr = i;
529 return -EFAULT;
530 }
531 } else {
532 struct drm_exynos_g2d_userptr g2d_userptr;
533
534 if (copy_from_user(&g2d_userptr, (void __user *)handle,
535 sizeof(struct drm_exynos_g2d_userptr))) {
536 node->map_nr = i;
537 return -EFAULT;
538 }
539
540 addr = g2d_userptr_get_dma_addr(drm_dev,
541 g2d_userptr.userptr,
542 g2d_userptr.size,
543 file,
544 &handle);
545 if (IS_ERR(addr)) {
546 node->map_nr = i;
547 return -EFAULT;
548 }
549 }
550
551 cmdlist->data[offset] = *addr;
552 node->handles[i] = handle;
553 }
554
555 return 0;
556 }
557
558 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
559 struct g2d_cmdlist_node *node,
560 struct drm_file *filp)
561 {
562 struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
563 int i;
564
565 for (i = 0; i < node->map_nr; i++) {
566 unsigned long handle = node->handles[i];
567
568 if (node->obj_type[i] == BUF_TYPE_GEM)
569 exynos_drm_gem_put_dma_addr(subdrv->drm_dev, handle,
570 filp);
571 else
572 g2d_userptr_put_dma_addr(subdrv->drm_dev, handle,
573 false);
574
575 node->handles[i] = 0;
576 }
577
578 node->map_nr = 0;
579 }
580
581 static void g2d_dma_start(struct g2d_data *g2d,
582 struct g2d_runqueue_node *runqueue_node)
583 {
584 struct g2d_cmdlist_node *node =
585 list_first_entry(&runqueue_node->run_cmdlist,
586 struct g2d_cmdlist_node, list);
587
588 pm_runtime_get_sync(g2d->dev);
589 clk_enable(g2d->gate_clk);
590
591 /* interrupt enable */
592 writel_relaxed(G2D_INTEN_ACF | G2D_INTEN_UCF | G2D_INTEN_GCF,
593 g2d->regs + G2D_INTEN);
594
595 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
596 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
597 }
598
599 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
600 {
601 struct g2d_runqueue_node *runqueue_node;
602
603 if (list_empty(&g2d->runqueue))
604 return NULL;
605
606 runqueue_node = list_first_entry(&g2d->runqueue,
607 struct g2d_runqueue_node, list);
608 list_del_init(&runqueue_node->list);
609 return runqueue_node;
610 }
611
612 static void g2d_free_runqueue_node(struct g2d_data *g2d,
613 struct g2d_runqueue_node *runqueue_node)
614 {
615 struct g2d_cmdlist_node *node;
616
617 if (!runqueue_node)
618 return;
619
620 mutex_lock(&g2d->cmdlist_mutex);
621 /*
622 * commands in run_cmdlist have been completed so unmap all gem
623 * objects in each command node so that they are unreferenced.
624 */
625 list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
626 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
627 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
628 mutex_unlock(&g2d->cmdlist_mutex);
629
630 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
631 }
632
633 static void g2d_exec_runqueue(struct g2d_data *g2d)
634 {
635 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
636 if (g2d->runqueue_node)
637 g2d_dma_start(g2d, g2d->runqueue_node);
638 }
639
640 static void g2d_runqueue_worker(struct work_struct *work)
641 {
642 struct g2d_data *g2d = container_of(work, struct g2d_data,
643 runqueue_work);
644
645
646 mutex_lock(&g2d->runqueue_mutex);
647 clk_disable(g2d->gate_clk);
648 pm_runtime_put_sync(g2d->dev);
649
650 complete(&g2d->runqueue_node->complete);
651 if (g2d->runqueue_node->async)
652 g2d_free_runqueue_node(g2d, g2d->runqueue_node);
653
654 if (g2d->suspended)
655 g2d->runqueue_node = NULL;
656 else
657 g2d_exec_runqueue(g2d);
658 mutex_unlock(&g2d->runqueue_mutex);
659 }
660
661 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
662 {
663 struct drm_device *drm_dev = g2d->subdrv.drm_dev;
664 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
665 struct drm_exynos_pending_g2d_event *e;
666 struct timeval now;
667 unsigned long flags;
668
669 if (list_empty(&runqueue_node->event_list))
670 return;
671
672 e = list_first_entry(&runqueue_node->event_list,
673 struct drm_exynos_pending_g2d_event, base.link);
674
675 do_gettimeofday(&now);
676 e->event.tv_sec = now.tv_sec;
677 e->event.tv_usec = now.tv_usec;
678 e->event.cmdlist_no = cmdlist_no;
679
680 spin_lock_irqsave(&drm_dev->event_lock, flags);
681 list_move_tail(&e->base.link, &e->base.file_priv->event_list);
682 wake_up_interruptible(&e->base.file_priv->event_wait);
683 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
684 }
685
686 static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
687 {
688 struct g2d_data *g2d = dev_id;
689 u32 pending;
690
691 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
692 if (pending)
693 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
694
695 if (pending & G2D_INTP_GCMD_FIN) {
696 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
697
698 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
699 G2D_DMA_LIST_DONE_COUNT_OFFSET;
700
701 g2d_finish_event(g2d, cmdlist_no);
702
703 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
704 if (!(pending & G2D_INTP_ACMD_FIN)) {
705 writel_relaxed(G2D_DMA_CONTINUE,
706 g2d->regs + G2D_DMA_COMMAND);
707 }
708 }
709
710 if (pending & G2D_INTP_ACMD_FIN)
711 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
712
713 return IRQ_HANDLED;
714 }
715
716 static int g2d_check_reg_offset(struct device *dev,
717 struct g2d_cmdlist_node *node,
718 int nr, bool for_addr)
719 {
720 struct g2d_cmdlist *cmdlist = node->cmdlist;
721 int reg_offset;
722 int index;
723 int i;
724
725 for (i = 0; i < nr; i++) {
726 index = cmdlist->last - 2 * (i + 1);
727
728 if (for_addr) {
729 /* check userptr buffer type. */
730 reg_offset = (cmdlist->data[index] &
731 ~0x7fffffff) >> 31;
732 if (reg_offset) {
733 node->obj_type[i] = BUF_TYPE_USERPTR;
734 cmdlist->data[index] &= ~G2D_BUF_USERPTR;
735 }
736 }
737
738 reg_offset = cmdlist->data[index] & ~0xfffff000;
739
740 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
741 goto err;
742 if (reg_offset % 4)
743 goto err;
744
745 switch (reg_offset) {
746 case G2D_SRC_BASE_ADDR:
747 case G2D_SRC_PLANE2_BASE_ADDR:
748 case G2D_DST_BASE_ADDR:
749 case G2D_DST_PLANE2_BASE_ADDR:
750 case G2D_PAT_BASE_ADDR:
751 case G2D_MSK_BASE_ADDR:
752 if (!for_addr)
753 goto err;
754
755 if (node->obj_type[i] != BUF_TYPE_USERPTR)
756 node->obj_type[i] = BUF_TYPE_GEM;
757 break;
758 default:
759 if (for_addr)
760 goto err;
761 break;
762 }
763 }
764
765 return 0;
766
767 err:
768 dev_err(dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
769 return -EINVAL;
770 }
771
772 /* ioctl functions */
773 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
774 struct drm_file *file)
775 {
776 struct drm_exynos_g2d_get_ver *ver = data;
777
778 ver->major = G2D_HW_MAJOR_VER;
779 ver->minor = G2D_HW_MINOR_VER;
780
781 return 0;
782 }
783 EXPORT_SYMBOL_GPL(exynos_g2d_get_ver_ioctl);
784
785 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
786 struct drm_file *file)
787 {
788 struct drm_exynos_file_private *file_priv = file->driver_priv;
789 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
790 struct device *dev = g2d_priv->dev;
791 struct g2d_data *g2d;
792 struct drm_exynos_g2d_set_cmdlist *req = data;
793 struct drm_exynos_g2d_cmd *cmd;
794 struct drm_exynos_pending_g2d_event *e;
795 struct g2d_cmdlist_node *node;
796 struct g2d_cmdlist *cmdlist;
797 unsigned long flags;
798 int size;
799 int ret;
800
801 if (!dev)
802 return -ENODEV;
803
804 g2d = dev_get_drvdata(dev);
805 if (!g2d)
806 return -EFAULT;
807
808 node = g2d_get_cmdlist(g2d);
809 if (!node)
810 return -ENOMEM;
811
812 node->event = NULL;
813
814 if (req->event_type != G2D_EVENT_NOT) {
815 spin_lock_irqsave(&drm_dev->event_lock, flags);
816 if (file->event_space < sizeof(e->event)) {
817 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
818 ret = -ENOMEM;
819 goto err;
820 }
821 file->event_space -= sizeof(e->event);
822 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
823
824 e = kzalloc(sizeof(*node->event), GFP_KERNEL);
825 if (!e) {
826 dev_err(dev, "failed to allocate event\n");
827
828 spin_lock_irqsave(&drm_dev->event_lock, flags);
829 file->event_space += sizeof(e->event);
830 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
831
832 ret = -ENOMEM;
833 goto err;
834 }
835
836 e->event.base.type = DRM_EXYNOS_G2D_EVENT;
837 e->event.base.length = sizeof(e->event);
838 e->event.user_data = req->user_data;
839 e->base.event = &e->event.base;
840 e->base.file_priv = file;
841 e->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
842
843 node->event = e;
844 }
845
846 cmdlist = node->cmdlist;
847
848 cmdlist->last = 0;
849
850 /*
851 * If don't clear SFR registers, the cmdlist is affected by register
852 * values of previous cmdlist. G2D hw executes SFR clear command and
853 * a next command at the same time then the next command is ignored and
854 * is executed rightly from next next command, so needs a dummy command
855 * to next command of SFR clear command.
856 */
857 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
858 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
859 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
860 cmdlist->data[cmdlist->last++] = 0;
861
862 if (node->event) {
863 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
864 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
865 }
866
867 /* Check size of cmdlist: last 2 is about G2D_BITBLT_START */
868 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
869 if (size > G2D_CMDLIST_DATA_NUM) {
870 dev_err(dev, "cmdlist size is too big\n");
871 ret = -EINVAL;
872 goto err_free_event;
873 }
874
875 cmd = (struct drm_exynos_g2d_cmd *)(uint32_t)req->cmd;
876
877 if (copy_from_user(cmdlist->data + cmdlist->last,
878 (void __user *)cmd,
879 sizeof(*cmd) * req->cmd_nr)) {
880 ret = -EFAULT;
881 goto err_free_event;
882 }
883 cmdlist->last += req->cmd_nr * 2;
884
885 ret = g2d_check_reg_offset(dev, node, req->cmd_nr, false);
886 if (ret < 0)
887 goto err_free_event;
888
889 node->map_nr = req->cmd_buf_nr;
890 if (req->cmd_buf_nr) {
891 struct drm_exynos_g2d_cmd *cmd_buf;
892
893 cmd_buf = (struct drm_exynos_g2d_cmd *)(uint32_t)req->cmd_buf;
894
895 if (copy_from_user(cmdlist->data + cmdlist->last,
896 (void __user *)cmd_buf,
897 sizeof(*cmd_buf) * req->cmd_buf_nr)) {
898 ret = -EFAULT;
899 goto err_free_event;
900 }
901 cmdlist->last += req->cmd_buf_nr * 2;
902
903 ret = g2d_check_reg_offset(dev, node, req->cmd_buf_nr, true);
904 if (ret < 0)
905 goto err_free_event;
906
907 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
908 if (ret < 0)
909 goto err_unmap;
910 }
911
912 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
913 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
914
915 /* head */
916 cmdlist->head = cmdlist->last / 2;
917
918 /* tail */
919 cmdlist->data[cmdlist->last] = 0;
920
921 g2d_add_cmdlist_to_inuse(g2d_priv, node);
922
923 return 0;
924
925 err_unmap:
926 g2d_unmap_cmdlist_gem(g2d, node, file);
927 err_free_event:
928 if (node->event) {
929 spin_lock_irqsave(&drm_dev->event_lock, flags);
930 file->event_space += sizeof(e->event);
931 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
932 kfree(node->event);
933 }
934 err:
935 g2d_put_cmdlist(g2d, node);
936 return ret;
937 }
938 EXPORT_SYMBOL_GPL(exynos_g2d_set_cmdlist_ioctl);
939
940 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
941 struct drm_file *file)
942 {
943 struct drm_exynos_file_private *file_priv = file->driver_priv;
944 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
945 struct device *dev = g2d_priv->dev;
946 struct g2d_data *g2d;
947 struct drm_exynos_g2d_exec *req = data;
948 struct g2d_runqueue_node *runqueue_node;
949 struct list_head *run_cmdlist;
950 struct list_head *event_list;
951
952 if (!dev)
953 return -ENODEV;
954
955 g2d = dev_get_drvdata(dev);
956 if (!g2d)
957 return -EFAULT;
958
959 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
960 if (!runqueue_node) {
961 dev_err(dev, "failed to allocate memory\n");
962 return -ENOMEM;
963 }
964 run_cmdlist = &runqueue_node->run_cmdlist;
965 event_list = &runqueue_node->event_list;
966 INIT_LIST_HEAD(run_cmdlist);
967 INIT_LIST_HEAD(event_list);
968 init_completion(&runqueue_node->complete);
969 runqueue_node->async = req->async;
970
971 list_splice_init(&g2d_priv->inuse_cmdlist, run_cmdlist);
972 list_splice_init(&g2d_priv->event_list, event_list);
973
974 if (list_empty(run_cmdlist)) {
975 dev_err(dev, "there is no inuse cmdlist\n");
976 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
977 return -EPERM;
978 }
979
980 mutex_lock(&g2d->runqueue_mutex);
981 runqueue_node->pid = current->pid;
982 runqueue_node->filp = file;
983 list_add_tail(&runqueue_node->list, &g2d->runqueue);
984 if (!g2d->runqueue_node)
985 g2d_exec_runqueue(g2d);
986 mutex_unlock(&g2d->runqueue_mutex);
987
988 if (runqueue_node->async)
989 goto out;
990
991 wait_for_completion(&runqueue_node->complete);
992 g2d_free_runqueue_node(g2d, runqueue_node);
993
994 out:
995 return 0;
996 }
997 EXPORT_SYMBOL_GPL(exynos_g2d_exec_ioctl);
998
999 static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
1000 {
1001 struct g2d_data *g2d;
1002 int ret;
1003
1004 g2d = dev_get_drvdata(dev);
1005 if (!g2d)
1006 return -EFAULT;
1007
1008 /* allocate dma-aware cmdlist buffer. */
1009 ret = g2d_init_cmdlist(g2d);
1010 if (ret < 0) {
1011 dev_err(dev, "cmdlist init failed\n");
1012 return ret;
1013 }
1014
1015 if (!is_drm_iommu_supported(drm_dev))
1016 return 0;
1017
1018 ret = drm_iommu_attach_device(drm_dev, dev);
1019 if (ret < 0) {
1020 dev_err(dev, "failed to enable iommu.\n");
1021 g2d_fini_cmdlist(g2d);
1022 }
1023
1024 return ret;
1025
1026 }
1027
1028 static void g2d_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
1029 {
1030 if (!is_drm_iommu_supported(drm_dev))
1031 return;
1032
1033 drm_iommu_detach_device(drm_dev, dev);
1034 }
1035
1036 static int g2d_open(struct drm_device *drm_dev, struct device *dev,
1037 struct drm_file *file)
1038 {
1039 struct drm_exynos_file_private *file_priv = file->driver_priv;
1040 struct exynos_drm_g2d_private *g2d_priv;
1041
1042 g2d_priv = kzalloc(sizeof(*g2d_priv), GFP_KERNEL);
1043 if (!g2d_priv) {
1044 dev_err(dev, "failed to allocate g2d private data\n");
1045 return -ENOMEM;
1046 }
1047
1048 g2d_priv->dev = dev;
1049 file_priv->g2d_priv = g2d_priv;
1050
1051 INIT_LIST_HEAD(&g2d_priv->inuse_cmdlist);
1052 INIT_LIST_HEAD(&g2d_priv->event_list);
1053 INIT_LIST_HEAD(&g2d_priv->userptr_list);
1054
1055 return 0;
1056 }
1057
1058 static void g2d_close(struct drm_device *drm_dev, struct device *dev,
1059 struct drm_file *file)
1060 {
1061 struct drm_exynos_file_private *file_priv = file->driver_priv;
1062 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
1063 struct g2d_data *g2d;
1064 struct g2d_cmdlist_node *node, *n;
1065
1066 if (!dev)
1067 return;
1068
1069 g2d = dev_get_drvdata(dev);
1070 if (!g2d)
1071 return;
1072
1073 mutex_lock(&g2d->cmdlist_mutex);
1074 list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) {
1075 /*
1076 * unmap all gem objects not completed.
1077 *
1078 * P.S. if current process was terminated forcely then
1079 * there may be some commands in inuse_cmdlist so unmap
1080 * them.
1081 */
1082 g2d_unmap_cmdlist_gem(g2d, node, file);
1083 list_move_tail(&node->list, &g2d->free_cmdlist);
1084 }
1085 mutex_unlock(&g2d->cmdlist_mutex);
1086
1087 /* release all g2d_userptr in pool. */
1088 g2d_userptr_free_all(drm_dev, g2d, file);
1089
1090 kfree(file_priv->g2d_priv);
1091 }
1092
1093 static int __devinit g2d_probe(struct platform_device *pdev)
1094 {
1095 struct device *dev = &pdev->dev;
1096 struct resource *res;
1097 struct g2d_data *g2d;
1098 struct exynos_drm_subdrv *subdrv;
1099 int ret;
1100
1101 g2d = devm_kzalloc(&pdev->dev, sizeof(*g2d), GFP_KERNEL);
1102 if (!g2d) {
1103 dev_err(dev, "failed to allocate driver data\n");
1104 return -ENOMEM;
1105 }
1106
1107 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1108 sizeof(struct g2d_runqueue_node), 0, 0, NULL);
1109 if (!g2d->runqueue_slab)
1110 return -ENOMEM;
1111
1112 g2d->dev = dev;
1113
1114 g2d->g2d_workq = create_singlethread_workqueue("g2d");
1115 if (!g2d->g2d_workq) {
1116 dev_err(dev, "failed to create workqueue\n");
1117 ret = -EINVAL;
1118 goto err_destroy_slab;
1119 }
1120
1121 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1122 INIT_LIST_HEAD(&g2d->free_cmdlist);
1123 INIT_LIST_HEAD(&g2d->runqueue);
1124
1125 mutex_init(&g2d->cmdlist_mutex);
1126 mutex_init(&g2d->runqueue_mutex);
1127
1128 g2d->gate_clk = clk_get(dev, "fimg2d");
1129 if (IS_ERR(g2d->gate_clk)) {
1130 dev_err(dev, "failed to get gate clock\n");
1131 ret = PTR_ERR(g2d->gate_clk);
1132 goto err_destroy_workqueue;
1133 }
1134
1135 pm_runtime_enable(dev);
1136
1137 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1138
1139 g2d->regs = devm_request_and_ioremap(&pdev->dev, res);
1140 if (!g2d->regs) {
1141 dev_err(dev, "failed to remap I/O memory\n");
1142 ret = -ENXIO;
1143 goto err_put_clk;
1144 }
1145
1146 g2d->irq = platform_get_irq(pdev, 0);
1147 if (g2d->irq < 0) {
1148 dev_err(dev, "failed to get irq\n");
1149 ret = g2d->irq;
1150 goto err_put_clk;
1151 }
1152
1153 ret = devm_request_irq(&pdev->dev, g2d->irq, g2d_irq_handler, 0,
1154 "drm_g2d", g2d);
1155 if (ret < 0) {
1156 dev_err(dev, "irq request failed\n");
1157 goto err_put_clk;
1158 }
1159
1160 g2d->max_pool = MAX_POOL;
1161
1162 platform_set_drvdata(pdev, g2d);
1163
1164 subdrv = &g2d->subdrv;
1165 subdrv->dev = dev;
1166 subdrv->probe = g2d_subdrv_probe;
1167 subdrv->remove = g2d_subdrv_remove;
1168 subdrv->open = g2d_open;
1169 subdrv->close = g2d_close;
1170
1171 ret = exynos_drm_subdrv_register(subdrv);
1172 if (ret < 0) {
1173 dev_err(dev, "failed to register drm g2d device\n");
1174 goto err_put_clk;
1175 }
1176
1177 dev_info(dev, "The exynos g2d(ver %d.%d) successfully probed\n",
1178 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1179
1180 return 0;
1181
1182 err_put_clk:
1183 pm_runtime_disable(dev);
1184 clk_put(g2d->gate_clk);
1185 err_destroy_workqueue:
1186 destroy_workqueue(g2d->g2d_workq);
1187 err_destroy_slab:
1188 kmem_cache_destroy(g2d->runqueue_slab);
1189 return ret;
1190 }
1191
1192 static int __devexit g2d_remove(struct platform_device *pdev)
1193 {
1194 struct g2d_data *g2d = platform_get_drvdata(pdev);
1195
1196 cancel_work_sync(&g2d->runqueue_work);
1197 exynos_drm_subdrv_unregister(&g2d->subdrv);
1198
1199 while (g2d->runqueue_node) {
1200 g2d_free_runqueue_node(g2d, g2d->runqueue_node);
1201 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
1202 }
1203
1204 pm_runtime_disable(&pdev->dev);
1205 clk_put(g2d->gate_clk);
1206
1207 g2d_fini_cmdlist(g2d);
1208 destroy_workqueue(g2d->g2d_workq);
1209 kmem_cache_destroy(g2d->runqueue_slab);
1210
1211 return 0;
1212 }
1213
1214 #ifdef CONFIG_PM_SLEEP
1215 static int g2d_suspend(struct device *dev)
1216 {
1217 struct g2d_data *g2d = dev_get_drvdata(dev);
1218
1219 mutex_lock(&g2d->runqueue_mutex);
1220 g2d->suspended = true;
1221 mutex_unlock(&g2d->runqueue_mutex);
1222
1223 while (g2d->runqueue_node)
1224 /* FIXME: good range? */
1225 usleep_range(500, 1000);
1226
1227 flush_work(&g2d->runqueue_work);
1228
1229 return 0;
1230 }
1231
1232 static int g2d_resume(struct device *dev)
1233 {
1234 struct g2d_data *g2d = dev_get_drvdata(dev);
1235
1236 g2d->suspended = false;
1237 g2d_exec_runqueue(g2d);
1238
1239 return 0;
1240 }
1241 #endif
1242
1243 static SIMPLE_DEV_PM_OPS(g2d_pm_ops, g2d_suspend, g2d_resume);
1244
1245 struct platform_driver g2d_driver = {
1246 .probe = g2d_probe,
1247 .remove = __devexit_p(g2d_remove),
1248 .driver = {
1249 .name = "s5p-g2d",
1250 .owner = THIS_MODULE,
1251 .pm = &g2d_pm_ops,
1252 },
1253 };
(deprecated) Btrfs devel tree