1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012 Samsung Electronics Co.Ltd
4 * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
7 #include <linux/kernel.h>
9 #include <linux/component.h>
10 #include <linux/err.h>
11 #include <linux/interrupt.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15 #include <linux/slab.h>
16 #include <linux/workqueue.h>
17 #include <linux/dma-mapping.h>
21 #include <drm/exynos_drm.h>
22 #include "exynos_drm_drv.h"
23 #include "exynos_drm_g2d.h"
24 #include "exynos_drm_gem.h"
26 #define G2D_HW_MAJOR_VER 4
27 #define G2D_HW_MINOR_VER 1
29 /* vaild register range set from user: 0x0104 ~ 0x0880 */
30 #define G2D_VALID_START 0x0104
31 #define G2D_VALID_END 0x0880
33 /* general registers */
34 #define G2D_SOFT_RESET 0x0000
35 #define G2D_INTEN 0x0004
36 #define G2D_INTC_PEND 0x000C
37 #define G2D_DMA_SFR_BASE_ADDR 0x0080
38 #define G2D_DMA_COMMAND 0x0084
39 #define G2D_DMA_STATUS 0x008C
40 #define G2D_DMA_HOLD_CMD 0x0090
42 /* command registers */
43 #define G2D_BITBLT_START 0x0100
45 /* registers for base address */
46 #define G2D_SRC_BASE_ADDR 0x0304
47 #define G2D_SRC_STRIDE 0x0308
48 #define G2D_SRC_COLOR_MODE 0x030C
49 #define G2D_SRC_LEFT_TOP 0x0310
50 #define G2D_SRC_RIGHT_BOTTOM 0x0314
51 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318
52 #define G2D_DST_BASE_ADDR 0x0404
53 #define G2D_DST_STRIDE 0x0408
54 #define G2D_DST_COLOR_MODE 0x040C
55 #define G2D_DST_LEFT_TOP 0x0410
56 #define G2D_DST_RIGHT_BOTTOM 0x0414
57 #define G2D_DST_PLANE2_BASE_ADDR 0x0418
58 #define G2D_PAT_BASE_ADDR 0x0500
59 #define G2D_MSK_BASE_ADDR 0x0520
62 #define G2D_SFRCLEAR (1 << 1)
63 #define G2D_R (1 << 0)
66 #define G2D_INTEN_ACF (1 << 3)
67 #define G2D_INTEN_UCF (1 << 2)
68 #define G2D_INTEN_GCF (1 << 1)
69 #define G2D_INTEN_SCF (1 << 0)
72 #define G2D_INTP_ACMD_FIN (1 << 3)
73 #define G2D_INTP_UCMD_FIN (1 << 2)
74 #define G2D_INTP_GCMD_FIN (1 << 1)
75 #define G2D_INTP_SCMD_FIN (1 << 0)
78 #define G2D_DMA_HALT (1 << 2)
79 #define G2D_DMA_CONTINUE (1 << 1)
80 #define G2D_DMA_START (1 << 0)
83 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17)
84 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1)
85 #define G2D_DMA_DONE (1 << 0)
86 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17
88 /* G2D_DMA_HOLD_CMD */
89 #define G2D_USER_HOLD (1 << 2)
90 #define G2D_LIST_HOLD (1 << 1)
91 #define G2D_BITBLT_HOLD (1 << 0)
93 /* G2D_BITBLT_START */
94 #define G2D_START_CASESEL (1 << 2)
95 #define G2D_START_NHOLT (1 << 1)
96 #define G2D_START_BITBLT (1 << 0)
98 /* buffer color format */
99 #define G2D_FMT_XRGB8888 0
100 #define G2D_FMT_ARGB8888 1
101 #define G2D_FMT_RGB565 2
102 #define G2D_FMT_XRGB1555 3
103 #define G2D_FMT_ARGB1555 4
104 #define G2D_FMT_XRGB4444 5
105 #define G2D_FMT_ARGB4444 6
106 #define G2D_FMT_PACKED_RGB888 7
107 #define G2D_FMT_A8 11
108 #define G2D_FMT_L8 12
110 /* buffer valid length */
111 #define G2D_LEN_MIN 1
112 #define G2D_LEN_MAX 8000
114 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4)
115 #define G2D_CMDLIST_NUM 64
116 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
117 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
119 /* maximum buffer pool size of userptr is 64MB as default */
120 #define MAX_POOL (64 * 1024 * 1024)
140 * If set, suspends the runqueue worker after the currently
141 * processed node is finished.
143 G2D_BIT_SUSPEND_RUNQUEUE,
145 * If set, indicates that the engine is currently busy.
150 /* cmdlist data structure */
153 unsigned long data[G2D_CMDLIST_DATA_NUM];
154 u32 last; /* last data offset */
158 * A structure of buffer description
160 * @format: color format
161 * @stride: buffer stride/pitch in bytes
162 * @left_x: the x coordinates of left top corner
163 * @top_y: the y coordinates of left top corner
164 * @right_x: the x coordinates of right bottom corner
165 * @bottom_y: the y coordinates of right bottom corner
168 struct g2d_buf_desc {
173 unsigned int right_x;
174 unsigned int bottom_y;
178 * A structure of buffer information
180 * @map_nr: manages the number of mapped buffers
181 * @reg_types: stores regitster type in the order of requested command
182 * @handles: stores buffer handle in its reg_type position
183 * @types: stores buffer type in its reg_type position
184 * @descs: stores buffer description in its reg_type position
187 struct g2d_buf_info {
189 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR];
190 void *obj[MAX_REG_TYPE_NR];
191 unsigned int types[MAX_REG_TYPE_NR];
192 struct g2d_buf_desc descs[MAX_REG_TYPE_NR];
195 struct drm_exynos_pending_g2d_event {
196 struct drm_pending_event base;
197 struct drm_exynos_g2d_event event;
200 struct g2d_cmdlist_userptr {
201 struct list_head list;
203 unsigned long userptr;
205 struct frame_vector *vec;
206 struct sg_table *sgt;
211 struct g2d_cmdlist_node {
212 struct list_head list;
213 struct g2d_cmdlist *cmdlist;
215 struct g2d_buf_info buf_info;
217 struct drm_exynos_pending_g2d_event *event;
220 struct g2d_runqueue_node {
221 struct list_head list;
222 struct list_head run_cmdlist;
223 struct list_head event_list;
224 struct drm_file *filp;
226 struct completion complete;
232 struct clk *gate_clk;
235 struct workqueue_struct *g2d_workq;
236 struct work_struct runqueue_work;
237 struct drm_device *drm_dev;
241 struct g2d_cmdlist_node *cmdlist_node;
242 struct list_head free_cmdlist;
243 struct mutex cmdlist_mutex;
244 dma_addr_t cmdlist_pool;
245 void *cmdlist_pool_virt;
246 unsigned long cmdlist_dma_attrs;
249 struct g2d_runqueue_node *runqueue_node;
250 struct list_head runqueue;
251 struct mutex runqueue_mutex;
252 struct kmem_cache *runqueue_slab;
254 unsigned long current_pool;
255 unsigned long max_pool;
258 static inline void g2d_hw_reset(struct g2d_data *g2d)
260 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);
261 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
264 static int g2d_init_cmdlist(struct g2d_data *g2d)
266 struct device *dev = g2d->dev;
267 struct g2d_cmdlist_node *node = g2d->cmdlist_node;
270 struct g2d_buf_info *buf_info;
272 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
274 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),
275 G2D_CMDLIST_POOL_SIZE,
276 &g2d->cmdlist_pool, GFP_KERNEL,
277 g2d->cmdlist_dma_attrs);
278 if (!g2d->cmdlist_pool_virt) {
279 dev_err(dev, "failed to allocate dma memory\n");
283 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
289 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
293 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
295 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
297 buf_info = &node[nr].buf_info;
298 for (i = 0; i < MAX_REG_TYPE_NR; i++)
299 buf_info->reg_types[i] = REG_TYPE_NONE;
301 list_add_tail(&node[nr].list, &g2d->free_cmdlist);
307 dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
308 g2d->cmdlist_pool_virt,
309 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
313 static void g2d_fini_cmdlist(struct g2d_data *g2d)
315 kfree(g2d->cmdlist_node);
317 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
318 dma_free_attrs(to_dma_dev(g2d->drm_dev),
319 G2D_CMDLIST_POOL_SIZE,
320 g2d->cmdlist_pool_virt,
321 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
325 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
327 struct device *dev = g2d->dev;
328 struct g2d_cmdlist_node *node;
330 mutex_lock(&g2d->cmdlist_mutex);
331 if (list_empty(&g2d->free_cmdlist)) {
332 dev_err(dev, "there is no free cmdlist\n");
333 mutex_unlock(&g2d->cmdlist_mutex);
337 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
339 list_del_init(&node->list);
340 mutex_unlock(&g2d->cmdlist_mutex);
345 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
347 mutex_lock(&g2d->cmdlist_mutex);
348 list_move_tail(&node->list, &g2d->free_cmdlist);
349 mutex_unlock(&g2d->cmdlist_mutex);
352 static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
353 struct g2d_cmdlist_node *node)
355 struct g2d_cmdlist_node *lnode;
357 if (list_empty(&file_priv->inuse_cmdlist))
360 /* this links to base address of new cmdlist */
361 lnode = list_entry(file_priv->inuse_cmdlist.prev,
362 struct g2d_cmdlist_node, list);
363 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
366 list_add_tail(&node->list, &file_priv->inuse_cmdlist);
369 list_add_tail(&node->event->base.link, &file_priv->event_list);
372 static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
376 struct g2d_cmdlist_userptr *g2d_userptr = obj;
385 atomic_dec(&g2d_userptr->refcount);
387 if (atomic_read(&g2d_userptr->refcount) > 0)
390 if (g2d_userptr->in_pool)
394 dma_unmap_sg(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt->sgl,
395 g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL);
397 pages = frame_vector_pages(g2d_userptr->vec);
398 if (!IS_ERR(pages)) {
401 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++)
402 set_page_dirty_lock(pages[i]);
404 put_vaddr_frames(g2d_userptr->vec);
405 frame_vector_destroy(g2d_userptr->vec);
407 if (!g2d_userptr->out_of_list)
408 list_del_init(&g2d_userptr->list);
410 sg_free_table(g2d_userptr->sgt);
411 kfree(g2d_userptr->sgt);
415 static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
416 unsigned long userptr,
418 struct drm_file *filp,
421 struct drm_exynos_file_private *file_priv = filp->driver_priv;
422 struct g2d_cmdlist_userptr *g2d_userptr;
423 struct sg_table *sgt;
424 unsigned long start, end;
425 unsigned int npages, offset;
429 DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
430 return ERR_PTR(-EINVAL);
433 /* check if userptr already exists in userptr_list. */
434 list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
435 if (g2d_userptr->userptr == userptr) {
437 * also check size because there could be same address
438 * and different size.
440 if (g2d_userptr->size == size) {
441 atomic_inc(&g2d_userptr->refcount);
444 return &g2d_userptr->dma_addr;
448 * at this moment, maybe g2d dma is accessing this
449 * g2d_userptr memory region so just remove this
450 * g2d_userptr object from userptr_list not to be
451 * referred again and also except it the userptr
452 * pool to be released after the dma access completion.
454 g2d_userptr->out_of_list = true;
455 g2d_userptr->in_pool = false;
456 list_del_init(&g2d_userptr->list);
462 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
464 return ERR_PTR(-ENOMEM);
466 atomic_set(&g2d_userptr->refcount, 1);
467 g2d_userptr->size = size;
469 start = userptr & PAGE_MASK;
470 offset = userptr & ~PAGE_MASK;
471 end = PAGE_ALIGN(userptr + size);
472 npages = (end - start) >> PAGE_SHIFT;
473 g2d_userptr->vec = frame_vector_create(npages);
474 if (!g2d_userptr->vec) {
479 ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
482 DRM_DEV_ERROR(g2d->dev,
483 "failed to get user pages from userptr.\n");
485 goto err_destroy_framevec;
487 goto err_put_framevec;
489 if (frame_vector_to_pages(g2d_userptr->vec) < 0) {
491 goto err_put_framevec;
494 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
497 goto err_put_framevec;
500 ret = sg_alloc_table_from_pages(sgt,
501 frame_vector_pages(g2d_userptr->vec),
502 npages, offset, size, GFP_KERNEL);
504 DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
508 g2d_userptr->sgt = sgt;
510 if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents,
511 DMA_BIDIRECTIONAL)) {
512 DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
514 goto err_sg_free_table;
517 g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
518 g2d_userptr->userptr = userptr;
520 list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);
522 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
523 g2d->current_pool += npages << PAGE_SHIFT;
524 g2d_userptr->in_pool = true;
529 return &g2d_userptr->dma_addr;
538 put_vaddr_frames(g2d_userptr->vec);
540 err_destroy_framevec:
541 frame_vector_destroy(g2d_userptr->vec);
549 static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
551 struct drm_exynos_file_private *file_priv = filp->driver_priv;
552 struct g2d_cmdlist_userptr *g2d_userptr, *n;
554 list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
555 if (g2d_userptr->in_pool)
556 g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);
558 g2d->current_pool = 0;
561 static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
563 enum g2d_reg_type reg_type;
565 switch (reg_offset) {
566 case G2D_SRC_BASE_ADDR:
568 case G2D_SRC_COLOR_MODE:
569 case G2D_SRC_LEFT_TOP:
570 case G2D_SRC_RIGHT_BOTTOM:
571 reg_type = REG_TYPE_SRC;
573 case G2D_SRC_PLANE2_BASE_ADDR:
574 reg_type = REG_TYPE_SRC_PLANE2;
576 case G2D_DST_BASE_ADDR:
578 case G2D_DST_COLOR_MODE:
579 case G2D_DST_LEFT_TOP:
580 case G2D_DST_RIGHT_BOTTOM:
581 reg_type = REG_TYPE_DST;
583 case G2D_DST_PLANE2_BASE_ADDR:
584 reg_type = REG_TYPE_DST_PLANE2;
586 case G2D_PAT_BASE_ADDR:
587 reg_type = REG_TYPE_PAT;
589 case G2D_MSK_BASE_ADDR:
590 reg_type = REG_TYPE_MSK;
593 reg_type = REG_TYPE_NONE;
594 DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
602 static unsigned long g2d_get_buf_bpp(unsigned int format)
607 case G2D_FMT_XRGB8888:
608 case G2D_FMT_ARGB8888:
612 case G2D_FMT_XRGB1555:
613 case G2D_FMT_ARGB1555:
614 case G2D_FMT_XRGB4444:
615 case G2D_FMT_ARGB4444:
618 case G2D_FMT_PACKED_RGB888:
629 static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
630 struct g2d_buf_desc *buf_desc,
631 enum g2d_reg_type reg_type,
635 unsigned long bpp, last_pos;
638 * check source and destination buffers only.
639 * so the others are always valid.
641 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)
644 /* This check also makes sure that right_x > left_x. */
645 width = (int)buf_desc->right_x - (int)buf_desc->left_x;
646 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
647 DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
651 /* This check also makes sure that bottom_y > top_y. */
652 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
653 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
654 DRM_DEV_ERROR(g2d->dev,
655 "height[%d] is out of range!\n", height);
659 bpp = g2d_get_buf_bpp(buf_desc->format);
661 /* Compute the position of the last byte that the engine accesses. */
662 last_pos = ((unsigned long)buf_desc->bottom_y - 1) *
663 (unsigned long)buf_desc->stride +
664 (unsigned long)buf_desc->right_x * bpp - 1;
667 * Since right_x > left_x and bottom_y > top_y we already know
668 * that the first_pos < last_pos (first_pos being the position
669 * of the first byte the engine accesses), it just remains to
670 * check if last_pos is smaller then the buffer size.
673 if (last_pos >= size) {
674 DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
675 "is out of range [%lu]!\n", last_pos, size);
682 static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
683 struct g2d_cmdlist_node *node,
684 struct drm_device *drm_dev,
685 struct drm_file *file)
687 struct g2d_cmdlist *cmdlist = node->cmdlist;
688 struct g2d_buf_info *buf_info = &node->buf_info;
693 for (i = 0; i < buf_info->map_nr; i++) {
694 struct g2d_buf_desc *buf_desc;
695 enum g2d_reg_type reg_type;
697 unsigned long handle;
700 reg_pos = cmdlist->last - 2 * (i + 1);
702 offset = cmdlist->data[reg_pos];
703 handle = cmdlist->data[reg_pos + 1];
705 reg_type = g2d_get_reg_type(g2d, offset);
706 if (reg_type == REG_TYPE_NONE) {
711 buf_desc = &buf_info->descs[reg_type];
713 if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
714 struct exynos_drm_gem *exynos_gem;
716 exynos_gem = exynos_drm_gem_get(file, handle);
722 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
723 reg_type, exynos_gem->size)) {
724 exynos_drm_gem_put(exynos_gem);
729 addr = &exynos_gem->dma_addr;
730 buf_info->obj[reg_type] = exynos_gem;
732 struct drm_exynos_g2d_userptr g2d_userptr;
734 if (copy_from_user(&g2d_userptr, (void __user *)handle,
735 sizeof(struct drm_exynos_g2d_userptr))) {
740 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
747 addr = g2d_userptr_get_dma_addr(g2d,
751 &buf_info->obj[reg_type]);
758 cmdlist->data[reg_pos + 1] = *addr;
759 buf_info->reg_types[i] = reg_type;
765 buf_info->map_nr = i;
769 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
770 struct g2d_cmdlist_node *node,
771 struct drm_file *filp)
773 struct g2d_buf_info *buf_info = &node->buf_info;
776 for (i = 0; i < buf_info->map_nr; i++) {
777 struct g2d_buf_desc *buf_desc;
778 enum g2d_reg_type reg_type;
781 reg_type = buf_info->reg_types[i];
783 buf_desc = &buf_info->descs[reg_type];
784 obj = buf_info->obj[reg_type];
786 if (buf_info->types[reg_type] == BUF_TYPE_GEM)
787 exynos_drm_gem_put(obj);
789 g2d_userptr_put_dma_addr(g2d, obj, false);
791 buf_info->reg_types[i] = REG_TYPE_NONE;
792 buf_info->obj[reg_type] = NULL;
793 buf_info->types[reg_type] = 0;
794 memset(buf_desc, 0x00, sizeof(*buf_desc));
797 buf_info->map_nr = 0;
800 static void g2d_dma_start(struct g2d_data *g2d,
801 struct g2d_runqueue_node *runqueue_node)
803 struct g2d_cmdlist_node *node =
804 list_first_entry(&runqueue_node->run_cmdlist,
805 struct g2d_cmdlist_node, list);
807 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
808 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
809 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
812 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
814 struct g2d_runqueue_node *runqueue_node;
816 if (list_empty(&g2d->runqueue))
819 runqueue_node = list_first_entry(&g2d->runqueue,
820 struct g2d_runqueue_node, list);
821 list_del_init(&runqueue_node->list);
822 return runqueue_node;
825 static void g2d_free_runqueue_node(struct g2d_data *g2d,
826 struct g2d_runqueue_node *runqueue_node)
828 struct g2d_cmdlist_node *node;
830 mutex_lock(&g2d->cmdlist_mutex);
832 * commands in run_cmdlist have been completed so unmap all gem
833 * objects in each command node so that they are unreferenced.
835 list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
836 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
837 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
838 mutex_unlock(&g2d->cmdlist_mutex);
840 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
844 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
845 * @g2d: G2D state object
846 * @file: if not zero, only remove items with this DRM file
848 * Has to be called under runqueue lock.
850 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
852 struct g2d_runqueue_node *node, *n;
854 if (list_empty(&g2d->runqueue))
857 list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
858 if (file && node->filp != file)
861 list_del_init(&node->list);
862 g2d_free_runqueue_node(g2d, node);
866 static void g2d_runqueue_worker(struct work_struct *work)
868 struct g2d_data *g2d = container_of(work, struct g2d_data,
870 struct g2d_runqueue_node *runqueue_node;
873 * The engine is busy and the completion of the current node is going
874 * to poke the runqueue worker, so nothing to do here.
876 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
879 mutex_lock(&g2d->runqueue_mutex);
881 runqueue_node = g2d->runqueue_node;
882 g2d->runqueue_node = NULL;
885 pm_runtime_mark_last_busy(g2d->dev);
886 pm_runtime_put_autosuspend(g2d->dev);
888 complete(&runqueue_node->complete);
889 if (runqueue_node->async)
890 g2d_free_runqueue_node(g2d, runqueue_node);
893 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
894 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
896 if (g2d->runqueue_node) {
897 pm_runtime_get_sync(g2d->dev);
898 g2d_dma_start(g2d, g2d->runqueue_node);
902 mutex_unlock(&g2d->runqueue_mutex);
905 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
907 struct drm_device *drm_dev = g2d->drm_dev;
908 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
909 struct drm_exynos_pending_g2d_event *e;
910 struct timespec64 now;
912 if (list_empty(&runqueue_node->event_list))
915 e = list_first_entry(&runqueue_node->event_list,
916 struct drm_exynos_pending_g2d_event, base.link);
918 ktime_get_ts64(&now);
919 e->event.tv_sec = now.tv_sec;
920 e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
921 e->event.cmdlist_no = cmdlist_no;
923 drm_send_event(drm_dev, &e->base);
926 static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
928 struct g2d_data *g2d = dev_id;
931 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
933 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
935 if (pending & G2D_INTP_GCMD_FIN) {
936 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
938 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
939 G2D_DMA_LIST_DONE_COUNT_OFFSET;
941 g2d_finish_event(g2d, cmdlist_no);
943 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
944 if (!(pending & G2D_INTP_ACMD_FIN)) {
945 writel_relaxed(G2D_DMA_CONTINUE,
946 g2d->regs + G2D_DMA_COMMAND);
950 if (pending & G2D_INTP_ACMD_FIN) {
951 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
952 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
959 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
960 * @g2d: G2D state object
961 * @file: if not zero, only wait if the current runqueue node belongs
964 * Should the engine not become idle after a 100ms timeout, a hardware
967 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
969 struct device *dev = g2d->dev;
971 struct g2d_runqueue_node *runqueue_node = NULL;
972 unsigned int tries = 10;
974 mutex_lock(&g2d->runqueue_mutex);
976 /* If no node is currently processed, we have nothing to do. */
977 if (!g2d->runqueue_node)
980 runqueue_node = g2d->runqueue_node;
982 /* Check if the currently processed item belongs to us. */
983 if (file && runqueue_node->filp != file)
986 mutex_unlock(&g2d->runqueue_mutex);
988 /* Wait for the G2D engine to finish. */
989 while (tries-- && (g2d->runqueue_node == runqueue_node))
992 mutex_lock(&g2d->runqueue_mutex);
994 if (g2d->runqueue_node != runqueue_node)
997 dev_err(dev, "wait timed out, resetting engine...\n");
1001 * After the hardware reset of the engine we are going to loose
1002 * the IRQ which triggers the PM runtime put().
1003 * So do this manually here.
1005 pm_runtime_mark_last_busy(dev);
1006 pm_runtime_put_autosuspend(dev);
1008 complete(&runqueue_node->complete);
1009 if (runqueue_node->async)
1010 g2d_free_runqueue_node(g2d, runqueue_node);
1013 mutex_unlock(&g2d->runqueue_mutex);
1016 static int g2d_check_reg_offset(struct g2d_data *g2d,
1017 struct g2d_cmdlist_node *node,
1018 int nr, bool for_addr)
1020 struct g2d_cmdlist *cmdlist = node->cmdlist;
1025 for (i = 0; i < nr; i++) {
1026 struct g2d_buf_info *buf_info = &node->buf_info;
1027 struct g2d_buf_desc *buf_desc;
1028 enum g2d_reg_type reg_type;
1029 unsigned long value;
1031 index = cmdlist->last - 2 * (i + 1);
1033 reg_offset = cmdlist->data[index] & ~0xfffff000;
1034 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
1039 switch (reg_offset) {
1040 case G2D_SRC_BASE_ADDR:
1041 case G2D_SRC_PLANE2_BASE_ADDR:
1042 case G2D_DST_BASE_ADDR:
1043 case G2D_DST_PLANE2_BASE_ADDR:
1044 case G2D_PAT_BASE_ADDR:
1045 case G2D_MSK_BASE_ADDR:
1049 reg_type = g2d_get_reg_type(g2d, reg_offset);
1051 /* check userptr buffer type. */
1052 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
1053 buf_info->types[reg_type] = BUF_TYPE_USERPTR;
1054 cmdlist->data[index] &= ~G2D_BUF_USERPTR;
1056 buf_info->types[reg_type] = BUF_TYPE_GEM;
1058 case G2D_SRC_STRIDE:
1059 case G2D_DST_STRIDE:
1063 reg_type = g2d_get_reg_type(g2d, reg_offset);
1065 buf_desc = &buf_info->descs[reg_type];
1066 buf_desc->stride = cmdlist->data[index + 1];
1068 case G2D_SRC_COLOR_MODE:
1069 case G2D_DST_COLOR_MODE:
1073 reg_type = g2d_get_reg_type(g2d, reg_offset);
1075 buf_desc = &buf_info->descs[reg_type];
1076 value = cmdlist->data[index + 1];
1078 buf_desc->format = value & 0xf;
1080 case G2D_SRC_LEFT_TOP:
1081 case G2D_DST_LEFT_TOP:
1085 reg_type = g2d_get_reg_type(g2d, reg_offset);
1087 buf_desc = &buf_info->descs[reg_type];
1088 value = cmdlist->data[index + 1];
1090 buf_desc->left_x = value & 0x1fff;
1091 buf_desc->top_y = (value & 0x1fff0000) >> 16;
1093 case G2D_SRC_RIGHT_BOTTOM:
1094 case G2D_DST_RIGHT_BOTTOM:
1098 reg_type = g2d_get_reg_type(g2d, reg_offset);
1100 buf_desc = &buf_info->descs[reg_type];
1101 value = cmdlist->data[index + 1];
1103 buf_desc->right_x = value & 0x1fff;
1104 buf_desc->bottom_y = (value & 0x1fff0000) >> 16;
1116 dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
1120 /* ioctl functions */
1121 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
1122 struct drm_file *file)
1124 struct drm_exynos_g2d_get_ver *ver = data;
1126 ver->major = G2D_HW_MAJOR_VER;
1127 ver->minor = G2D_HW_MINOR_VER;
1132 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
1133 struct drm_file *file)
1135 struct drm_exynos_file_private *file_priv = file->driver_priv;
1136 struct exynos_drm_private *priv = drm_dev->dev_private;
1137 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1138 struct drm_exynos_g2d_set_cmdlist *req = data;
1139 struct drm_exynos_g2d_cmd *cmd;
1140 struct drm_exynos_pending_g2d_event *e;
1141 struct g2d_cmdlist_node *node;
1142 struct g2d_cmdlist *cmdlist;
1146 node = g2d_get_cmdlist(g2d);
1151 * To avoid an integer overflow for the later size computations, we
1152 * enforce a maximum number of submitted commands here. This limit is
1153 * sufficient for all conceivable usage cases of the G2D.
1155 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
1156 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
1157 dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
1163 if (req->event_type != G2D_EVENT_NOT) {
1164 e = kzalloc(sizeof(*node->event), GFP_KERNEL);
1170 e->event.base.type = DRM_EXYNOS_G2D_EVENT;
1171 e->event.base.length = sizeof(e->event);
1172 e->event.user_data = req->user_data;
1174 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base);
1183 cmdlist = node->cmdlist;
1188 * If don't clear SFR registers, the cmdlist is affected by register
1189 * values of previous cmdlist. G2D hw executes SFR clear command and
1190 * a next command at the same time then the next command is ignored and
1191 * is executed rightly from next next command, so needs a dummy command
1192 * to next command of SFR clear command.
1194 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
1195 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
1196 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
1197 cmdlist->data[cmdlist->last++] = 0;
1200 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
1201 * and GCF bit should be set to INTEN register if user wants
1202 * G2D interrupt event once current command list execution is
1204 * Otherwise only ACF bit should be set to INTEN register so
1205 * that one interrupt is occurred after all command lists
1206 * have been completed.
1209 cmdlist->data[cmdlist->last++] = G2D_INTEN;
1210 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
1211 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
1212 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
1214 cmdlist->data[cmdlist->last++] = G2D_INTEN;
1215 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
1219 * Check the size of cmdlist. The 2 that is added last comes from
1220 * the implicit G2D_BITBLT_START that is appended once we have
1221 * checked all the submitted commands.
1223 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
1224 if (size > G2D_CMDLIST_DATA_NUM) {
1225 dev_err(g2d->dev, "cmdlist size is too big\n");
1227 goto err_free_event;
1230 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;
1232 if (copy_from_user(cmdlist->data + cmdlist->last,
1234 sizeof(*cmd) * req->cmd_nr)) {
1236 goto err_free_event;
1238 cmdlist->last += req->cmd_nr * 2;
1240 ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);
1242 goto err_free_event;
1244 node->buf_info.map_nr = req->cmd_buf_nr;
1245 if (req->cmd_buf_nr) {
1246 struct drm_exynos_g2d_cmd *cmd_buf;
1248 cmd_buf = (struct drm_exynos_g2d_cmd *)
1249 (unsigned long)req->cmd_buf;
1251 if (copy_from_user(cmdlist->data + cmdlist->last,
1252 (void __user *)cmd_buf,
1253 sizeof(*cmd_buf) * req->cmd_buf_nr)) {
1255 goto err_free_event;
1257 cmdlist->last += req->cmd_buf_nr * 2;
1259 ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);
1261 goto err_free_event;
1263 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
1268 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
1269 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
1272 cmdlist->head = cmdlist->last / 2;
1275 cmdlist->data[cmdlist->last] = 0;
1277 g2d_add_cmdlist_to_inuse(file_priv, node);
1282 g2d_unmap_cmdlist_gem(g2d, node, file);
1285 drm_event_cancel_free(drm_dev, &node->event->base);
1287 g2d_put_cmdlist(g2d, node);
1291 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
1292 struct drm_file *file)
1294 struct drm_exynos_file_private *file_priv = file->driver_priv;
1295 struct exynos_drm_private *priv = drm_dev->dev_private;
1296 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1297 struct drm_exynos_g2d_exec *req = data;
1298 struct g2d_runqueue_node *runqueue_node;
1299 struct list_head *run_cmdlist;
1300 struct list_head *event_list;
1302 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
1306 run_cmdlist = &runqueue_node->run_cmdlist;
1307 event_list = &runqueue_node->event_list;
1308 INIT_LIST_HEAD(run_cmdlist);
1309 INIT_LIST_HEAD(event_list);
1310 init_completion(&runqueue_node->complete);
1311 runqueue_node->async = req->async;
1313 list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);
1314 list_splice_init(&file_priv->event_list, event_list);
1316 if (list_empty(run_cmdlist)) {
1317 dev_err(g2d->dev, "there is no inuse cmdlist\n");
1318 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
1322 mutex_lock(&g2d->runqueue_mutex);
1323 runqueue_node->pid = current->pid;
1324 runqueue_node->filp = file;
1325 list_add_tail(&runqueue_node->list, &g2d->runqueue);
1326 mutex_unlock(&g2d->runqueue_mutex);
1328 /* Let the runqueue know that there is work to do. */
1329 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1331 if (runqueue_node->async)
1334 wait_for_completion(&runqueue_node->complete);
1335 g2d_free_runqueue_node(g2d, runqueue_node);
1341 int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
1343 struct drm_exynos_file_private *file_priv = file->driver_priv;
1345 INIT_LIST_HEAD(&file_priv->inuse_cmdlist);
1346 INIT_LIST_HEAD(&file_priv->event_list);
1347 INIT_LIST_HEAD(&file_priv->userptr_list);
1352 void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
1354 struct drm_exynos_file_private *file_priv = file->driver_priv;
1355 struct exynos_drm_private *priv = drm_dev->dev_private;
1356 struct g2d_data *g2d;
1357 struct g2d_cmdlist_node *node, *n;
1362 g2d = dev_get_drvdata(priv->g2d_dev);
1364 /* Remove the runqueue nodes that belong to us. */
1365 mutex_lock(&g2d->runqueue_mutex);
1366 g2d_remove_runqueue_nodes(g2d, file);
1367 mutex_unlock(&g2d->runqueue_mutex);
1370 * Wait for the runqueue worker to finish its current node.
1371 * After this the engine should no longer be accessing any
1372 * memory belonging to us.
1374 g2d_wait_finish(g2d, file);
1377 * Even after the engine is idle, there might still be stale cmdlists
1378 * (i.e. cmdlisst which we submitted but never executed) around, with
1379 * their corresponding GEM/userptr buffers.
1380 * Properly unmap these buffers here.
1382 mutex_lock(&g2d->cmdlist_mutex);
1383 list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
1384 g2d_unmap_cmdlist_gem(g2d, node, file);
1385 list_move_tail(&node->list, &g2d->free_cmdlist);
1387 mutex_unlock(&g2d->cmdlist_mutex);
1389 /* release all g2d_userptr in pool. */
1390 g2d_userptr_free_all(g2d, file);
1393 static int g2d_bind(struct device *dev, struct device *master, void *data)
1395 struct g2d_data *g2d = dev_get_drvdata(dev);
1396 struct drm_device *drm_dev = data;
1397 struct exynos_drm_private *priv = drm_dev->dev_private;
1400 g2d->drm_dev = drm_dev;
1402 /* allocate dma-aware cmdlist buffer. */
1403 ret = g2d_init_cmdlist(g2d);
1405 dev_err(dev, "cmdlist init failed\n");
1409 ret = exynos_drm_register_dma(drm_dev, dev);
1411 dev_err(dev, "failed to enable iommu.\n");
1412 g2d_fini_cmdlist(g2d);
1415 priv->g2d_dev = dev;
1417 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
1418 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1422 static void g2d_unbind(struct device *dev, struct device *master, void *data)
1424 struct g2d_data *g2d = dev_get_drvdata(dev);
1425 struct drm_device *drm_dev = data;
1426 struct exynos_drm_private *priv = drm_dev->dev_private;
1428 /* Suspend operation and wait for engine idle. */
1429 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1430 g2d_wait_finish(g2d, NULL);
1431 priv->g2d_dev = NULL;
1433 cancel_work_sync(&g2d->runqueue_work);
1434 exynos_drm_unregister_dma(g2d->drm_dev, dev);
1437 static const struct component_ops g2d_component_ops = {
1439 .unbind = g2d_unbind,
1442 static int g2d_probe(struct platform_device *pdev)
1444 struct device *dev = &pdev->dev;
1445 struct resource *res;
1446 struct g2d_data *g2d;
1449 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);
1453 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1454 sizeof(struct g2d_runqueue_node), 0, 0, NULL);
1455 if (!g2d->runqueue_slab)
1460 g2d->g2d_workq = create_singlethread_workqueue("g2d");
1461 if (!g2d->g2d_workq) {
1462 dev_err(dev, "failed to create workqueue\n");
1464 goto err_destroy_slab;
1467 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1468 INIT_LIST_HEAD(&g2d->free_cmdlist);
1469 INIT_LIST_HEAD(&g2d->runqueue);
1471 mutex_init(&g2d->cmdlist_mutex);
1472 mutex_init(&g2d->runqueue_mutex);
1474 g2d->gate_clk = devm_clk_get(dev, "fimg2d");
1475 if (IS_ERR(g2d->gate_clk)) {
1476 dev_err(dev, "failed to get gate clock\n");
1477 ret = PTR_ERR(g2d->gate_clk);
1478 goto err_destroy_workqueue;
1481 pm_runtime_use_autosuspend(dev);
1482 pm_runtime_set_autosuspend_delay(dev, 2000);
1483 pm_runtime_enable(dev);
1484 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1485 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
1487 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1489 g2d->regs = devm_ioremap_resource(dev, res);
1490 if (IS_ERR(g2d->regs)) {
1491 ret = PTR_ERR(g2d->regs);
1495 g2d->irq = platform_get_irq(pdev, 0);
1497 dev_err(dev, "failed to get irq\n");
1502 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0,
1505 dev_err(dev, "irq request failed\n");
1509 g2d->max_pool = MAX_POOL;
1511 platform_set_drvdata(pdev, g2d);
1513 ret = component_add(dev, &g2d_component_ops);
1515 dev_err(dev, "failed to register drm g2d device\n");
1522 pm_runtime_disable(dev);
1523 err_destroy_workqueue:
1524 destroy_workqueue(g2d->g2d_workq);
1526 kmem_cache_destroy(g2d->runqueue_slab);
1530 static int g2d_remove(struct platform_device *pdev)
1532 struct g2d_data *g2d = platform_get_drvdata(pdev);
1534 component_del(&pdev->dev, &g2d_component_ops);
1536 /* There should be no locking needed here. */
1537 g2d_remove_runqueue_nodes(g2d, NULL);
1539 pm_runtime_dont_use_autosuspend(&pdev->dev);
1540 pm_runtime_disable(&pdev->dev);
1542 g2d_fini_cmdlist(g2d);
1543 destroy_workqueue(g2d->g2d_workq);
1544 kmem_cache_destroy(g2d->runqueue_slab);
1549 #ifdef CONFIG_PM_SLEEP
1550 static int g2d_suspend(struct device *dev)
1552 struct g2d_data *g2d = dev_get_drvdata(dev);
1555 * Suspend the runqueue worker operation and wait until the G2D
1558 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1559 g2d_wait_finish(g2d, NULL);
1560 flush_work(&g2d->runqueue_work);
1565 static int g2d_resume(struct device *dev)
1567 struct g2d_data *g2d = dev_get_drvdata(dev);
1569 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1570 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1577 static int g2d_runtime_suspend(struct device *dev)
1579 struct g2d_data *g2d = dev_get_drvdata(dev);
1581 clk_disable_unprepare(g2d->gate_clk);
1586 static int g2d_runtime_resume(struct device *dev)
1588 struct g2d_data *g2d = dev_get_drvdata(dev);
1591 ret = clk_prepare_enable(g2d->gate_clk);
1593 dev_warn(dev, "failed to enable clock.\n");
1599 static const struct dev_pm_ops g2d_pm_ops = {
1600 SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)
1601 SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)
1604 static const struct of_device_id exynos_g2d_match[] = {
1605 { .compatible = "samsung,exynos5250-g2d" },
1606 { .compatible = "samsung,exynos4212-g2d" },
1609 MODULE_DEVICE_TABLE(of, exynos_g2d_match);
1611 struct platform_driver g2d_driver = {
1613 .remove = g2d_remove,
1615 .name = "exynos-drm-g2d",
1616 .owner = THIS_MODULE,
1618 .of_match_table = exynos_g2d_match,