1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2019 Collabora Ltd */
4 #include <drm/drm_file.h>
5 #include <drm/drm_gem_shmem_helper.h>
6 #include <drm/panfrost_drm.h>
7 #include <linux/completion.h>
8 #include <linux/iopoll.h>
9 #include <linux/pm_runtime.h>
10 #include <linux/slab.h>
11 #include <linux/uaccess.h>
13 #include "panfrost_device.h"
14 #include "panfrost_features.h"
15 #include "panfrost_gem.h"
16 #include "panfrost_issues.h"
17 #include "panfrost_job.h"
18 #include "panfrost_mmu.h"
19 #include "panfrost_perfcnt.h"
20 #include "panfrost_regs.h"
22 #define COUNTERS_PER_BLOCK 64
23 #define BYTES_PER_COUNTER 4
24 #define BLOCKS_PER_COREGROUP 8
25 #define V4_SHADERS_PER_COREGROUP 4
27 struct panfrost_perfcnt {
28 struct panfrost_gem_mapping *mapping;
31 struct panfrost_file_priv *user;
33 struct completion dump_comp;
36 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
38 complete(&pfdev->perfcnt->dump_comp);
41 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
43 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
46 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
51 reinit_completion(&pfdev->perfcnt->dump_comp);
52 gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
53 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
54 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
55 gpu_write(pfdev, GPU_INT_CLEAR,
56 GPU_IRQ_CLEAN_CACHES_COMPLETED |
57 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
58 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
59 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
60 msecs_to_jiffies(1000));
69 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
70 struct drm_file *file_priv,
71 unsigned int counterset)
73 struct panfrost_file_priv *user = file_priv->driver_priv;
74 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
75 struct drm_gem_shmem_object *bo;
79 if (user == perfcnt->user)
81 else if (perfcnt->user)
84 ret = pm_runtime_get_sync(pfdev->dev);
88 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
92 /* Map the perfcnt buf in the address space attached to file_priv. */
93 ret = panfrost_gem_open(&bo->base, file_priv);
97 perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
99 if (!perfcnt->mapping) {
104 perfcnt->buf = drm_gem_shmem_vmap(&bo->base);
105 if (IS_ERR(perfcnt->buf)) {
106 ret = PTR_ERR(perfcnt->buf);
107 goto err_put_mapping;
111 * Invalidate the cache and clear the counters to start from a fresh
114 reinit_completion(&pfdev->perfcnt->dump_comp);
115 gpu_write(pfdev, GPU_INT_CLEAR,
116 GPU_IRQ_CLEAN_CACHES_COMPLETED |
117 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
118 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
119 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
120 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
121 msecs_to_jiffies(1000));
127 perfcnt->user = user;
129 as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
130 cfg = GPU_PERFCNT_CFG_AS(as) |
131 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
134 * Bifrost GPUs have 2 set of counters, but we're only interested by
135 * the first one for now.
137 if (panfrost_model_is_bifrost(pfdev))
138 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
140 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
141 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
142 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
145 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
148 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
149 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
151 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
153 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
155 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
156 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
158 /* The BO ref is retained by the mapping. */
159 drm_gem_object_put_unlocked(&bo->base);
164 drm_gem_shmem_vunmap(&bo->base, perfcnt->buf);
166 panfrost_gem_mapping_put(perfcnt->mapping);
168 panfrost_gem_close(&bo->base, file_priv);
170 drm_gem_object_put_unlocked(&bo->base);
174 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
175 struct drm_file *file_priv)
177 struct panfrost_file_priv *user = file_priv->driver_priv;
178 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
180 if (user != perfcnt->user)
183 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
184 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
185 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
186 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
187 gpu_write(pfdev, GPU_PERFCNT_CFG,
188 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
190 perfcnt->user = NULL;
191 drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, perfcnt->buf);
193 panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
194 panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
195 panfrost_gem_mapping_put(perfcnt->mapping);
196 perfcnt->mapping = NULL;
197 pm_runtime_mark_last_busy(pfdev->dev);
198 pm_runtime_put_autosuspend(pfdev->dev);
203 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
204 struct drm_file *file_priv)
206 struct panfrost_device *pfdev = dev->dev_private;
207 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
208 struct drm_panfrost_perfcnt_enable *req = data;
211 ret = panfrost_unstable_ioctl_check();
215 /* Only Bifrost GPUs have 2 set of counters. */
216 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
219 mutex_lock(&perfcnt->lock);
221 ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
224 ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
225 mutex_unlock(&perfcnt->lock);
230 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
231 struct drm_file *file_priv)
233 struct panfrost_device *pfdev = dev->dev_private;
234 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
235 struct drm_panfrost_perfcnt_dump *req = data;
236 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
239 ret = panfrost_unstable_ioctl_check();
243 mutex_lock(&perfcnt->lock);
244 if (perfcnt->user != file_priv->driver_priv) {
249 ret = panfrost_perfcnt_dump_locked(pfdev);
253 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
257 mutex_unlock(&perfcnt->lock);
262 void panfrost_perfcnt_close(struct drm_file *file_priv)
264 struct panfrost_file_priv *pfile = file_priv->driver_priv;
265 struct panfrost_device *pfdev = pfile->pfdev;
266 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
268 pm_runtime_get_sync(pfdev->dev);
269 mutex_lock(&perfcnt->lock);
270 if (perfcnt->user == pfile)
271 panfrost_perfcnt_disable_locked(pfdev, file_priv);
272 mutex_unlock(&perfcnt->lock);
273 pm_runtime_mark_last_busy(pfdev->dev);
274 pm_runtime_put_autosuspend(pfdev->dev);
277 int panfrost_perfcnt_init(struct panfrost_device *pfdev)
279 struct panfrost_perfcnt *perfcnt;
282 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
283 unsigned int ncoregroups;
285 ncoregroups = hweight64(pfdev->features.l2_present);
286 size = ncoregroups * BLOCKS_PER_COREGROUP *
287 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
289 unsigned int nl2c, ncores;
292 * TODO: define a macro to extract the number of l2 caches from
295 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
298 * shader_present might be sparse, but the counters layout
299 * forces to dump unused regions too, hence the fls64() call
300 * instead of hweight64().
302 ncores = fls64(pfdev->features.shader_present);
305 * There's always one JM and one Tiler block, hence the '+ 2'
308 size = (nl2c + ncores + 2) *
309 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
312 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
316 perfcnt->bosize = size;
318 /* Start with everything disabled. */
319 gpu_write(pfdev, GPU_PERFCNT_CFG,
320 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
321 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
322 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
323 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
324 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
326 init_completion(&perfcnt->dump_comp);
327 mutex_init(&perfcnt->lock);
328 pfdev->perfcnt = perfcnt;
333 void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
335 /* Disable everything before leaving. */
336 gpu_write(pfdev, GPU_PERFCNT_CFG,
337 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
338 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
339 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
340 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
341 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);