2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
23 * Cmdstream submission:
26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
27 #define BO_VALID 0x8000 /* is current addr in cmdstream correct/valid? */
28 #define BO_LOCKED 0x4000
29 #define BO_PINNED 0x2000
31 static struct msm_gem_submit *submit_create(struct drm_device *dev,
32 struct msm_gpu *gpu, int nr)
34 struct msm_gem_submit *submit;
35 int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0]));
37 submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
44 submit->pid = get_pid(task_pid(current));
46 /* initially, until copy_from_user() and bo lookup succeeds: */
50 INIT_LIST_HEAD(&submit->node);
51 INIT_LIST_HEAD(&submit->bo_list);
52 ww_acquire_init(&submit->ticket, &reservation_ww_class);
57 void msm_gem_submit_free(struct msm_gem_submit *submit)
59 fence_put(submit->fence);
60 list_del(&submit->node);
65 static int submit_lookup_objects(struct msm_gem_submit *submit,
66 struct drm_msm_gem_submit *args, struct drm_file *file)
71 spin_lock(&file->table_lock);
73 for (i = 0; i < args->nr_bos; i++) {
74 struct drm_msm_gem_submit_bo submit_bo;
75 struct drm_gem_object *obj;
76 struct msm_gem_object *msm_obj;
77 void __user *userptr =
78 u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
80 /* make sure we don't have garbage flags, in case we hit
81 * error path before flags is initialized:
83 submit->bos[i].flags = 0;
85 ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
91 if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
92 DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
97 submit->bos[i].flags = submit_bo.flags;
98 /* in validate_objects() we figure out if this is true: */
99 submit->bos[i].iova = submit_bo.presumed;
101 /* normally use drm_gem_object_lookup(), but for bulk lookup
102 * all under single table_lock just hit object_idr directly:
104 obj = idr_find(&file->object_idr, submit_bo.handle);
106 DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
111 msm_obj = to_msm_bo(obj);
113 if (!list_empty(&msm_obj->submit_entry)) {
114 DRM_ERROR("handle %u at index %u already on submit list\n",
115 submit_bo.handle, i);
120 drm_gem_object_reference(obj);
122 submit->bos[i].obj = msm_obj;
124 list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
129 spin_unlock(&file->table_lock);
134 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
136 struct msm_gem_object *msm_obj = submit->bos[i].obj;
138 if (submit->bos[i].flags & BO_PINNED)
139 msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
141 if (submit->bos[i].flags & BO_LOCKED)
142 ww_mutex_unlock(&msm_obj->resv->lock);
144 if (!(submit->bos[i].flags & BO_VALID))
145 submit->bos[i].iova = 0;
147 submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
150 /* This is where we make sure all the bo's are reserved and pin'd: */
151 static int submit_lock_objects(struct msm_gem_submit *submit)
153 int contended, slow_locked = -1, i, ret = 0;
156 for (i = 0; i < submit->nr_bos; i++) {
157 struct msm_gem_object *msm_obj = submit->bos[i].obj;
159 if (slow_locked == i)
164 if (!(submit->bos[i].flags & BO_LOCKED)) {
165 ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
169 submit->bos[i].flags |= BO_LOCKED;
173 ww_acquire_done(&submit->ticket);
179 submit_unlock_unpin_bo(submit, i);
182 submit_unlock_unpin_bo(submit, slow_locked);
184 if (ret == -EDEADLK) {
185 struct msm_gem_object *msm_obj = submit->bos[contended].obj;
186 /* we lost out in a seqno race, lock and retry.. */
187 ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
190 submit->bos[contended].flags |= BO_LOCKED;
191 slow_locked = contended;
199 static int submit_fence_sync(struct msm_gem_submit *submit)
203 for (i = 0; i < submit->nr_bos; i++) {
204 struct msm_gem_object *msm_obj = submit->bos[i].obj;
205 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
207 ret = msm_gem_sync_object(&msm_obj->base, submit->gpu->fctx, write);
215 static int submit_pin_objects(struct msm_gem_submit *submit)
219 submit->valid = true;
221 for (i = 0; i < submit->nr_bos; i++) {
222 struct msm_gem_object *msm_obj = submit->bos[i].obj;
225 /* if locking succeeded, pin bo: */
226 ret = msm_gem_get_iova_locked(&msm_obj->base,
227 submit->gpu->id, &iova);
232 submit->bos[i].flags |= BO_PINNED;
234 if (iova == submit->bos[i].iova) {
235 submit->bos[i].flags |= BO_VALID;
237 submit->bos[i].iova = iova;
238 /* iova changed, so address in cmdstream is not valid: */
239 submit->bos[i].flags &= ~BO_VALID;
240 submit->valid = false;
247 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
248 struct msm_gem_object **obj, uint32_t *iova, bool *valid)
250 if (idx >= submit->nr_bos) {
251 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
252 idx, submit->nr_bos);
257 *obj = submit->bos[idx].obj;
259 *iova = submit->bos[idx].iova;
261 *valid = !!(submit->bos[idx].flags & BO_VALID);
266 /* process the reloc's and patch up the cmdstream as needed: */
267 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
268 uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
270 uint32_t i, last_offset = 0;
275 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
279 /* For now, just map the entire thing. Eventually we probably
280 * to do it page-by-page, w/ kmap() if not vmap()d..
282 ptr = msm_gem_vaddr_locked(&obj->base);
286 DBG("failed to map: %d", ret);
290 for (i = 0; i < nr_relocs; i++) {
291 struct drm_msm_gem_submit_reloc submit_reloc;
292 void __user *userptr =
293 u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
297 ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
301 if (submit_reloc.submit_offset % 4) {
302 DRM_ERROR("non-aligned reloc offset: %u\n",
303 submit_reloc.submit_offset);
307 /* offset in dwords: */
308 off = submit_reloc.submit_offset / 4;
310 if ((off >= (obj->base.size / 4)) ||
311 (off < last_offset)) {
312 DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
316 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
323 iova += submit_reloc.reloc_offset;
325 if (submit_reloc.shift < 0)
326 iova >>= -submit_reloc.shift;
328 iova <<= submit_reloc.shift;
330 ptr[off] = iova | submit_reloc.or;
338 static void submit_cleanup(struct msm_gem_submit *submit)
342 for (i = 0; i < submit->nr_bos; i++) {
343 struct msm_gem_object *msm_obj = submit->bos[i].obj;
344 submit_unlock_unpin_bo(submit, i);
345 list_del_init(&msm_obj->submit_entry);
346 drm_gem_object_unreference(&msm_obj->base);
349 ww_acquire_fini(&submit->ticket);
352 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
353 struct drm_file *file)
355 struct msm_drm_private *priv = dev->dev_private;
356 struct drm_msm_gem_submit *args = data;
357 struct msm_file_private *ctx = file->driver_priv;
358 struct msm_gem_submit *submit;
359 struct msm_gpu *gpu = priv->gpu;
366 /* for now, we just have 3d pipe.. eventually this would need to
367 * be more clever to dispatch to appropriate gpu module:
369 if (args->pipe != MSM_PIPE_3D0)
372 if (args->nr_cmds > MAX_CMDS)
375 submit = submit_create(dev, gpu, args->nr_bos);
379 mutex_lock(&dev->struct_mutex);
381 ret = submit_lookup_objects(submit, args, file);
385 ret = submit_lock_objects(submit);
389 ret = submit_fence_sync(submit);
393 ret = submit_pin_objects(submit);
397 for (i = 0; i < args->nr_cmds; i++) {
398 struct drm_msm_gem_submit_cmd submit_cmd;
399 void __user *userptr =
400 u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
401 struct msm_gem_object *msm_obj;
404 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
410 /* validate input from userspace: */
411 switch (submit_cmd.type) {
412 case MSM_SUBMIT_CMD_BUF:
413 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
414 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
417 DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
422 ret = submit_bo(submit, submit_cmd.submit_idx,
423 &msm_obj, &iova, NULL);
427 if (submit_cmd.size % 4) {
428 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
434 if ((submit_cmd.size + submit_cmd.submit_offset) >=
435 msm_obj->base.size) {
436 DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
441 submit->cmd[i].type = submit_cmd.type;
442 submit->cmd[i].size = submit_cmd.size / 4;
443 submit->cmd[i].iova = iova + submit_cmd.submit_offset;
444 submit->cmd[i].idx = submit_cmd.submit_idx;
449 ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
450 submit_cmd.nr_relocs, submit_cmd.relocs);
457 ret = msm_gpu_submit(gpu, submit, ctx);
459 args->fence = submit->fence->seqno;
462 submit_cleanup(submit);
464 msm_gem_submit_free(submit);
465 mutex_unlock(&dev->struct_mutex);