2 * Copyright 2017 Red Hat
3 * Parts ported from amdgpu (fence wait code).
4 * Copyright 2016 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
33 * persistent objects that contain an optional fence. The fence can be updated
34 * with a new fence, or be NULL.
36 * syncobj's can be waited upon, where it will wait for the underlying
39 * syncobj's can be export to fd's and back, these fd's are opaque and
40 * have no other use case, except passing the syncobj between processes.
42 * Their primary use-case is to implement Vulkan fences and semaphores.
44 * syncobj have a kref reference count, but also have an optional file.
45 * The file is only created once the syncobj is exported.
46 * The file takes a reference on the kref.
50 #include <linux/file.h>
52 #include <linux/anon_inodes.h>
53 #include <linux/sync_file.h>
54 #include <linux/sched/signal.h>
56 #include "drm_internal.h"
57 #include <drm/drm_syncobj.h>
59 struct syncobj_wait_entry {
60 struct list_head node;
61 struct task_struct *task;
62 struct dma_fence *fence;
63 struct dma_fence_cb fence_cb;
67 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
68 struct syncobj_wait_entry *wait);
71 * drm_syncobj_find - lookup and reference a sync object.
72 * @file_private: drm file private pointer
73 * @handle: sync object handle to lookup.
75 * Returns a reference to the syncobj pointed to by handle or NULL. The
76 * reference must be released by calling drm_syncobj_put().
78 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
81 struct drm_syncobj *syncobj;
83 spin_lock(&file_private->syncobj_table_lock);
85 /* Check if we currently have a reference on the object */
86 syncobj = idr_find(&file_private->syncobj_idr, handle);
88 drm_syncobj_get(syncobj);
90 spin_unlock(&file_private->syncobj_table_lock);
94 EXPORT_SYMBOL(drm_syncobj_find);
96 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
97 struct syncobj_wait_entry *wait)
99 struct dma_fence *fence;
104 spin_lock(&syncobj->lock);
105 /* We've already tried once to get a fence and failed. Now that we
106 * have the lock, try one more time just to be sure we don't add a
107 * callback when a fence has already been set.
109 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
110 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
111 dma_fence_put(fence);
112 list_add_tail(&wait->node, &syncobj->cb_list);
114 wait->fence = dma_fence_get_stub();
118 spin_unlock(&syncobj->lock);
121 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
122 struct syncobj_wait_entry *wait)
124 if (!wait->node.next)
127 spin_lock(&syncobj->lock);
128 list_del_init(&wait->node);
129 spin_unlock(&syncobj->lock);
133 * drm_syncobj_add_point - add new timeline point to the syncobj
134 * @syncobj: sync object to add timeline point do
135 * @chain: chain node to use to add the point
136 * @fence: fence to encapsulate in the chain node
137 * @point: sequence number to use for the point
139 * Add the chain node as new timeline point to the syncobj.
141 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
142 struct dma_fence_chain *chain,
143 struct dma_fence *fence,
146 struct syncobj_wait_entry *cur, *tmp;
147 struct dma_fence *prev;
149 dma_fence_get(fence);
151 spin_lock(&syncobj->lock);
153 prev = drm_syncobj_fence_get(syncobj);
154 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
155 if (prev && prev->seqno >= point)
156 DRM_ERROR("You are adding an unorder point to timeline!\n");
157 dma_fence_chain_init(chain, prev, fence, point);
158 rcu_assign_pointer(syncobj->fence, &chain->base);
160 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
161 syncobj_wait_syncobj_func(syncobj, cur);
162 spin_unlock(&syncobj->lock);
164 /* Walk the chain once to trigger garbage collection */
165 dma_fence_chain_for_each(fence, prev);
168 EXPORT_SYMBOL(drm_syncobj_add_point);
171 * drm_syncobj_replace_fence - replace fence in a sync object.
172 * @syncobj: Sync object to replace fence in
173 * @fence: fence to install in sync file.
175 * This replaces the fence on a sync object.
177 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
178 struct dma_fence *fence)
180 struct dma_fence *old_fence;
181 struct syncobj_wait_entry *cur, *tmp;
184 dma_fence_get(fence);
186 spin_lock(&syncobj->lock);
188 old_fence = rcu_dereference_protected(syncobj->fence,
189 lockdep_is_held(&syncobj->lock));
190 rcu_assign_pointer(syncobj->fence, fence);
192 if (fence != old_fence) {
193 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
194 syncobj_wait_syncobj_func(syncobj, cur);
197 spin_unlock(&syncobj->lock);
199 dma_fence_put(old_fence);
201 EXPORT_SYMBOL(drm_syncobj_replace_fence);
204 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
205 * @syncobj: sync object to assign the fence on
207 * Assign a already signaled stub fence to the sync object.
209 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
211 struct dma_fence *fence = dma_fence_get_stub();
213 drm_syncobj_replace_fence(syncobj, fence);
214 dma_fence_put(fence);
217 /* 5s default for wait submission */
218 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
220 * drm_syncobj_find_fence - lookup and reference the fence in a sync object
221 * @file_private: drm file private pointer
222 * @handle: sync object handle to lookup.
223 * @point: timeline point
224 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
225 * @fence: out parameter for the fence
227 * This is just a convenience function that combines drm_syncobj_find() and
228 * drm_syncobj_fence_get().
230 * Returns 0 on success or a negative error value on failure. On success @fence
231 * contains a reference to the fence, which must be released by calling
234 int drm_syncobj_find_fence(struct drm_file *file_private,
235 u32 handle, u64 point, u64 flags,
236 struct dma_fence **fence)
238 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
239 struct syncobj_wait_entry wait;
240 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
246 *fence = drm_syncobj_fence_get(syncobj);
247 drm_syncobj_put(syncobj);
250 ret = dma_fence_chain_find_seqno(fence, point);
253 dma_fence_put(*fence);
258 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
261 memset(&wait, 0, sizeof(wait));
264 drm_syncobj_fence_add_wait(syncobj, &wait);
267 set_current_state(TASK_INTERRUPTIBLE);
277 if (signal_pending(current)) {
282 timeout = schedule_timeout(timeout);
285 __set_current_state(TASK_RUNNING);
289 drm_syncobj_remove_wait(syncobj, &wait);
293 EXPORT_SYMBOL(drm_syncobj_find_fence);
296 * drm_syncobj_free - free a sync object.
297 * @kref: kref to free.
299 * Only to be called from kref_put in drm_syncobj_put.
301 void drm_syncobj_free(struct kref *kref)
303 struct drm_syncobj *syncobj = container_of(kref,
306 drm_syncobj_replace_fence(syncobj, NULL);
309 EXPORT_SYMBOL(drm_syncobj_free);
312 * drm_syncobj_create - create a new syncobj
313 * @out_syncobj: returned syncobj
314 * @flags: DRM_SYNCOBJ_* flags
315 * @fence: if non-NULL, the syncobj will represent this fence
317 * This is the first function to create a sync object. After creating, drivers
318 * probably want to make it available to userspace, either through
319 * drm_syncobj_get_handle() or drm_syncobj_get_fd().
321 * Returns 0 on success or a negative error value on failure.
323 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
324 struct dma_fence *fence)
326 struct drm_syncobj *syncobj;
328 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
332 kref_init(&syncobj->refcount);
333 INIT_LIST_HEAD(&syncobj->cb_list);
334 spin_lock_init(&syncobj->lock);
336 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
337 drm_syncobj_assign_null_handle(syncobj);
340 drm_syncobj_replace_fence(syncobj, fence);
342 *out_syncobj = syncobj;
345 EXPORT_SYMBOL(drm_syncobj_create);
348 * drm_syncobj_get_handle - get a handle from a syncobj
349 * @file_private: drm file private pointer
350 * @syncobj: Sync object to export
351 * @handle: out parameter with the new handle
353 * Exports a sync object created with drm_syncobj_create() as a handle on
354 * @file_private to userspace.
356 * Returns 0 on success or a negative error value on failure.
358 int drm_syncobj_get_handle(struct drm_file *file_private,
359 struct drm_syncobj *syncobj, u32 *handle)
363 /* take a reference to put in the idr */
364 drm_syncobj_get(syncobj);
366 idr_preload(GFP_KERNEL);
367 spin_lock(&file_private->syncobj_table_lock);
368 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
369 spin_unlock(&file_private->syncobj_table_lock);
374 drm_syncobj_put(syncobj);
381 EXPORT_SYMBOL(drm_syncobj_get_handle);
383 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
384 u32 *handle, uint32_t flags)
387 struct drm_syncobj *syncobj;
389 ret = drm_syncobj_create(&syncobj, flags, NULL);
393 ret = drm_syncobj_get_handle(file_private, syncobj, handle);
394 drm_syncobj_put(syncobj);
398 static int drm_syncobj_destroy(struct drm_file *file_private,
401 struct drm_syncobj *syncobj;
403 spin_lock(&file_private->syncobj_table_lock);
404 syncobj = idr_remove(&file_private->syncobj_idr, handle);
405 spin_unlock(&file_private->syncobj_table_lock);
410 drm_syncobj_put(syncobj);
414 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
416 struct drm_syncobj *syncobj = file->private_data;
418 drm_syncobj_put(syncobj);
422 static const struct file_operations drm_syncobj_file_fops = {
423 .release = drm_syncobj_file_release,
427 * drm_syncobj_get_fd - get a file descriptor from a syncobj
428 * @syncobj: Sync object to export
429 * @p_fd: out parameter with the new file descriptor
431 * Exports a sync object created with drm_syncobj_create() as a file descriptor.
433 * Returns 0 on success or a negative error value on failure.
435 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
440 fd = get_unused_fd_flags(O_CLOEXEC);
444 file = anon_inode_getfile("syncobj_file",
445 &drm_syncobj_file_fops,
449 return PTR_ERR(file);
452 drm_syncobj_get(syncobj);
453 fd_install(fd, file);
458 EXPORT_SYMBOL(drm_syncobj_get_fd);
460 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
461 u32 handle, int *p_fd)
463 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
469 ret = drm_syncobj_get_fd(syncobj, p_fd);
470 drm_syncobj_put(syncobj);
474 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
477 struct drm_syncobj *syncobj;
478 struct fd f = fdget(fd);
484 if (f.file->f_op != &drm_syncobj_file_fops) {
489 /* take a reference to put in the idr */
490 syncobj = f.file->private_data;
491 drm_syncobj_get(syncobj);
493 idr_preload(GFP_KERNEL);
494 spin_lock(&file_private->syncobj_table_lock);
495 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
496 spin_unlock(&file_private->syncobj_table_lock);
503 drm_syncobj_put(syncobj);
509 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
512 struct dma_fence *fence = sync_file_get_fence(fd);
513 struct drm_syncobj *syncobj;
518 syncobj = drm_syncobj_find(file_private, handle);
520 dma_fence_put(fence);
524 drm_syncobj_replace_fence(syncobj, fence);
525 dma_fence_put(fence);
526 drm_syncobj_put(syncobj);
530 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
531 int handle, int *p_fd)
534 struct dma_fence *fence;
535 struct sync_file *sync_file;
536 int fd = get_unused_fd_flags(O_CLOEXEC);
541 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
545 sync_file = sync_file_create(fence);
547 dma_fence_put(fence);
554 fd_install(fd, sync_file->file);
563 * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
564 * @file_private: drm file-private structure to set up
566 * Called at device open time, sets up the structure for handling refcounting
570 drm_syncobj_open(struct drm_file *file_private)
572 idr_init_base(&file_private->syncobj_idr, 1);
573 spin_lock_init(&file_private->syncobj_table_lock);
577 drm_syncobj_release_handle(int id, void *ptr, void *data)
579 struct drm_syncobj *syncobj = ptr;
581 drm_syncobj_put(syncobj);
586 * drm_syncobj_release - release file-private sync object resources
587 * @file_private: drm file-private structure to clean up
589 * Called at close time when the filp is going away.
591 * Releases any remaining references on objects by this filp.
594 drm_syncobj_release(struct drm_file *file_private)
596 idr_for_each(&file_private->syncobj_idr,
597 &drm_syncobj_release_handle, file_private);
598 idr_destroy(&file_private->syncobj_idr);
602 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
603 struct drm_file *file_private)
605 struct drm_syncobj_create *args = data;
607 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
610 /* no valid flags yet */
611 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
614 return drm_syncobj_create_as_handle(file_private,
615 &args->handle, args->flags);
619 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
620 struct drm_file *file_private)
622 struct drm_syncobj_destroy *args = data;
624 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
627 /* make sure padding is empty */
630 return drm_syncobj_destroy(file_private, args->handle);
634 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
635 struct drm_file *file_private)
637 struct drm_syncobj_handle *args = data;
639 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
645 if (args->flags != 0 &&
646 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
649 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
650 return drm_syncobj_export_sync_file(file_private, args->handle,
653 return drm_syncobj_handle_to_fd(file_private, args->handle,
658 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
659 struct drm_file *file_private)
661 struct drm_syncobj_handle *args = data;
663 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
669 if (args->flags != 0 &&
670 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
673 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
674 return drm_syncobj_import_sync_file_fence(file_private,
678 return drm_syncobj_fd_to_handle(file_private, args->fd,
682 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
683 struct drm_syncobj_transfer *args)
685 struct drm_syncobj *timeline_syncobj = NULL;
686 struct dma_fence *fence;
687 struct dma_fence_chain *chain;
690 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
691 if (!timeline_syncobj) {
694 ret = drm_syncobj_find_fence(file_private, args->src_handle,
695 args->src_point, args->flags,
699 chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
704 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
706 dma_fence_put(fence);
708 drm_syncobj_put(timeline_syncobj);
714 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
715 struct drm_syncobj_transfer *args)
717 struct drm_syncobj *binary_syncobj = NULL;
718 struct dma_fence *fence;
721 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
724 ret = drm_syncobj_find_fence(file_private, args->src_handle,
725 args->src_point, args->flags, &fence);
728 drm_syncobj_replace_fence(binary_syncobj, fence);
729 dma_fence_put(fence);
731 drm_syncobj_put(binary_syncobj);
736 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
737 struct drm_file *file_private)
739 struct drm_syncobj_transfer *args = data;
742 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
749 ret = drm_syncobj_transfer_to_timeline(file_private, args);
751 ret = drm_syncobj_transfer_to_binary(file_private, args);
756 static void syncobj_wait_fence_func(struct dma_fence *fence,
757 struct dma_fence_cb *cb)
759 struct syncobj_wait_entry *wait =
760 container_of(cb, struct syncobj_wait_entry, fence_cb);
762 wake_up_process(wait->task);
765 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
766 struct syncobj_wait_entry *wait)
768 struct dma_fence *fence;
770 /* This happens inside the syncobj lock */
771 fence = rcu_dereference_protected(syncobj->fence,
772 lockdep_is_held(&syncobj->lock));
773 dma_fence_get(fence);
774 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
775 dma_fence_put(fence);
778 wait->fence = dma_fence_get_stub();
783 wake_up_process(wait->task);
784 list_del_init(&wait->node);
787 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
788 void __user *user_points,
794 struct syncobj_wait_entry *entries;
795 struct dma_fence *fence;
797 uint32_t signaled_count, i;
799 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
804 memset(points, 0, count * sizeof(uint64_t));
806 } else if (copy_from_user(points, user_points,
807 sizeof(uint64_t) * count)) {
809 goto err_free_points;
812 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
815 goto err_free_points;
817 /* Walk the list of sync objects and initialize entries. We do
818 * this up-front so that we can properly return -EINVAL if there is
819 * a syncobj with a missing fence and then never have the chance of
820 * returning -EINVAL again.
823 for (i = 0; i < count; ++i) {
824 struct dma_fence *fence;
826 entries[i].task = current;
827 entries[i].point = points[i];
828 fence = drm_syncobj_fence_get(syncobjs[i]);
829 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
830 dma_fence_put(fence);
831 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
835 goto cleanup_entries;
840 entries[i].fence = fence;
842 entries[i].fence = dma_fence_get_stub();
844 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
845 dma_fence_is_signaled(entries[i].fence)) {
846 if (signaled_count == 0 && idx)
852 if (signaled_count == count ||
853 (signaled_count > 0 &&
854 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
855 goto cleanup_entries;
857 /* There's a very annoying laxness in the dma_fence API here, in
858 * that backends are not required to automatically report when a
859 * fence is signaled prior to fence->ops->enable_signaling() being
860 * called. So here if we fail to match signaled_count, we need to
861 * fallthough and try a 0 timeout wait!
864 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
865 for (i = 0; i < count; ++i)
866 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
870 set_current_state(TASK_INTERRUPTIBLE);
873 for (i = 0; i < count; ++i) {
874 fence = entries[i].fence;
878 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
879 dma_fence_is_signaled(fence) ||
880 (!entries[i].fence_cb.func &&
881 dma_fence_add_callback(fence,
882 &entries[i].fence_cb,
883 syncobj_wait_fence_func))) {
884 /* The fence has been signaled */
885 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
895 if (signaled_count == count)
903 if (signal_pending(current)) {
904 timeout = -ERESTARTSYS;
908 timeout = schedule_timeout(timeout);
912 __set_current_state(TASK_RUNNING);
915 for (i = 0; i < count; ++i) {
916 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
917 if (entries[i].fence_cb.func)
918 dma_fence_remove_callback(entries[i].fence,
919 &entries[i].fence_cb);
920 dma_fence_put(entries[i].fence);
931 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
933 * @timeout_nsec: timeout nsec component in ns, 0 for poll
935 * Calculate the timeout in jiffies from an absolute time in sec/nsec.
937 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
939 ktime_t abs_timeout, now;
940 u64 timeout_ns, timeout_jiffies64;
942 /* make 0 timeout means poll - absolute 0 doesn't seem valid */
943 if (timeout_nsec == 0)
946 abs_timeout = ns_to_ktime(timeout_nsec);
949 if (!ktime_after(abs_timeout, now))
952 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
954 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
955 /* clamp timeout to avoid infinite timeout */
956 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
957 return MAX_SCHEDULE_TIMEOUT - 1;
959 return timeout_jiffies64 + 1;
961 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
963 static int drm_syncobj_array_wait(struct drm_device *dev,
964 struct drm_file *file_private,
965 struct drm_syncobj_wait *wait,
966 struct drm_syncobj_timeline_wait *timeline_wait,
967 struct drm_syncobj **syncobjs, bool timeline)
969 signed long timeout = 0;
973 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
974 timeout = drm_syncobj_array_wait_timeout(syncobjs,
981 wait->first_signaled = first;
983 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
984 timeout = drm_syncobj_array_wait_timeout(syncobjs,
985 u64_to_user_ptr(timeline_wait->points),
986 timeline_wait->count_handles,
987 timeline_wait->flags,
991 timeline_wait->first_signaled = first;
996 static int drm_syncobj_array_find(struct drm_file *file_private,
997 void __user *user_handles,
998 uint32_t count_handles,
999 struct drm_syncobj ***syncobjs_out)
1001 uint32_t i, *handles;
1002 struct drm_syncobj **syncobjs;
1005 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1006 if (handles == NULL)
1009 if (copy_from_user(handles, user_handles,
1010 sizeof(uint32_t) * count_handles)) {
1012 goto err_free_handles;
1015 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1016 if (syncobjs == NULL) {
1018 goto err_free_handles;
1021 for (i = 0; i < count_handles; i++) {
1022 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1025 goto err_put_syncobjs;
1030 *syncobjs_out = syncobjs;
1035 drm_syncobj_put(syncobjs[i]);
1043 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1047 for (i = 0; i < count; i++)
1048 drm_syncobj_put(syncobjs[i]);
1053 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1054 struct drm_file *file_private)
1056 struct drm_syncobj_wait *args = data;
1057 struct drm_syncobj **syncobjs;
1060 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1063 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1064 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1067 if (args->count_handles == 0)
1070 ret = drm_syncobj_array_find(file_private,
1071 u64_to_user_ptr(args->handles),
1072 args->count_handles,
1077 ret = drm_syncobj_array_wait(dev, file_private,
1078 args, NULL, syncobjs, false);
1080 drm_syncobj_array_free(syncobjs, args->count_handles);
1086 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1087 struct drm_file *file_private)
1089 struct drm_syncobj_timeline_wait *args = data;
1090 struct drm_syncobj **syncobjs;
1093 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1096 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1097 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1098 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1101 if (args->count_handles == 0)
1104 ret = drm_syncobj_array_find(file_private,
1105 u64_to_user_ptr(args->handles),
1106 args->count_handles,
1111 ret = drm_syncobj_array_wait(dev, file_private,
1112 NULL, args, syncobjs, true);
1114 drm_syncobj_array_free(syncobjs, args->count_handles);
1121 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1122 struct drm_file *file_private)
1124 struct drm_syncobj_array *args = data;
1125 struct drm_syncobj **syncobjs;
1129 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1135 if (args->count_handles == 0)
1138 ret = drm_syncobj_array_find(file_private,
1139 u64_to_user_ptr(args->handles),
1140 args->count_handles,
1145 for (i = 0; i < args->count_handles; i++)
1146 drm_syncobj_replace_fence(syncobjs[i], NULL);
1148 drm_syncobj_array_free(syncobjs, args->count_handles);
1154 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1155 struct drm_file *file_private)
1157 struct drm_syncobj_array *args = data;
1158 struct drm_syncobj **syncobjs;
1162 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1168 if (args->count_handles == 0)
1171 ret = drm_syncobj_array_find(file_private,
1172 u64_to_user_ptr(args->handles),
1173 args->count_handles,
1178 for (i = 0; i < args->count_handles; i++)
1179 drm_syncobj_assign_null_handle(syncobjs[i]);
1181 drm_syncobj_array_free(syncobjs, args->count_handles);
1187 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1188 struct drm_file *file_private)
1190 struct drm_syncobj_timeline_array *args = data;
1191 struct drm_syncobj **syncobjs;
1192 struct dma_fence_chain **chains;
1197 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1203 if (args->count_handles == 0)
1206 ret = drm_syncobj_array_find(file_private,
1207 u64_to_user_ptr(args->handles),
1208 args->count_handles,
1213 points = kmalloc_array(args->count_handles, sizeof(*points),
1219 if (!u64_to_user_ptr(args->points)) {
1220 memset(points, 0, args->count_handles * sizeof(uint64_t));
1221 } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1222 sizeof(uint64_t) * args->count_handles)) {
1227 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1232 for (i = 0; i < args->count_handles; i++) {
1233 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1235 for (j = 0; j < i; j++)
1242 for (i = 0; i < args->count_handles; i++) {
1243 struct dma_fence *fence = dma_fence_get_stub();
1245 drm_syncobj_add_point(syncobjs[i], chains[i],
1247 dma_fence_put(fence);
1254 drm_syncobj_array_free(syncobjs, args->count_handles);
1259 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1260 struct drm_file *file_private)
1262 struct drm_syncobj_timeline_array *args = data;
1263 struct drm_syncobj **syncobjs;
1264 uint64_t __user *points = u64_to_user_ptr(args->points);
1268 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1274 if (args->count_handles == 0)
1277 ret = drm_syncobj_array_find(file_private,
1278 u64_to_user_ptr(args->handles),
1279 args->count_handles,
1284 for (i = 0; i < args->count_handles; i++) {
1285 struct dma_fence_chain *chain;
1286 struct dma_fence *fence;
1289 fence = drm_syncobj_fence_get(syncobjs[i]);
1290 chain = to_dma_fence_chain(fence);
1292 struct dma_fence *iter, *last_signaled = NULL;
1294 dma_fence_chain_for_each(iter, fence) {
1297 dma_fence_put(last_signaled);
1298 last_signaled = dma_fence_get(iter);
1299 if (!to_dma_fence_chain(last_signaled)->prev_seqno)
1300 /* It is most likely that timeline has
1301 * unorder points. */
1304 point = dma_fence_is_signaled(last_signaled) ?
1305 last_signaled->seqno :
1306 to_dma_fence_chain(last_signaled)->prev_seqno;
1307 dma_fence_put(last_signaled);
1311 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1312 ret = ret ? -EFAULT : 0;
1316 drm_syncobj_array_free(syncobjs, args->count_handles);