drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 struct syncobj_wait_entry {
  60         struct list_head node;
  61         struct task_struct *task;
  62         struct dma_fence *fence;
  63         struct dma_fence_cb fence_cb;
  64         u64    point;
  65 };
  66
  67 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
  68                                       struct syncobj_wait_entry *wait);
  69
  70 /**
  71  * drm_syncobj_find - lookup and reference a sync object.
  72  * @file_private: drm file private pointer
  73  * @handle: sync object handle to lookup.
  74  *
  75  * Returns a reference to the syncobj pointed to by handle or NULL. The
  76  * reference must be released by calling drm_syncobj_put().
  77  */
  78 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  79                                      u32 handle)
  80 {
  81         struct drm_syncobj *syncobj;
  82
  83         spin_lock(&file_private->syncobj_table_lock);
  84
  85         /* Check if we currently have a reference on the object */
  86         syncobj = idr_find(&file_private->syncobj_idr, handle);
  87         if (syncobj)
  88                 drm_syncobj_get(syncobj);
  89
  90         spin_unlock(&file_private->syncobj_table_lock);
  91
  92         return syncobj;
  93 }
  94 EXPORT_SYMBOL(drm_syncobj_find);
  95
  96 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
  97                                        struct syncobj_wait_entry *wait)
  98 {
  99         struct dma_fence *fence;
 100
 101         if (wait->fence)
 102                 return;
 103
 104         spin_lock(&syncobj->lock);
 105         /* We've already tried once to get a fence and failed.  Now that we
 106          * have the lock, try one more time just to be sure we don't add a
 107          * callback when a fence has already been set.
 108          */
 109         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 110         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 111                 dma_fence_put(fence);
 112                 list_add_tail(&wait->node, &syncobj->cb_list);
 113         } else if (!fence) {
 114                 wait->fence = dma_fence_get_stub();
 115         } else {
 116                 wait->fence = fence;
 117         }
 118         spin_unlock(&syncobj->lock);
 119 }
 120
 121 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 122                                     struct syncobj_wait_entry *wait)
 123 {
 124         if (!wait->node.next)
 125                 return;
 126
 127         spin_lock(&syncobj->lock);
 128         list_del_init(&wait->node);
 129         spin_unlock(&syncobj->lock);
 130 }
 131
 132 /**
 133  * drm_syncobj_add_point - add new timeline point to the syncobj
 134  * @syncobj: sync object to add timeline point do
 135  * @chain: chain node to use to add the point
 136  * @fence: fence to encapsulate in the chain node
 137  * @point: sequence number to use for the point
 138  *
 139  * Add the chain node as new timeline point to the syncobj.
 140  */
 141 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 142                            struct dma_fence_chain *chain,
 143                            struct dma_fence *fence,
 144                            uint64_t point)
 145 {
 146         struct syncobj_wait_entry *cur, *tmp;
 147         struct dma_fence *prev;
 148
 149         dma_fence_get(fence);
 150
 151         spin_lock(&syncobj->lock);
 152
 153         prev = drm_syncobj_fence_get(syncobj);
 154         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 155         if (prev && prev->seqno >= point)
 156                 DRM_ERROR("You are adding an unorder point to timeline!\n");
 157         dma_fence_chain_init(chain, prev, fence, point);
 158         rcu_assign_pointer(syncobj->fence, &chain->base);
 159
 160         list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 161                 syncobj_wait_syncobj_func(syncobj, cur);
 162         spin_unlock(&syncobj->lock);
 163
 164         /* Walk the chain once to trigger garbage collection */
 165         dma_fence_chain_for_each(fence, prev);
 166         dma_fence_put(prev);
 167 }
 168 EXPORT_SYMBOL(drm_syncobj_add_point);
 169
 170 /**
 171  * drm_syncobj_replace_fence - replace fence in a sync object.
 172  * @syncobj: Sync object to replace fence in
 173  * @fence: fence to install in sync file.
 174  *
 175  * This replaces the fence on a sync object.
 176  */
 177 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 178                                struct dma_fence *fence)
 179 {
 180         struct dma_fence *old_fence;
 181         struct syncobj_wait_entry *cur, *tmp;
 182
 183         if (fence)
 184                 dma_fence_get(fence);
 185
 186         spin_lock(&syncobj->lock);
 187
 188         old_fence = rcu_dereference_protected(syncobj->fence,
 189                                               lockdep_is_held(&syncobj->lock));
 190         rcu_assign_pointer(syncobj->fence, fence);
 191
 192         if (fence != old_fence) {
 193                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 194                         syncobj_wait_syncobj_func(syncobj, cur);
 195         }
 196
 197         spin_unlock(&syncobj->lock);
 198
 199         dma_fence_put(old_fence);
 200 }
 201 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 202
 203 /**
 204  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 205  * @syncobj: sync object to assign the fence on
 206  *
 207  * Assign a already signaled stub fence to the sync object.
 208  */
 209 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 210 {
 211         struct dma_fence *fence = dma_fence_get_stub();
 212
 213         drm_syncobj_replace_fence(syncobj, fence);
 214         dma_fence_put(fence);
 215 }
 216
 217 /* 5s default for wait submission */
 218 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 219 /**
 220  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 221  * @file_private: drm file private pointer
 222  * @handle: sync object handle to lookup.
 223  * @point: timeline point
 224  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 225  * @fence: out parameter for the fence
 226  *
 227  * This is just a convenience function that combines drm_syncobj_find() and
 228  * drm_syncobj_fence_get().
 229  *
 230  * Returns 0 on success or a negative error value on failure. On success @fence
 231  * contains a reference to the fence, which must be released by calling
 232  * dma_fence_put().
 233  */
 234 int drm_syncobj_find_fence(struct drm_file *file_private,
 235                            u32 handle, u64 point, u64 flags,
 236                            struct dma_fence **fence)
 237 {
 238         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 239         struct syncobj_wait_entry wait;
 240         u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 241         int ret;
 242
 243         if (!syncobj)
 244                 return -ENOENT;
 245
 246         *fence = drm_syncobj_fence_get(syncobj);
 247         drm_syncobj_put(syncobj);
 248
 249         if (*fence) {
 250                 ret = dma_fence_chain_find_seqno(fence, point);
 251                 if (!ret)
 252                         return 0;
 253                 dma_fence_put(*fence);
 254         } else {
 255                 ret = -EINVAL;
 256         }
 257
 258         if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 259                 return ret;
 260
 261         memset(&wait, 0, sizeof(wait));
 262         wait.task = current;
 263         wait.point = point;
 264         drm_syncobj_fence_add_wait(syncobj, &wait);
 265
 266         do {
 267                 set_current_state(TASK_INTERRUPTIBLE);
 268                 if (wait.fence) {
 269                         ret = 0;
 270                         break;
 271                 }
 272                 if (timeout == 0) {
 273                         ret = -ETIME;
 274                         break;
 275                 }
 276
 277                 if (signal_pending(current)) {
 278                         ret = -ERESTARTSYS;
 279                         break;
 280                 }
 281
 282                 timeout = schedule_timeout(timeout);
 283         } while (1);
 284
 285         __set_current_state(TASK_RUNNING);
 286         *fence = wait.fence;
 287
 288         if (wait.node.next)
 289                 drm_syncobj_remove_wait(syncobj, &wait);
 290
 291         return ret;
 292 }
 293 EXPORT_SYMBOL(drm_syncobj_find_fence);
 294
 295 /**
 296  * drm_syncobj_free - free a sync object.
 297  * @kref: kref to free.
 298  *
 299  * Only to be called from kref_put in drm_syncobj_put.
 300  */
 301 void drm_syncobj_free(struct kref *kref)
 302 {
 303         struct drm_syncobj *syncobj = container_of(kref,
 304                                                    struct drm_syncobj,
 305                                                    refcount);
 306         drm_syncobj_replace_fence(syncobj, NULL);
 307         kfree(syncobj);
 308 }
 309 EXPORT_SYMBOL(drm_syncobj_free);
 310
 311 /**
 312  * drm_syncobj_create - create a new syncobj
 313  * @out_syncobj: returned syncobj
 314  * @flags: DRM_SYNCOBJ_* flags
 315  * @fence: if non-NULL, the syncobj will represent this fence
 316  *
 317  * This is the first function to create a sync object. After creating, drivers
 318  * probably want to make it available to userspace, either through
 319  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 320  *
 321  * Returns 0 on success or a negative error value on failure.
 322  */
 323 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 324                        struct dma_fence *fence)
 325 {
 326         struct drm_syncobj *syncobj;
 327
 328         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 329         if (!syncobj)
 330                 return -ENOMEM;
 331
 332         kref_init(&syncobj->refcount);
 333         INIT_LIST_HEAD(&syncobj->cb_list);
 334         spin_lock_init(&syncobj->lock);
 335
 336         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
 337                 drm_syncobj_assign_null_handle(syncobj);
 338
 339         if (fence)
 340                 drm_syncobj_replace_fence(syncobj, fence);
 341
 342         *out_syncobj = syncobj;
 343         return 0;
 344 }
 345 EXPORT_SYMBOL(drm_syncobj_create);
 346
 347 /**
 348  * drm_syncobj_get_handle - get a handle from a syncobj
 349  * @file_private: drm file private pointer
 350  * @syncobj: Sync object to export
 351  * @handle: out parameter with the new handle
 352  *
 353  * Exports a sync object created with drm_syncobj_create() as a handle on
 354  * @file_private to userspace.
 355  *
 356  * Returns 0 on success or a negative error value on failure.
 357  */
 358 int drm_syncobj_get_handle(struct drm_file *file_private,
 359                            struct drm_syncobj *syncobj, u32 *handle)
 360 {
 361         int ret;
 362
 363         /* take a reference to put in the idr */
 364         drm_syncobj_get(syncobj);
 365
 366         idr_preload(GFP_KERNEL);
 367         spin_lock(&file_private->syncobj_table_lock);
 368         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 369         spin_unlock(&file_private->syncobj_table_lock);
 370
 371         idr_preload_end();
 372
 373         if (ret < 0) {
 374                 drm_syncobj_put(syncobj);
 375                 return ret;
 376         }
 377
 378         *handle = ret;
 379         return 0;
 380 }
 381 EXPORT_SYMBOL(drm_syncobj_get_handle);
 382
 383 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 384                                         u32 *handle, uint32_t flags)
 385 {
 386         int ret;
 387         struct drm_syncobj *syncobj;
 388
 389         ret = drm_syncobj_create(&syncobj, flags, NULL);
 390         if (ret)
 391                 return ret;
 392
 393         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 394         drm_syncobj_put(syncobj);
 395         return ret;
 396 }
 397
 398 static int drm_syncobj_destroy(struct drm_file *file_private,
 399                                u32 handle)
 400 {
 401         struct drm_syncobj *syncobj;
 402
 403         spin_lock(&file_private->syncobj_table_lock);
 404         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 405         spin_unlock(&file_private->syncobj_table_lock);
 406
 407         if (!syncobj)
 408                 return -EINVAL;
 409
 410         drm_syncobj_put(syncobj);
 411         return 0;
 412 }
 413
 414 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 415 {
 416         struct drm_syncobj *syncobj = file->private_data;
 417
 418         drm_syncobj_put(syncobj);
 419         return 0;
 420 }
 421
 422 static const struct file_operations drm_syncobj_file_fops = {
 423         .release = drm_syncobj_file_release,
 424 };
 425
 426 /**
 427  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 428  * @syncobj: Sync object to export
 429  * @p_fd: out parameter with the new file descriptor
 430  *
 431  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 432  *
 433  * Returns 0 on success or a negative error value on failure.
 434  */
 435 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 436 {
 437         struct file *file;
 438         int fd;
 439
 440         fd = get_unused_fd_flags(O_CLOEXEC);
 441         if (fd < 0)
 442                 return fd;
 443
 444         file = anon_inode_getfile("syncobj_file",
 445                                   &drm_syncobj_file_fops,
 446                                   syncobj, 0);
 447         if (IS_ERR(file)) {
 448                 put_unused_fd(fd);
 449                 return PTR_ERR(file);
 450         }
 451
 452         drm_syncobj_get(syncobj);
 453         fd_install(fd, file);
 454
 455         *p_fd = fd;
 456         return 0;
 457 }
 458 EXPORT_SYMBOL(drm_syncobj_get_fd);
 459
 460 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 461                                     u32 handle, int *p_fd)
 462 {
 463         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 464         int ret;
 465
 466         if (!syncobj)
 467                 return -EINVAL;
 468
 469         ret = drm_syncobj_get_fd(syncobj, p_fd);
 470         drm_syncobj_put(syncobj);
 471         return ret;
 472 }
 473
 474 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 475                                     int fd, u32 *handle)
 476 {
 477         struct drm_syncobj *syncobj;
 478         struct file *file;
 479         int ret;
 480
 481         file = fget(fd);
 482         if (!file)
 483                 return -EINVAL;
 484
 485         if (file->f_op != &drm_syncobj_file_fops) {
 486                 fput(file);
 487                 return -EINVAL;
 488         }
 489
 490         /* take a reference to put in the idr */
 491         syncobj = file->private_data;
 492         drm_syncobj_get(syncobj);
 493
 494         idr_preload(GFP_KERNEL);
 495         spin_lock(&file_private->syncobj_table_lock);
 496         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 497         spin_unlock(&file_private->syncobj_table_lock);
 498         idr_preload_end();
 499
 500         if (ret > 0) {
 501                 *handle = ret;
 502                 ret = 0;
 503         } else
 504                 drm_syncobj_put(syncobj);
 505
 506         fput(file);
 507         return ret;
 508 }
 509
 510 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 511                                               int fd, int handle)
 512 {
 513         struct dma_fence *fence = sync_file_get_fence(fd);
 514         struct drm_syncobj *syncobj;
 515
 516         if (!fence)
 517                 return -EINVAL;
 518
 519         syncobj = drm_syncobj_find(file_private, handle);
 520         if (!syncobj) {
 521                 dma_fence_put(fence);
 522                 return -ENOENT;
 523         }
 524
 525         drm_syncobj_replace_fence(syncobj, fence);
 526         dma_fence_put(fence);
 527         drm_syncobj_put(syncobj);
 528         return 0;
 529 }
 530
 531 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 532                                         int handle, int *p_fd)
 533 {
 534         int ret;
 535         struct dma_fence *fence;
 536         struct sync_file *sync_file;
 537         int fd = get_unused_fd_flags(O_CLOEXEC);
 538
 539         if (fd < 0)
 540                 return fd;
 541
 542         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 543         if (ret)
 544                 goto err_put_fd;
 545
 546         sync_file = sync_file_create(fence);
 547
 548         dma_fence_put(fence);
 549
 550         if (!sync_file) {
 551                 ret = -EINVAL;
 552                 goto err_put_fd;
 553         }
 554
 555         fd_install(fd, sync_file->file);
 556
 557         *p_fd = fd;
 558         return 0;
 559 err_put_fd:
 560         put_unused_fd(fd);
 561         return ret;
 562 }
 563 /**
 564  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 565  * @file_private: drm file-private structure to set up
 566  *
 567  * Called at device open time, sets up the structure for handling refcounting
 568  * of sync objects.
 569  */
 570 void
 571 drm_syncobj_open(struct drm_file *file_private)
 572 {
 573         idr_init_base(&file_private->syncobj_idr, 1);
 574         spin_lock_init(&file_private->syncobj_table_lock);
 575 }
 576
 577 static int
 578 drm_syncobj_release_handle(int id, void *ptr, void *data)
 579 {
 580         struct drm_syncobj *syncobj = ptr;
 581
 582         drm_syncobj_put(syncobj);
 583         return 0;
 584 }
 585
 586 /**
 587  * drm_syncobj_release - release file-private sync object resources
 588  * @file_private: drm file-private structure to clean up
 589  *
 590  * Called at close time when the filp is going away.
 591  *
 592  * Releases any remaining references on objects by this filp.
 593  */
 594 void
 595 drm_syncobj_release(struct drm_file *file_private)
 596 {
 597         idr_for_each(&file_private->syncobj_idr,
 598                      &drm_syncobj_release_handle, file_private);
 599         idr_destroy(&file_private->syncobj_idr);
 600 }
 601
 602 int
 603 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 604                          struct drm_file *file_private)
 605 {
 606         struct drm_syncobj_create *args = data;
 607
 608         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 609                 return -EOPNOTSUPP;
 610
 611         /* no valid flags yet */
 612         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 613                 return -EINVAL;
 614
 615         return drm_syncobj_create_as_handle(file_private,
 616                                             &args->handle, args->flags);
 617 }
 618
 619 int
 620 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 621                           struct drm_file *file_private)
 622 {
 623         struct drm_syncobj_destroy *args = data;
 624
 625         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 626                 return -EOPNOTSUPP;
 627
 628         /* make sure padding is empty */
 629         if (args->pad)
 630                 return -EINVAL;
 631         return drm_syncobj_destroy(file_private, args->handle);
 632 }
 633
 634 int
 635 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 636                                    struct drm_file *file_private)
 637 {
 638         struct drm_syncobj_handle *args = data;
 639
 640         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 641                 return -EOPNOTSUPP;
 642
 643         if (args->pad)
 644                 return -EINVAL;
 645
 646         if (args->flags != 0 &&
 647             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 648                 return -EINVAL;
 649
 650         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 651                 return drm_syncobj_export_sync_file(file_private, args->handle,
 652                                                     &args->fd);
 653
 654         return drm_syncobj_handle_to_fd(file_private, args->handle,
 655                                         &args->fd);
 656 }
 657
 658 int
 659 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 660                                    struct drm_file *file_private)
 661 {
 662         struct drm_syncobj_handle *args = data;
 663
 664         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 665                 return -EOPNOTSUPP;
 666
 667         if (args->pad)
 668                 return -EINVAL;
 669
 670         if (args->flags != 0 &&
 671             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 672                 return -EINVAL;
 673
 674         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 675                 return drm_syncobj_import_sync_file_fence(file_private,
 676                                                           args->fd,
 677                                                           args->handle);
 678
 679         return drm_syncobj_fd_to_handle(file_private, args->fd,
 680                                         &args->handle);
 681 }
 682
 683 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
 684                                             struct drm_syncobj_transfer *args)
 685 {
 686         struct drm_syncobj *timeline_syncobj = NULL;
 687         struct dma_fence *fence;
 688         struct dma_fence_chain *chain;
 689         int ret;
 690
 691         timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 692         if (!timeline_syncobj) {
 693                 return -ENOENT;
 694         }
 695         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 696                                      args->src_point, args->flags,
 697                                      &fence);
 698         if (ret)
 699                 goto err;
 700         chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
 701         if (!chain) {
 702                 ret = -ENOMEM;
 703                 goto err1;
 704         }
 705         drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
 706 err1:
 707         dma_fence_put(fence);
 708 err:
 709         drm_syncobj_put(timeline_syncobj);
 710
 711         return ret;
 712 }
 713
 714 static int
 715 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
 716                                struct drm_syncobj_transfer *args)
 717 {
 718         struct drm_syncobj *binary_syncobj = NULL;
 719         struct dma_fence *fence;
 720         int ret;
 721
 722         binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 723         if (!binary_syncobj)
 724                 return -ENOENT;
 725         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 726                                      args->src_point, args->flags, &fence);
 727         if (ret)
 728                 goto err;
 729         drm_syncobj_replace_fence(binary_syncobj, fence);
 730         dma_fence_put(fence);
 731 err:
 732         drm_syncobj_put(binary_syncobj);
 733
 734         return ret;
 735 }
 736 int
 737 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
 738                            struct drm_file *file_private)
 739 {
 740         struct drm_syncobj_transfer *args = data;
 741         int ret;
 742
 743         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 744                 return -ENODEV;
 745
 746         if (args->pad)
 747                 return -EINVAL;
 748
 749         if (args->dst_point)
 750                 ret = drm_syncobj_transfer_to_timeline(file_private, args);
 751         else
 752                 ret = drm_syncobj_transfer_to_binary(file_private, args);
 753
 754         return ret;
 755 }
 756
 757 static void syncobj_wait_fence_func(struct dma_fence *fence,
 758                                     struct dma_fence_cb *cb)
 759 {
 760         struct syncobj_wait_entry *wait =
 761                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 762
 763         wake_up_process(wait->task);
 764 }
 765
 766 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 767                                       struct syncobj_wait_entry *wait)
 768 {
 769         struct dma_fence *fence;
 770
 771         /* This happens inside the syncobj lock */
 772         fence = rcu_dereference_protected(syncobj->fence,
 773                                           lockdep_is_held(&syncobj->lock));
 774         dma_fence_get(fence);
 775         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 776                 dma_fence_put(fence);
 777                 return;
 778         } else if (!fence) {
 779                 wait->fence = dma_fence_get_stub();
 780         } else {
 781                 wait->fence = fence;
 782         }
 783
 784         wake_up_process(wait->task);
 785         list_del_init(&wait->node);
 786 }
 787
 788 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 789                                                   void __user *user_points,
 790                                                   uint32_t count,
 791                                                   uint32_t flags,
 792                                                   signed long timeout,
 793                                                   uint32_t *idx)
 794 {
 795         struct syncobj_wait_entry *entries;
 796         struct dma_fence *fence;
 797         uint64_t *points;
 798         uint32_t signaled_count, i;
 799
 800         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
 801         if (points == NULL)
 802                 return -ENOMEM;
 803
 804         if (!user_points) {
 805                 memset(points, 0, count * sizeof(uint64_t));
 806
 807         } else if (copy_from_user(points, user_points,
 808                                   sizeof(uint64_t) * count)) {
 809                 timeout = -EFAULT;
 810                 goto err_free_points;
 811         }
 812
 813         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 814         if (!entries) {
 815                 timeout = -ENOMEM;
 816                 goto err_free_points;
 817         }
 818         /* Walk the list of sync objects and initialize entries.  We do
 819          * this up-front so that we can properly return -EINVAL if there is
 820          * a syncobj with a missing fence and then never have the chance of
 821          * returning -EINVAL again.
 822          */
 823         signaled_count = 0;
 824         for (i = 0; i < count; ++i) {
 825                 struct dma_fence *fence;
 826
 827                 entries[i].task = current;
 828                 entries[i].point = points[i];
 829                 fence = drm_syncobj_fence_get(syncobjs[i]);
 830                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
 831                         dma_fence_put(fence);
 832                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 833                                 continue;
 834                         } else {
 835                                 timeout = -EINVAL;
 836                                 goto cleanup_entries;
 837                         }
 838                 }
 839
 840                 if (fence)
 841                         entries[i].fence = fence;
 842                 else
 843                         entries[i].fence = dma_fence_get_stub();
 844
 845                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
 846                     dma_fence_is_signaled(entries[i].fence)) {
 847                         if (signaled_count == 0 && idx)
 848                                 *idx = i;
 849                         signaled_count++;
 850                 }
 851         }
 852
 853         if (signaled_count == count ||
 854             (signaled_count > 0 &&
 855              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 856                 goto cleanup_entries;
 857
 858         /* There's a very annoying laxness in the dma_fence API here, in
 859          * that backends are not required to automatically report when a
 860          * fence is signaled prior to fence->ops->enable_signaling() being
 861          * called.  So here if we fail to match signaled_count, we need to
 862          * fallthough and try a 0 timeout wait!
 863          */
 864
 865         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 866                 for (i = 0; i < count; ++i)
 867                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
 868         }
 869
 870         do {
 871                 set_current_state(TASK_INTERRUPTIBLE);
 872
 873                 signaled_count = 0;
 874                 for (i = 0; i < count; ++i) {
 875                         fence = entries[i].fence;
 876                         if (!fence)
 877                                 continue;
 878
 879                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
 880                             dma_fence_is_signaled(fence) ||
 881                             (!entries[i].fence_cb.func &&
 882                              dma_fence_add_callback(fence,
 883                                                     &entries[i].fence_cb,
 884                                                     syncobj_wait_fence_func))) {
 885                                 /* The fence has been signaled */
 886                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 887                                         signaled_count++;
 888                                 } else {
 889                                         if (idx)
 890                                                 *idx = i;
 891                                         goto done_waiting;
 892                                 }
 893                         }
 894                 }
 895
 896                 if (signaled_count == count)
 897                         goto done_waiting;
 898
 899                 if (timeout == 0) {
 900                         timeout = -ETIME;
 901                         goto done_waiting;
 902                 }
 903
 904                 if (signal_pending(current)) {
 905                         timeout = -ERESTARTSYS;
 906                         goto done_waiting;
 907                 }
 908
 909                 timeout = schedule_timeout(timeout);
 910         } while (1);
 911
 912 done_waiting:
 913         __set_current_state(TASK_RUNNING);
 914
 915 cleanup_entries:
 916         for (i = 0; i < count; ++i) {
 917                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
 918                 if (entries[i].fence_cb.func)
 919                         dma_fence_remove_callback(entries[i].fence,
 920                                                   &entries[i].fence_cb);
 921                 dma_fence_put(entries[i].fence);
 922         }
 923         kfree(entries);
 924
 925 err_free_points:
 926         kfree(points);
 927
 928         return timeout;
 929 }
 930
 931 /**
 932  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 933  *
 934  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 935  *
 936  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 937  */
 938 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 939 {
 940         ktime_t abs_timeout, now;
 941         u64 timeout_ns, timeout_jiffies64;
 942
 943         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 944         if (timeout_nsec == 0)
 945                 return 0;
 946
 947         abs_timeout = ns_to_ktime(timeout_nsec);
 948         now = ktime_get();
 949
 950         if (!ktime_after(abs_timeout, now))
 951                 return 0;
 952
 953         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 954
 955         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 956         /*  clamp timeout to avoid infinite timeout */
 957         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 958                 return MAX_SCHEDULE_TIMEOUT - 1;
 959
 960         return timeout_jiffies64 + 1;
 961 }
 962 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
 963
 964 static int drm_syncobj_array_wait(struct drm_device *dev,
 965                                   struct drm_file *file_private,
 966                                   struct drm_syncobj_wait *wait,
 967                                   struct drm_syncobj_timeline_wait *timeline_wait,
 968                                   struct drm_syncobj **syncobjs, bool timeline)
 969 {
 970         signed long timeout = 0;
 971         uint32_t first = ~0;
 972
 973         if (!timeline) {
 974                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 975                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
 976                                                          NULL,
 977                                                          wait->count_handles,
 978                                                          wait->flags,
 979                                                          timeout, &first);
 980                 if (timeout < 0)
 981                         return timeout;
 982                 wait->first_signaled = first;
 983         } else {
 984                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
 985                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
 986                                                          u64_to_user_ptr(timeline_wait->points),
 987                                                          timeline_wait->count_handles,
 988                                                          timeline_wait->flags,
 989                                                          timeout, &first);
 990                 if (timeout < 0)
 991                         return timeout;
 992                 timeline_wait->first_signaled = first;
 993         }
 994         return 0;
 995 }
 996
 997 static int drm_syncobj_array_find(struct drm_file *file_private,
 998                                   void __user *user_handles,
 999                                   uint32_t count_handles,
1000                                   struct drm_syncobj ***syncobjs_out)
1001 {
1002         uint32_t i, *handles;
1003         struct drm_syncobj **syncobjs;
1004         int ret;
1005
1006         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1007         if (handles == NULL)
1008                 return -ENOMEM;
1009
1010         if (copy_from_user(handles, user_handles,
1011                            sizeof(uint32_t) * count_handles)) {
1012                 ret = -EFAULT;
1013                 goto err_free_handles;
1014         }
1015
1016         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1017         if (syncobjs == NULL) {
1018                 ret = -ENOMEM;
1019                 goto err_free_handles;
1020         }
1021
1022         for (i = 0; i < count_handles; i++) {
1023                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1024                 if (!syncobjs[i]) {
1025                         ret = -ENOENT;
1026                         goto err_put_syncobjs;
1027                 }
1028         }
1029
1030         kfree(handles);
1031         *syncobjs_out = syncobjs;
1032         return 0;
1033
1034 err_put_syncobjs:
1035         while (i-- > 0)
1036                 drm_syncobj_put(syncobjs[i]);
1037         kfree(syncobjs);
1038 err_free_handles:
1039         kfree(handles);
1040
1041         return ret;
1042 }
1043
1044 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1045                                    uint32_t count)
1046 {
1047         uint32_t i;
1048         for (i = 0; i < count; i++)
1049                 drm_syncobj_put(syncobjs[i]);
1050         kfree(syncobjs);
1051 }
1052
1053 int
1054 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1055                        struct drm_file *file_private)
1056 {
1057         struct drm_syncobj_wait *args = data;
1058         struct drm_syncobj **syncobjs;
1059         int ret = 0;
1060
1061         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1062                 return -EOPNOTSUPP;
1063
1064         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1065                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1066                 return -EINVAL;
1067
1068         if (args->count_handles == 0)
1069                 return -EINVAL;
1070
1071         ret = drm_syncobj_array_find(file_private,
1072                                      u64_to_user_ptr(args->handles),
1073                                      args->count_handles,
1074                                      &syncobjs);
1075         if (ret < 0)
1076                 return ret;
1077
1078         ret = drm_syncobj_array_wait(dev, file_private,
1079                                      args, NULL, syncobjs, false);
1080
1081         drm_syncobj_array_free(syncobjs, args->count_handles);
1082
1083         return ret;
1084 }
1085
1086 int
1087 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1088                                 struct drm_file *file_private)
1089 {
1090         struct drm_syncobj_timeline_wait *args = data;
1091         struct drm_syncobj **syncobjs;
1092         int ret = 0;
1093
1094         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1095                 return -ENODEV;
1096
1097         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1098                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1099                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1100                 return -EINVAL;
1101
1102         if (args->count_handles == 0)
1103                 return -EINVAL;
1104
1105         ret = drm_syncobj_array_find(file_private,
1106                                      u64_to_user_ptr(args->handles),
1107                                      args->count_handles,
1108                                      &syncobjs);
1109         if (ret < 0)
1110                 return ret;
1111
1112         ret = drm_syncobj_array_wait(dev, file_private,
1113                                      NULL, args, syncobjs, true);
1114
1115         drm_syncobj_array_free(syncobjs, args->count_handles);
1116
1117         return ret;
1118 }
1119
1120
1121 int
1122 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1123                         struct drm_file *file_private)
1124 {
1125         struct drm_syncobj_array *args = data;
1126         struct drm_syncobj **syncobjs;
1127         uint32_t i;
1128         int ret;
1129
1130         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1131                 return -EOPNOTSUPP;
1132
1133         if (args->pad != 0)
1134                 return -EINVAL;
1135
1136         if (args->count_handles == 0)
1137                 return -EINVAL;
1138
1139         ret = drm_syncobj_array_find(file_private,
1140                                      u64_to_user_ptr(args->handles),
1141                                      args->count_handles,
1142                                      &syncobjs);
1143         if (ret < 0)
1144                 return ret;
1145
1146         for (i = 0; i < args->count_handles; i++)
1147                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1148
1149         drm_syncobj_array_free(syncobjs, args->count_handles);
1150
1151         return 0;
1152 }
1153
1154 int
1155 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1156                          struct drm_file *file_private)
1157 {
1158         struct drm_syncobj_array *args = data;
1159         struct drm_syncobj **syncobjs;
1160         uint32_t i;
1161         int ret;
1162
1163         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1164                 return -EOPNOTSUPP;
1165
1166         if (args->pad != 0)
1167                 return -EINVAL;
1168
1169         if (args->count_handles == 0)
1170                 return -EINVAL;
1171
1172         ret = drm_syncobj_array_find(file_private,
1173                                      u64_to_user_ptr(args->handles),
1174                                      args->count_handles,
1175                                      &syncobjs);
1176         if (ret < 0)
1177                 return ret;
1178
1179         for (i = 0; i < args->count_handles; i++)
1180                 drm_syncobj_assign_null_handle(syncobjs[i]);
1181
1182         drm_syncobj_array_free(syncobjs, args->count_handles);
1183
1184         return ret;
1185 }
1186
1187 int
1188 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1189                                   struct drm_file *file_private)
1190 {
1191         struct drm_syncobj_timeline_array *args = data;
1192         struct drm_syncobj **syncobjs;
1193         struct dma_fence_chain **chains;
1194         uint64_t *points;
1195         uint32_t i, j;
1196         int ret;
1197
1198         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1199                 return -EOPNOTSUPP;
1200
1201         if (args->pad != 0)
1202                 return -EINVAL;
1203
1204         if (args->count_handles == 0)
1205                 return -EINVAL;
1206
1207         ret = drm_syncobj_array_find(file_private,
1208                                      u64_to_user_ptr(args->handles),
1209                                      args->count_handles,
1210                                      &syncobjs);
1211         if (ret < 0)
1212                 return ret;
1213
1214         points = kmalloc_array(args->count_handles, sizeof(*points),
1215                                GFP_KERNEL);
1216         if (!points) {
1217                 ret = -ENOMEM;
1218                 goto out;
1219         }
1220         if (!u64_to_user_ptr(args->points)) {
1221                 memset(points, 0, args->count_handles * sizeof(uint64_t));
1222         } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1223                                   sizeof(uint64_t) * args->count_handles)) {
1224                 ret = -EFAULT;
1225                 goto err_points;
1226         }
1227
1228         chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1229         if (!chains) {
1230                 ret = -ENOMEM;
1231                 goto err_points;
1232         }
1233         for (i = 0; i < args->count_handles; i++) {
1234                 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1235                 if (!chains[i]) {
1236                         for (j = 0; j < i; j++)
1237                                 kfree(chains[j]);
1238                         ret = -ENOMEM;
1239                         goto err_chains;
1240                 }
1241         }
1242
1243         for (i = 0; i < args->count_handles; i++) {
1244                 struct dma_fence *fence = dma_fence_get_stub();
1245
1246                 drm_syncobj_add_point(syncobjs[i], chains[i],
1247                                       fence, points[i]);
1248                 dma_fence_put(fence);
1249         }
1250 err_chains:
1251         kfree(chains);
1252 err_points:
1253         kfree(points);
1254 out:
1255         drm_syncobj_array_free(syncobjs, args->count_handles);
1256
1257         return ret;
1258 }
1259
1260 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1261                             struct drm_file *file_private)
1262 {
1263         struct drm_syncobj_timeline_array *args = data;
1264         struct drm_syncobj **syncobjs;
1265         uint64_t __user *points = u64_to_user_ptr(args->points);
1266         uint32_t i;
1267         int ret;
1268
1269         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1270                 return -ENODEV;
1271
1272         if (args->pad != 0)
1273                 return -EINVAL;
1274
1275         if (args->count_handles == 0)
1276                 return -EINVAL;
1277
1278         ret = drm_syncobj_array_find(file_private,
1279                                      u64_to_user_ptr(args->handles),
1280                                      args->count_handles,
1281                                      &syncobjs);
1282         if (ret < 0)
1283                 return ret;
1284
1285         for (i = 0; i < args->count_handles; i++) {
1286                 struct dma_fence_chain *chain;
1287                 struct dma_fence *fence;
1288                 uint64_t point;
1289
1290                 fence = drm_syncobj_fence_get(syncobjs[i]);
1291                 chain = to_dma_fence_chain(fence);
1292                 if (chain) {
1293                         struct dma_fence *iter, *last_signaled = NULL;
1294
1295                         dma_fence_chain_for_each(iter, fence) {
1296                                 if (!iter)
1297                                         break;
1298                                 dma_fence_put(last_signaled);
1299                                 last_signaled = dma_fence_get(iter);
1300                                 if (!to_dma_fence_chain(last_signaled)->prev_seqno)
1301                                         /* It is most likely that timeline has
1302                                          * unorder points. */
1303                                         break;
1304                         }
1305                         point = dma_fence_is_signaled(last_signaled) ?
1306                                 last_signaled->seqno :
1307                                 to_dma_fence_chain(last_signaled)->prev_seqno;
1308                         dma_fence_put(last_signaled);
1309                 } else {
1310                         point = 0;
1311                 }
1312                 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1313                 ret = ret ? -EFAULT : 0;
1314                 if (ret)
1315                         break;
1316         }
1317         drm_syncobj_array_free(syncobjs, args->count_handles);
1318
1319         return ret;
1320 }