drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 struct syncobj_wait_entry {
  60         struct list_head node;
  61         struct task_struct *task;
  62         struct dma_fence *fence;
  63         struct dma_fence_cb fence_cb;
  64 };
  65
  66 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
  67                                       struct syncobj_wait_entry *wait);
  68
  69 /**
  70  * drm_syncobj_find - lookup and reference a sync object.
  71  * @file_private: drm file private pointer
  72  * @handle: sync object handle to lookup.
  73  *
  74  * Returns a reference to the syncobj pointed to by handle or NULL. The
  75  * reference must be released by calling drm_syncobj_put().
  76  */
  77 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  78                                      u32 handle)
  79 {
  80         struct drm_syncobj *syncobj;
  81
  82         spin_lock(&file_private->syncobj_table_lock);
  83
  84         /* Check if we currently have a reference on the object */
  85         syncobj = idr_find(&file_private->syncobj_idr, handle);
  86         if (syncobj)
  87                 drm_syncobj_get(syncobj);
  88
  89         spin_unlock(&file_private->syncobj_table_lock);
  90
  91         return syncobj;
  92 }
  93 EXPORT_SYMBOL(drm_syncobj_find);
  94
  95 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
  96                                        struct syncobj_wait_entry *wait)
  97 {
  98         if (wait->fence)
  99                 return;
 100
 101         spin_lock(&syncobj->lock);
 102         /* We've already tried once to get a fence and failed.  Now that we
 103          * have the lock, try one more time just to be sure we don't add a
 104          * callback when a fence has already been set.
 105          */
 106         if (syncobj->fence)
 107                 wait->fence = dma_fence_get(
 108                         rcu_dereference_protected(syncobj->fence, 1));
 109         else
 110                 list_add_tail(&wait->node, &syncobj->cb_list);
 111         spin_unlock(&syncobj->lock);
 112 }
 113
 114 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 115                                     struct syncobj_wait_entry *wait)
 116 {
 117         if (!wait->node.next)
 118                 return;
 119
 120         spin_lock(&syncobj->lock);
 121         list_del_init(&wait->node);
 122         spin_unlock(&syncobj->lock);
 123 }
 124
 125 /**
 126  * drm_syncobj_replace_fence - replace fence in a sync object.
 127  * @syncobj: Sync object to replace fence in
 128  * @fence: fence to install in sync file.
 129  *
 130  * This replaces the fence on a sync object.
 131  */
 132 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 133                                struct dma_fence *fence)
 134 {
 135         struct dma_fence *old_fence;
 136         struct syncobj_wait_entry *cur, *tmp;
 137
 138         if (fence)
 139                 dma_fence_get(fence);
 140
 141         spin_lock(&syncobj->lock);
 142
 143         old_fence = rcu_dereference_protected(syncobj->fence,
 144                                               lockdep_is_held(&syncobj->lock));
 145         rcu_assign_pointer(syncobj->fence, fence);
 146
 147         if (fence != old_fence) {
 148                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
 149                         list_del_init(&cur->node);
 150                         syncobj_wait_syncobj_func(syncobj, cur);
 151                 }
 152         }
 153
 154         spin_unlock(&syncobj->lock);
 155
 156         dma_fence_put(old_fence);
 157 }
 158 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 159
 160 /**
 161  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 162  * @syncobj: sync object to assign the fence on
 163  *
 164  * Assign a already signaled stub fence to the sync object.
 165  */
 166 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 167 {
 168         struct dma_fence *fence = dma_fence_get_stub();
 169
 170         drm_syncobj_replace_fence(syncobj, fence);
 171         dma_fence_put(fence);
 172 }
 173
 174 /**
 175  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 176  * @file_private: drm file private pointer
 177  * @handle: sync object handle to lookup.
 178  * @point: timeline point
 179  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 180  * @fence: out parameter for the fence
 181  *
 182  * This is just a convenience function that combines drm_syncobj_find() and
 183  * drm_syncobj_fence_get().
 184  *
 185  * Returns 0 on success or a negative error value on failure. On success @fence
 186  * contains a reference to the fence, which must be released by calling
 187  * dma_fence_put().
 188  */
 189 int drm_syncobj_find_fence(struct drm_file *file_private,
 190                            u32 handle, u64 point, u64 flags,
 191                            struct dma_fence **fence)
 192 {
 193         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 194         int ret = 0;
 195
 196         if (!syncobj)
 197                 return -ENOENT;
 198
 199         *fence = drm_syncobj_fence_get(syncobj);
 200         if (!*fence) {
 201                 ret = -EINVAL;
 202         }
 203         drm_syncobj_put(syncobj);
 204         return ret;
 205 }
 206 EXPORT_SYMBOL(drm_syncobj_find_fence);
 207
 208 /**
 209  * drm_syncobj_free - free a sync object.
 210  * @kref: kref to free.
 211  *
 212  * Only to be called from kref_put in drm_syncobj_put.
 213  */
 214 void drm_syncobj_free(struct kref *kref)
 215 {
 216         struct drm_syncobj *syncobj = container_of(kref,
 217                                                    struct drm_syncobj,
 218                                                    refcount);
 219         drm_syncobj_replace_fence(syncobj, NULL);
 220         kfree(syncobj);
 221 }
 222 EXPORT_SYMBOL(drm_syncobj_free);
 223
 224 /**
 225  * drm_syncobj_create - create a new syncobj
 226  * @out_syncobj: returned syncobj
 227  * @flags: DRM_SYNCOBJ_* flags
 228  * @fence: if non-NULL, the syncobj will represent this fence
 229  *
 230  * This is the first function to create a sync object. After creating, drivers
 231  * probably want to make it available to userspace, either through
 232  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 233  *
 234  * Returns 0 on success or a negative error value on failure.
 235  */
 236 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 237                        struct dma_fence *fence)
 238 {
 239         struct drm_syncobj *syncobj;
 240
 241         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 242         if (!syncobj)
 243                 return -ENOMEM;
 244
 245         kref_init(&syncobj->refcount);
 246         INIT_LIST_HEAD(&syncobj->cb_list);
 247         spin_lock_init(&syncobj->lock);
 248
 249         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
 250                 drm_syncobj_assign_null_handle(syncobj);
 251
 252         if (fence)
 253                 drm_syncobj_replace_fence(syncobj, fence);
 254
 255         *out_syncobj = syncobj;
 256         return 0;
 257 }
 258 EXPORT_SYMBOL(drm_syncobj_create);
 259
 260 /**
 261  * drm_syncobj_get_handle - get a handle from a syncobj
 262  * @file_private: drm file private pointer
 263  * @syncobj: Sync object to export
 264  * @handle: out parameter with the new handle
 265  *
 266  * Exports a sync object created with drm_syncobj_create() as a handle on
 267  * @file_private to userspace.
 268  *
 269  * Returns 0 on success or a negative error value on failure.
 270  */
 271 int drm_syncobj_get_handle(struct drm_file *file_private,
 272                            struct drm_syncobj *syncobj, u32 *handle)
 273 {
 274         int ret;
 275
 276         /* take a reference to put in the idr */
 277         drm_syncobj_get(syncobj);
 278
 279         idr_preload(GFP_KERNEL);
 280         spin_lock(&file_private->syncobj_table_lock);
 281         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 282         spin_unlock(&file_private->syncobj_table_lock);
 283
 284         idr_preload_end();
 285
 286         if (ret < 0) {
 287                 drm_syncobj_put(syncobj);
 288                 return ret;
 289         }
 290
 291         *handle = ret;
 292         return 0;
 293 }
 294 EXPORT_SYMBOL(drm_syncobj_get_handle);
 295
 296 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 297                                         u32 *handle, uint32_t flags)
 298 {
 299         int ret;
 300         struct drm_syncobj *syncobj;
 301
 302         ret = drm_syncobj_create(&syncobj, flags, NULL);
 303         if (ret)
 304                 return ret;
 305
 306         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 307         drm_syncobj_put(syncobj);
 308         return ret;
 309 }
 310
 311 static int drm_syncobj_destroy(struct drm_file *file_private,
 312                                u32 handle)
 313 {
 314         struct drm_syncobj *syncobj;
 315
 316         spin_lock(&file_private->syncobj_table_lock);
 317         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 318         spin_unlock(&file_private->syncobj_table_lock);
 319
 320         if (!syncobj)
 321                 return -EINVAL;
 322
 323         drm_syncobj_put(syncobj);
 324         return 0;
 325 }
 326
 327 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 328 {
 329         struct drm_syncobj *syncobj = file->private_data;
 330
 331         drm_syncobj_put(syncobj);
 332         return 0;
 333 }
 334
 335 static const struct file_operations drm_syncobj_file_fops = {
 336         .release = drm_syncobj_file_release,
 337 };
 338
 339 /**
 340  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 341  * @syncobj: Sync object to export
 342  * @p_fd: out parameter with the new file descriptor
 343  *
 344  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 345  *
 346  * Returns 0 on success or a negative error value on failure.
 347  */
 348 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 349 {
 350         struct file *file;
 351         int fd;
 352
 353         fd = get_unused_fd_flags(O_CLOEXEC);
 354         if (fd < 0)
 355                 return fd;
 356
 357         file = anon_inode_getfile("syncobj_file",
 358                                   &drm_syncobj_file_fops,
 359                                   syncobj, 0);
 360         if (IS_ERR(file)) {
 361                 put_unused_fd(fd);
 362                 return PTR_ERR(file);
 363         }
 364
 365         drm_syncobj_get(syncobj);
 366         fd_install(fd, file);
 367
 368         *p_fd = fd;
 369         return 0;
 370 }
 371 EXPORT_SYMBOL(drm_syncobj_get_fd);
 372
 373 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 374                                     u32 handle, int *p_fd)
 375 {
 376         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 377         int ret;
 378
 379         if (!syncobj)
 380                 return -EINVAL;
 381
 382         ret = drm_syncobj_get_fd(syncobj, p_fd);
 383         drm_syncobj_put(syncobj);
 384         return ret;
 385 }
 386
 387 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 388                                     int fd, u32 *handle)
 389 {
 390         struct drm_syncobj *syncobj;
 391         struct file *file;
 392         int ret;
 393
 394         file = fget(fd);
 395         if (!file)
 396                 return -EINVAL;
 397
 398         if (file->f_op != &drm_syncobj_file_fops) {
 399                 fput(file);
 400                 return -EINVAL;
 401         }
 402
 403         /* take a reference to put in the idr */
 404         syncobj = file->private_data;
 405         drm_syncobj_get(syncobj);
 406
 407         idr_preload(GFP_KERNEL);
 408         spin_lock(&file_private->syncobj_table_lock);
 409         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 410         spin_unlock(&file_private->syncobj_table_lock);
 411         idr_preload_end();
 412
 413         if (ret > 0) {
 414                 *handle = ret;
 415                 ret = 0;
 416         } else
 417                 drm_syncobj_put(syncobj);
 418
 419         fput(file);
 420         return ret;
 421 }
 422
 423 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 424                                               int fd, int handle)
 425 {
 426         struct dma_fence *fence = sync_file_get_fence(fd);
 427         struct drm_syncobj *syncobj;
 428
 429         if (!fence)
 430                 return -EINVAL;
 431
 432         syncobj = drm_syncobj_find(file_private, handle);
 433         if (!syncobj) {
 434                 dma_fence_put(fence);
 435                 return -ENOENT;
 436         }
 437
 438         drm_syncobj_replace_fence(syncobj, fence);
 439         dma_fence_put(fence);
 440         drm_syncobj_put(syncobj);
 441         return 0;
 442 }
 443
 444 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 445                                         int handle, int *p_fd)
 446 {
 447         int ret;
 448         struct dma_fence *fence;
 449         struct sync_file *sync_file;
 450         int fd = get_unused_fd_flags(O_CLOEXEC);
 451
 452         if (fd < 0)
 453                 return fd;
 454
 455         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 456         if (ret)
 457                 goto err_put_fd;
 458
 459         sync_file = sync_file_create(fence);
 460
 461         dma_fence_put(fence);
 462
 463         if (!sync_file) {
 464                 ret = -EINVAL;
 465                 goto err_put_fd;
 466         }
 467
 468         fd_install(fd, sync_file->file);
 469
 470         *p_fd = fd;
 471         return 0;
 472 err_put_fd:
 473         put_unused_fd(fd);
 474         return ret;
 475 }
 476 /**
 477  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 478  * @file_private: drm file-private structure to set up
 479  *
 480  * Called at device open time, sets up the structure for handling refcounting
 481  * of sync objects.
 482  */
 483 void
 484 drm_syncobj_open(struct drm_file *file_private)
 485 {
 486         idr_init_base(&file_private->syncobj_idr, 1);
 487         spin_lock_init(&file_private->syncobj_table_lock);
 488 }
 489
 490 static int
 491 drm_syncobj_release_handle(int id, void *ptr, void *data)
 492 {
 493         struct drm_syncobj *syncobj = ptr;
 494
 495         drm_syncobj_put(syncobj);
 496         return 0;
 497 }
 498
 499 /**
 500  * drm_syncobj_release - release file-private sync object resources
 501  * @file_private: drm file-private structure to clean up
 502  *
 503  * Called at close time when the filp is going away.
 504  *
 505  * Releases any remaining references on objects by this filp.
 506  */
 507 void
 508 drm_syncobj_release(struct drm_file *file_private)
 509 {
 510         idr_for_each(&file_private->syncobj_idr,
 511                      &drm_syncobj_release_handle, file_private);
 512         idr_destroy(&file_private->syncobj_idr);
 513 }
 514
 515 int
 516 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 517                          struct drm_file *file_private)
 518 {
 519         struct drm_syncobj_create *args = data;
 520
 521         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 522                 return -EOPNOTSUPP;
 523
 524         /* no valid flags yet */
 525         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 526                 return -EINVAL;
 527
 528         return drm_syncobj_create_as_handle(file_private,
 529                                             &args->handle, args->flags);
 530 }
 531
 532 int
 533 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 534                           struct drm_file *file_private)
 535 {
 536         struct drm_syncobj_destroy *args = data;
 537
 538         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 539                 return -EOPNOTSUPP;
 540
 541         /* make sure padding is empty */
 542         if (args->pad)
 543                 return -EINVAL;
 544         return drm_syncobj_destroy(file_private, args->handle);
 545 }
 546
 547 int
 548 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 549                                    struct drm_file *file_private)
 550 {
 551         struct drm_syncobj_handle *args = data;
 552
 553         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 554                 return -EOPNOTSUPP;
 555
 556         if (args->pad)
 557                 return -EINVAL;
 558
 559         if (args->flags != 0 &&
 560             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 561                 return -EINVAL;
 562
 563         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 564                 return drm_syncobj_export_sync_file(file_private, args->handle,
 565                                                     &args->fd);
 566
 567         return drm_syncobj_handle_to_fd(file_private, args->handle,
 568                                         &args->fd);
 569 }
 570
 571 int
 572 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 573                                    struct drm_file *file_private)
 574 {
 575         struct drm_syncobj_handle *args = data;
 576
 577         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 578                 return -EOPNOTSUPP;
 579
 580         if (args->pad)
 581                 return -EINVAL;
 582
 583         if (args->flags != 0 &&
 584             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 585                 return -EINVAL;
 586
 587         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 588                 return drm_syncobj_import_sync_file_fence(file_private,
 589                                                           args->fd,
 590                                                           args->handle);
 591
 592         return drm_syncobj_fd_to_handle(file_private, args->fd,
 593                                         &args->handle);
 594 }
 595
 596 static void syncobj_wait_fence_func(struct dma_fence *fence,
 597                                     struct dma_fence_cb *cb)
 598 {
 599         struct syncobj_wait_entry *wait =
 600                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 601
 602         wake_up_process(wait->task);
 603 }
 604
 605 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 606                                       struct syncobj_wait_entry *wait)
 607 {
 608         /* This happens inside the syncobj lock */
 609         wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 610                                                               lockdep_is_held(&syncobj->lock)));
 611         wake_up_process(wait->task);
 612 }
 613
 614 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 615                                                   uint32_t count,
 616                                                   uint32_t flags,
 617                                                   signed long timeout,
 618                                                   uint32_t *idx)
 619 {
 620         struct syncobj_wait_entry *entries;
 621         struct dma_fence *fence;
 622         uint32_t signaled_count, i;
 623
 624         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 625         if (!entries)
 626                 return -ENOMEM;
 627
 628         /* Walk the list of sync objects and initialize entries.  We do
 629          * this up-front so that we can properly return -EINVAL if there is
 630          * a syncobj with a missing fence and then never have the chance of
 631          * returning -EINVAL again.
 632          */
 633         signaled_count = 0;
 634         for (i = 0; i < count; ++i) {
 635                 entries[i].task = current;
 636                 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
 637                 if (!entries[i].fence) {
 638                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 639                                 continue;
 640                         } else {
 641                                 timeout = -EINVAL;
 642                                 goto cleanup_entries;
 643                         }
 644                 }
 645
 646                 if (dma_fence_is_signaled(entries[i].fence)) {
 647                         if (signaled_count == 0 && idx)
 648                                 *idx = i;
 649                         signaled_count++;
 650                 }
 651         }
 652
 653         if (signaled_count == count ||
 654             (signaled_count > 0 &&
 655              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 656                 goto cleanup_entries;
 657
 658         /* There's a very annoying laxness in the dma_fence API here, in
 659          * that backends are not required to automatically report when a
 660          * fence is signaled prior to fence->ops->enable_signaling() being
 661          * called.  So here if we fail to match signaled_count, we need to
 662          * fallthough and try a 0 timeout wait!
 663          */
 664
 665         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 666                 for (i = 0; i < count; ++i)
 667                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
 668         }
 669
 670         do {
 671                 set_current_state(TASK_INTERRUPTIBLE);
 672
 673                 signaled_count = 0;
 674                 for (i = 0; i < count; ++i) {
 675                         fence = entries[i].fence;
 676                         if (!fence)
 677                                 continue;
 678
 679                         if (dma_fence_is_signaled(fence) ||
 680                             (!entries[i].fence_cb.func &&
 681                              dma_fence_add_callback(fence,
 682                                                     &entries[i].fence_cb,
 683                                                     syncobj_wait_fence_func))) {
 684                                 /* The fence has been signaled */
 685                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 686                                         signaled_count++;
 687                                 } else {
 688                                         if (idx)
 689                                                 *idx = i;
 690                                         goto done_waiting;
 691                                 }
 692                         }
 693                 }
 694
 695                 if (signaled_count == count)
 696                         goto done_waiting;
 697
 698                 if (timeout == 0) {
 699                         timeout = -ETIME;
 700                         goto done_waiting;
 701                 }
 702
 703                 if (signal_pending(current)) {
 704                         timeout = -ERESTARTSYS;
 705                         goto done_waiting;
 706                 }
 707
 708                 timeout = schedule_timeout(timeout);
 709         } while (1);
 710
 711 done_waiting:
 712         __set_current_state(TASK_RUNNING);
 713
 714 cleanup_entries:
 715         for (i = 0; i < count; ++i) {
 716                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
 717                 if (entries[i].fence_cb.func)
 718                         dma_fence_remove_callback(entries[i].fence,
 719                                                   &entries[i].fence_cb);
 720                 dma_fence_put(entries[i].fence);
 721         }
 722         kfree(entries);
 723
 724         return timeout;
 725 }
 726
 727 /**
 728  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 729  *
 730  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 731  *
 732  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 733  */
 734 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 735 {
 736         ktime_t abs_timeout, now;
 737         u64 timeout_ns, timeout_jiffies64;
 738
 739         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 740         if (timeout_nsec == 0)
 741                 return 0;
 742
 743         abs_timeout = ns_to_ktime(timeout_nsec);
 744         now = ktime_get();
 745
 746         if (!ktime_after(abs_timeout, now))
 747                 return 0;
 748
 749         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 750
 751         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 752         /*  clamp timeout to avoid infinite timeout */
 753         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 754                 return MAX_SCHEDULE_TIMEOUT - 1;
 755
 756         return timeout_jiffies64 + 1;
 757 }
 758 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
 759
 760 static int drm_syncobj_array_wait(struct drm_device *dev,
 761                                   struct drm_file *file_private,
 762                                   struct drm_syncobj_wait *wait,
 763                                   struct drm_syncobj **syncobjs)
 764 {
 765         signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 766         uint32_t first = ~0;
 767
 768         timeout = drm_syncobj_array_wait_timeout(syncobjs,
 769                                                  wait->count_handles,
 770                                                  wait->flags,
 771                                                  timeout, &first);
 772         if (timeout < 0)
 773                 return timeout;
 774
 775         wait->first_signaled = first;
 776         return 0;
 777 }
 778
 779 static int drm_syncobj_array_find(struct drm_file *file_private,
 780                                   void __user *user_handles,
 781                                   uint32_t count_handles,
 782                                   struct drm_syncobj ***syncobjs_out)
 783 {
 784         uint32_t i, *handles;
 785         struct drm_syncobj **syncobjs;
 786         int ret;
 787
 788         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 789         if (handles == NULL)
 790                 return -ENOMEM;
 791
 792         if (copy_from_user(handles, user_handles,
 793                            sizeof(uint32_t) * count_handles)) {
 794                 ret = -EFAULT;
 795                 goto err_free_handles;
 796         }
 797
 798         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 799         if (syncobjs == NULL) {
 800                 ret = -ENOMEM;
 801                 goto err_free_handles;
 802         }
 803
 804         for (i = 0; i < count_handles; i++) {
 805                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 806                 if (!syncobjs[i]) {
 807                         ret = -ENOENT;
 808                         goto err_put_syncobjs;
 809                 }
 810         }
 811
 812         kfree(handles);
 813         *syncobjs_out = syncobjs;
 814         return 0;
 815
 816 err_put_syncobjs:
 817         while (i-- > 0)
 818                 drm_syncobj_put(syncobjs[i]);
 819         kfree(syncobjs);
 820 err_free_handles:
 821         kfree(handles);
 822
 823         return ret;
 824 }
 825
 826 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 827                                    uint32_t count)
 828 {
 829         uint32_t i;
 830         for (i = 0; i < count; i++)
 831                 drm_syncobj_put(syncobjs[i]);
 832         kfree(syncobjs);
 833 }
 834
 835 int
 836 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 837                        struct drm_file *file_private)
 838 {
 839         struct drm_syncobj_wait *args = data;
 840         struct drm_syncobj **syncobjs;
 841         int ret = 0;
 842
 843         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 844                 return -EOPNOTSUPP;
 845
 846         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 847                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 848                 return -EINVAL;
 849
 850         if (args->count_handles == 0)
 851                 return -EINVAL;
 852
 853         ret = drm_syncobj_array_find(file_private,
 854                                      u64_to_user_ptr(args->handles),
 855                                      args->count_handles,
 856                                      &syncobjs);
 857         if (ret < 0)
 858                 return ret;
 859
 860         ret = drm_syncobj_array_wait(dev, file_private,
 861                                      args, syncobjs);
 862
 863         drm_syncobj_array_free(syncobjs, args->count_handles);
 864
 865         return ret;
 866 }
 867
 868 int
 869 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 870                         struct drm_file *file_private)
 871 {
 872         struct drm_syncobj_array *args = data;
 873         struct drm_syncobj **syncobjs;
 874         uint32_t i;
 875         int ret;
 876
 877         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 878                 return -EOPNOTSUPP;
 879
 880         if (args->pad != 0)
 881                 return -EINVAL;
 882
 883         if (args->count_handles == 0)
 884                 return -EINVAL;
 885
 886         ret = drm_syncobj_array_find(file_private,
 887                                      u64_to_user_ptr(args->handles),
 888                                      args->count_handles,
 889                                      &syncobjs);
 890         if (ret < 0)
 891                 return ret;
 892
 893         for (i = 0; i < args->count_handles; i++)
 894                 drm_syncobj_replace_fence(syncobjs[i], NULL);
 895
 896         drm_syncobj_array_free(syncobjs, args->count_handles);
 897
 898         return 0;
 899 }
 900
 901 int
 902 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 903                          struct drm_file *file_private)
 904 {
 905         struct drm_syncobj_array *args = data;
 906         struct drm_syncobj **syncobjs;
 907         uint32_t i;
 908         int ret;
 909
 910         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 911                 return -EOPNOTSUPP;
 912
 913         if (args->pad != 0)
 914                 return -EINVAL;
 915
 916         if (args->count_handles == 0)
 917                 return -EINVAL;
 918
 919         ret = drm_syncobj_array_find(file_private,
 920                                      u64_to_user_ptr(args->handles),
 921                                      args->count_handles,
 922                                      &syncobjs);
 923         if (ret < 0)
 924                 return ret;
 925
 926         for (i = 0; i < args->count_handles; i++)
 927                 drm_syncobj_assign_null_handle(syncobjs[i]);
 928
 929         drm_syncobj_array_free(syncobjs, args->count_handles);
 930
 931         return ret;
 932 }