drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 struct syncobj_wait_entry {
  60         struct list_head node;
  61         struct task_struct *task;
  62         struct dma_fence *fence;
  63         struct dma_fence_cb fence_cb;
  64         u64    point;
  65 };
  66
  67 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
  68                                       struct syncobj_wait_entry *wait);
  69
  70 /**
  71  * drm_syncobj_find - lookup and reference a sync object.
  72  * @file_private: drm file private pointer
  73  * @handle: sync object handle to lookup.
  74  *
  75  * Returns a reference to the syncobj pointed to by handle or NULL. The
  76  * reference must be released by calling drm_syncobj_put().
  77  */
  78 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  79                                      u32 handle)
  80 {
  81         struct drm_syncobj *syncobj;
  82
  83         spin_lock(&file_private->syncobj_table_lock);
  84
  85         /* Check if we currently have a reference on the object */
  86         syncobj = idr_find(&file_private->syncobj_idr, handle);
  87         if (syncobj)
  88                 drm_syncobj_get(syncobj);
  89
  90         spin_unlock(&file_private->syncobj_table_lock);
  91
  92         return syncobj;
  93 }
  94 EXPORT_SYMBOL(drm_syncobj_find);
  95
  96 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
  97                                        struct syncobj_wait_entry *wait)
  98 {
  99         struct dma_fence *fence;
 100
 101         if (wait->fence)
 102                 return;
 103
 104         spin_lock(&syncobj->lock);
 105         /* We've already tried once to get a fence and failed.  Now that we
 106          * have the lock, try one more time just to be sure we don't add a
 107          * callback when a fence has already been set.
 108          */
 109         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 110         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 111                 dma_fence_put(fence);
 112                 list_add_tail(&wait->node, &syncobj->cb_list);
 113         } else if (!fence) {
 114                 wait->fence = dma_fence_get_stub();
 115         } else {
 116                 wait->fence = fence;
 117         }
 118         spin_unlock(&syncobj->lock);
 119 }
 120
 121 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 122                                     struct syncobj_wait_entry *wait)
 123 {
 124         if (!wait->node.next)
 125                 return;
 126
 127         spin_lock(&syncobj->lock);
 128         list_del_init(&wait->node);
 129         spin_unlock(&syncobj->lock);
 130 }
 131
 132 /**
 133  * drm_syncobj_add_point - add new timeline point to the syncobj
 134  * @syncobj: sync object to add timeline point do
 135  * @chain: chain node to use to add the point
 136  * @fence: fence to encapsulate in the chain node
 137  * @point: sequence number to use for the point
 138  *
 139  * Add the chain node as new timeline point to the syncobj.
 140  */
 141 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 142                            struct dma_fence_chain *chain,
 143                            struct dma_fence *fence,
 144                            uint64_t point)
 145 {
 146         struct syncobj_wait_entry *cur, *tmp;
 147         struct dma_fence *prev;
 148
 149         dma_fence_get(fence);
 150
 151         spin_lock(&syncobj->lock);
 152
 153         prev = drm_syncobj_fence_get(syncobj);
 154         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 155         if (prev && prev->seqno >= point)
 156                 DRM_ERROR("You are adding an unorder point to timeline!\n");
 157         dma_fence_chain_init(chain, prev, fence, point);
 158         rcu_assign_pointer(syncobj->fence, &chain->base);
 159
 160         list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 161                 syncobj_wait_syncobj_func(syncobj, cur);
 162         spin_unlock(&syncobj->lock);
 163
 164         /* Walk the chain once to trigger garbage collection */
 165         dma_fence_chain_for_each(fence, prev);
 166         dma_fence_put(prev);
 167 }
 168 EXPORT_SYMBOL(drm_syncobj_add_point);
 169
 170 /**
 171  * drm_syncobj_replace_fence - replace fence in a sync object.
 172  * @syncobj: Sync object to replace fence in
 173  * @fence: fence to install in sync file.
 174  *
 175  * This replaces the fence on a sync object.
 176  */
 177 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 178                                struct dma_fence *fence)
 179 {
 180         struct dma_fence *old_fence;
 181         struct syncobj_wait_entry *cur, *tmp;
 182
 183         if (fence)
 184                 dma_fence_get(fence);
 185
 186         spin_lock(&syncobj->lock);
 187
 188         old_fence = rcu_dereference_protected(syncobj->fence,
 189                                               lockdep_is_held(&syncobj->lock));
 190         rcu_assign_pointer(syncobj->fence, fence);
 191
 192         if (fence != old_fence) {
 193                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 194                         syncobj_wait_syncobj_func(syncobj, cur);
 195         }
 196
 197         spin_unlock(&syncobj->lock);
 198
 199         dma_fence_put(old_fence);
 200 }
 201 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 202
 203 /**
 204  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 205  * @syncobj: sync object to assign the fence on
 206  *
 207  * Assign a already signaled stub fence to the sync object.
 208  */
 209 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 210 {
 211         struct dma_fence *fence = dma_fence_get_stub();
 212
 213         drm_syncobj_replace_fence(syncobj, fence);
 214         dma_fence_put(fence);
 215 }
 216
 217 /**
 218  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 219  * @file_private: drm file private pointer
 220  * @handle: sync object handle to lookup.
 221  * @point: timeline point
 222  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 223  * @fence: out parameter for the fence
 224  *
 225  * This is just a convenience function that combines drm_syncobj_find() and
 226  * drm_syncobj_fence_get().
 227  *
 228  * Returns 0 on success or a negative error value on failure. On success @fence
 229  * contains a reference to the fence, which must be released by calling
 230  * dma_fence_put().
 231  */
 232 int drm_syncobj_find_fence(struct drm_file *file_private,
 233                            u32 handle, u64 point, u64 flags,
 234                            struct dma_fence **fence)
 235 {
 236         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 237         int ret = 0;
 238
 239         if (!syncobj)
 240                 return -ENOENT;
 241
 242         *fence = drm_syncobj_fence_get(syncobj);
 243         if (!*fence) {
 244                 ret = -EINVAL;
 245         }
 246         drm_syncobj_put(syncobj);
 247         return ret;
 248 }
 249 EXPORT_SYMBOL(drm_syncobj_find_fence);
 250
 251 /**
 252  * drm_syncobj_free - free a sync object.
 253  * @kref: kref to free.
 254  *
 255  * Only to be called from kref_put in drm_syncobj_put.
 256  */
 257 void drm_syncobj_free(struct kref *kref)
 258 {
 259         struct drm_syncobj *syncobj = container_of(kref,
 260                                                    struct drm_syncobj,
 261                                                    refcount);
 262         drm_syncobj_replace_fence(syncobj, NULL);
 263         kfree(syncobj);
 264 }
 265 EXPORT_SYMBOL(drm_syncobj_free);
 266
 267 /**
 268  * drm_syncobj_create - create a new syncobj
 269  * @out_syncobj: returned syncobj
 270  * @flags: DRM_SYNCOBJ_* flags
 271  * @fence: if non-NULL, the syncobj will represent this fence
 272  *
 273  * This is the first function to create a sync object. After creating, drivers
 274  * probably want to make it available to userspace, either through
 275  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 276  *
 277  * Returns 0 on success or a negative error value on failure.
 278  */
 279 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 280                        struct dma_fence *fence)
 281 {
 282         struct drm_syncobj *syncobj;
 283
 284         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 285         if (!syncobj)
 286                 return -ENOMEM;
 287
 288         kref_init(&syncobj->refcount);
 289         INIT_LIST_HEAD(&syncobj->cb_list);
 290         spin_lock_init(&syncobj->lock);
 291
 292         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
 293                 drm_syncobj_assign_null_handle(syncobj);
 294
 295         if (fence)
 296                 drm_syncobj_replace_fence(syncobj, fence);
 297
 298         *out_syncobj = syncobj;
 299         return 0;
 300 }
 301 EXPORT_SYMBOL(drm_syncobj_create);
 302
 303 /**
 304  * drm_syncobj_get_handle - get a handle from a syncobj
 305  * @file_private: drm file private pointer
 306  * @syncobj: Sync object to export
 307  * @handle: out parameter with the new handle
 308  *
 309  * Exports a sync object created with drm_syncobj_create() as a handle on
 310  * @file_private to userspace.
 311  *
 312  * Returns 0 on success or a negative error value on failure.
 313  */
 314 int drm_syncobj_get_handle(struct drm_file *file_private,
 315                            struct drm_syncobj *syncobj, u32 *handle)
 316 {
 317         int ret;
 318
 319         /* take a reference to put in the idr */
 320         drm_syncobj_get(syncobj);
 321
 322         idr_preload(GFP_KERNEL);
 323         spin_lock(&file_private->syncobj_table_lock);
 324         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 325         spin_unlock(&file_private->syncobj_table_lock);
 326
 327         idr_preload_end();
 328
 329         if (ret < 0) {
 330                 drm_syncobj_put(syncobj);
 331                 return ret;
 332         }
 333
 334         *handle = ret;
 335         return 0;
 336 }
 337 EXPORT_SYMBOL(drm_syncobj_get_handle);
 338
 339 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 340                                         u32 *handle, uint32_t flags)
 341 {
 342         int ret;
 343         struct drm_syncobj *syncobj;
 344
 345         ret = drm_syncobj_create(&syncobj, flags, NULL);
 346         if (ret)
 347                 return ret;
 348
 349         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 350         drm_syncobj_put(syncobj);
 351         return ret;
 352 }
 353
 354 static int drm_syncobj_destroy(struct drm_file *file_private,
 355                                u32 handle)
 356 {
 357         struct drm_syncobj *syncobj;
 358
 359         spin_lock(&file_private->syncobj_table_lock);
 360         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 361         spin_unlock(&file_private->syncobj_table_lock);
 362
 363         if (!syncobj)
 364                 return -EINVAL;
 365
 366         drm_syncobj_put(syncobj);
 367         return 0;
 368 }
 369
 370 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 371 {
 372         struct drm_syncobj *syncobj = file->private_data;
 373
 374         drm_syncobj_put(syncobj);
 375         return 0;
 376 }
 377
 378 static const struct file_operations drm_syncobj_file_fops = {
 379         .release = drm_syncobj_file_release,
 380 };
 381
 382 /**
 383  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 384  * @syncobj: Sync object to export
 385  * @p_fd: out parameter with the new file descriptor
 386  *
 387  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 388  *
 389  * Returns 0 on success or a negative error value on failure.
 390  */
 391 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 392 {
 393         struct file *file;
 394         int fd;
 395
 396         fd = get_unused_fd_flags(O_CLOEXEC);
 397         if (fd < 0)
 398                 return fd;
 399
 400         file = anon_inode_getfile("syncobj_file",
 401                                   &drm_syncobj_file_fops,
 402                                   syncobj, 0);
 403         if (IS_ERR(file)) {
 404                 put_unused_fd(fd);
 405                 return PTR_ERR(file);
 406         }
 407
 408         drm_syncobj_get(syncobj);
 409         fd_install(fd, file);
 410
 411         *p_fd = fd;
 412         return 0;
 413 }
 414 EXPORT_SYMBOL(drm_syncobj_get_fd);
 415
 416 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 417                                     u32 handle, int *p_fd)
 418 {
 419         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 420         int ret;
 421
 422         if (!syncobj)
 423                 return -EINVAL;
 424
 425         ret = drm_syncobj_get_fd(syncobj, p_fd);
 426         drm_syncobj_put(syncobj);
 427         return ret;
 428 }
 429
 430 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 431                                     int fd, u32 *handle)
 432 {
 433         struct drm_syncobj *syncobj;
 434         struct file *file;
 435         int ret;
 436
 437         file = fget(fd);
 438         if (!file)
 439                 return -EINVAL;
 440
 441         if (file->f_op != &drm_syncobj_file_fops) {
 442                 fput(file);
 443                 return -EINVAL;
 444         }
 445
 446         /* take a reference to put in the idr */
 447         syncobj = file->private_data;
 448         drm_syncobj_get(syncobj);
 449
 450         idr_preload(GFP_KERNEL);
 451         spin_lock(&file_private->syncobj_table_lock);
 452         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 453         spin_unlock(&file_private->syncobj_table_lock);
 454         idr_preload_end();
 455
 456         if (ret > 0) {
 457                 *handle = ret;
 458                 ret = 0;
 459         } else
 460                 drm_syncobj_put(syncobj);
 461
 462         fput(file);
 463         return ret;
 464 }
 465
 466 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 467                                               int fd, int handle)
 468 {
 469         struct dma_fence *fence = sync_file_get_fence(fd);
 470         struct drm_syncobj *syncobj;
 471
 472         if (!fence)
 473                 return -EINVAL;
 474
 475         syncobj = drm_syncobj_find(file_private, handle);
 476         if (!syncobj) {
 477                 dma_fence_put(fence);
 478                 return -ENOENT;
 479         }
 480
 481         drm_syncobj_replace_fence(syncobj, fence);
 482         dma_fence_put(fence);
 483         drm_syncobj_put(syncobj);
 484         return 0;
 485 }
 486
 487 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 488                                         int handle, int *p_fd)
 489 {
 490         int ret;
 491         struct dma_fence *fence;
 492         struct sync_file *sync_file;
 493         int fd = get_unused_fd_flags(O_CLOEXEC);
 494
 495         if (fd < 0)
 496                 return fd;
 497
 498         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 499         if (ret)
 500                 goto err_put_fd;
 501
 502         sync_file = sync_file_create(fence);
 503
 504         dma_fence_put(fence);
 505
 506         if (!sync_file) {
 507                 ret = -EINVAL;
 508                 goto err_put_fd;
 509         }
 510
 511         fd_install(fd, sync_file->file);
 512
 513         *p_fd = fd;
 514         return 0;
 515 err_put_fd:
 516         put_unused_fd(fd);
 517         return ret;
 518 }
 519 /**
 520  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 521  * @file_private: drm file-private structure to set up
 522  *
 523  * Called at device open time, sets up the structure for handling refcounting
 524  * of sync objects.
 525  */
 526 void
 527 drm_syncobj_open(struct drm_file *file_private)
 528 {
 529         idr_init_base(&file_private->syncobj_idr, 1);
 530         spin_lock_init(&file_private->syncobj_table_lock);
 531 }
 532
 533 static int
 534 drm_syncobj_release_handle(int id, void *ptr, void *data)
 535 {
 536         struct drm_syncobj *syncobj = ptr;
 537
 538         drm_syncobj_put(syncobj);
 539         return 0;
 540 }
 541
 542 /**
 543  * drm_syncobj_release - release file-private sync object resources
 544  * @file_private: drm file-private structure to clean up
 545  *
 546  * Called at close time when the filp is going away.
 547  *
 548  * Releases any remaining references on objects by this filp.
 549  */
 550 void
 551 drm_syncobj_release(struct drm_file *file_private)
 552 {
 553         idr_for_each(&file_private->syncobj_idr,
 554                      &drm_syncobj_release_handle, file_private);
 555         idr_destroy(&file_private->syncobj_idr);
 556 }
 557
 558 int
 559 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 560                          struct drm_file *file_private)
 561 {
 562         struct drm_syncobj_create *args = data;
 563
 564         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 565                 return -EOPNOTSUPP;
 566
 567         /* no valid flags yet */
 568         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 569                 return -EINVAL;
 570
 571         return drm_syncobj_create_as_handle(file_private,
 572                                             &args->handle, args->flags);
 573 }
 574
 575 int
 576 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 577                           struct drm_file *file_private)
 578 {
 579         struct drm_syncobj_destroy *args = data;
 580
 581         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 582                 return -EOPNOTSUPP;
 583
 584         /* make sure padding is empty */
 585         if (args->pad)
 586                 return -EINVAL;
 587         return drm_syncobj_destroy(file_private, args->handle);
 588 }
 589
 590 int
 591 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 592                                    struct drm_file *file_private)
 593 {
 594         struct drm_syncobj_handle *args = data;
 595
 596         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 597                 return -EOPNOTSUPP;
 598
 599         if (args->pad)
 600                 return -EINVAL;
 601
 602         if (args->flags != 0 &&
 603             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 604                 return -EINVAL;
 605
 606         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 607                 return drm_syncobj_export_sync_file(file_private, args->handle,
 608                                                     &args->fd);
 609
 610         return drm_syncobj_handle_to_fd(file_private, args->handle,
 611                                         &args->fd);
 612 }
 613
 614 int
 615 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 616                                    struct drm_file *file_private)
 617 {
 618         struct drm_syncobj_handle *args = data;
 619
 620         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 621                 return -EOPNOTSUPP;
 622
 623         if (args->pad)
 624                 return -EINVAL;
 625
 626         if (args->flags != 0 &&
 627             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 628                 return -EINVAL;
 629
 630         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 631                 return drm_syncobj_import_sync_file_fence(file_private,
 632                                                           args->fd,
 633                                                           args->handle);
 634
 635         return drm_syncobj_fd_to_handle(file_private, args->fd,
 636                                         &args->handle);
 637 }
 638
 639 static void syncobj_wait_fence_func(struct dma_fence *fence,
 640                                     struct dma_fence_cb *cb)
 641 {
 642         struct syncobj_wait_entry *wait =
 643                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 644
 645         wake_up_process(wait->task);
 646 }
 647
 648 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 649                                       struct syncobj_wait_entry *wait)
 650 {
 651         struct dma_fence *fence;
 652
 653         /* This happens inside the syncobj lock */
 654         fence = rcu_dereference_protected(syncobj->fence,
 655                                           lockdep_is_held(&syncobj->lock));
 656         dma_fence_get(fence);
 657         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 658                 dma_fence_put(fence);
 659                 return;
 660         } else if (!fence) {
 661                 wait->fence = dma_fence_get_stub();
 662         } else {
 663                 wait->fence = fence;
 664         }
 665
 666         wake_up_process(wait->task);
 667         list_del_init(&wait->node);
 668 }
 669
 670 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 671                                                   void __user *user_points,
 672                                                   uint32_t count,
 673                                                   uint32_t flags,
 674                                                   signed long timeout,
 675                                                   uint32_t *idx)
 676 {
 677         struct syncobj_wait_entry *entries;
 678         struct dma_fence *fence;
 679         uint64_t *points;
 680         uint32_t signaled_count, i;
 681
 682         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
 683         if (points == NULL)
 684                 return -ENOMEM;
 685
 686         if (!user_points) {
 687                 memset(points, 0, count * sizeof(uint64_t));
 688
 689         } else if (copy_from_user(points, user_points,
 690                                   sizeof(uint64_t) * count)) {
 691                 timeout = -EFAULT;
 692                 goto err_free_points;
 693         }
 694
 695         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 696         if (!entries) {
 697                 timeout = -ENOMEM;
 698                 goto err_free_points;
 699         }
 700         /* Walk the list of sync objects and initialize entries.  We do
 701          * this up-front so that we can properly return -EINVAL if there is
 702          * a syncobj with a missing fence and then never have the chance of
 703          * returning -EINVAL again.
 704          */
 705         signaled_count = 0;
 706         for (i = 0; i < count; ++i) {
 707                 struct dma_fence *fence;
 708
 709                 entries[i].task = current;
 710                 entries[i].point = points[i];
 711                 fence = drm_syncobj_fence_get(syncobjs[i]);
 712                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
 713                         dma_fence_put(fence);
 714                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 715                                 continue;
 716                         } else {
 717                                 timeout = -EINVAL;
 718                                 goto cleanup_entries;
 719                         }
 720                 }
 721
 722                 if (fence)
 723                         entries[i].fence = fence;
 724                 else
 725                         entries[i].fence = dma_fence_get_stub();
 726
 727                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
 728                     dma_fence_is_signaled(entries[i].fence)) {
 729                         if (signaled_count == 0 && idx)
 730                                 *idx = i;
 731                         signaled_count++;
 732                 }
 733         }
 734
 735         if (signaled_count == count ||
 736             (signaled_count > 0 &&
 737              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 738                 goto cleanup_entries;
 739
 740         /* There's a very annoying laxness in the dma_fence API here, in
 741          * that backends are not required to automatically report when a
 742          * fence is signaled prior to fence->ops->enable_signaling() being
 743          * called.  So here if we fail to match signaled_count, we need to
 744          * fallthough and try a 0 timeout wait!
 745          */
 746
 747         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 748                 for (i = 0; i < count; ++i)
 749                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
 750         }
 751
 752         do {
 753                 set_current_state(TASK_INTERRUPTIBLE);
 754
 755                 signaled_count = 0;
 756                 for (i = 0; i < count; ++i) {
 757                         fence = entries[i].fence;
 758                         if (!fence)
 759                                 continue;
 760
 761                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
 762                             dma_fence_is_signaled(fence) ||
 763                             (!entries[i].fence_cb.func &&
 764                              dma_fence_add_callback(fence,
 765                                                     &entries[i].fence_cb,
 766                                                     syncobj_wait_fence_func))) {
 767                                 /* The fence has been signaled */
 768                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 769                                         signaled_count++;
 770                                 } else {
 771                                         if (idx)
 772                                                 *idx = i;
 773                                         goto done_waiting;
 774                                 }
 775                         }
 776                 }
 777
 778                 if (signaled_count == count)
 779                         goto done_waiting;
 780
 781                 if (timeout == 0) {
 782                         timeout = -ETIME;
 783                         goto done_waiting;
 784                 }
 785
 786                 if (signal_pending(current)) {
 787                         timeout = -ERESTARTSYS;
 788                         goto done_waiting;
 789                 }
 790
 791                 timeout = schedule_timeout(timeout);
 792         } while (1);
 793
 794 done_waiting:
 795         __set_current_state(TASK_RUNNING);
 796
 797 cleanup_entries:
 798         for (i = 0; i < count; ++i) {
 799                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
 800                 if (entries[i].fence_cb.func)
 801                         dma_fence_remove_callback(entries[i].fence,
 802                                                   &entries[i].fence_cb);
 803                 dma_fence_put(entries[i].fence);
 804         }
 805         kfree(entries);
 806
 807 err_free_points:
 808         kfree(points);
 809
 810         return timeout;
 811 }
 812
 813 /**
 814  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 815  *
 816  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 817  *
 818  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 819  */
 820 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 821 {
 822         ktime_t abs_timeout, now;
 823         u64 timeout_ns, timeout_jiffies64;
 824
 825         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 826         if (timeout_nsec == 0)
 827                 return 0;
 828
 829         abs_timeout = ns_to_ktime(timeout_nsec);
 830         now = ktime_get();
 831
 832         if (!ktime_after(abs_timeout, now))
 833                 return 0;
 834
 835         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 836
 837         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 838         /*  clamp timeout to avoid infinite timeout */
 839         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 840                 return MAX_SCHEDULE_TIMEOUT - 1;
 841
 842         return timeout_jiffies64 + 1;
 843 }
 844 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
 845
 846 static int drm_syncobj_array_wait(struct drm_device *dev,
 847                                   struct drm_file *file_private,
 848                                   struct drm_syncobj_wait *wait,
 849                                   struct drm_syncobj_timeline_wait *timeline_wait,
 850                                   struct drm_syncobj **syncobjs, bool timeline)
 851 {
 852         signed long timeout = 0;
 853         uint32_t first = ~0;
 854
 855         if (!timeline) {
 856                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 857                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
 858                                                          NULL,
 859                                                          wait->count_handles,
 860                                                          wait->flags,
 861                                                          timeout, &first);
 862                 if (timeout < 0)
 863                         return timeout;
 864                 wait->first_signaled = first;
 865         } else {
 866                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
 867                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
 868                                                          u64_to_user_ptr(timeline_wait->points),
 869                                                          timeline_wait->count_handles,
 870                                                          timeline_wait->flags,
 871                                                          timeout, &first);
 872                 if (timeout < 0)
 873                         return timeout;
 874                 timeline_wait->first_signaled = first;
 875         }
 876         return 0;
 877 }
 878
 879 static int drm_syncobj_array_find(struct drm_file *file_private,
 880                                   void __user *user_handles,
 881                                   uint32_t count_handles,
 882                                   struct drm_syncobj ***syncobjs_out)
 883 {
 884         uint32_t i, *handles;
 885         struct drm_syncobj **syncobjs;
 886         int ret;
 887
 888         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 889         if (handles == NULL)
 890                 return -ENOMEM;
 891
 892         if (copy_from_user(handles, user_handles,
 893                            sizeof(uint32_t) * count_handles)) {
 894                 ret = -EFAULT;
 895                 goto err_free_handles;
 896         }
 897
 898         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 899         if (syncobjs == NULL) {
 900                 ret = -ENOMEM;
 901                 goto err_free_handles;
 902         }
 903
 904         for (i = 0; i < count_handles; i++) {
 905                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 906                 if (!syncobjs[i]) {
 907                         ret = -ENOENT;
 908                         goto err_put_syncobjs;
 909                 }
 910         }
 911
 912         kfree(handles);
 913         *syncobjs_out = syncobjs;
 914         return 0;
 915
 916 err_put_syncobjs:
 917         while (i-- > 0)
 918                 drm_syncobj_put(syncobjs[i]);
 919         kfree(syncobjs);
 920 err_free_handles:
 921         kfree(handles);
 922
 923         return ret;
 924 }
 925
 926 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 927                                    uint32_t count)
 928 {
 929         uint32_t i;
 930         for (i = 0; i < count; i++)
 931                 drm_syncobj_put(syncobjs[i]);
 932         kfree(syncobjs);
 933 }
 934
 935 int
 936 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 937                        struct drm_file *file_private)
 938 {
 939         struct drm_syncobj_wait *args = data;
 940         struct drm_syncobj **syncobjs;
 941         int ret = 0;
 942
 943         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 944                 return -EOPNOTSUPP;
 945
 946         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 947                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 948                 return -EINVAL;
 949
 950         if (args->count_handles == 0)
 951                 return -EINVAL;
 952
 953         ret = drm_syncobj_array_find(file_private,
 954                                      u64_to_user_ptr(args->handles),
 955                                      args->count_handles,
 956                                      &syncobjs);
 957         if (ret < 0)
 958                 return ret;
 959
 960         ret = drm_syncobj_array_wait(dev, file_private,
 961                                      args, NULL, syncobjs, false);
 962
 963         drm_syncobj_array_free(syncobjs, args->count_handles);
 964
 965         return ret;
 966 }
 967
 968 int
 969 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
 970                                 struct drm_file *file_private)
 971 {
 972         struct drm_syncobj_timeline_wait *args = data;
 973         struct drm_syncobj **syncobjs;
 974         int ret = 0;
 975
 976         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 977                 return -ENODEV;
 978
 979         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 980                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
 981                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
 982                 return -EINVAL;
 983
 984         if (args->count_handles == 0)
 985                 return -EINVAL;
 986
 987         ret = drm_syncobj_array_find(file_private,
 988                                      u64_to_user_ptr(args->handles),
 989                                      args->count_handles,
 990                                      &syncobjs);
 991         if (ret < 0)
 992                 return ret;
 993
 994         ret = drm_syncobj_array_wait(dev, file_private,
 995                                      NULL, args, syncobjs, true);
 996
 997         drm_syncobj_array_free(syncobjs, args->count_handles);
 998
 999         return ret;
1000 }
1001
1002
1003 int
1004 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1005                         struct drm_file *file_private)
1006 {
1007         struct drm_syncobj_array *args = data;
1008         struct drm_syncobj **syncobjs;
1009         uint32_t i;
1010         int ret;
1011
1012         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1013                 return -EOPNOTSUPP;
1014
1015         if (args->pad != 0)
1016                 return -EINVAL;
1017
1018         if (args->count_handles == 0)
1019                 return -EINVAL;
1020
1021         ret = drm_syncobj_array_find(file_private,
1022                                      u64_to_user_ptr(args->handles),
1023                                      args->count_handles,
1024                                      &syncobjs);
1025         if (ret < 0)
1026                 return ret;
1027
1028         for (i = 0; i < args->count_handles; i++)
1029                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1030
1031         drm_syncobj_array_free(syncobjs, args->count_handles);
1032
1033         return 0;
1034 }
1035
1036 int
1037 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1038                          struct drm_file *file_private)
1039 {
1040         struct drm_syncobj_array *args = data;
1041         struct drm_syncobj **syncobjs;
1042         uint32_t i;
1043         int ret;
1044
1045         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1046                 return -EOPNOTSUPP;
1047
1048         if (args->pad != 0)
1049                 return -EINVAL;
1050
1051         if (args->count_handles == 0)
1052                 return -EINVAL;
1053
1054         ret = drm_syncobj_array_find(file_private,
1055                                      u64_to_user_ptr(args->handles),
1056                                      args->count_handles,
1057                                      &syncobjs);
1058         if (ret < 0)
1059                 return ret;
1060
1061         for (i = 0; i < args->count_handles; i++)
1062                 drm_syncobj_assign_null_handle(syncobjs[i]);
1063
1064         drm_syncobj_array_free(syncobjs, args->count_handles);
1065
1066         return ret;
1067 }