drivers/gpu/drm/i915/gem/i915_gem_wait.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016 Intel Corporation
   5  */
   6
   7 #include <linux/dma-fence-array.h>
   8 #include <linux/jiffies.h>
   9
  10 #include "gt/intel_engine.h"
  11
  12 #include "i915_gem_ioctls.h"
  13 #include "i915_gem_object.h"
  14
  15 static long
  16 i915_gem_object_wait_fence(struct dma_fence *fence,
  17                            unsigned int flags,
  18                            long timeout)
  19 {
  20         BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
  21
  22         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  23                 return timeout;
  24
  25         if (dma_fence_is_i915(fence))
  26                 return i915_request_wait(to_request(fence), flags, timeout);
  27
  28         return dma_fence_wait_timeout(fence,
  29                                       flags & I915_WAIT_INTERRUPTIBLE,
  30                                       timeout);
  31 }
  32
  33 static long
  34 i915_gem_object_wait_reservation(struct reservation_object *resv,
  35                                  unsigned int flags,
  36                                  long timeout)
  37 {
  38         unsigned int seq = __read_seqcount_begin(&resv->seq);
  39         struct dma_fence *excl;
  40         bool prune_fences = false;
  41
  42         if (flags & I915_WAIT_ALL) {
  43                 struct dma_fence **shared;
  44                 unsigned int count, i;
  45                 int ret;
  46
  47                 ret = reservation_object_get_fences_rcu(resv,
  48                                                         &excl, &count, &shared);
  49                 if (ret)
  50                         return ret;
  51
  52                 for (i = 0; i < count; i++) {
  53                         timeout = i915_gem_object_wait_fence(shared[i],
  54                                                              flags, timeout);
  55                         if (timeout < 0)
  56                                 break;
  57
  58                         dma_fence_put(shared[i]);
  59                 }
  60
  61                 for (; i < count; i++)
  62                         dma_fence_put(shared[i]);
  63                 kfree(shared);
  64
  65                 /*
  66                  * If both shared fences and an exclusive fence exist,
  67                  * then by construction the shared fences must be later
  68                  * than the exclusive fence. If we successfully wait for
  69                  * all the shared fences, we know that the exclusive fence
  70                  * must all be signaled. If all the shared fences are
  71                  * signaled, we can prune the array and recover the
  72                  * floating references on the fences/requests.
  73                  */
  74                 prune_fences = count && timeout >= 0;
  75         } else {
  76                 excl = reservation_object_get_excl_rcu(resv);
  77         }
  78
  79         if (excl && timeout >= 0)
  80                 timeout = i915_gem_object_wait_fence(excl, flags, timeout);
  81
  82         dma_fence_put(excl);
  83
  84         /*
  85          * Opportunistically prune the fences iff we know they have *all* been
  86          * signaled and that the reservation object has not been changed (i.e.
  87          * no new fences have been added).
  88          */
  89         if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
  90                 if (reservation_object_trylock(resv)) {
  91                         if (!__read_seqcount_retry(&resv->seq, seq))
  92                                 reservation_object_add_excl_fence(resv, NULL);
  93                         reservation_object_unlock(resv);
  94                 }
  95         }
  96
  97         return timeout;
  98 }
  99
 100 static void __fence_set_priority(struct dma_fence *fence,
 101                                  const struct i915_sched_attr *attr)
 102 {
 103         struct i915_request *rq;
 104         struct intel_engine_cs *engine;
 105
 106         if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
 107                 return;
 108
 109         rq = to_request(fence);
 110         engine = rq->engine;
 111
 112         local_bh_disable();
 113         rcu_read_lock(); /* RCU serialisation for set-wedged protection */
 114         if (engine->schedule)
 115                 engine->schedule(rq, attr);
 116         rcu_read_unlock();
 117         local_bh_enable(); /* kick the tasklets if queues were reprioritised */
 118 }
 119
 120 static void fence_set_priority(struct dma_fence *fence,
 121                                const struct i915_sched_attr *attr)
 122 {
 123         /* Recurse once into a fence-array */
 124         if (dma_fence_is_array(fence)) {
 125                 struct dma_fence_array *array = to_dma_fence_array(fence);
 126                 int i;
 127
 128                 for (i = 0; i < array->num_fences; i++)
 129                         __fence_set_priority(array->fences[i], attr);
 130         } else {
 131                 __fence_set_priority(fence, attr);
 132         }
 133 }
 134
 135 int
 136 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 137                               unsigned int flags,
 138                               const struct i915_sched_attr *attr)
 139 {
 140         struct dma_fence *excl;
 141
 142         if (flags & I915_WAIT_ALL) {
 143                 struct dma_fence **shared;
 144                 unsigned int count, i;
 145                 int ret;
 146
 147                 ret = reservation_object_get_fences_rcu(obj->base.resv,
 148                                                         &excl, &count, &shared);
 149                 if (ret)
 150                         return ret;
 151
 152                 for (i = 0; i < count; i++) {
 153                         fence_set_priority(shared[i], attr);
 154                         dma_fence_put(shared[i]);
 155                 }
 156
 157                 kfree(shared);
 158         } else {
 159                 excl = reservation_object_get_excl_rcu(obj->base.resv);
 160         }
 161
 162         if (excl) {
 163                 fence_set_priority(excl, attr);
 164                 dma_fence_put(excl);
 165         }
 166         return 0;
 167 }
 168
 169 /**
 170  * Waits for rendering to the object to be completed
 171  * @obj: i915 gem object
 172  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 173  * @timeout: how long to wait
 174  */
 175 int
 176 i915_gem_object_wait(struct drm_i915_gem_object *obj,
 177                      unsigned int flags,
 178                      long timeout)
 179 {
 180         might_sleep();
 181         GEM_BUG_ON(timeout < 0);
 182
 183         timeout = i915_gem_object_wait_reservation(obj->base.resv,
 184                                                    flags, timeout);
 185         return timeout < 0 ? timeout : 0;
 186 }
 187
 188 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
 189 {
 190         /* nsecs_to_jiffies64() does not guard against overflow */
 191         if (NSEC_PER_SEC % HZ &&
 192             div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
 193                 return MAX_JIFFY_OFFSET;
 194
 195         return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
 196 }
 197
 198 static unsigned long to_wait_timeout(s64 timeout_ns)
 199 {
 200         if (timeout_ns < 0)
 201                 return MAX_SCHEDULE_TIMEOUT;
 202
 203         if (timeout_ns == 0)
 204                 return 0;
 205
 206         return nsecs_to_jiffies_timeout(timeout_ns);
 207 }
 208
 209 /**
 210  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 211  * @dev: drm device pointer
 212  * @data: ioctl data blob
 213  * @file: drm file pointer
 214  *
 215  * Returns 0 if successful, else an error is returned with the remaining time in
 216  * the timeout parameter.
 217  *  -ETIME: object is still busy after timeout
 218  *  -ERESTARTSYS: signal interrupted the wait
 219  *  -ENONENT: object doesn't exist
 220  * Also possible, but rare:
 221  *  -EAGAIN: incomplete, restart syscall
 222  *  -ENOMEM: damn
 223  *  -ENODEV: Internal IRQ fail
 224  *  -E?: The add request failed
 225  *
 226  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 227  * non-zero timeout parameter the wait ioctl will wait for the given number of
 228  * nanoseconds on an object becoming unbusy. Since the wait itself does so
 229  * without holding struct_mutex the object may become re-busied before this
 230  * function completes. A similar but shorter * race condition exists in the busy
 231  * ioctl
 232  */
 233 int
 234 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 235 {
 236         struct drm_i915_gem_wait *args = data;
 237         struct drm_i915_gem_object *obj;
 238         ktime_t start;
 239         long ret;
 240
 241         if (args->flags != 0)
 242                 return -EINVAL;
 243
 244         obj = i915_gem_object_lookup(file, args->bo_handle);
 245         if (!obj)
 246                 return -ENOENT;
 247
 248         start = ktime_get();
 249
 250         ret = i915_gem_object_wait(obj,
 251                                    I915_WAIT_INTERRUPTIBLE |
 252                                    I915_WAIT_PRIORITY |
 253                                    I915_WAIT_ALL,
 254                                    to_wait_timeout(args->timeout_ns));
 255
 256         if (args->timeout_ns > 0) {
 257                 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
 258                 if (args->timeout_ns < 0)
 259                         args->timeout_ns = 0;
 260
 261                 /*
 262                  * Apparently ktime isn't accurate enough and occasionally has a
 263                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
 264                  * things up to make the test happy. We allow up to 1 jiffy.
 265                  *
 266                  * This is a regression from the timespec->ktime conversion.
 267                  */
 268                 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
 269                         args->timeout_ns = 0;
 270
 271                 /* Asked to wait beyond the jiffie/scheduler precision? */
 272                 if (ret == -ETIME && args->timeout_ns)
 273                         ret = -EAGAIN;
 274         }
 275
 276         i915_gem_object_put(obj);
 277         return ret;
 278 }