drivers/gpu/drm/i915/gem/i915_gem_client_blt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "gt/intel_context.h"
   8 #include "gt/intel_engine_pm.h"
   9 #include "gt/intel_engine_pool.h"
  10 #include "i915_gem_client_blt.h"
  11 #include "i915_gem_object_blt.h"
  12
  13 struct i915_sleeve {
  14         struct i915_vma *vma;
  15         struct drm_i915_gem_object *obj;
  16         struct sg_table *pages;
  17         struct i915_page_sizes page_sizes;
  18 };
  19
  20 static int vma_set_pages(struct i915_vma *vma)
  21 {
  22         struct i915_sleeve *sleeve = vma->private;
  23
  24         vma->pages = sleeve->pages;
  25         vma->page_sizes = sleeve->page_sizes;
  26
  27         return 0;
  28 }
  29
  30 static void vma_clear_pages(struct i915_vma *vma)
  31 {
  32         GEM_BUG_ON(!vma->pages);
  33         vma->pages = NULL;
  34 }
  35
  36 static int vma_bind(struct i915_vma *vma,
  37                     enum i915_cache_level cache_level,
  38                     u32 flags)
  39 {
  40         return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
  41 }
  42
  43 static void vma_unbind(struct i915_vma *vma)
  44 {
  45         vma->vm->vma_ops.unbind_vma(vma);
  46 }
  47
  48 static const struct i915_vma_ops proxy_vma_ops = {
  49         .set_pages = vma_set_pages,
  50         .clear_pages = vma_clear_pages,
  51         .bind_vma = vma_bind,
  52         .unbind_vma = vma_unbind,
  53 };
  54
  55 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
  56                                          struct drm_i915_gem_object *obj,
  57                                          struct sg_table *pages,
  58                                          struct i915_page_sizes *page_sizes)
  59 {
  60         struct i915_sleeve *sleeve;
  61         struct i915_vma *vma;
  62         int err;
  63
  64         sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
  65         if (!sleeve)
  66                 return ERR_PTR(-ENOMEM);
  67
  68         vma = i915_vma_instance(obj, vm, NULL);
  69         if (IS_ERR(vma)) {
  70                 err = PTR_ERR(vma);
  71                 goto err_free;
  72         }
  73
  74         vma->private = sleeve;
  75         vma->ops = &proxy_vma_ops;
  76
  77         sleeve->vma = vma;
  78         sleeve->pages = pages;
  79         sleeve->page_sizes = *page_sizes;
  80
  81         return sleeve;
  82
  83 err_free:
  84         kfree(sleeve);
  85         return ERR_PTR(err);
  86 }
  87
  88 static void destroy_sleeve(struct i915_sleeve *sleeve)
  89 {
  90         kfree(sleeve);
  91 }
  92
  93 struct clear_pages_work {
  94         struct dma_fence dma;
  95         struct dma_fence_cb cb;
  96         struct i915_sw_fence wait;
  97         struct work_struct work;
  98         struct irq_work irq_work;
  99         struct i915_sleeve *sleeve;
 100         struct intel_context *ce;
 101         u32 value;
 102 };
 103
 104 static const char *clear_pages_work_driver_name(struct dma_fence *fence)
 105 {
 106         return DRIVER_NAME;
 107 }
 108
 109 static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
 110 {
 111         return "clear";
 112 }
 113
 114 static void clear_pages_work_release(struct dma_fence *fence)
 115 {
 116         struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
 117
 118         destroy_sleeve(w->sleeve);
 119
 120         i915_sw_fence_fini(&w->wait);
 121
 122         BUILD_BUG_ON(offsetof(typeof(*w), dma));
 123         dma_fence_free(&w->dma);
 124 }
 125
 126 static const struct dma_fence_ops clear_pages_work_ops = {
 127         .get_driver_name = clear_pages_work_driver_name,
 128         .get_timeline_name = clear_pages_work_timeline_name,
 129         .release = clear_pages_work_release,
 130 };
 131
 132 static void clear_pages_signal_irq_worker(struct irq_work *work)
 133 {
 134         struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
 135
 136         dma_fence_signal(&w->dma);
 137         dma_fence_put(&w->dma);
 138 }
 139
 140 static void clear_pages_dma_fence_cb(struct dma_fence *fence,
 141                                      struct dma_fence_cb *cb)
 142 {
 143         struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
 144
 145         if (fence->error)
 146                 dma_fence_set_error(&w->dma, fence->error);
 147
 148         /*
 149          * Push the signalling of the fence into yet another worker to avoid
 150          * the nightmare locking around the fence spinlock.
 151          */
 152         irq_work_queue(&w->irq_work);
 153 }
 154
 155 static void clear_pages_worker(struct work_struct *work)
 156 {
 157         struct clear_pages_work *w = container_of(work, typeof(*w), work);
 158         struct drm_i915_private *i915 = w->ce->engine->i915;
 159         struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
 160         struct i915_vma *vma = w->sleeve->vma;
 161         struct i915_request *rq;
 162         struct i915_vma *batch;
 163         int err = w->dma.error;
 164
 165         if (unlikely(err))
 166                 goto out_signal;
 167
 168         if (obj->cache_dirty) {
 169                 if (i915_gem_object_has_struct_page(obj))
 170                         drm_clflush_sg(w->sleeve->pages);
 171                 obj->cache_dirty = false;
 172         }
 173         obj->read_domains = I915_GEM_GPU_DOMAINS;
 174         obj->write_domain = 0;
 175
 176         /* XXX: we need to kill this */
 177         mutex_lock(&i915->drm.struct_mutex);
 178         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 179         if (unlikely(err))
 180                 goto out_unlock;
 181
 182         batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
 183         if (IS_ERR(batch)) {
 184                 err = PTR_ERR(batch);
 185                 goto out_unpin;
 186         }
 187
 188         rq = intel_context_create_request(w->ce);
 189         if (IS_ERR(rq)) {
 190                 err = PTR_ERR(rq);
 191                 goto out_batch;
 192         }
 193
 194         /* There's no way the fence has signalled */
 195         if (dma_fence_add_callback(&rq->fence, &w->cb,
 196                                    clear_pages_dma_fence_cb))
 197                 GEM_BUG_ON(1);
 198
 199         err = intel_emit_vma_mark_active(batch, rq);
 200         if (unlikely(err))
 201                 goto out_request;
 202
 203         if (w->ce->engine->emit_init_breadcrumb) {
 204                 err = w->ce->engine->emit_init_breadcrumb(rq);
 205                 if (unlikely(err))
 206                         goto out_request;
 207         }
 208
 209         /*
 210          * w->dma is already exported via (vma|obj)->resv we need only
 211          * keep track of the GPU activity within this vma/request, and
 212          * propagate the signal from the request to w->dma.
 213          */
 214         err = i915_active_ref(&vma->active, rq->timeline, rq);
 215         if (err)
 216                 goto out_request;
 217
 218         err = w->ce->engine->emit_bb_start(rq,
 219                                            batch->node.start, batch->node.size,
 220                                            0);
 221 out_request:
 222         if (unlikely(err)) {
 223                 i915_request_skip(rq, err);
 224                 err = 0;
 225         }
 226
 227         i915_request_add(rq);
 228 out_batch:
 229         intel_emit_vma_release(w->ce, batch);
 230 out_unpin:
 231         i915_vma_unpin(vma);
 232 out_unlock:
 233         mutex_unlock(&i915->drm.struct_mutex);
 234 out_signal:
 235         if (unlikely(err)) {
 236                 dma_fence_set_error(&w->dma, err);
 237                 dma_fence_signal(&w->dma);
 238                 dma_fence_put(&w->dma);
 239         }
 240 }
 241
 242 static int __i915_sw_fence_call
 243 clear_pages_work_notify(struct i915_sw_fence *fence,
 244                         enum i915_sw_fence_notify state)
 245 {
 246         struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
 247
 248         switch (state) {
 249         case FENCE_COMPLETE:
 250                 schedule_work(&w->work);
 251                 break;
 252
 253         case FENCE_FREE:
 254                 dma_fence_put(&w->dma);
 255                 break;
 256         }
 257
 258         return NOTIFY_DONE;
 259 }
 260
 261 static DEFINE_SPINLOCK(fence_lock);
 262
 263 /* XXX: better name please */
 264 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
 265                                      struct intel_context *ce,
 266                                      struct sg_table *pages,
 267                                      struct i915_page_sizes *page_sizes,
 268                                      u32 value)
 269 {
 270         struct clear_pages_work *work;
 271         struct i915_sleeve *sleeve;
 272         int err;
 273
 274         sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
 275         if (IS_ERR(sleeve))
 276                 return PTR_ERR(sleeve);
 277
 278         work = kmalloc(sizeof(*work), GFP_KERNEL);
 279         if (!work) {
 280                 destroy_sleeve(sleeve);
 281                 return -ENOMEM;
 282         }
 283
 284         work->value = value;
 285         work->sleeve = sleeve;
 286         work->ce = ce;
 287
 288         INIT_WORK(&work->work, clear_pages_worker);
 289
 290         init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
 291
 292         dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
 293         i915_sw_fence_init(&work->wait, clear_pages_work_notify);
 294
 295         i915_gem_object_lock(obj);
 296         err = i915_sw_fence_await_reservation(&work->wait,
 297                                               obj->base.resv, NULL,
 298                                               true, I915_FENCE_TIMEOUT,
 299                                               I915_FENCE_GFP);
 300         if (err < 0) {
 301                 dma_fence_set_error(&work->dma, err);
 302         } else {
 303                 dma_resv_add_excl_fence(obj->base.resv, &work->dma);
 304                 err = 0;
 305         }
 306         i915_gem_object_unlock(obj);
 307
 308         dma_fence_get(&work->dma);
 309         i915_sw_fence_commit(&work->wait);
 310
 311         return err;
 312 }
 313
 314 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 315 #include "selftests/i915_gem_client_blt.c"
 316 #endif