2 * Copyright © 2008-2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
28 #include <drm/drm_vma_manager.h>
29 #include <drm/drm_pci.h>
30 #include <drm/i915_drm.h>
31 #include <linux/dma-fence-array.h>
32 #include <linux/kthread.h>
33 #include <linux/reservation.h>
34 #include <linux/shmem_fs.h>
35 #include <linux/slab.h>
36 #include <linux/stop_machine.h>
37 #include <linux/swap.h>
38 #include <linux/pci.h>
39 #include <linux/dma-buf.h>
40 #include <linux/mman.h>
42 #include "gem/i915_gem_ioctls.h"
43 #include "gt/intel_engine_pm.h"
44 #include "gt/intel_gt_pm.h"
45 #include "gt/intel_mocs.h"
46 #include "gt/intel_reset.h"
47 #include "gt/intel_workarounds.h"
50 #include "i915_gem_clflush.h"
51 #include "i915_gemfs.h"
52 #include "i915_gem_pm.h"
53 #include "i915_trace.h"
54 #include "i915_vgpu.h"
56 #include "intel_display.h"
57 #include "intel_drv.h"
58 #include "intel_frontbuffer.h"
62 insert_mappable_node(struct i915_ggtt *ggtt,
63 struct drm_mm_node *node, u32 size)
65 memset(node, 0, sizeof(*node));
66 return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
67 size, 0, I915_COLOR_UNEVICTABLE,
68 0, ggtt->mappable_end,
73 remove_mappable_node(struct drm_mm_node *node)
75 drm_mm_remove_node(node);
79 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
80 struct drm_file *file)
82 struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
83 struct drm_i915_gem_get_aperture *args = data;
87 mutex_lock(&ggtt->vm.mutex);
89 pinned = ggtt->vm.reserved;
90 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
91 if (i915_vma_is_pinned(vma))
92 pinned += vma->node.size;
94 mutex_unlock(&ggtt->vm.mutex);
96 args->aper_size = ggtt->vm.total;
97 args->aper_available_size = args->aper_size - pinned;
102 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
104 struct address_space *mapping = obj->base.filp->f_mapping;
105 drm_dma_handle_t *phys;
107 struct scatterlist *sg;
112 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
115 /* Always aligning to the object size, allows a single allocation
116 * to handle all possible callers, and given typical object sizes,
117 * the alignment of the buddy allocation will naturally match.
119 phys = drm_pci_alloc(obj->base.dev,
120 roundup_pow_of_two(obj->base.size),
121 roundup_pow_of_two(obj->base.size));
126 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
130 page = shmem_read_mapping_page(mapping, i);
136 src = kmap_atomic(page);
137 memcpy(vaddr, src, PAGE_SIZE);
138 drm_clflush_virt_range(vaddr, PAGE_SIZE);
145 i915_gem_chipset_flush(to_i915(obj->base.dev));
147 st = kmalloc(sizeof(*st), GFP_KERNEL);
153 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
161 sg->length = obj->base.size;
163 sg_dma_address(sg) = phys->busaddr;
164 sg_dma_len(sg) = obj->base.size;
166 obj->phys_handle = phys;
168 __i915_gem_object_set_pages(obj, st, sg->length);
173 drm_pci_free(obj->base.dev, phys);
179 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
180 struct sg_table *pages)
182 __i915_gem_object_release_shmem(obj, pages, false);
185 struct address_space *mapping = obj->base.filp->f_mapping;
186 char *vaddr = obj->phys_handle->vaddr;
189 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
193 page = shmem_read_mapping_page(mapping, i);
197 dst = kmap_atomic(page);
198 drm_clflush_virt_range(vaddr, PAGE_SIZE);
199 memcpy(dst, vaddr, PAGE_SIZE);
202 set_page_dirty(page);
203 if (obj->mm.madv == I915_MADV_WILLNEED)
204 mark_page_accessed(page);
208 obj->mm.dirty = false;
211 sg_free_table(pages);
214 drm_pci_free(obj->base.dev, obj->phys_handle);
218 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
220 i915_gem_object_unpin_pages(obj);
223 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
224 .get_pages = i915_gem_object_get_pages_phys,
225 .put_pages = i915_gem_object_put_pages_phys,
226 .release = i915_gem_object_release_phys,
229 int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
231 struct i915_vma *vma;
232 LIST_HEAD(still_in_list);
235 lockdep_assert_held(&obj->base.dev->struct_mutex);
237 /* Closed vma are removed from the obj->vma_list - but they may
238 * still have an active binding on the object. To remove those we
239 * must wait for all rendering to complete to the object (as unbinding
240 * must anyway), and retire the requests.
242 ret = i915_gem_object_set_to_cpu_domain(obj, false);
246 spin_lock(&obj->vma.lock);
247 while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
250 list_move_tail(&vma->obj_link, &still_in_list);
251 spin_unlock(&obj->vma.lock);
253 ret = i915_vma_unbind(vma);
255 spin_lock(&obj->vma.lock);
257 list_splice(&still_in_list, &obj->vma.list);
258 spin_unlock(&obj->vma.lock);
264 i915_gem_object_wait_fence(struct dma_fence *fence,
268 struct i915_request *rq;
270 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
272 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
275 if (!dma_fence_is_i915(fence))
276 return dma_fence_wait_timeout(fence,
277 flags & I915_WAIT_INTERRUPTIBLE,
280 rq = to_request(fence);
281 if (i915_request_completed(rq))
284 timeout = i915_request_wait(rq, flags, timeout);
287 if (flags & I915_WAIT_LOCKED && i915_request_completed(rq))
288 i915_request_retire_upto(rq);
294 i915_gem_object_wait_reservation(struct reservation_object *resv,
298 unsigned int seq = __read_seqcount_begin(&resv->seq);
299 struct dma_fence *excl;
300 bool prune_fences = false;
302 if (flags & I915_WAIT_ALL) {
303 struct dma_fence **shared;
304 unsigned int count, i;
307 ret = reservation_object_get_fences_rcu(resv,
308 &excl, &count, &shared);
312 for (i = 0; i < count; i++) {
313 timeout = i915_gem_object_wait_fence(shared[i],
318 dma_fence_put(shared[i]);
321 for (; i < count; i++)
322 dma_fence_put(shared[i]);
326 * If both shared fences and an exclusive fence exist,
327 * then by construction the shared fences must be later
328 * than the exclusive fence. If we successfully wait for
329 * all the shared fences, we know that the exclusive fence
330 * must all be signaled. If all the shared fences are
331 * signaled, we can prune the array and recover the
332 * floating references on the fences/requests.
334 prune_fences = count && timeout >= 0;
336 excl = reservation_object_get_excl_rcu(resv);
339 if (excl && timeout >= 0)
340 timeout = i915_gem_object_wait_fence(excl, flags, timeout);
345 * Opportunistically prune the fences iff we know they have *all* been
346 * signaled and that the reservation object has not been changed (i.e.
347 * no new fences have been added).
349 if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
350 if (reservation_object_trylock(resv)) {
351 if (!__read_seqcount_retry(&resv->seq, seq))
352 reservation_object_add_excl_fence(resv, NULL);
353 reservation_object_unlock(resv);
360 static void __fence_set_priority(struct dma_fence *fence,
361 const struct i915_sched_attr *attr)
363 struct i915_request *rq;
364 struct intel_engine_cs *engine;
366 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
369 rq = to_request(fence);
373 rcu_read_lock(); /* RCU serialisation for set-wedged protection */
374 if (engine->schedule)
375 engine->schedule(rq, attr);
377 local_bh_enable(); /* kick the tasklets if queues were reprioritised */
380 static void fence_set_priority(struct dma_fence *fence,
381 const struct i915_sched_attr *attr)
383 /* Recurse once into a fence-array */
384 if (dma_fence_is_array(fence)) {
385 struct dma_fence_array *array = to_dma_fence_array(fence);
388 for (i = 0; i < array->num_fences; i++)
389 __fence_set_priority(array->fences[i], attr);
391 __fence_set_priority(fence, attr);
396 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
398 const struct i915_sched_attr *attr)
400 struct dma_fence *excl;
402 if (flags & I915_WAIT_ALL) {
403 struct dma_fence **shared;
404 unsigned int count, i;
407 ret = reservation_object_get_fences_rcu(obj->resv,
408 &excl, &count, &shared);
412 for (i = 0; i < count; i++) {
413 fence_set_priority(shared[i], attr);
414 dma_fence_put(shared[i]);
419 excl = reservation_object_get_excl_rcu(obj->resv);
423 fence_set_priority(excl, attr);
430 * Waits for rendering to the object to be completed
431 * @obj: i915 gem object
432 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
433 * @timeout: how long to wait
436 i915_gem_object_wait(struct drm_i915_gem_object *obj,
441 GEM_BUG_ON(timeout < 0);
443 timeout = i915_gem_object_wait_reservation(obj->resv, flags, timeout);
444 return timeout < 0 ? timeout : 0;
448 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
449 struct drm_i915_gem_pwrite *args,
450 struct drm_file *file)
452 void *vaddr = obj->phys_handle->vaddr + args->offset;
453 char __user *user_data = u64_to_user_ptr(args->data_ptr);
455 /* We manually control the domain here and pretend that it
456 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
458 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
459 if (copy_from_user(vaddr, user_data, args->size))
462 drm_clflush_virt_range(vaddr, args->size);
463 i915_gem_chipset_flush(to_i915(obj->base.dev));
465 intel_fb_obj_flush(obj, ORIGIN_CPU);
470 i915_gem_create(struct drm_file *file,
471 struct drm_i915_private *dev_priv,
475 struct drm_i915_gem_object *obj;
480 size = round_up(*size_p, PAGE_SIZE);
484 /* Allocate the new object */
485 obj = i915_gem_object_create_shmem(dev_priv, size);
489 ret = drm_gem_handle_create(file, &obj->base, &handle);
490 /* drop reference from allocate - handle holds it now */
491 i915_gem_object_put(obj);
501 i915_gem_dumb_create(struct drm_file *file,
502 struct drm_device *dev,
503 struct drm_mode_create_dumb *args)
505 int cpp = DIV_ROUND_UP(args->bpp, 8);
510 format = DRM_FORMAT_C8;
513 format = DRM_FORMAT_RGB565;
516 format = DRM_FORMAT_XRGB8888;
522 /* have to work out size/pitch and return them */
523 args->pitch = ALIGN(args->width * cpp, 64);
525 /* align stride to page size so that we can remap */
526 if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format,
527 DRM_FORMAT_MOD_LINEAR))
528 args->pitch = ALIGN(args->pitch, 4096);
530 args->size = args->pitch * args->height;
531 return i915_gem_create(file, to_i915(dev),
532 &args->size, &args->handle);
535 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
537 return !(obj->cache_level == I915_CACHE_NONE ||
538 obj->cache_level == I915_CACHE_WT);
542 * Creates a new mm object and returns a handle to it.
543 * @dev: drm device pointer
544 * @data: ioctl data blob
545 * @file: drm file pointer
548 i915_gem_create_ioctl(struct drm_device *dev, void *data,
549 struct drm_file *file)
551 struct drm_i915_private *dev_priv = to_i915(dev);
552 struct drm_i915_gem_create *args = data;
554 i915_gem_flush_free_objects(dev_priv);
556 return i915_gem_create(file, dev_priv,
557 &args->size, &args->handle);
560 static inline enum fb_op_origin
561 fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
563 return (domain == I915_GEM_DOMAIN_GTT ?
564 obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
567 void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv)
569 intel_wakeref_t wakeref;
572 * No actual flushing is required for the GTT write domain for reads
573 * from the GTT domain. Writes to it "immediately" go to main memory
574 * as far as we know, so there's no chipset flush. It also doesn't
575 * land in the GPU render cache.
577 * However, we do have to enforce the order so that all writes through
578 * the GTT land before any writes to the device, such as updates to
581 * We also have to wait a bit for the writes to land from the GTT.
582 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
583 * timing. This issue has only been observed when switching quickly
584 * between GTT writes and CPU reads from inside the kernel on recent hw,
585 * and it appears to only affect discrete GTT blocks (i.e. on LLC
586 * system agents we cannot reproduce this behaviour, until Cannonlake
592 if (INTEL_INFO(dev_priv)->has_coherent_ggtt)
595 i915_gem_chipset_flush(dev_priv);
597 with_intel_runtime_pm(dev_priv, wakeref) {
598 spin_lock_irq(&dev_priv->uncore.lock);
600 POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE));
602 spin_unlock_irq(&dev_priv->uncore.lock);
607 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
609 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
610 struct i915_vma *vma;
612 if (!(obj->write_domain & flush_domains))
615 switch (obj->write_domain) {
616 case I915_GEM_DOMAIN_GTT:
617 i915_gem_flush_ggtt_writes(dev_priv);
619 intel_fb_obj_flush(obj,
620 fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
622 for_each_ggtt_vma(vma, obj) {
626 i915_vma_unset_ggtt_write(vma);
630 case I915_GEM_DOMAIN_WC:
634 case I915_GEM_DOMAIN_CPU:
635 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
638 case I915_GEM_DOMAIN_RENDER:
639 if (gpu_write_needs_clflush(obj))
640 obj->cache_dirty = true;
644 obj->write_domain = 0;
648 * Pins the specified object's pages and synchronizes the object with
649 * GPU accesses. Sets needs_clflush to non-zero if the caller should
650 * flush the object from the CPU cache.
652 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
653 unsigned int *needs_clflush)
657 lockdep_assert_held(&obj->base.dev->struct_mutex);
660 if (!i915_gem_object_has_struct_page(obj))
663 ret = i915_gem_object_wait(obj,
664 I915_WAIT_INTERRUPTIBLE |
666 MAX_SCHEDULE_TIMEOUT);
670 ret = i915_gem_object_pin_pages(obj);
674 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
675 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
676 ret = i915_gem_object_set_to_cpu_domain(obj, false);
683 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
685 /* If we're not in the cpu read domain, set ourself into the gtt
686 * read domain and manually flush cachelines (if required). This
687 * optimizes for the case when the gpu will dirty the data
688 * anyway again before the next pread happens.
690 if (!obj->cache_dirty &&
691 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
692 *needs_clflush = CLFLUSH_BEFORE;
695 /* return with the pages pinned */
699 i915_gem_object_unpin_pages(obj);
703 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
704 unsigned int *needs_clflush)
708 lockdep_assert_held(&obj->base.dev->struct_mutex);
711 if (!i915_gem_object_has_struct_page(obj))
714 ret = i915_gem_object_wait(obj,
715 I915_WAIT_INTERRUPTIBLE |
718 MAX_SCHEDULE_TIMEOUT);
722 ret = i915_gem_object_pin_pages(obj);
726 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
727 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
728 ret = i915_gem_object_set_to_cpu_domain(obj, true);
735 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
737 /* If we're not in the cpu write domain, set ourself into the
738 * gtt write domain and manually flush cachelines (as required).
739 * This optimizes for the case when the gpu will use the data
740 * right away and we therefore have to clflush anyway.
742 if (!obj->cache_dirty) {
743 *needs_clflush |= CLFLUSH_AFTER;
746 * Same trick applies to invalidate partially written
747 * cachelines read before writing.
749 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
750 *needs_clflush |= CLFLUSH_BEFORE;
754 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
755 obj->mm.dirty = true;
756 /* return with the pages pinned */
760 i915_gem_object_unpin_pages(obj);
765 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
774 drm_clflush_virt_range(vaddr + offset, len);
776 ret = __copy_to_user(user_data, vaddr + offset, len);
780 return ret ? -EFAULT : 0;
784 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
785 struct drm_i915_gem_pread *args)
787 char __user *user_data;
789 unsigned int needs_clflush;
790 unsigned int idx, offset;
793 ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
797 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
798 mutex_unlock(&obj->base.dev->struct_mutex);
803 user_data = u64_to_user_ptr(args->data_ptr);
804 offset = offset_in_page(args->offset);
805 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
806 struct page *page = i915_gem_object_get_page(obj, idx);
807 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
809 ret = shmem_pread(page, offset, length, user_data,
819 i915_gem_obj_finish_shmem_access(obj);
824 gtt_user_read(struct io_mapping *mapping,
825 loff_t base, int offset,
826 char __user *user_data, int length)
829 unsigned long unwritten;
831 /* We can use the cpu mem copy function because this is X86. */
832 vaddr = io_mapping_map_atomic_wc(mapping, base);
833 unwritten = __copy_to_user_inatomic(user_data,
834 (void __force *)vaddr + offset,
836 io_mapping_unmap_atomic(vaddr);
838 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
839 unwritten = copy_to_user(user_data,
840 (void __force *)vaddr + offset,
842 io_mapping_unmap(vaddr);
848 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
849 const struct drm_i915_gem_pread *args)
851 struct drm_i915_private *i915 = to_i915(obj->base.dev);
852 struct i915_ggtt *ggtt = &i915->ggtt;
853 intel_wakeref_t wakeref;
854 struct drm_mm_node node;
855 struct i915_vma *vma;
856 void __user *user_data;
860 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
864 wakeref = intel_runtime_pm_get(i915);
865 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
870 node.start = i915_ggtt_offset(vma);
871 node.allocated = false;
872 ret = i915_vma_put_fence(vma);
879 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
882 GEM_BUG_ON(!node.allocated);
885 ret = i915_gem_object_set_to_gtt_domain(obj, false);
889 mutex_unlock(&i915->drm.struct_mutex);
891 user_data = u64_to_user_ptr(args->data_ptr);
893 offset = args->offset;
896 /* Operation in this page
898 * page_base = page offset within aperture
899 * page_offset = offset within page
900 * page_length = bytes to copy for this page
902 u32 page_base = node.start;
903 unsigned page_offset = offset_in_page(offset);
904 unsigned page_length = PAGE_SIZE - page_offset;
905 page_length = remain < page_length ? remain : page_length;
906 if (node.allocated) {
908 ggtt->vm.insert_page(&ggtt->vm,
909 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
910 node.start, I915_CACHE_NONE, 0);
913 page_base += offset & PAGE_MASK;
916 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
917 user_data, page_length)) {
922 remain -= page_length;
923 user_data += page_length;
924 offset += page_length;
927 mutex_lock(&i915->drm.struct_mutex);
929 if (node.allocated) {
931 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
932 remove_mappable_node(&node);
937 intel_runtime_pm_put(i915, wakeref);
938 mutex_unlock(&i915->drm.struct_mutex);
944 * Reads data from the object referenced by handle.
945 * @dev: drm device pointer
946 * @data: ioctl data blob
947 * @file: drm file pointer
949 * On error, the contents of *data are undefined.
952 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
953 struct drm_file *file)
955 struct drm_i915_gem_pread *args = data;
956 struct drm_i915_gem_object *obj;
962 if (!access_ok(u64_to_user_ptr(args->data_ptr),
966 obj = i915_gem_object_lookup(file, args->handle);
970 /* Bounds check source. */
971 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
976 trace_i915_gem_object_pread(obj, args->offset, args->size);
978 ret = i915_gem_object_wait(obj,
979 I915_WAIT_INTERRUPTIBLE,
980 MAX_SCHEDULE_TIMEOUT);
984 ret = i915_gem_object_pin_pages(obj);
988 ret = i915_gem_shmem_pread(obj, args);
989 if (ret == -EFAULT || ret == -ENODEV)
990 ret = i915_gem_gtt_pread(obj, args);
992 i915_gem_object_unpin_pages(obj);
994 i915_gem_object_put(obj);
998 /* This is the fast write path which cannot handle
999 * page faults in the source data
1003 ggtt_write(struct io_mapping *mapping,
1004 loff_t base, int offset,
1005 char __user *user_data, int length)
1007 void __iomem *vaddr;
1008 unsigned long unwritten;
1010 /* We can use the cpu mem copy function because this is X86. */
1011 vaddr = io_mapping_map_atomic_wc(mapping, base);
1012 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
1014 io_mapping_unmap_atomic(vaddr);
1016 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1017 unwritten = copy_from_user((void __force *)vaddr + offset,
1019 io_mapping_unmap(vaddr);
1026 * This is the fast pwrite path, where we copy the data directly from the
1027 * user into the GTT, uncached.
1028 * @obj: i915 GEM object
1029 * @args: pwrite arguments structure
1032 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
1033 const struct drm_i915_gem_pwrite *args)
1035 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1036 struct i915_ggtt *ggtt = &i915->ggtt;
1037 intel_wakeref_t wakeref;
1038 struct drm_mm_node node;
1039 struct i915_vma *vma;
1041 void __user *user_data;
1044 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1048 if (i915_gem_object_has_struct_page(obj)) {
1050 * Avoid waking the device up if we can fallback, as
1051 * waking/resuming is very slow (worst-case 10-100 ms
1052 * depending on PCI sleeps and our own resume time).
1053 * This easily dwarfs any performance advantage from
1054 * using the cache bypass of indirect GGTT access.
1056 wakeref = intel_runtime_pm_get_if_in_use(i915);
1062 /* No backing pages, no fallback, we must force GGTT access */
1063 wakeref = intel_runtime_pm_get(i915);
1066 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1071 node.start = i915_ggtt_offset(vma);
1072 node.allocated = false;
1073 ret = i915_vma_put_fence(vma);
1075 i915_vma_unpin(vma);
1080 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1083 GEM_BUG_ON(!node.allocated);
1086 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1090 mutex_unlock(&i915->drm.struct_mutex);
1092 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1094 user_data = u64_to_user_ptr(args->data_ptr);
1095 offset = args->offset;
1096 remain = args->size;
1098 /* Operation in this page
1100 * page_base = page offset within aperture
1101 * page_offset = offset within page
1102 * page_length = bytes to copy for this page
1104 u32 page_base = node.start;
1105 unsigned int page_offset = offset_in_page(offset);
1106 unsigned int page_length = PAGE_SIZE - page_offset;
1107 page_length = remain < page_length ? remain : page_length;
1108 if (node.allocated) {
1109 wmb(); /* flush the write before we modify the GGTT */
1110 ggtt->vm.insert_page(&ggtt->vm,
1111 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1112 node.start, I915_CACHE_NONE, 0);
1113 wmb(); /* flush modifications to the GGTT (insert_page) */
1115 page_base += offset & PAGE_MASK;
1117 /* If we get a fault while copying data, then (presumably) our
1118 * source page isn't available. Return the error and we'll
1119 * retry in the slow path.
1120 * If the object is non-shmem backed, we retry again with the
1121 * path that handles page fault.
1123 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
1124 user_data, page_length)) {
1129 remain -= page_length;
1130 user_data += page_length;
1131 offset += page_length;
1133 intel_fb_obj_flush(obj, ORIGIN_CPU);
1135 mutex_lock(&i915->drm.struct_mutex);
1137 if (node.allocated) {
1139 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
1140 remove_mappable_node(&node);
1142 i915_vma_unpin(vma);
1145 intel_runtime_pm_put(i915, wakeref);
1147 mutex_unlock(&i915->drm.struct_mutex);
1151 /* Per-page copy function for the shmem pwrite fastpath.
1152 * Flushes invalid cachelines before writing to the target if
1153 * needs_clflush_before is set and flushes out any written cachelines after
1154 * writing if needs_clflush is set.
1157 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
1158 bool needs_clflush_before,
1159 bool needs_clflush_after)
1166 if (needs_clflush_before)
1167 drm_clflush_virt_range(vaddr + offset, len);
1169 ret = __copy_from_user(vaddr + offset, user_data, len);
1170 if (!ret && needs_clflush_after)
1171 drm_clflush_virt_range(vaddr + offset, len);
1175 return ret ? -EFAULT : 0;
1179 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
1180 const struct drm_i915_gem_pwrite *args)
1182 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1183 void __user *user_data;
1185 unsigned int partial_cacheline_write;
1186 unsigned int needs_clflush;
1187 unsigned int offset, idx;
1190 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1194 ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1195 mutex_unlock(&i915->drm.struct_mutex);
1199 /* If we don't overwrite a cacheline completely we need to be
1200 * careful to have up-to-date data by first clflushing. Don't
1201 * overcomplicate things and flush the entire patch.
1203 partial_cacheline_write = 0;
1204 if (needs_clflush & CLFLUSH_BEFORE)
1205 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
1207 user_data = u64_to_user_ptr(args->data_ptr);
1208 remain = args->size;
1209 offset = offset_in_page(args->offset);
1210 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1211 struct page *page = i915_gem_object_get_page(obj, idx);
1212 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
1214 ret = shmem_pwrite(page, offset, length, user_data,
1215 (offset | length) & partial_cacheline_write,
1216 needs_clflush & CLFLUSH_AFTER);
1221 user_data += length;
1225 intel_fb_obj_flush(obj, ORIGIN_CPU);
1226 i915_gem_obj_finish_shmem_access(obj);
1231 * Writes data to the object referenced by handle.
1233 * @data: ioctl data blob
1236 * On error, the contents of the buffer that were to be modified are undefined.
1239 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1240 struct drm_file *file)
1242 struct drm_i915_gem_pwrite *args = data;
1243 struct drm_i915_gem_object *obj;
1246 if (args->size == 0)
1249 if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
1252 obj = i915_gem_object_lookup(file, args->handle);
1256 /* Bounds check destination. */
1257 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
1262 /* Writes not allowed into this read-only object */
1263 if (i915_gem_object_is_readonly(obj)) {
1268 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1271 if (obj->ops->pwrite)
1272 ret = obj->ops->pwrite(obj, args);
1276 ret = i915_gem_object_wait(obj,
1277 I915_WAIT_INTERRUPTIBLE |
1279 MAX_SCHEDULE_TIMEOUT);
1283 ret = i915_gem_object_pin_pages(obj);
1288 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1289 * it would end up going through the fenced access, and we'll get
1290 * different detiling behavior between reading and writing.
1291 * pread/pwrite currently are reading and writing from the CPU
1292 * perspective, requiring manual detiling by the client.
1294 if (!i915_gem_object_has_struct_page(obj) ||
1295 cpu_write_needs_clflush(obj))
1296 /* Note that the gtt paths might fail with non-page-backed user
1297 * pointers (e.g. gtt mappings when moving data between
1298 * textures). Fallback to the shmem path in that case.
1300 ret = i915_gem_gtt_pwrite_fast(obj, args);
1302 if (ret == -EFAULT || ret == -ENOSPC) {
1303 if (obj->phys_handle)
1304 ret = i915_gem_phys_pwrite(obj, args, file);
1306 ret = i915_gem_shmem_pwrite(obj, args);
1309 i915_gem_object_unpin_pages(obj);
1311 i915_gem_object_put(obj);
1315 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
1317 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1318 struct list_head *list;
1319 struct i915_vma *vma;
1321 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1323 mutex_lock(&i915->ggtt.vm.mutex);
1324 for_each_ggtt_vma(vma, obj) {
1325 if (!drm_mm_node_allocated(&vma->node))
1328 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1330 mutex_unlock(&i915->ggtt.vm.mutex);
1332 spin_lock(&i915->mm.obj_lock);
1333 list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
1334 list_move_tail(&obj->mm.link, list);
1335 spin_unlock(&i915->mm.obj_lock);
1339 * Called when user space prepares to use an object with the CPU, either
1340 * through the mmap ioctl's mapping or a GTT mapping.
1342 * @data: ioctl data blob
1346 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1347 struct drm_file *file)
1349 struct drm_i915_gem_set_domain *args = data;
1350 struct drm_i915_gem_object *obj;
1351 u32 read_domains = args->read_domains;
1352 u32 write_domain = args->write_domain;
1355 /* Only handle setting domains to types used by the CPU. */
1356 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1360 * Having something in the write domain implies it's in the read
1361 * domain, and only that read domain. Enforce that in the request.
1363 if (write_domain && read_domains != write_domain)
1369 obj = i915_gem_object_lookup(file, args->handle);
1374 * Already in the desired write domain? Nothing for us to do!
1376 * We apply a little bit of cunning here to catch a broader set of
1377 * no-ops. If obj->write_domain is set, we must be in the same
1378 * obj->read_domains, and only that domain. Therefore, if that
1379 * obj->write_domain matches the request read_domains, we are
1380 * already in the same read/write domain and can skip the operation,
1381 * without having to further check the requested write_domain.
1383 if (READ_ONCE(obj->write_domain) == read_domains) {
1389 * Try to flush the object off the GPU without holding the lock.
1390 * We will repeat the flush holding the lock in the normal manner
1391 * to catch cases where we are gazumped.
1393 err = i915_gem_object_wait(obj,
1394 I915_WAIT_INTERRUPTIBLE |
1395 I915_WAIT_PRIORITY |
1396 (write_domain ? I915_WAIT_ALL : 0),
1397 MAX_SCHEDULE_TIMEOUT);
1402 * Proxy objects do not control access to the backing storage, ergo
1403 * they cannot be used as a means to manipulate the cache domain
1404 * tracking for that backing storage. The proxy object is always
1405 * considered to be outside of any cache domain.
1407 if (i915_gem_object_is_proxy(obj)) {
1413 * Flush and acquire obj->pages so that we are coherent through
1414 * direct access in memory with previous cached writes through
1415 * shmemfs and that our cache domain tracking remains valid.
1416 * For example, if the obj->filp was moved to swap without us
1417 * being notified and releasing the pages, we would mistakenly
1418 * continue to assume that the obj remained out of the CPU cached
1421 err = i915_gem_object_pin_pages(obj);
1425 err = i915_mutex_lock_interruptible(dev);
1429 if (read_domains & I915_GEM_DOMAIN_WC)
1430 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
1431 else if (read_domains & I915_GEM_DOMAIN_GTT)
1432 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
1434 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
1436 /* And bump the LRU for this access */
1437 i915_gem_object_bump_inactive_ggtt(obj);
1439 mutex_unlock(&dev->struct_mutex);
1441 if (write_domain != 0)
1442 intel_fb_obj_invalidate(obj,
1443 fb_write_origin(obj, write_domain));
1446 i915_gem_object_unpin_pages(obj);
1448 i915_gem_object_put(obj);
1453 * Called when user space has done writes to this buffer
1455 * @data: ioctl data blob
1459 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1460 struct drm_file *file)
1462 struct drm_i915_gem_sw_finish *args = data;
1463 struct drm_i915_gem_object *obj;
1465 obj = i915_gem_object_lookup(file, args->handle);
1470 * Proxy objects are barred from CPU access, so there is no
1471 * need to ban sw_finish as it is a nop.
1474 /* Pinned buffers may be scanout, so flush the cache */
1475 i915_gem_object_flush_if_display(obj);
1476 i915_gem_object_put(obj);
1482 __vma_matches(struct vm_area_struct *vma, struct file *filp,
1483 unsigned long addr, unsigned long size)
1485 if (vma->vm_file != filp)
1488 return vma->vm_start == addr &&
1489 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
1493 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1496 * @data: ioctl data blob
1499 * While the mapping holds a reference on the contents of the object, it doesn't
1500 * imply a ref on the object itself.
1504 * DRM driver writers who look a this function as an example for how to do GEM
1505 * mmap support, please don't implement mmap support like here. The modern way
1506 * to implement DRM mmap support is with an mmap offset ioctl (like
1507 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1508 * That way debug tooling like valgrind will understand what's going on, hiding
1509 * the mmap call in a driver private ioctl will break that. The i915 driver only
1510 * does cpu mmaps this way because we didn't know better.
1513 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1514 struct drm_file *file)
1516 struct drm_i915_gem_mmap *args = data;
1517 struct drm_i915_gem_object *obj;
1520 if (args->flags & ~(I915_MMAP_WC))
1523 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1526 obj = i915_gem_object_lookup(file, args->handle);
1530 /* prime objects have no backing filp to GEM mmap
1533 if (!obj->base.filp) {
1538 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
1543 addr = vm_mmap(obj->base.filp, 0, args->size,
1544 PROT_READ | PROT_WRITE, MAP_SHARED,
1546 if (IS_ERR_VALUE(addr))
1549 if (args->flags & I915_MMAP_WC) {
1550 struct mm_struct *mm = current->mm;
1551 struct vm_area_struct *vma;
1553 if (down_write_killable(&mm->mmap_sem)) {
1557 vma = find_vma(mm, addr);
1558 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
1560 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1563 up_write(&mm->mmap_sem);
1564 if (IS_ERR_VALUE(addr))
1567 /* This may race, but that's ok, it only gets set */
1568 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1570 i915_gem_object_put(obj);
1572 args->addr_ptr = (u64)addr;
1576 i915_gem_object_put(obj);
1580 static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
1582 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
1586 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
1588 * A history of the GTT mmap interface:
1590 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
1591 * aligned and suitable for fencing, and still fit into the available
1592 * mappable space left by the pinned display objects. A classic problem
1593 * we called the page-fault-of-doom where we would ping-pong between
1594 * two objects that could not fit inside the GTT and so the memcpy
1595 * would page one object in at the expense of the other between every
1598 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
1599 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
1600 * object is too large for the available space (or simply too large
1601 * for the mappable aperture!), a view is created instead and faulted
1602 * into userspace. (This view is aligned and sized appropriately for
1605 * 2 - Recognise WC as a separate cache domain so that we can flush the
1606 * delayed writes via GTT before performing direct access via WC.
1608 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
1609 * pagefault; swapin remains transparent.
1613 * * snoopable objects cannot be accessed via the GTT. It can cause machine
1614 * hangs on some architectures, corruption on others. An attempt to service
1615 * a GTT page fault from a snoopable object will generate a SIGBUS.
1617 * * the object must be able to fit into RAM (physical memory, though no
1618 * limited to the mappable aperture).
1623 * * a new GTT page fault will synchronize rendering from the GPU and flush
1624 * all data to system memory. Subsequent access will not be synchronized.
1626 * * all mappings are revoked on runtime device suspend.
1628 * * there are only 8, 16 or 32 fence registers to share between all users
1629 * (older machines require fence register for display and blitter access
1630 * as well). Contention of the fence registers will cause the previous users
1631 * to be unmapped and any new access will generate new page faults.
1633 * * running out of memory while servicing a fault may generate a SIGBUS,
1634 * rather than the expected SIGSEGV.
1636 int i915_gem_mmap_gtt_version(void)
1641 static inline struct i915_ggtt_view
1642 compute_partial_view(const struct drm_i915_gem_object *obj,
1643 pgoff_t page_offset,
1646 struct i915_ggtt_view view;
1648 if (i915_gem_object_is_tiled(obj))
1649 chunk = roundup(chunk, tile_row_pages(obj));
1651 view.type = I915_GGTT_VIEW_PARTIAL;
1652 view.partial.offset = rounddown(page_offset, chunk);
1654 min_t(unsigned int, chunk,
1655 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
1657 /* If the partial covers the entire object, just create a normal VMA. */
1658 if (chunk >= obj->base.size >> PAGE_SHIFT)
1659 view.type = I915_GGTT_VIEW_NORMAL;
1665 * i915_gem_fault - fault a page into the GTT
1668 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1669 * from userspace. The fault handler takes care of binding the object to
1670 * the GTT (if needed), allocating and programming a fence register (again,
1671 * only if needed based on whether the old reg is still valid or the object
1672 * is tiled) and inserting a new PTE into the faulting process.
1674 * Note that the faulting process may involve evicting existing objects
1675 * from the GTT and/or fence registers to make room. So performance may
1676 * suffer if the GTT working set is large or there are few fence registers
1679 * The current feature set supported by i915_gem_fault() and thus GTT mmaps
1680 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
1682 vm_fault_t i915_gem_fault(struct vm_fault *vmf)
1684 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
1685 struct vm_area_struct *area = vmf->vma;
1686 struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
1687 struct drm_device *dev = obj->base.dev;
1688 struct drm_i915_private *dev_priv = to_i915(dev);
1689 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1690 bool write = area->vm_flags & VM_WRITE;
1691 intel_wakeref_t wakeref;
1692 struct i915_vma *vma;
1693 pgoff_t page_offset;
1697 /* Sanity check that we allow writing into this object */
1698 if (i915_gem_object_is_readonly(obj) && write)
1699 return VM_FAULT_SIGBUS;
1701 /* We don't use vmf->pgoff since that has the fake offset */
1702 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
1704 trace_i915_gem_object_fault(obj, page_offset, true, write);
1706 ret = i915_gem_object_pin_pages(obj);
1710 wakeref = intel_runtime_pm_get(dev_priv);
1712 srcu = i915_reset_trylock(dev_priv);
1718 ret = i915_mutex_lock_interruptible(dev);
1722 /* Access to snoopable pages through the GTT is incoherent. */
1723 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
1728 /* Now pin it into the GTT as needed */
1729 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1734 /* Use a partial view if it is bigger than available space */
1735 struct i915_ggtt_view view =
1736 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
1739 flags = PIN_MAPPABLE;
1740 if (view.type == I915_GGTT_VIEW_NORMAL)
1741 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
1744 * Userspace is now writing through an untracked VMA, abandon
1745 * all hope that the hardware is able to track future writes.
1747 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
1749 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
1750 if (IS_ERR(vma) && !view.type) {
1751 flags = PIN_MAPPABLE;
1752 view.type = I915_GGTT_VIEW_PARTIAL;
1753 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
1761 ret = i915_vma_pin_fence(vma);
1765 /* Finally, remap it using the new GTT offset */
1766 ret = remap_io_mapping(area,
1767 area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
1768 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
1769 min_t(u64, vma->size, area->vm_end - area->vm_start),
1774 /* Mark as being mmapped into userspace for later revocation */
1775 assert_rpm_wakelock_held(dev_priv);
1776 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
1777 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
1778 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
1779 intel_wakeref_auto(&dev_priv->mm.userfault_wakeref,
1780 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
1781 GEM_BUG_ON(!obj->userfault_count);
1783 i915_vma_set_ggtt_write(vma);
1786 i915_vma_unpin_fence(vma);
1788 __i915_vma_unpin(vma);
1790 mutex_unlock(&dev->struct_mutex);
1792 i915_reset_unlock(dev_priv, srcu);
1794 intel_runtime_pm_put(dev_priv, wakeref);
1795 i915_gem_object_unpin_pages(obj);
1800 * We eat errors when the gpu is terminally wedged to avoid
1801 * userspace unduly crashing (gl has no provisions for mmaps to
1802 * fail). But any other -EIO isn't ours (e.g. swap in failure)
1803 * and so needs to be reported.
1805 if (!i915_terminally_wedged(dev_priv))
1806 return VM_FAULT_SIGBUS;
1807 /* else: fall through */
1810 * EAGAIN means the gpu is hung and we'll wait for the error
1811 * handler to reset everything when re-faulting in
1812 * i915_mutex_lock_interruptible.
1819 * EBUSY is ok: this just means that another thread
1820 * already did the job.
1822 return VM_FAULT_NOPAGE;
1824 return VM_FAULT_OOM;
1827 return VM_FAULT_SIGBUS;
1829 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1830 return VM_FAULT_SIGBUS;
1834 static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
1836 struct i915_vma *vma;
1838 GEM_BUG_ON(!obj->userfault_count);
1840 obj->userfault_count = 0;
1841 list_del(&obj->userfault_link);
1842 drm_vma_node_unmap(&obj->base.vma_node,
1843 obj->base.dev->anon_inode->i_mapping);
1845 for_each_ggtt_vma(vma, obj)
1846 i915_vma_unset_userfault(vma);
1850 * i915_gem_release_mmap - remove physical page mappings
1851 * @obj: obj in question
1853 * Preserve the reservation of the mmapping with the DRM core code, but
1854 * relinquish ownership of the pages back to the system.
1856 * It is vital that we remove the page mapping if we have mapped a tiled
1857 * object through the GTT and then lose the fence register due to
1858 * resource pressure. Similarly if the object has been moved out of the
1859 * aperture, than pages mapped into userspace must be revoked. Removing the
1860 * mapping will then trigger a page fault on the next user access, allowing
1861 * fixup by i915_gem_fault().
1864 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1866 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1867 intel_wakeref_t wakeref;
1869 /* Serialisation between user GTT access and our code depends upon
1870 * revoking the CPU's PTE whilst the mutex is held. The next user
1871 * pagefault then has to wait until we release the mutex.
1873 * Note that RPM complicates somewhat by adding an additional
1874 * requirement that operations to the GGTT be made holding the RPM
1877 lockdep_assert_held(&i915->drm.struct_mutex);
1878 wakeref = intel_runtime_pm_get(i915);
1880 if (!obj->userfault_count)
1883 __i915_gem_object_release_mmap(obj);
1885 /* Ensure that the CPU's PTE are revoked and there are not outstanding
1886 * memory transactions from userspace before we return. The TLB
1887 * flushing implied above by changing the PTE above *should* be
1888 * sufficient, an extra barrier here just provides us with a bit
1889 * of paranoid documentation about our requirement to serialise
1890 * memory writes before touching registers / GSM.
1895 intel_runtime_pm_put(i915, wakeref);
1898 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
1900 struct drm_i915_gem_object *obj, *on;
1904 * Only called during RPM suspend. All users of the userfault_list
1905 * must be holding an RPM wakeref to ensure that this can not
1906 * run concurrently with themselves (and use the struct_mutex for
1907 * protection between themselves).
1910 list_for_each_entry_safe(obj, on,
1911 &dev_priv->mm.userfault_list, userfault_link)
1912 __i915_gem_object_release_mmap(obj);
1914 /* The fence will be lost when the device powers down. If any were
1915 * in use by hardware (i.e. they are pinned), we should not be powering
1916 * down! All other fences will be reacquired by the user upon waking.
1918 for (i = 0; i < dev_priv->num_fence_regs; i++) {
1919 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1921 /* Ideally we want to assert that the fence register is not
1922 * live at this point (i.e. that no piece of code will be
1923 * trying to write through fence + GTT, as that both violates
1924 * our tracking of activity and associated locking/barriers,
1925 * but also is illegal given that the hw is powered down).
1927 * Previously we used reg->pin_count as a "liveness" indicator.
1928 * That is not sufficient, and we need a more fine-grained
1929 * tool if we want to have a sanity check here.
1935 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
1940 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1942 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
1945 err = drm_gem_create_mmap_offset(&obj->base);
1949 /* Attempt to reap some mmap space from dead objects */
1951 err = i915_gem_wait_for_idle(dev_priv,
1952 I915_WAIT_INTERRUPTIBLE,
1953 MAX_SCHEDULE_TIMEOUT);
1957 i915_gem_drain_freed_objects(dev_priv);
1958 err = drm_gem_create_mmap_offset(&obj->base);
1962 } while (flush_delayed_work(&dev_priv->gem.retire_work));
1967 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1969 drm_gem_free_mmap_offset(&obj->base);
1973 i915_gem_mmap_gtt(struct drm_file *file,
1974 struct drm_device *dev,
1978 struct drm_i915_gem_object *obj;
1981 obj = i915_gem_object_lookup(file, handle);
1985 ret = i915_gem_object_create_mmap_offset(obj);
1987 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1989 i915_gem_object_put(obj);
1994 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1996 * @data: GTT mapping ioctl data
1997 * @file: GEM object info
1999 * Simply returns the fake offset to userspace so it can mmap it.
2000 * The mmap call will end up in drm_gem_mmap(), which will set things
2001 * up so we can get faults in the handler above.
2003 * The fault handler will take care of binding the object into the GTT
2004 * (since it may have been evicted to make room for something), allocating
2005 * a fence register, and mapping the appropriate aperture address into
2009 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2010 struct drm_file *file)
2012 struct drm_i915_gem_mmap_gtt *args = data;
2014 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2017 /* Immediately discard the backing storage */
2018 void __i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2020 i915_gem_object_free_mmap_offset(obj);
2022 if (obj->base.filp == NULL)
2025 /* Our goal here is to return as much of the memory as
2026 * is possible back to the system as we are called from OOM.
2027 * To do this we must instruct the shmfs to drop all of its
2028 * backing pages, *now*.
2030 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2031 obj->mm.madv = __I915_MADV_PURGED;
2032 obj->mm.pages = ERR_PTR(-EFAULT);
2035 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
2037 struct radix_tree_iter iter;
2041 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
2042 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
2046 static struct sg_table *
2047 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
2049 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2050 struct sg_table *pages;
2052 pages = fetch_and_zero(&obj->mm.pages);
2053 if (IS_ERR_OR_NULL(pages))
2056 spin_lock(&i915->mm.obj_lock);
2057 list_del(&obj->mm.link);
2058 spin_unlock(&i915->mm.obj_lock);
2060 if (obj->mm.mapping) {
2063 ptr = page_mask_bits(obj->mm.mapping);
2064 if (is_vmalloc_addr(ptr))
2067 kunmap(kmap_to_page(ptr));
2069 obj->mm.mapping = NULL;
2072 __i915_gem_object_reset_page_iter(obj);
2073 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
2078 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
2079 enum i915_mm_subclass subclass)
2081 struct sg_table *pages;
2084 if (i915_gem_object_has_pinned_pages(obj))
2087 GEM_BUG_ON(obj->bind_count);
2089 /* May be called by shrinker from within get_pages() (on another bo) */
2090 mutex_lock_nested(&obj->mm.lock, subclass);
2091 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) {
2097 * ->put_pages might need to allocate memory for the bit17 swizzle
2098 * array, hence protect them from being reaped by removing them from gtt
2101 pages = __i915_gem_object_unset_pages(obj);
2104 * XXX Temporary hijinx to avoid updating all backends to handle
2105 * NULL pages. In the future, when we have more asynchronous
2106 * get_pages backends we should be better able to handle the
2107 * cancellation of the async task in a more uniform manner.
2109 if (!pages && !i915_gem_object_needs_async_cancel(obj))
2110 pages = ERR_PTR(-EINVAL);
2113 obj->ops->put_pages(obj, pages);
2117 mutex_unlock(&obj->mm.lock);
2122 bool i915_sg_trim(struct sg_table *orig_st)
2124 struct sg_table new_st;
2125 struct scatterlist *sg, *new_sg;
2128 if (orig_st->nents == orig_st->orig_nents)
2131 if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
2134 new_sg = new_st.sgl;
2135 for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2136 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2137 sg_dma_address(new_sg) = sg_dma_address(sg);
2138 sg_dma_len(new_sg) = sg_dma_len(sg);
2140 new_sg = sg_next(new_sg);
2142 GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
2144 sg_free_table(orig_st);
2150 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
2151 struct sg_table *pages,
2152 unsigned int sg_page_sizes)
2154 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2155 unsigned long supported = INTEL_INFO(i915)->page_sizes;
2158 lockdep_assert_held(&obj->mm.lock);
2160 /* Make the pages coherent with the GPU (flushing any swapin). */
2161 if (obj->cache_dirty) {
2162 obj->write_domain = 0;
2163 if (i915_gem_object_has_struct_page(obj))
2164 drm_clflush_sg(pages);
2165 obj->cache_dirty = false;
2168 obj->mm.get_page.sg_pos = pages->sgl;
2169 obj->mm.get_page.sg_idx = 0;
2171 obj->mm.pages = pages;
2173 if (i915_gem_object_is_tiled(obj) &&
2174 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
2175 GEM_BUG_ON(obj->mm.quirked);
2176 __i915_gem_object_pin_pages(obj);
2177 obj->mm.quirked = true;
2180 GEM_BUG_ON(!sg_page_sizes);
2181 obj->mm.page_sizes.phys = sg_page_sizes;
2184 * Calculate the supported page-sizes which fit into the given
2185 * sg_page_sizes. This will give us the page-sizes which we may be able
2186 * to use opportunistically when later inserting into the GTT. For
2187 * example if phys=2G, then in theory we should be able to use 1G, 2M,
2188 * 64K or 4K pages, although in practice this will depend on a number of
2191 obj->mm.page_sizes.sg = 0;
2192 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
2193 if (obj->mm.page_sizes.phys & ~0u << i)
2194 obj->mm.page_sizes.sg |= BIT(i);
2196 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
2198 spin_lock(&i915->mm.obj_lock);
2199 list_add(&obj->mm.link, &i915->mm.unbound_list);
2200 spin_unlock(&i915->mm.obj_lock);
2203 static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2207 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
2208 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2212 err = obj->ops->get_pages(obj);
2213 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
2218 /* Ensure that the associated pages are gathered from the backing storage
2219 * and pinned into our object. i915_gem_object_pin_pages() may be called
2220 * multiple times before they are released by a single call to
2221 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
2222 * either as a result of memory pressure (reaping pages under the shrinker)
2223 * or as the object is itself released.
2225 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2229 err = mutex_lock_interruptible(&obj->mm.lock);
2233 if (unlikely(!i915_gem_object_has_pages(obj))) {
2234 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2236 err = ____i915_gem_object_get_pages(obj);
2240 smp_mb__before_atomic();
2242 atomic_inc(&obj->mm.pages_pin_count);
2245 mutex_unlock(&obj->mm.lock);
2249 /* The 'mapping' part of i915_gem_object_pin_map() below */
2250 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2251 enum i915_map_type type)
2253 unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2254 struct sg_table *sgt = obj->mm.pages;
2255 struct sgt_iter sgt_iter;
2257 struct page *stack_pages[32];
2258 struct page **pages = stack_pages;
2259 unsigned long i = 0;
2263 /* A single page can always be kmapped */
2264 if (n_pages == 1 && type == I915_MAP_WB)
2265 return kmap(sg_page(sgt->sgl));
2267 if (n_pages > ARRAY_SIZE(stack_pages)) {
2268 /* Too big for stack -- allocate temporary array instead */
2269 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
2274 for_each_sgt_page(page, sgt_iter, sgt)
2277 /* Check that we have the expected number of pages */
2278 GEM_BUG_ON(i != n_pages);
2283 /* fallthrough to use PAGE_KERNEL anyway */
2285 pgprot = PAGE_KERNEL;
2288 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2291 addr = vmap(pages, n_pages, 0, pgprot);
2293 if (pages != stack_pages)
2299 /* get, pin, and map the pages of the object into kernel space */
2300 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2301 enum i915_map_type type)
2303 enum i915_map_type has_type;
2308 if (unlikely(!i915_gem_object_has_struct_page(obj)))
2309 return ERR_PTR(-ENXIO);
2311 ret = mutex_lock_interruptible(&obj->mm.lock);
2313 return ERR_PTR(ret);
2315 pinned = !(type & I915_MAP_OVERRIDE);
2316 type &= ~I915_MAP_OVERRIDE;
2318 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
2319 if (unlikely(!i915_gem_object_has_pages(obj))) {
2320 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2322 ret = ____i915_gem_object_get_pages(obj);
2326 smp_mb__before_atomic();
2328 atomic_inc(&obj->mm.pages_pin_count);
2331 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
2333 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
2334 if (ptr && has_type != type) {
2340 if (is_vmalloc_addr(ptr))
2343 kunmap(kmap_to_page(ptr));
2345 ptr = obj->mm.mapping = NULL;
2349 ptr = i915_gem_object_map(obj, type);
2355 obj->mm.mapping = page_pack_bits(ptr, type);
2359 mutex_unlock(&obj->mm.lock);
2363 atomic_dec(&obj->mm.pages_pin_count);
2369 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
2370 unsigned long offset,
2373 enum i915_map_type has_type;
2376 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
2377 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
2378 offset, size, obj->base.size));
2380 obj->mm.dirty = true;
2382 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
2385 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
2386 if (has_type == I915_MAP_WC)
2389 drm_clflush_virt_range(ptr + offset, size);
2390 if (size == obj->base.size) {
2391 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
2392 obj->cache_dirty = false;
2396 static unsigned long to_wait_timeout(s64 timeout_ns)
2399 return MAX_SCHEDULE_TIMEOUT;
2401 if (timeout_ns == 0)
2404 return nsecs_to_jiffies_timeout(timeout_ns);
2408 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2409 * @dev: drm device pointer
2410 * @data: ioctl data blob
2411 * @file: drm file pointer
2413 * Returns 0 if successful, else an error is returned with the remaining time in
2414 * the timeout parameter.
2415 * -ETIME: object is still busy after timeout
2416 * -ERESTARTSYS: signal interrupted the wait
2417 * -ENONENT: object doesn't exist
2418 * Also possible, but rare:
2419 * -EAGAIN: incomplete, restart syscall
2421 * -ENODEV: Internal IRQ fail
2422 * -E?: The add request failed
2424 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2425 * non-zero timeout parameter the wait ioctl will wait for the given number of
2426 * nanoseconds on an object becoming unbusy. Since the wait itself does so
2427 * without holding struct_mutex the object may become re-busied before this
2428 * function completes. A similar but shorter * race condition exists in the busy
2432 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2434 struct drm_i915_gem_wait *args = data;
2435 struct drm_i915_gem_object *obj;
2439 if (args->flags != 0)
2442 obj = i915_gem_object_lookup(file, args->bo_handle);
2446 start = ktime_get();
2448 ret = i915_gem_object_wait(obj,
2449 I915_WAIT_INTERRUPTIBLE |
2450 I915_WAIT_PRIORITY |
2452 to_wait_timeout(args->timeout_ns));
2454 if (args->timeout_ns > 0) {
2455 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
2456 if (args->timeout_ns < 0)
2457 args->timeout_ns = 0;
2460 * Apparently ktime isn't accurate enough and occasionally has a
2461 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
2462 * things up to make the test happy. We allow up to 1 jiffy.
2464 * This is a regression from the timespec->ktime conversion.
2466 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
2467 args->timeout_ns = 0;
2469 /* Asked to wait beyond the jiffie/scheduler precision? */
2470 if (ret == -ETIME && args->timeout_ns)
2474 i915_gem_object_put(obj);
2478 static int wait_for_engines(struct drm_i915_private *i915)
2480 if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
2481 dev_err(i915->drm.dev,
2482 "Failed to idle engines, declaring wedged!\n");
2484 i915_gem_set_wedged(i915);
2492 wait_for_timelines(struct drm_i915_private *i915,
2493 unsigned int flags, long timeout)
2495 struct i915_gt_timelines *gt = &i915->gt.timelines;
2496 struct i915_timeline *tl;
2498 mutex_lock(>->mutex);
2499 list_for_each_entry(tl, >->active_list, link) {
2500 struct i915_request *rq;
2502 rq = i915_active_request_get_unlocked(&tl->last_request);
2506 mutex_unlock(>->mutex);
2511 * Switching to the kernel context is often used a synchronous
2512 * step prior to idling, e.g. in suspend for flushing all
2513 * current operations to memory before sleeping. These we
2514 * want to complete as quickly as possible to avoid prolonged
2515 * stalls, so allow the gpu to boost to maximum clocks.
2517 if (flags & I915_WAIT_FOR_IDLE_BOOST)
2520 timeout = i915_request_wait(rq, flags, timeout);
2521 i915_request_put(rq);
2525 /* restart after reacquiring the lock */
2526 mutex_lock(>->mutex);
2527 tl = list_entry(>->active_list, typeof(*tl), link);
2529 mutex_unlock(>->mutex);
2534 int i915_gem_wait_for_idle(struct drm_i915_private *i915,
2535 unsigned int flags, long timeout)
2537 GEM_TRACE("flags=%x (%s), timeout=%ld%s, awake?=%s\n",
2538 flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
2539 timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "",
2540 yesno(i915->gt.awake));
2542 /* If the device is asleep, we have no requests outstanding */
2543 if (!READ_ONCE(i915->gt.awake))
2546 timeout = wait_for_timelines(i915, flags, timeout);
2550 if (flags & I915_WAIT_LOCKED) {
2553 lockdep_assert_held(&i915->drm.struct_mutex);
2555 err = wait_for_engines(i915);
2559 i915_retire_requests(i915);
2565 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
2568 * We manually flush the CPU domain so that we can override and
2569 * force the flush for the display, and perform it asyncrhonously.
2571 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
2572 if (obj->cache_dirty)
2573 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
2574 obj->write_domain = 0;
2577 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
2579 if (!READ_ONCE(obj->pin_global))
2582 mutex_lock(&obj->base.dev->struct_mutex);
2583 __i915_gem_object_flush_for_display(obj);
2584 mutex_unlock(&obj->base.dev->struct_mutex);
2588 * Moves a single object to the WC read, and possibly write domain.
2589 * @obj: object to act on
2590 * @write: ask for write access or read only
2592 * This function returns when the move is complete, including waiting on
2596 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
2600 lockdep_assert_held(&obj->base.dev->struct_mutex);
2602 ret = i915_gem_object_wait(obj,
2603 I915_WAIT_INTERRUPTIBLE |
2605 (write ? I915_WAIT_ALL : 0),
2606 MAX_SCHEDULE_TIMEOUT);
2610 if (obj->write_domain == I915_GEM_DOMAIN_WC)
2613 /* Flush and acquire obj->pages so that we are coherent through
2614 * direct access in memory with previous cached writes through
2615 * shmemfs and that our cache domain tracking remains valid.
2616 * For example, if the obj->filp was moved to swap without us
2617 * being notified and releasing the pages, we would mistakenly
2618 * continue to assume that the obj remained out of the CPU cached
2621 ret = i915_gem_object_pin_pages(obj);
2625 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
2627 /* Serialise direct access to this object with the barriers for
2628 * coherent writes from the GPU, by effectively invalidating the
2629 * WC domain upon first access.
2631 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
2634 /* It should now be out of any other write domains, and we can update
2635 * the domain values for our changes.
2637 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
2638 obj->read_domains |= I915_GEM_DOMAIN_WC;
2640 obj->read_domains = I915_GEM_DOMAIN_WC;
2641 obj->write_domain = I915_GEM_DOMAIN_WC;
2642 obj->mm.dirty = true;
2645 i915_gem_object_unpin_pages(obj);
2650 * Moves a single object to the GTT read, and possibly write domain.
2651 * @obj: object to act on
2652 * @write: ask for write access or read only
2654 * This function returns when the move is complete, including waiting on
2658 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2662 lockdep_assert_held(&obj->base.dev->struct_mutex);
2664 ret = i915_gem_object_wait(obj,
2665 I915_WAIT_INTERRUPTIBLE |
2667 (write ? I915_WAIT_ALL : 0),
2668 MAX_SCHEDULE_TIMEOUT);
2672 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
2675 /* Flush and acquire obj->pages so that we are coherent through
2676 * direct access in memory with previous cached writes through
2677 * shmemfs and that our cache domain tracking remains valid.
2678 * For example, if the obj->filp was moved to swap without us
2679 * being notified and releasing the pages, we would mistakenly
2680 * continue to assume that the obj remained out of the CPU cached
2683 ret = i915_gem_object_pin_pages(obj);
2687 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
2689 /* Serialise direct access to this object with the barriers for
2690 * coherent writes from the GPU, by effectively invalidating the
2691 * GTT domain upon first access.
2693 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
2696 /* It should now be out of any other write domains, and we can update
2697 * the domain values for our changes.
2699 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2700 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2702 obj->read_domains = I915_GEM_DOMAIN_GTT;
2703 obj->write_domain = I915_GEM_DOMAIN_GTT;
2704 obj->mm.dirty = true;
2707 i915_gem_object_unpin_pages(obj);
2712 * Changes the cache-level of an object across all VMA.
2713 * @obj: object to act on
2714 * @cache_level: new cache level to set for the object
2716 * After this function returns, the object will be in the new cache-level
2717 * across all GTT and the contents of the backing storage will be coherent,
2718 * with respect to the new cache-level. In order to keep the backing storage
2719 * coherent for all users, we only allow a single cache level to be set
2720 * globally on the object and prevent it from being changed whilst the
2721 * hardware is reading from the object. That is if the object is currently
2722 * on the scanout it will be set to uncached (or equivalent display
2723 * cache coherency) and all non-MOCS GPU access will also be uncached so
2724 * that all direct access to the scanout remains coherent.
2726 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2727 enum i915_cache_level cache_level)
2729 struct i915_vma *vma;
2732 lockdep_assert_held(&obj->base.dev->struct_mutex);
2734 if (obj->cache_level == cache_level)
2737 /* Inspect the list of currently bound VMA and unbind any that would
2738 * be invalid given the new cache-level. This is principally to
2739 * catch the issue of the CS prefetch crossing page boundaries and
2740 * reading an invalid PTE on older architectures.
2743 list_for_each_entry(vma, &obj->vma.list, obj_link) {
2744 if (!drm_mm_node_allocated(&vma->node))
2747 if (i915_vma_is_pinned(vma)) {
2748 DRM_DEBUG("can not change the cache level of pinned objects\n");
2752 if (!i915_vma_is_closed(vma) &&
2753 i915_gem_valid_gtt_space(vma, cache_level))
2756 ret = i915_vma_unbind(vma);
2760 /* As unbinding may affect other elements in the
2761 * obj->vma_list (due to side-effects from retiring
2762 * an active vma), play safe and restart the iterator.
2767 /* We can reuse the existing drm_mm nodes but need to change the
2768 * cache-level on the PTE. We could simply unbind them all and
2769 * rebind with the correct cache-level on next use. However since
2770 * we already have a valid slot, dma mapping, pages etc, we may as
2771 * rewrite the PTE in the belief that doing so tramples upon less
2772 * state and so involves less work.
2774 if (obj->bind_count) {
2775 /* Before we change the PTE, the GPU must not be accessing it.
2776 * If we wait upon the object, we know that all the bound
2777 * VMA are no longer active.
2779 ret = i915_gem_object_wait(obj,
2780 I915_WAIT_INTERRUPTIBLE |
2783 MAX_SCHEDULE_TIMEOUT);
2787 if (!HAS_LLC(to_i915(obj->base.dev)) &&
2788 cache_level != I915_CACHE_NONE) {
2789 /* Access to snoopable pages through the GTT is
2790 * incoherent and on some machines causes a hard
2791 * lockup. Relinquish the CPU mmaping to force
2792 * userspace to refault in the pages and we can
2793 * then double check if the GTT mapping is still
2794 * valid for that pointer access.
2796 i915_gem_release_mmap(obj);
2798 /* As we no longer need a fence for GTT access,
2799 * we can relinquish it now (and so prevent having
2800 * to steal a fence from someone else on the next
2801 * fence request). Note GPU activity would have
2802 * dropped the fence as all snoopable access is
2803 * supposed to be linear.
2805 for_each_ggtt_vma(vma, obj) {
2806 ret = i915_vma_put_fence(vma);
2811 /* We either have incoherent backing store and
2812 * so no GTT access or the architecture is fully
2813 * coherent. In such cases, existing GTT mmaps
2814 * ignore the cache bit in the PTE and we can
2815 * rewrite it without confusing the GPU or having
2816 * to force userspace to fault back in its mmaps.
2820 list_for_each_entry(vma, &obj->vma.list, obj_link) {
2821 if (!drm_mm_node_allocated(&vma->node))
2824 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
2830 list_for_each_entry(vma, &obj->vma.list, obj_link)
2831 vma->node.color = cache_level;
2832 i915_gem_object_set_cache_coherency(obj, cache_level);
2833 obj->cache_dirty = true; /* Always invalidate stale cachelines */
2838 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
2839 struct drm_file *file)
2841 struct drm_i915_gem_caching *args = data;
2842 struct drm_i915_gem_object *obj;
2846 obj = i915_gem_object_lookup_rcu(file, args->handle);
2852 switch (obj->cache_level) {
2853 case I915_CACHE_LLC:
2854 case I915_CACHE_L3_LLC:
2855 args->caching = I915_CACHING_CACHED;
2859 args->caching = I915_CACHING_DISPLAY;
2863 args->caching = I915_CACHING_NONE;
2871 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
2872 struct drm_file *file)
2874 struct drm_i915_private *i915 = to_i915(dev);
2875 struct drm_i915_gem_caching *args = data;
2876 struct drm_i915_gem_object *obj;
2877 enum i915_cache_level level;
2880 switch (args->caching) {
2881 case I915_CACHING_NONE:
2882 level = I915_CACHE_NONE;
2884 case I915_CACHING_CACHED:
2886 * Due to a HW issue on BXT A stepping, GPU stores via a
2887 * snooped mapping may leave stale data in a corresponding CPU
2888 * cacheline, whereas normally such cachelines would get
2891 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
2894 level = I915_CACHE_LLC;
2896 case I915_CACHING_DISPLAY:
2897 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
2903 obj = i915_gem_object_lookup(file, args->handle);
2908 * The caching mode of proxy object is handled by its generator, and
2909 * not allowed to be changed by userspace.
2911 if (i915_gem_object_is_proxy(obj)) {
2916 if (obj->cache_level == level)
2919 ret = i915_gem_object_wait(obj,
2920 I915_WAIT_INTERRUPTIBLE,
2921 MAX_SCHEDULE_TIMEOUT);
2925 ret = i915_mutex_lock_interruptible(dev);
2929 ret = i915_gem_object_set_cache_level(obj, level);
2930 mutex_unlock(&dev->struct_mutex);
2933 i915_gem_object_put(obj);
2938 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
2939 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
2940 * (for pageflips). We only flush the caches while preparing the buffer for
2941 * display, the callers are responsible for frontbuffer flush.
2944 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2946 const struct i915_ggtt_view *view,
2949 struct i915_vma *vma;
2952 lockdep_assert_held(&obj->base.dev->struct_mutex);
2954 /* Mark the global pin early so that we account for the
2955 * display coherency whilst setting up the cache domains.
2959 /* The display engine is not coherent with the LLC cache on gen6. As
2960 * a result, we make sure that the pinning that is about to occur is
2961 * done with uncached PTEs. This is lowest common denominator for all
2964 * However for gen6+, we could do better by using the GFDT bit instead
2965 * of uncaching, which would allow us to flush all the LLC-cached data
2966 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2968 ret = i915_gem_object_set_cache_level(obj,
2969 HAS_WT(to_i915(obj->base.dev)) ?
2970 I915_CACHE_WT : I915_CACHE_NONE);
2973 goto err_unpin_global;
2976 /* As the user may map the buffer once pinned in the display plane
2977 * (e.g. libkms for the bootup splash), we have to ensure that we
2978 * always use map_and_fenceable for all scanout buffers. However,
2979 * it may simply be too big to fit into mappable, in which case
2980 * put it anyway and hope that userspace can cope (but always first
2981 * try to preserve the existing ABI).
2983 vma = ERR_PTR(-ENOSPC);
2984 if ((flags & PIN_MAPPABLE) == 0 &&
2985 (!view || view->type == I915_GGTT_VIEW_NORMAL))
2986 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
2991 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
2993 goto err_unpin_global;
2995 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
2997 __i915_gem_object_flush_for_display(obj);
2999 /* It should now be out of any other write domains, and we can update
3000 * the domain values for our changes.
3002 obj->read_domains |= I915_GEM_DOMAIN_GTT;
3012 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
3014 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
3016 if (WARN_ON(vma->obj->pin_global == 0))
3019 if (--vma->obj->pin_global == 0)
3020 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
3022 /* Bump the LRU to try and avoid premature eviction whilst flipping */
3023 i915_gem_object_bump_inactive_ggtt(vma->obj);
3025 i915_vma_unpin(vma);
3029 * Moves a single object to the CPU read, and possibly write domain.
3030 * @obj: object to act on
3031 * @write: requesting write or read-only access
3033 * This function returns when the move is complete, including waiting on
3037 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3041 lockdep_assert_held(&obj->base.dev->struct_mutex);
3043 ret = i915_gem_object_wait(obj,
3044 I915_WAIT_INTERRUPTIBLE |
3046 (write ? I915_WAIT_ALL : 0),
3047 MAX_SCHEDULE_TIMEOUT);
3051 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
3053 /* Flush the CPU cache if it's still invalid. */
3054 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3055 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
3056 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3059 /* It should now be out of any other write domains, and we can update
3060 * the domain values for our changes.
3062 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
3064 /* If we're writing through the CPU, then the GPU read domains will
3065 * need to be invalidated at next use.
3068 __start_cpu_write(obj);
3073 /* Throttle our rendering by waiting until the ring has completed our requests
3074 * emitted over 20 msec ago.
3076 * Note that if we were to use the current jiffies each time around the loop,
3077 * we wouldn't escape the function with any frames outstanding if the time to
3078 * render a frame was over 20ms.
3080 * This should get us reasonable parallelism between CPU and GPU but also
3081 * relatively low latency when blocking on a particular request to finish.
3084 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3086 struct drm_i915_private *dev_priv = to_i915(dev);
3087 struct drm_i915_file_private *file_priv = file->driver_priv;
3088 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3089 struct i915_request *request, *target = NULL;
3092 /* ABI: return -EIO if already wedged */
3093 ret = i915_terminally_wedged(dev_priv);
3097 spin_lock(&file_priv->mm.lock);
3098 list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
3099 if (time_after_eq(request->emitted_jiffies, recent_enough))
3103 list_del(&target->client_link);
3104 target->file_priv = NULL;
3110 i915_request_get(target);
3111 spin_unlock(&file_priv->mm.lock);
3116 ret = i915_request_wait(target,
3117 I915_WAIT_INTERRUPTIBLE,
3118 MAX_SCHEDULE_TIMEOUT);
3119 i915_request_put(target);
3121 return ret < 0 ? ret : 0;
3125 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3126 const struct i915_ggtt_view *view,
3131 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3132 struct i915_address_space *vm = &dev_priv->ggtt.vm;
3133 struct i915_vma *vma;
3136 lockdep_assert_held(&obj->base.dev->struct_mutex);
3138 if (flags & PIN_MAPPABLE &&
3139 (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
3140 /* If the required space is larger than the available
3141 * aperture, we will not able to find a slot for the
3142 * object and unbinding the object now will be in
3143 * vain. Worse, doing so may cause us to ping-pong
3144 * the object in and out of the Global GTT and
3145 * waste a lot of cycles under the mutex.
3147 if (obj->base.size > dev_priv->ggtt.mappable_end)
3148 return ERR_PTR(-E2BIG);
3150 /* If NONBLOCK is set the caller is optimistically
3151 * trying to cache the full object within the mappable
3152 * aperture, and *must* have a fallback in place for
3153 * situations where we cannot bind the object. We
3154 * can be a little more lax here and use the fallback
3155 * more often to avoid costly migrations of ourselves
3156 * and other objects within the aperture.
3158 * Half-the-aperture is used as a simple heuristic.
3159 * More interesting would to do search for a free
3160 * block prior to making the commitment to unbind.
3161 * That caters for the self-harm case, and with a
3162 * little more heuristics (e.g. NOFAULT, NOEVICT)
3163 * we could try to minimise harm to others.
3165 if (flags & PIN_NONBLOCK &&
3166 obj->base.size > dev_priv->ggtt.mappable_end / 2)
3167 return ERR_PTR(-ENOSPC);
3170 vma = i915_vma_instance(obj, vm, view);
3174 if (i915_vma_misplaced(vma, size, alignment, flags)) {
3175 if (flags & PIN_NONBLOCK) {
3176 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
3177 return ERR_PTR(-ENOSPC);
3179 if (flags & PIN_MAPPABLE &&
3180 vma->fence_size > dev_priv->ggtt.mappable_end / 2)
3181 return ERR_PTR(-ENOSPC);
3184 WARN(i915_vma_is_pinned(vma),
3185 "bo is already pinned in ggtt with incorrect alignment:"
3186 " offset=%08x, req.alignment=%llx,"
3187 " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
3188 i915_ggtt_offset(vma), alignment,
3189 !!(flags & PIN_MAPPABLE),
3190 i915_vma_is_map_and_fenceable(vma));
3191 ret = i915_vma_unbind(vma);
3193 return ERR_PTR(ret);
3196 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
3198 return ERR_PTR(ret);
3203 static __always_inline u32 __busy_read_flag(u8 id)
3205 if (id == (u8)I915_ENGINE_CLASS_INVALID)
3208 GEM_BUG_ON(id >= 16);
3209 return 0x10000u << id;
3212 static __always_inline u32 __busy_write_id(u8 id)
3215 * The uABI guarantees an active writer is also amongst the read
3216 * engines. This would be true if we accessed the activity tracking
3217 * under the lock, but as we perform the lookup of the object and
3218 * its activity locklessly we can not guarantee that the last_write
3219 * being active implies that we have set the same engine flag from
3220 * last_read - hence we always set both read and write busy for
3223 if (id == (u8)I915_ENGINE_CLASS_INVALID)
3226 return (id + 1) | __busy_read_flag(id);
3229 static __always_inline unsigned int
3230 __busy_set_if_active(const struct dma_fence *fence, u32 (*flag)(u8 id))
3232 const struct i915_request *rq;
3235 * We have to check the current hw status of the fence as the uABI
3236 * guarantees forward progress. We could rely on the idle worker
3237 * to eventually flush us, but to minimise latency just ask the
3240 * Note we only report on the status of native fences.
3242 if (!dma_fence_is_i915(fence))
3245 /* opencode to_request() in order to avoid const warnings */
3246 rq = container_of(fence, const struct i915_request, fence);
3247 if (i915_request_completed(rq))
3250 /* Beware type-expansion follies! */
3251 BUILD_BUG_ON(!typecheck(u8, rq->engine->uabi_class));
3252 return flag(rq->engine->uabi_class);
3255 static __always_inline unsigned int
3256 busy_check_reader(const struct dma_fence *fence)
3258 return __busy_set_if_active(fence, __busy_read_flag);
3261 static __always_inline unsigned int
3262 busy_check_writer(const struct dma_fence *fence)
3267 return __busy_set_if_active(fence, __busy_write_id);
3271 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3272 struct drm_file *file)
3274 struct drm_i915_gem_busy *args = data;
3275 struct drm_i915_gem_object *obj;
3276 struct reservation_object_list *list;
3282 obj = i915_gem_object_lookup_rcu(file, args->handle);
3287 * A discrepancy here is that we do not report the status of
3288 * non-i915 fences, i.e. even though we may report the object as idle,
3289 * a call to set-domain may still stall waiting for foreign rendering.
3290 * This also means that wait-ioctl may report an object as busy,
3291 * where busy-ioctl considers it idle.
3293 * We trade the ability to warn of foreign fences to report on which
3294 * i915 engines are active for the object.
3296 * Alternatively, we can trade that extra information on read/write
3299 * !reservation_object_test_signaled_rcu(obj->resv, true);
3300 * to report the overall busyness. This is what the wait-ioctl does.
3304 seq = raw_read_seqcount(&obj->resv->seq);
3306 /* Translate the exclusive fence to the READ *and* WRITE engine */
3307 args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
3309 /* Translate shared fences to READ set of engines */
3310 list = rcu_dereference(obj->resv->fence);
3312 unsigned int shared_count = list->shared_count, i;
3314 for (i = 0; i < shared_count; ++i) {
3315 struct dma_fence *fence =
3316 rcu_dereference(list->shared[i]);
3318 args->busy |= busy_check_reader(fence);
3322 if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
3332 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3333 struct drm_file *file_priv)
3335 return i915_gem_ring_throttle(dev, file_priv);
3339 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3340 struct drm_file *file_priv)
3342 struct drm_i915_private *dev_priv = to_i915(dev);
3343 struct drm_i915_gem_madvise *args = data;
3344 struct drm_i915_gem_object *obj;
3347 switch (args->madv) {
3348 case I915_MADV_DONTNEED:
3349 case I915_MADV_WILLNEED:
3355 obj = i915_gem_object_lookup(file_priv, args->handle);
3359 err = mutex_lock_interruptible(&obj->mm.lock);
3363 if (i915_gem_object_has_pages(obj) &&
3364 i915_gem_object_is_tiled(obj) &&
3365 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
3366 if (obj->mm.madv == I915_MADV_WILLNEED) {
3367 GEM_BUG_ON(!obj->mm.quirked);
3368 __i915_gem_object_unpin_pages(obj);
3369 obj->mm.quirked = false;
3371 if (args->madv == I915_MADV_WILLNEED) {
3372 GEM_BUG_ON(obj->mm.quirked);
3373 __i915_gem_object_pin_pages(obj);
3374 obj->mm.quirked = true;
3378 if (obj->mm.madv != __I915_MADV_PURGED)
3379 obj->mm.madv = args->madv;
3381 /* if the object is no longer attached, discard its backing storage */
3382 if (obj->mm.madv == I915_MADV_DONTNEED &&
3383 !i915_gem_object_has_pages(obj))
3384 __i915_gem_object_truncate(obj);
3386 args->retained = obj->mm.madv != __I915_MADV_PURGED;
3387 mutex_unlock(&obj->mm.lock);
3390 i915_gem_object_put(obj);
3394 void i915_gem_sanitize(struct drm_i915_private *i915)
3396 intel_wakeref_t wakeref;
3400 wakeref = intel_runtime_pm_get(i915);
3401 intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
3404 * As we have just resumed the machine and woken the device up from
3405 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
3406 * back to defaults, recovering from whatever wedged state we left it
3407 * in and so worth trying to use the device once more.
3409 if (i915_terminally_wedged(i915))
3410 i915_gem_unset_wedged(i915);
3413 * If we inherit context state from the BIOS or earlier occupants
3414 * of the GPU, the GPU may be in an inconsistent state when we
3415 * try to take over. The only way to remove the earlier state
3416 * is by resetting. However, resetting on earlier gen is tricky as
3417 * it may impact the display and we are uncertain about the stability
3418 * of the reset, so this could be applied to even earlier gen.
3420 intel_gt_sanitize(i915, false);
3422 intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
3423 intel_runtime_pm_put(i915, wakeref);
3425 mutex_lock(&i915->drm.struct_mutex);
3426 i915_gem_contexts_lost(i915);
3427 mutex_unlock(&i915->drm.struct_mutex);
3430 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
3432 if (INTEL_GEN(dev_priv) < 5 ||
3433 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3436 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3437 DISP_TILE_SURFACE_SWIZZLING);
3439 if (IS_GEN(dev_priv, 5))
3442 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3443 if (IS_GEN(dev_priv, 6))
3444 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3445 else if (IS_GEN(dev_priv, 7))
3446 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3447 else if (IS_GEN(dev_priv, 8))
3448 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
3453 static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
3455 I915_WRITE(RING_CTL(base), 0);
3456 I915_WRITE(RING_HEAD(base), 0);
3457 I915_WRITE(RING_TAIL(base), 0);
3458 I915_WRITE(RING_START(base), 0);
3461 static void init_unused_rings(struct drm_i915_private *dev_priv)
3463 if (IS_I830(dev_priv)) {
3464 init_unused_ring(dev_priv, PRB1_BASE);
3465 init_unused_ring(dev_priv, SRB0_BASE);
3466 init_unused_ring(dev_priv, SRB1_BASE);
3467 init_unused_ring(dev_priv, SRB2_BASE);
3468 init_unused_ring(dev_priv, SRB3_BASE);
3469 } else if (IS_GEN(dev_priv, 2)) {
3470 init_unused_ring(dev_priv, SRB0_BASE);
3471 init_unused_ring(dev_priv, SRB1_BASE);
3472 } else if (IS_GEN(dev_priv, 3)) {
3473 init_unused_ring(dev_priv, PRB1_BASE);
3474 init_unused_ring(dev_priv, PRB2_BASE);
3478 int i915_gem_init_hw(struct drm_i915_private *dev_priv)
3482 dev_priv->gt.last_init_time = ktime_get();
3484 /* Double layer security blanket, see i915_gem_init() */
3485 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
3487 if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
3488 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
3490 if (IS_HASWELL(dev_priv))
3491 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
3492 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
3494 /* Apply the GT workarounds... */
3495 intel_gt_apply_workarounds(dev_priv);
3496 /* ...and determine whether they are sticking. */
3497 intel_gt_verify_workarounds(dev_priv, "init");
3499 i915_gem_init_swizzling(dev_priv);
3502 * At least 830 can leave some of the unused rings
3503 * "active" (ie. head != tail) after resume which
3504 * will prevent c3 entry. Makes sure all unused rings
3507 init_unused_rings(dev_priv);
3509 BUG_ON(!dev_priv->kernel_context);
3510 ret = i915_terminally_wedged(dev_priv);
3514 ret = i915_ppgtt_init_hw(dev_priv);
3516 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
3520 ret = intel_wopcm_init_hw(&dev_priv->wopcm);
3522 DRM_ERROR("Enabling WOPCM failed (%d)\n", ret);
3526 /* We can't enable contexts until all firmware is loaded */
3527 ret = intel_uc_init_hw(dev_priv);
3529 DRM_ERROR("Enabling uc failed (%d)\n", ret);
3533 intel_mocs_init_l3cc_table(dev_priv);
3535 /* Only when the HW is re-initialised, can we replay the requests */
3536 ret = intel_engines_resume(dev_priv);
3540 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
3542 intel_engines_set_scheduler_caps(dev_priv);
3546 intel_uc_fini_hw(dev_priv);
3548 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
3553 static int __intel_engines_record_defaults(struct drm_i915_private *i915)
3555 struct intel_engine_cs *engine;
3556 struct i915_gem_context *ctx;
3557 struct i915_gem_engines *e;
3558 enum intel_engine_id id;
3562 * As we reset the gpu during very early sanitisation, the current
3563 * register state on the GPU should reflect its defaults values.
3564 * We load a context onto the hw (with restore-inhibit), then switch
3565 * over to a second context to save that default register state. We
3566 * can then prime every new context with that state so they all start
3567 * from the same default HW values.
3570 ctx = i915_gem_context_create_kernel(i915, 0);
3572 return PTR_ERR(ctx);
3574 e = i915_gem_context_lock_engines(ctx);
3576 for_each_engine(engine, i915, id) {
3577 struct intel_context *ce = e->engines[id];
3578 struct i915_request *rq;
3580 rq = intel_context_create_request(ce);
3587 if (rq->engine->init_context)
3588 err = rq->engine->init_context(rq);
3590 i915_request_add(rq);
3595 /* Flush the default context image to memory, and enable powersaving. */
3596 if (!i915_gem_load_power_context(i915)) {
3601 for_each_engine(engine, i915, id) {
3602 struct intel_context *ce = e->engines[id];
3603 struct i915_vma *state = ce->state;
3609 GEM_BUG_ON(intel_context_is_pinned(ce));
3612 * As we will hold a reference to the logical state, it will
3613 * not be torn down with the context, and importantly the
3614 * object will hold onto its vma (making it possible for a
3615 * stray GTT write to corrupt our defaults). Unmap the vma
3616 * from the GTT to prevent such accidents and reclaim the
3619 err = i915_vma_unbind(state);
3623 err = i915_gem_object_set_to_cpu_domain(state->obj, false);
3627 engine->default_state = i915_gem_object_get(state->obj);
3628 i915_gem_object_set_cache_coherency(engine->default_state,
3631 /* Check we can acquire the image of the context state */
3632 vaddr = i915_gem_object_pin_map(engine->default_state,
3634 if (IS_ERR(vaddr)) {
3635 err = PTR_ERR(vaddr);
3639 i915_gem_object_unpin_map(engine->default_state);
3642 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
3643 unsigned int found = intel_engines_has_context_isolation(i915);
3646 * Make sure that classes with multiple engine instances all
3647 * share the same basic configuration.
3649 for_each_engine(engine, i915, id) {
3650 unsigned int bit = BIT(engine->uabi_class);
3651 unsigned int expected = engine->default_state ? bit : 0;
3653 if ((found & bit) != expected) {
3654 DRM_ERROR("mismatching default context state for class %d on engine %s\n",
3655 engine->uabi_class, engine->name);
3661 i915_gem_context_unlock_engines(ctx);
3662 i915_gem_context_set_closed(ctx);
3663 i915_gem_context_put(ctx);
3668 * If we have to abandon now, we expect the engines to be idle
3669 * and ready to be torn-down. The quickest way we can accomplish
3670 * this is by declaring ourselves wedged.
3672 i915_gem_set_wedged(i915);
3677 i915_gem_init_scratch(struct drm_i915_private *i915, unsigned int size)
3679 struct drm_i915_gem_object *obj;
3680 struct i915_vma *vma;
3683 obj = i915_gem_object_create_stolen(i915, size);
3685 obj = i915_gem_object_create_internal(i915, size);
3687 DRM_ERROR("Failed to allocate scratch page\n");
3688 return PTR_ERR(obj);
3691 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
3697 ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
3701 i915->gt.scratch = vma;
3705 i915_gem_object_put(obj);
3709 static void i915_gem_fini_scratch(struct drm_i915_private *i915)
3711 i915_vma_unpin_and_release(&i915->gt.scratch, 0);
3714 static int intel_engines_verify_workarounds(struct drm_i915_private *i915)
3716 struct intel_engine_cs *engine;
3717 enum intel_engine_id id;
3720 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
3723 for_each_engine(engine, i915, id) {
3724 if (intel_engine_verify_workarounds(engine, "load"))
3731 int i915_gem_init(struct drm_i915_private *dev_priv)
3735 /* We need to fallback to 4K pages if host doesn't support huge gtt. */
3736 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
3737 mkwrite_device_info(dev_priv)->page_sizes =
3738 I915_GTT_PAGE_SIZE_4K;
3740 dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
3742 i915_timelines_init(dev_priv);
3744 ret = i915_gem_init_userptr(dev_priv);
3748 ret = intel_uc_init_misc(dev_priv);
3752 ret = intel_wopcm_init(&dev_priv->wopcm);
3756 /* This is just a security blanket to placate dragons.
3757 * On some systems, we very sporadically observe that the first TLBs
3758 * used by the CS may be stale, despite us poking the TLB reset. If
3759 * we hold the forcewake during initialisation these problems
3760 * just magically go away.
3762 mutex_lock(&dev_priv->drm.struct_mutex);
3763 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
3765 ret = i915_gem_init_ggtt(dev_priv);
3767 GEM_BUG_ON(ret == -EIO);
3771 ret = i915_gem_init_scratch(dev_priv,
3772 IS_GEN(dev_priv, 2) ? SZ_256K : PAGE_SIZE);
3774 GEM_BUG_ON(ret == -EIO);
3778 ret = intel_engines_setup(dev_priv);
3780 GEM_BUG_ON(ret == -EIO);
3784 ret = i915_gem_contexts_init(dev_priv);
3786 GEM_BUG_ON(ret == -EIO);
3790 ret = intel_engines_init(dev_priv);
3792 GEM_BUG_ON(ret == -EIO);
3796 intel_init_gt_powersave(dev_priv);
3798 ret = intel_uc_init(dev_priv);
3802 ret = i915_gem_init_hw(dev_priv);
3807 * Despite its name intel_init_clock_gating applies both display
3808 * clock gating workarounds; GT mmio workarounds and the occasional
3809 * GT power context workaround. Worse, sometimes it includes a context
3810 * register workaround which we need to apply before we record the
3811 * default HW state for all contexts.
3813 * FIXME: break up the workarounds and apply them at the right time!
3815 intel_init_clock_gating(dev_priv);
3817 ret = intel_engines_verify_workarounds(dev_priv);
3821 ret = __intel_engines_record_defaults(dev_priv);
3825 if (i915_inject_load_failure()) {
3830 if (i915_inject_load_failure()) {
3835 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
3836 mutex_unlock(&dev_priv->drm.struct_mutex);
3841 * Unwinding is complicated by that we want to handle -EIO to mean
3842 * disable GPU submission but keep KMS alive. We want to mark the
3843 * HW as irrevisibly wedged, but keep enough state around that the
3844 * driver doesn't explode during runtime.
3847 mutex_unlock(&dev_priv->drm.struct_mutex);
3849 i915_gem_set_wedged(dev_priv);
3850 i915_gem_suspend(dev_priv);
3851 i915_gem_suspend_late(dev_priv);
3853 i915_gem_drain_workqueue(dev_priv);
3855 mutex_lock(&dev_priv->drm.struct_mutex);
3856 intel_uc_fini_hw(dev_priv);
3858 intel_uc_fini(dev_priv);
3861 intel_cleanup_gt_powersave(dev_priv);
3862 intel_engines_cleanup(dev_priv);
3866 i915_gem_contexts_fini(dev_priv);
3868 i915_gem_fini_scratch(dev_priv);
3871 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
3872 mutex_unlock(&dev_priv->drm.struct_mutex);
3875 intel_uc_fini_misc(dev_priv);
3878 i915_gem_cleanup_userptr(dev_priv);
3879 i915_timelines_fini(dev_priv);
3883 mutex_lock(&dev_priv->drm.struct_mutex);
3886 * Allow engine initialisation to fail by marking the GPU as
3887 * wedged. But we only want to do this where the GPU is angry,
3888 * for all other failure, such as an allocation failure, bail.
3890 if (!i915_reset_failed(dev_priv)) {
3891 i915_load_error(dev_priv,
3892 "Failed to initialize GPU, declaring it wedged!\n");
3893 i915_gem_set_wedged(dev_priv);
3896 /* Minimal basic recovery for KMS */
3897 ret = i915_ggtt_enable_hw(dev_priv);
3898 i915_gem_restore_gtt_mappings(dev_priv);
3899 i915_gem_restore_fences(dev_priv);
3900 intel_init_clock_gating(dev_priv);
3902 mutex_unlock(&dev_priv->drm.struct_mutex);
3905 i915_gem_drain_freed_objects(dev_priv);
3909 void i915_gem_fini(struct drm_i915_private *dev_priv)
3911 GEM_BUG_ON(dev_priv->gt.awake);
3913 intel_wakeref_auto_fini(&dev_priv->mm.userfault_wakeref);
3915 i915_gem_suspend_late(dev_priv);
3916 intel_disable_gt_powersave(dev_priv);
3918 /* Flush any outstanding unpin_work. */
3919 i915_gem_drain_workqueue(dev_priv);
3921 mutex_lock(&dev_priv->drm.struct_mutex);
3922 intel_uc_fini_hw(dev_priv);
3923 intel_uc_fini(dev_priv);
3924 intel_engines_cleanup(dev_priv);
3925 i915_gem_contexts_fini(dev_priv);
3926 i915_gem_fini_scratch(dev_priv);
3927 mutex_unlock(&dev_priv->drm.struct_mutex);
3929 intel_wa_list_free(&dev_priv->gt_wa_list);
3931 intel_cleanup_gt_powersave(dev_priv);
3933 intel_uc_fini_misc(dev_priv);
3934 i915_gem_cleanup_userptr(dev_priv);
3935 i915_timelines_fini(dev_priv);
3937 i915_gem_drain_freed_objects(dev_priv);
3939 WARN_ON(!list_empty(&dev_priv->contexts.list));
3942 void i915_gem_init_mmio(struct drm_i915_private *i915)
3944 i915_gem_sanitize(i915);
3948 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
3952 if (INTEL_GEN(dev_priv) >= 7 && !IS_VALLEYVIEW(dev_priv) &&
3953 !IS_CHERRYVIEW(dev_priv))
3954 dev_priv->num_fence_regs = 32;
3955 else if (INTEL_GEN(dev_priv) >= 4 ||
3956 IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
3957 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
3958 dev_priv->num_fence_regs = 16;
3960 dev_priv->num_fence_regs = 8;
3962 if (intel_vgpu_active(dev_priv))
3963 dev_priv->num_fence_regs =
3964 I915_READ(vgtif_reg(avail_rs.fence_num));
3966 /* Initialize fence registers to zero */
3967 for (i = 0; i < dev_priv->num_fence_regs; i++) {
3968 struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
3970 fence->i915 = dev_priv;
3972 list_add_tail(&fence->link, &dev_priv->mm.fence_list);
3974 i915_gem_restore_fences(dev_priv);
3976 i915_gem_detect_bit_6_swizzle(dev_priv);
3979 static void i915_gem_init__mm(struct drm_i915_private *i915)
3981 spin_lock_init(&i915->mm.object_stat_lock);
3982 spin_lock_init(&i915->mm.obj_lock);
3983 spin_lock_init(&i915->mm.free_lock);
3985 init_llist_head(&i915->mm.free_list);
3987 INIT_LIST_HEAD(&i915->mm.unbound_list);
3988 INIT_LIST_HEAD(&i915->mm.bound_list);
3989 INIT_LIST_HEAD(&i915->mm.fence_list);
3991 INIT_LIST_HEAD(&i915->mm.userfault_list);
3992 intel_wakeref_auto_init(&i915->mm.userfault_wakeref, i915);
3994 i915_gem_init__objects(i915);
3997 int i915_gem_init_early(struct drm_i915_private *dev_priv)
4001 intel_gt_pm_init(dev_priv);
4003 INIT_LIST_HEAD(&dev_priv->gt.active_rings);
4004 INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
4006 i915_gem_init__mm(dev_priv);
4007 i915_gem_init__pm(dev_priv);
4009 init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4010 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4011 mutex_init(&dev_priv->gpu_error.wedge_mutex);
4012 init_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu);
4014 atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
4016 spin_lock_init(&dev_priv->fb_tracking.lock);
4018 err = i915_gemfs_init(dev_priv);
4020 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
4025 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
4027 i915_gem_drain_freed_objects(dev_priv);
4028 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
4029 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
4030 WARN_ON(dev_priv->mm.object_count);
4032 cleanup_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu);
4034 i915_gemfs_fini(dev_priv);
4037 int i915_gem_freeze(struct drm_i915_private *dev_priv)
4039 /* Discard all purgeable objects, let userspace recover those as
4040 * required after resuming.
4042 i915_gem_shrink_all(dev_priv);
4047 int i915_gem_freeze_late(struct drm_i915_private *i915)
4049 struct drm_i915_gem_object *obj;
4050 struct list_head *phases[] = {
4051 &i915->mm.unbound_list,
4052 &i915->mm.bound_list,
4057 * Called just before we write the hibernation image.
4059 * We need to update the domain tracking to reflect that the CPU
4060 * will be accessing all the pages to create and restore from the
4061 * hibernation, and so upon restoration those pages will be in the
4064 * To make sure the hibernation image contains the latest state,
4065 * we update that state just before writing out the image.
4067 * To try and reduce the hibernation image, we manually shrink
4068 * the objects as well, see i915_gem_freeze()
4071 i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_UNBOUND);
4072 i915_gem_drain_freed_objects(i915);
4074 mutex_lock(&i915->drm.struct_mutex);
4075 for (phase = phases; *phase; phase++) {
4076 list_for_each_entry(obj, *phase, mm.link)
4077 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
4079 mutex_unlock(&i915->drm.struct_mutex);
4084 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4086 struct drm_i915_file_private *file_priv = file->driver_priv;
4087 struct i915_request *request;
4089 /* Clean up our request list when the client is going away, so that
4090 * later retire_requests won't dereference our soon-to-be-gone
4093 spin_lock(&file_priv->mm.lock);
4094 list_for_each_entry(request, &file_priv->mm.request_list, client_link)
4095 request->file_priv = NULL;
4096 spin_unlock(&file_priv->mm.lock);
4099 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
4101 struct drm_i915_file_private *file_priv;
4106 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4110 file->driver_priv = file_priv;
4111 file_priv->dev_priv = i915;
4112 file_priv->file = file;
4114 spin_lock_init(&file_priv->mm.lock);
4115 INIT_LIST_HEAD(&file_priv->mm.request_list);
4117 file_priv->bsd_engine = -1;
4118 file_priv->hang_timestamp = jiffies;
4120 ret = i915_gem_context_open(i915, file);
4128 * i915_gem_track_fb - update frontbuffer tracking
4129 * @old: current GEM buffer for the frontbuffer slots
4130 * @new: new GEM buffer for the frontbuffer slots
4131 * @frontbuffer_bits: bitmask of frontbuffer slots
4133 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4134 * from @old and setting them in @new. Both @old and @new can be NULL.
4136 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4137 struct drm_i915_gem_object *new,
4138 unsigned frontbuffer_bits)
4140 /* Control of individual bits within the mask are guarded by
4141 * the owning plane->mutex, i.e. we can never see concurrent
4142 * manipulation of individual bits. But since the bitfield as a whole
4143 * is updated using RMW, we need to use atomics in order to update
4146 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
4147 BITS_PER_TYPE(atomic_t));
4150 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
4151 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
4155 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
4156 atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
4160 struct scatterlist *
4161 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
4163 unsigned int *offset)
4165 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
4166 struct scatterlist *sg;
4167 unsigned int idx, count;
4170 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
4171 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
4173 /* As we iterate forward through the sg, we record each entry in a
4174 * radixtree for quick repeated (backwards) lookups. If we have seen
4175 * this index previously, we will have an entry for it.
4177 * Initial lookup is O(N), but this is amortized to O(1) for
4178 * sequential page access (where each new request is consecutive
4179 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
4180 * i.e. O(1) with a large constant!
4182 if (n < READ_ONCE(iter->sg_idx))
4185 mutex_lock(&iter->lock);
4187 /* We prefer to reuse the last sg so that repeated lookup of this
4188 * (or the subsequent) sg are fast - comparing against the last
4189 * sg is faster than going through the radixtree.
4194 count = __sg_page_count(sg);
4196 while (idx + count <= n) {
4201 /* If we cannot allocate and insert this entry, or the
4202 * individual pages from this range, cancel updating the
4203 * sg_idx so that on this lookup we are forced to linearly
4204 * scan onwards, but on future lookups we will try the
4205 * insertion again (in which case we need to be careful of
4206 * the error return reporting that we have already inserted
4209 ret = radix_tree_insert(&iter->radix, idx, sg);
4210 if (ret && ret != -EEXIST)
4213 entry = xa_mk_value(idx);
4214 for (i = 1; i < count; i++) {
4215 ret = radix_tree_insert(&iter->radix, idx + i, entry);
4216 if (ret && ret != -EEXIST)
4221 sg = ____sg_next(sg);
4222 count = __sg_page_count(sg);
4229 mutex_unlock(&iter->lock);
4231 if (unlikely(n < idx)) /* insertion completed by another thread */
4234 /* In case we failed to insert the entry into the radixtree, we need
4235 * to look beyond the current sg.
4237 while (idx + count <= n) {
4239 sg = ____sg_next(sg);
4240 count = __sg_page_count(sg);
4249 sg = radix_tree_lookup(&iter->radix, n);
4252 /* If this index is in the middle of multi-page sg entry,
4253 * the radix tree will contain a value entry that points
4254 * to the start of that range. We will return the pointer to
4255 * the base page and the offset of this page within the
4259 if (unlikely(xa_is_value(sg))) {
4260 unsigned long base = xa_to_value(sg);
4262 sg = radix_tree_lookup(&iter->radix, base);
4274 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
4276 struct scatterlist *sg;
4277 unsigned int offset;
4279 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
4281 sg = i915_gem_object_get_sg(obj, n, &offset);
4282 return nth_page(sg_page(sg), offset);
4285 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4287 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
4292 page = i915_gem_object_get_page(obj, n);
4294 set_page_dirty(page);
4300 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
4304 struct scatterlist *sg;
4305 unsigned int offset;
4307 sg = i915_gem_object_get_sg(obj, n, &offset);
4310 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
4312 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
4316 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
4319 return i915_gem_object_get_dma_address_len(obj, n, NULL);
4323 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
4325 struct sg_table *pages;
4328 if (align > obj->base.size)
4331 if (obj->ops == &i915_gem_phys_ops)
4334 if (obj->ops != &i915_gem_shmem_ops)
4337 err = i915_gem_object_unbind(obj);
4341 mutex_lock(&obj->mm.lock);
4343 if (obj->mm.madv != I915_MADV_WILLNEED) {
4348 if (obj->mm.quirked) {
4353 if (obj->mm.mapping) {
4358 pages = __i915_gem_object_unset_pages(obj);
4360 obj->ops = &i915_gem_phys_ops;
4362 err = ____i915_gem_object_get_pages(obj);
4366 /* Perma-pin (until release) the physical set of pages */
4367 __i915_gem_object_pin_pages(obj);
4369 if (!IS_ERR_OR_NULL(pages))
4370 i915_gem_shmem_ops.put_pages(obj, pages);
4371 mutex_unlock(&obj->mm.lock);
4375 obj->ops = &i915_gem_shmem_ops;
4376 if (!IS_ERR_OR_NULL(pages)) {
4377 unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
4379 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
4382 mutex_unlock(&obj->mm.lock);
4386 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4387 #include "selftests/scatterlist.c"
4388 #include "selftests/mock_gem_device.c"
4389 #include "selftests/huge_gem_object.c"
4390 #include "selftests/huge_pages.c"
4391 #include "selftests/i915_gem_object.c"
4392 #include "selftests/i915_gem_coherency.c"
4393 #include "selftests/i915_gem.c"