2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include <linux/mman.h>
8 #include <linux/sizes.h>
11 #include "i915_gem_gtt.h"
12 #include "i915_gem_ioctls.h"
13 #include "i915_gem_object.h"
15 #include "intel_drv.h"
18 __vma_matches(struct vm_area_struct *vma, struct file *filp,
19 unsigned long addr, unsigned long size)
21 if (vma->vm_file != filp)
24 return vma->vm_start == addr &&
25 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
29 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
32 * @data: ioctl data blob
35 * While the mapping holds a reference on the contents of the object, it doesn't
36 * imply a ref on the object itself.
40 * DRM driver writers who look a this function as an example for how to do GEM
41 * mmap support, please don't implement mmap support like here. The modern way
42 * to implement DRM mmap support is with an mmap offset ioctl (like
43 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
44 * That way debug tooling like valgrind will understand what's going on, hiding
45 * the mmap call in a driver private ioctl will break that. The i915 driver only
46 * does cpu mmaps this way because we didn't know better.
49 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
50 struct drm_file *file)
52 struct drm_i915_gem_mmap *args = data;
53 struct drm_i915_gem_object *obj;
56 if (args->flags & ~(I915_MMAP_WC))
59 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
62 obj = i915_gem_object_lookup(file, args->handle);
66 /* prime objects have no backing filp to GEM mmap
69 if (!obj->base.filp) {
74 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
79 addr = vm_mmap(obj->base.filp, 0, args->size,
80 PROT_READ | PROT_WRITE, MAP_SHARED,
82 if (IS_ERR_VALUE(addr))
85 if (args->flags & I915_MMAP_WC) {
86 struct mm_struct *mm = current->mm;
87 struct vm_area_struct *vma;
89 if (down_write_killable(&mm->mmap_sem)) {
93 vma = find_vma(mm, addr);
94 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
96 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
99 up_write(&mm->mmap_sem);
100 if (IS_ERR_VALUE(addr))
103 /* This may race, but that's ok, it only gets set */
104 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
106 i915_gem_object_put(obj);
108 args->addr_ptr = (u64)addr;
112 i915_gem_object_put(obj);
116 static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
118 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
122 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
124 * A history of the GTT mmap interface:
126 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
127 * aligned and suitable for fencing, and still fit into the available
128 * mappable space left by the pinned display objects. A classic problem
129 * we called the page-fault-of-doom where we would ping-pong between
130 * two objects that could not fit inside the GTT and so the memcpy
131 * would page one object in at the expense of the other between every
134 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
135 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
136 * object is too large for the available space (or simply too large
137 * for the mappable aperture!), a view is created instead and faulted
138 * into userspace. (This view is aligned and sized appropriately for
141 * 2 - Recognise WC as a separate cache domain so that we can flush the
142 * delayed writes via GTT before performing direct access via WC.
144 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
145 * pagefault; swapin remains transparent.
149 * * snoopable objects cannot be accessed via the GTT. It can cause machine
150 * hangs on some architectures, corruption on others. An attempt to service
151 * a GTT page fault from a snoopable object will generate a SIGBUS.
153 * * the object must be able to fit into RAM (physical memory, though no
154 * limited to the mappable aperture).
159 * * a new GTT page fault will synchronize rendering from the GPU and flush
160 * all data to system memory. Subsequent access will not be synchronized.
162 * * all mappings are revoked on runtime device suspend.
164 * * there are only 8, 16 or 32 fence registers to share between all users
165 * (older machines require fence register for display and blitter access
166 * as well). Contention of the fence registers will cause the previous users
167 * to be unmapped and any new access will generate new page faults.
169 * * running out of memory while servicing a fault may generate a SIGBUS,
170 * rather than the expected SIGSEGV.
172 int i915_gem_mmap_gtt_version(void)
177 static inline struct i915_ggtt_view
178 compute_partial_view(const struct drm_i915_gem_object *obj,
182 struct i915_ggtt_view view;
184 if (i915_gem_object_is_tiled(obj))
185 chunk = roundup(chunk, tile_row_pages(obj));
187 view.type = I915_GGTT_VIEW_PARTIAL;
188 view.partial.offset = rounddown(page_offset, chunk);
190 min_t(unsigned int, chunk,
191 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
193 /* If the partial covers the entire object, just create a normal VMA. */
194 if (chunk >= obj->base.size >> PAGE_SHIFT)
195 view.type = I915_GGTT_VIEW_NORMAL;
201 * i915_gem_fault - fault a page into the GTT
204 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
205 * from userspace. The fault handler takes care of binding the object to
206 * the GTT (if needed), allocating and programming a fence register (again,
207 * only if needed based on whether the old reg is still valid or the object
208 * is tiled) and inserting a new PTE into the faulting process.
210 * Note that the faulting process may involve evicting existing objects
211 * from the GTT and/or fence registers to make room. So performance may
212 * suffer if the GTT working set is large or there are few fence registers
215 * The current feature set supported by i915_gem_fault() and thus GTT mmaps
216 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
218 vm_fault_t i915_gem_fault(struct vm_fault *vmf)
220 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
221 struct vm_area_struct *area = vmf->vma;
222 struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
223 struct drm_device *dev = obj->base.dev;
224 struct drm_i915_private *i915 = to_i915(dev);
225 struct intel_runtime_pm *rpm = &i915->runtime_pm;
226 struct i915_ggtt *ggtt = &i915->ggtt;
227 bool write = area->vm_flags & VM_WRITE;
228 intel_wakeref_t wakeref;
229 struct i915_vma *vma;
234 /* Sanity check that we allow writing into this object */
235 if (i915_gem_object_is_readonly(obj) && write)
236 return VM_FAULT_SIGBUS;
238 /* We don't use vmf->pgoff since that has the fake offset */
239 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
241 trace_i915_gem_object_fault(obj, page_offset, true, write);
243 ret = i915_gem_object_pin_pages(obj);
247 wakeref = intel_runtime_pm_get(rpm);
249 srcu = i915_reset_trylock(i915);
255 ret = i915_mutex_lock_interruptible(dev);
259 /* Access to snoopable pages through the GTT is incoherent. */
260 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
265 /* Now pin it into the GTT as needed */
266 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
271 /* Use a partial view if it is bigger than available space */
272 struct i915_ggtt_view view =
273 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
276 flags = PIN_MAPPABLE;
277 if (view.type == I915_GGTT_VIEW_NORMAL)
278 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
281 * Userspace is now writing through an untracked VMA, abandon
282 * all hope that the hardware is able to track future writes.
284 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
286 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
287 if (IS_ERR(vma) && !view.type) {
288 flags = PIN_MAPPABLE;
289 view.type = I915_GGTT_VIEW_PARTIAL;
290 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
298 ret = i915_vma_pin_fence(vma);
302 /* Finally, remap it using the new GTT offset */
303 ret = remap_io_mapping(area,
304 area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
305 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
306 min_t(u64, vma->size, area->vm_end - area->vm_start),
311 /* Mark as being mmapped into userspace for later revocation */
312 assert_rpm_wakelock_held(rpm);
313 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
314 list_add(&obj->userfault_link, &i915->ggtt.userfault_list);
315 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
316 intel_wakeref_auto(&i915->ggtt.userfault_wakeref,
317 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
318 GEM_BUG_ON(!obj->userfault_count);
320 i915_vma_set_ggtt_write(vma);
323 i915_vma_unpin_fence(vma);
325 __i915_vma_unpin(vma);
327 mutex_unlock(&dev->struct_mutex);
329 i915_reset_unlock(i915, srcu);
331 intel_runtime_pm_put(rpm, wakeref);
332 i915_gem_object_unpin_pages(obj);
337 * We eat errors when the gpu is terminally wedged to avoid
338 * userspace unduly crashing (gl has no provisions for mmaps to
339 * fail). But any other -EIO isn't ours (e.g. swap in failure)
340 * and so needs to be reported.
342 if (!i915_terminally_wedged(i915))
343 return VM_FAULT_SIGBUS;
344 /* else, fall through */
347 * EAGAIN means the gpu is hung and we'll wait for the error
348 * handler to reset everything when re-faulting in
349 * i915_mutex_lock_interruptible.
356 * EBUSY is ok: this just means that another thread
357 * already did the job.
359 return VM_FAULT_NOPAGE;
364 return VM_FAULT_SIGBUS;
366 WARN_ONCE(ret, "unhandled error in %s: %i\n", __func__, ret);
367 return VM_FAULT_SIGBUS;
371 void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
373 struct i915_vma *vma;
375 GEM_BUG_ON(!obj->userfault_count);
377 obj->userfault_count = 0;
378 list_del(&obj->userfault_link);
379 drm_vma_node_unmap(&obj->base.vma_node,
380 obj->base.dev->anon_inode->i_mapping);
382 for_each_ggtt_vma(vma, obj)
383 i915_vma_unset_userfault(vma);
387 * i915_gem_object_release_mmap - remove physical page mappings
388 * @obj: obj in question
390 * Preserve the reservation of the mmapping with the DRM core code, but
391 * relinquish ownership of the pages back to the system.
393 * It is vital that we remove the page mapping if we have mapped a tiled
394 * object through the GTT and then lose the fence register due to
395 * resource pressure. Similarly if the object has been moved out of the
396 * aperture, than pages mapped into userspace must be revoked. Removing the
397 * mapping will then trigger a page fault on the next user access, allowing
398 * fixup by i915_gem_fault().
400 void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
402 struct drm_i915_private *i915 = to_i915(obj->base.dev);
403 intel_wakeref_t wakeref;
405 /* Serialisation between user GTT access and our code depends upon
406 * revoking the CPU's PTE whilst the mutex is held. The next user
407 * pagefault then has to wait until we release the mutex.
409 * Note that RPM complicates somewhat by adding an additional
410 * requirement that operations to the GGTT be made holding the RPM
413 lockdep_assert_held(&i915->drm.struct_mutex);
414 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
416 if (!obj->userfault_count)
419 __i915_gem_object_release_mmap(obj);
421 /* Ensure that the CPU's PTE are revoked and there are not outstanding
422 * memory transactions from userspace before we return. The TLB
423 * flushing implied above by changing the PTE above *should* be
424 * sufficient, an extra barrier here just provides us with a bit
425 * of paranoid documentation about our requirement to serialise
426 * memory writes before touching registers / GSM.
431 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
434 static int create_mmap_offset(struct drm_i915_gem_object *obj)
436 struct drm_i915_private *i915 = to_i915(obj->base.dev);
439 err = drm_gem_create_mmap_offset(&obj->base);
443 /* Attempt to reap some mmap space from dead objects */
445 err = i915_gem_wait_for_idle(i915,
446 I915_WAIT_INTERRUPTIBLE,
447 MAX_SCHEDULE_TIMEOUT);
451 i915_gem_drain_freed_objects(i915);
452 err = drm_gem_create_mmap_offset(&obj->base);
456 } while (flush_delayed_work(&i915->gem.retire_work));
462 i915_gem_mmap_gtt(struct drm_file *file,
463 struct drm_device *dev,
467 struct drm_i915_gem_object *obj;
470 obj = i915_gem_object_lookup(file, handle);
474 ret = create_mmap_offset(obj);
476 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
478 i915_gem_object_put(obj);
483 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
485 * @data: GTT mapping ioctl data
486 * @file: GEM object info
488 * Simply returns the fake offset to userspace so it can mmap it.
489 * The mmap call will end up in drm_gem_mmap(), which will set things
490 * up so we can get faults in the handler above.
492 * The fault handler will take care of binding the object into the GTT
493 * (since it may have been evicted to make room for something), allocating
494 * a fence register, and mapping the appropriate aperture address into
498 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
499 struct drm_file *file)
501 struct drm_i915_gem_mmap_gtt *args = data;
503 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
506 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
507 #include "selftests/i915_gem_mman.c"