2 * SPDX-License-Identifier: MIT
4 * Copyright © 2019 Intel Corporation
7 #include "gem/i915_gem_pm.h"
8 #include "gt/intel_gt.h"
9 #include "gt/intel_gt_pm.h"
12 #include "i915_globals.h"
14 static void call_idle_barriers(struct intel_engine_cs *engine)
16 struct llist_node *node, *next;
18 llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
19 struct i915_active_request *active =
20 container_of((struct list_head *)node,
21 typeof(*active), link);
23 INIT_LIST_HEAD(&active->link);
24 RCU_INIT_POINTER(active->request, NULL);
26 active->retire(active, NULL);
30 static void i915_gem_park(struct drm_i915_private *i915)
32 struct intel_engine_cs *engine;
33 enum intel_engine_id id;
35 lockdep_assert_held(&i915->drm.struct_mutex);
37 for_each_engine(engine, i915, id)
38 call_idle_barriers(engine); /* cleanup after wedging */
40 i915_vma_parked(i915);
45 static void idle_work_handler(struct work_struct *work)
47 struct drm_i915_private *i915 =
48 container_of(work, typeof(*i915), gem.idle_work);
51 cancel_delayed_work_sync(&i915->gem.retire_work);
52 mutex_lock(&i915->drm.struct_mutex);
54 intel_wakeref_lock(&i915->gt.wakeref);
55 park = (!intel_wakeref_is_active(&i915->gt.wakeref) &&
57 intel_wakeref_unlock(&i915->gt.wakeref);
61 queue_delayed_work(i915->wq,
62 &i915->gem.retire_work,
63 round_jiffies_up_relative(HZ));
65 mutex_unlock(&i915->drm.struct_mutex);
68 static void retire_work_handler(struct work_struct *work)
70 struct drm_i915_private *i915 =
71 container_of(work, typeof(*i915), gem.retire_work.work);
73 /* Come back later if the device is busy... */
74 if (mutex_trylock(&i915->drm.struct_mutex)) {
75 i915_retire_requests(i915);
76 mutex_unlock(&i915->drm.struct_mutex);
79 queue_delayed_work(i915->wq,
80 &i915->gem.retire_work,
81 round_jiffies_up_relative(HZ));
84 static int pm_notifier(struct notifier_block *nb,
88 struct drm_i915_private *i915 =
89 container_of(nb, typeof(*i915), gem.pm_notifier);
93 i915_globals_unpark();
94 queue_delayed_work(i915->wq,
95 &i915->gem.retire_work,
96 round_jiffies_up_relative(HZ));
100 queue_work(i915->wq, &i915->gem.idle_work);
107 static bool switch_to_kernel_context_sync(struct intel_gt *gt)
109 bool result = !intel_gt_is_wedged(gt);
112 if (i915_gem_wait_for_idle(gt->i915,
114 I915_WAIT_FOR_IDLE_BOOST,
115 I915_GEM_IDLE_TIMEOUT) == -ETIME) {
116 /* XXX hide warning from gem_eio */
117 if (i915_modparams.reset) {
118 dev_err(gt->i915->drm.dev,
119 "Failed to idle engines, declaring wedged!\n");
124 * Forcibly cancel outstanding work and leave
127 intel_gt_set_wedged(gt);
130 } while (i915_retire_requests(gt->i915) && result);
132 if (intel_gt_pm_wait_for_idle(gt))
138 bool i915_gem_load_power_context(struct drm_i915_private *i915)
140 return switch_to_kernel_context_sync(&i915->gt);
143 void i915_gem_suspend(struct drm_i915_private *i915)
147 intel_wakeref_auto(&i915->ggtt.userfault_wakeref, 0);
148 flush_workqueue(i915->wq);
150 mutex_lock(&i915->drm.struct_mutex);
153 * We have to flush all the executing contexts to main memory so
154 * that they can saved in the hibernation image. To ensure the last
155 * context image is coherent, we have to switch away from it. That
156 * leaves the i915->kernel_context still active when
157 * we actually suspend, and its image in memory may not match the GPU
158 * state. Fortunately, the kernel_context is disposable and we do
159 * not rely on its state.
161 switch_to_kernel_context_sync(&i915->gt);
163 mutex_unlock(&i915->drm.struct_mutex);
165 cancel_delayed_work_sync(&i915->gt.hangcheck.work);
167 i915_gem_drain_freed_objects(i915);
169 intel_uc_suspend(&i915->gt.uc);
172 static struct drm_i915_gem_object *first_mm_object(struct list_head *list)
174 return list_first_entry_or_null(list,
175 struct drm_i915_gem_object,
179 void i915_gem_suspend_late(struct drm_i915_private *i915)
181 struct drm_i915_gem_object *obj;
182 struct list_head *phases[] = {
183 &i915->mm.shrink_list,
184 &i915->mm.purge_list,
190 * Neither the BIOS, ourselves or any other kernel
191 * expects the system to be in execlists mode on startup,
192 * so we need to reset the GPU back to legacy mode. And the only
193 * known way to disable logical contexts is through a GPU reset.
195 * So in order to leave the system in a known default configuration,
196 * always reset the GPU upon unload and suspend. Afterwards we then
197 * clean up the GEM state tracking, flushing off the requests and
198 * leaving the system in a known idle state.
200 * Note that is of the upmost importance that the GPU is idle and
201 * all stray writes are flushed *before* we dismantle the backing
202 * storage for the pinned objects.
204 * However, since we are uncertain that resetting the GPU on older
205 * machines is a good idea, we don't - just in case it leaves the
206 * machine in an unusable condition.
209 spin_lock_irqsave(&i915->mm.obj_lock, flags);
210 for (phase = phases; *phase; phase++) {
213 while ((obj = first_mm_object(*phase))) {
214 list_move_tail(&obj->mm.link, &keep);
216 /* Beware the background _i915_gem_free_objects */
217 if (!kref_get_unless_zero(&obj->base.refcount))
220 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
222 i915_gem_object_lock(obj);
223 WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
224 i915_gem_object_unlock(obj);
225 i915_gem_object_put(obj);
227 spin_lock_irqsave(&i915->mm.obj_lock, flags);
230 list_splice_tail(&keep, *phase);
232 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
234 i915_gem_sanitize(i915);
237 void i915_gem_resume(struct drm_i915_private *i915)
241 mutex_lock(&i915->drm.struct_mutex);
242 intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
244 i915_gem_restore_gtt_mappings(i915);
245 i915_gem_restore_fences(i915);
247 if (i915_gem_init_hw(i915))
251 * As we didn't flush the kernel context before suspend, we cannot
252 * guarantee that the context image is complete. So let's just reset
253 * it and start again.
255 if (intel_gt_resume(&i915->gt))
258 intel_uc_resume(&i915->gt.uc);
260 /* Always reload a context for powersaving. */
261 if (!i915_gem_load_power_context(i915))
265 intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
266 mutex_unlock(&i915->drm.struct_mutex);
270 if (!intel_gt_is_wedged(&i915->gt)) {
271 dev_err(i915->drm.dev,
272 "Failed to re-initialize GPU, declaring it wedged!\n");
273 intel_gt_set_wedged(&i915->gt);
278 void i915_gem_init__pm(struct drm_i915_private *i915)
280 INIT_WORK(&i915->gem.idle_work, idle_work_handler);
281 INIT_DELAYED_WORK(&i915->gem.retire_work, retire_work_handler);
283 i915->gem.pm_notifier.notifier_call = pm_notifier;
284 blocking_notifier_chain_register(&i915->gt.pm_notifications,
285 &i915->gem.pm_notifier);