1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include "gem/i915_gem_pm.h"
8 #include "gem/i915_gem_ttm_pm.h"
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_pm.h"
11 #include "gt/intel_gt_requests.h"
12
13 #include "i915_driver.h"
14 #include "i915_drv.h"
15
16 #if defined(CONFIG_X86)
17 #include <asm/smp.h>
18 #else
19 #define wbinvd_on_all_cpus() \
20 pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
21 #endif
22
i915_gem_suspend(struct drm_i915_private * i915)23 void i915_gem_suspend(struct drm_i915_private *i915)
24 {
25 struct intel_gt *gt;
26 unsigned int i;
27
28 GEM_TRACE("%s\n", dev_name(i915->drm.dev));
29
30 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0);
31 /*
32 * On rare occasions, we've observed the fence completion triggers
33 * free_engines asynchronously via rcu_call. Ensure those are done.
34 * This path is only called on suspend, so it's an acceptable cost.
35 */
36 rcu_barrier();
37
38 flush_workqueue(i915->wq);
39
40 /*
41 * We have to flush all the executing contexts to main memory so
42 * that they can saved in the hibernation image. To ensure the last
43 * context image is coherent, we have to switch away from it. That
44 * leaves the i915->kernel_context still active when
45 * we actually suspend, and its image in memory may not match the GPU
46 * state. Fortunately, the kernel_context is disposable and we do
47 * not rely on its state.
48 */
49 for_each_gt(gt, i915, i)
50 intel_gt_suspend_prepare(gt);
51
52 i915_gem_drain_freed_objects(i915);
53 }
54
lmem_restore(struct drm_i915_private * i915,u32 flags)55 static int lmem_restore(struct drm_i915_private *i915, u32 flags)
56 {
57 struct intel_memory_region *mr;
58 int ret = 0, id;
59
60 for_each_memory_region(mr, i915, id) {
61 if (mr->type == INTEL_MEMORY_LOCAL) {
62 ret = i915_ttm_restore_region(mr, flags);
63 if (ret)
64 break;
65 }
66 }
67
68 return ret;
69 }
70
lmem_suspend(struct drm_i915_private * i915,u32 flags)71 static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
72 {
73 struct intel_memory_region *mr;
74 int ret = 0, id;
75
76 for_each_memory_region(mr, i915, id) {
77 if (mr->type == INTEL_MEMORY_LOCAL) {
78 ret = i915_ttm_backup_region(mr, flags);
79 if (ret)
80 break;
81 }
82 }
83
84 return ret;
85 }
86
lmem_recover(struct drm_i915_private * i915)87 static void lmem_recover(struct drm_i915_private *i915)
88 {
89 struct intel_memory_region *mr;
90 int id;
91
92 for_each_memory_region(mr, i915, id)
93 if (mr->type == INTEL_MEMORY_LOCAL)
94 i915_ttm_recover_region(mr);
95 }
96
i915_gem_backup_suspend(struct drm_i915_private * i915)97 int i915_gem_backup_suspend(struct drm_i915_private *i915)
98 {
99 int ret;
100
101 /* Opportunistically try to evict unpinned objects */
102 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
103 if (ret)
104 goto out_recover;
105
106 i915_gem_suspend(i915);
107
108 /*
109 * More objects may have become unpinned as requests were
110 * retired. Now try to evict again. The gt may be wedged here
111 * in which case we automatically fall back to memcpy.
112 * We allow also backing up pinned objects that have not been
113 * marked for early recover, and that may contain, for example,
114 * page-tables for the migrate context.
115 */
116 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
117 I915_TTM_BACKUP_PINNED);
118 if (ret)
119 goto out_recover;
120
121 /*
122 * Remaining objects are backed up using memcpy once we've stopped
123 * using the migrate context.
124 */
125 ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
126 if (ret)
127 goto out_recover;
128
129 return 0;
130
131 out_recover:
132 lmem_recover(i915);
133
134 return ret;
135 }
136
i915_gem_suspend_late(struct drm_i915_private * i915)137 void i915_gem_suspend_late(struct drm_i915_private *i915)
138 {
139 struct drm_i915_gem_object *obj;
140 struct list_head *phases[] = {
141 &i915->mm.shrink_list,
142 &i915->mm.purge_list,
143 NULL
144 }, **phase;
145 struct intel_gt *gt;
146 unsigned long flags;
147 unsigned int i;
148 bool flush = false;
149
150 /*
151 * Neither the BIOS, ourselves or any other kernel
152 * expects the system to be in execlists mode on startup,
153 * so we need to reset the GPU back to legacy mode. And the only
154 * known way to disable logical contexts is through a GPU reset.
155 *
156 * So in order to leave the system in a known default configuration,
157 * always reset the GPU upon unload and suspend. Afterwards we then
158 * clean up the GEM state tracking, flushing off the requests and
159 * leaving the system in a known idle state.
160 *
161 * Note that is of the upmost importance that the GPU is idle and
162 * all stray writes are flushed *before* we dismantle the backing
163 * storage for the pinned objects.
164 *
165 * However, since we are uncertain that resetting the GPU on older
166 * machines is a good idea, we don't - just in case it leaves the
167 * machine in an unusable condition.
168 */
169
170 /* Like i915_gem_suspend, flush tasks staged from fence triggers */
171 rcu_barrier();
172
173 for_each_gt(gt, i915, i)
174 intel_gt_suspend_late(gt);
175
176 spin_lock_irqsave(&i915->mm.obj_lock, flags);
177 for (phase = phases; *phase; phase++) {
178 list_for_each_entry(obj, *phase, mm.link) {
179 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
180 flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
181 __start_cpu_write(obj); /* presume auto-hibernate */
182 }
183 }
184 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
185 if (flush)
186 wbinvd_on_all_cpus();
187 }
188
i915_gem_freeze(struct drm_i915_private * i915)189 int i915_gem_freeze(struct drm_i915_private *i915)
190 {
191 /* Discard all purgeable objects, let userspace recover those as
192 * required after resuming.
193 */
194 i915_gem_shrink_all(i915);
195
196 return 0;
197 }
198
i915_gem_freeze_late(struct drm_i915_private * i915)199 int i915_gem_freeze_late(struct drm_i915_private *i915)
200 {
201 struct drm_i915_gem_object *obj;
202 intel_wakeref_t wakeref;
203
204 /*
205 * Called just before we write the hibernation image.
206 *
207 * We need to update the domain tracking to reflect that the CPU
208 * will be accessing all the pages to create and restore from the
209 * hibernation, and so upon restoration those pages will be in the
210 * CPU domain.
211 *
212 * To make sure the hibernation image contains the latest state,
213 * we update that state just before writing out the image.
214 *
215 * To try and reduce the hibernation image, we manually shrink
216 * the objects as well, see i915_gem_freeze()
217 */
218
219 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
220 i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
221 i915_gem_drain_freed_objects(i915);
222
223 wbinvd_on_all_cpus();
224 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
225 __start_cpu_write(obj);
226
227 return 0;
228 }
229
i915_gem_resume(struct drm_i915_private * i915)230 void i915_gem_resume(struct drm_i915_private *i915)
231 {
232 struct intel_gt *gt;
233 int ret, i, j;
234
235 GEM_TRACE("%s\n", dev_name(i915->drm.dev));
236
237 ret = lmem_restore(i915, 0);
238 GEM_WARN_ON(ret);
239
240 /*
241 * As we didn't flush the kernel context before suspend, we cannot
242 * guarantee that the context image is complete. So let's just reset
243 * it and start again.
244 */
245 for_each_gt(gt, i915, i)
246 if (intel_gt_resume(gt))
247 goto err_wedged;
248
249 ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
250 GEM_WARN_ON(ret);
251
252 return;
253
254 err_wedged:
255 for_each_gt(gt, i915, j) {
256 if (!intel_gt_is_wedged(gt)) {
257 dev_err(i915->drm.dev,
258 "Failed to re-initialize GPU[%u], declaring it wedged!\n",
259 j);
260 intel_gt_set_wedged(gt);
261 }
262
263 if (j == i)
264 break;
265 }
266 }
267