1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_pm.h" 7 #include "gem/i915_gem_ttm_pm.h" 8 #include "gt/intel_gt.h" 9 #include "gt/intel_gt_pm.h" 10 #include "gt/intel_gt_requests.h" 11 12 #include "i915_driver.h" 13 #include "i915_drv.h" 14 15 #if IS_ENABLED(CONFIG_X86) 16 #include <asm/smp.h> 17 #else 18 #define wbinvd_on_all_cpus() \ 19 pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__) 20 #endif 21 22 void i915_gem_suspend(struct drm_i915_private *i915) 23 { 24 struct intel_gt *gt; 25 unsigned int i; 26 27 GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 28 29 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0); 30 /* 31 * On rare occasions, we've observed the fence completion triggers 32 * free_engines asynchronously via rcu_call. Ensure those are done. 33 * This path is only called on suspend, so it's an acceptable cost. 34 */ 35 rcu_barrier(); 36 37 flush_workqueue(i915->wq); 38 39 /* 40 * We have to flush all the executing contexts to main memory so 41 * that they can saved in the hibernation image. To ensure the last 42 * context image is coherent, we have to switch away from it. That 43 * leaves the i915->kernel_context still active when 44 * we actually suspend, and its image in memory may not match the GPU 45 * state. Fortunately, the kernel_context is disposable and we do 46 * not rely on its state. 47 */ 48 for_each_gt(gt, i915, i) 49 intel_gt_suspend_prepare(gt); 50 51 i915_gem_drain_freed_objects(i915); 52 } 53 54 static int lmem_restore(struct drm_i915_private *i915, u32 flags) 55 { 56 struct intel_memory_region *mr; 57 int ret = 0, id; 58 59 for_each_memory_region(mr, i915, id) { 60 if (mr->type == INTEL_MEMORY_LOCAL) { 61 ret = i915_ttm_restore_region(mr, flags); 62 if (ret) 63 break; 64 } 65 } 66 67 return ret; 68 } 69 70 static int lmem_suspend(struct drm_i915_private *i915, u32 flags) 71 { 72 struct intel_memory_region *mr; 73 int ret = 0, id; 74 75 for_each_memory_region(mr, i915, id) { 76 if (mr->type == INTEL_MEMORY_LOCAL) { 77 ret = i915_ttm_backup_region(mr, flags); 78 if (ret) 79 break; 80 } 81 } 82 83 return ret; 84 } 85 86 static void lmem_recover(struct drm_i915_private *i915) 87 { 88 struct intel_memory_region *mr; 89 int id; 90 91 for_each_memory_region(mr, i915, id) 92 if (mr->type == INTEL_MEMORY_LOCAL) 93 i915_ttm_recover_region(mr); 94 } 95 96 int i915_gem_backup_suspend(struct drm_i915_private *i915) 97 { 98 int ret; 99 100 /* Opportunistically try to evict unpinned objects */ 101 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU); 102 if (ret) 103 goto out_recover; 104 105 i915_gem_suspend(i915); 106 107 /* 108 * More objects may have become unpinned as requests were 109 * retired. Now try to evict again. The gt may be wedged here 110 * in which case we automatically fall back to memcpy. 111 * We allow also backing up pinned objects that have not been 112 * marked for early recover, and that may contain, for example, 113 * page-tables for the migrate context. 114 */ 115 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU | 116 I915_TTM_BACKUP_PINNED); 117 if (ret) 118 goto out_recover; 119 120 /* 121 * Remaining objects are backed up using memcpy once we've stopped 122 * using the migrate context. 123 */ 124 ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED); 125 if (ret) 126 goto out_recover; 127 128 return 0; 129 130 out_recover: 131 lmem_recover(i915); 132 133 return ret; 134 } 135 136 void i915_gem_suspend_late(struct drm_i915_private *i915) 137 { 138 struct drm_i915_gem_object *obj; 139 struct list_head *phases[] = { 140 &i915->mm.shrink_list, 141 &i915->mm.purge_list, 142 NULL 143 }, **phase; 144 struct intel_gt *gt; 145 unsigned long flags; 146 unsigned int i; 147 bool flush = false; 148 149 /* 150 * Neither the BIOS, ourselves or any other kernel 151 * expects the system to be in execlists mode on startup, 152 * so we need to reset the GPU back to legacy mode. And the only 153 * known way to disable logical contexts is through a GPU reset. 154 * 155 * So in order to leave the system in a known default configuration, 156 * always reset the GPU upon unload and suspend. Afterwards we then 157 * clean up the GEM state tracking, flushing off the requests and 158 * leaving the system in a known idle state. 159 * 160 * Note that is of the upmost importance that the GPU is idle and 161 * all stray writes are flushed *before* we dismantle the backing 162 * storage for the pinned objects. 163 * 164 * However, since we are uncertain that resetting the GPU on older 165 * machines is a good idea, we don't - just in case it leaves the 166 * machine in an unusable condition. 167 */ 168 169 /* Like i915_gem_suspend, flush tasks staged from fence triggers */ 170 rcu_barrier(); 171 172 for_each_gt(gt, i915, i) 173 intel_gt_suspend_late(gt); 174 175 spin_lock_irqsave(&i915->mm.obj_lock, flags); 176 for (phase = phases; *phase; phase++) { 177 list_for_each_entry(obj, *phase, mm.link) { 178 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 179 flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0; 180 __start_cpu_write(obj); /* presume auto-hibernate */ 181 } 182 } 183 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 184 if (flush) 185 wbinvd_on_all_cpus(); 186 } 187 188 int i915_gem_freeze(struct drm_i915_private *i915) 189 { 190 /* Discard all purgeable objects, let userspace recover those as 191 * required after resuming. 192 */ 193 i915_gem_shrink_all(i915); 194 195 return 0; 196 } 197 198 int i915_gem_freeze_late(struct drm_i915_private *i915) 199 { 200 struct drm_i915_gem_object *obj; 201 intel_wakeref_t wakeref; 202 203 /* 204 * Called just before we write the hibernation image. 205 * 206 * We need to update the domain tracking to reflect that the CPU 207 * will be accessing all the pages to create and restore from the 208 * hibernation, and so upon restoration those pages will be in the 209 * CPU domain. 210 * 211 * To make sure the hibernation image contains the latest state, 212 * we update that state just before writing out the image. 213 * 214 * To try and reduce the hibernation image, we manually shrink 215 * the objects as well, see i915_gem_freeze() 216 */ 217 218 with_intel_runtime_pm(&i915->runtime_pm, wakeref) 219 i915_gem_shrink(NULL, i915, -1UL, NULL, ~0); 220 i915_gem_drain_freed_objects(i915); 221 222 wbinvd_on_all_cpus(); 223 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) 224 __start_cpu_write(obj); 225 226 return 0; 227 } 228 229 void i915_gem_resume(struct drm_i915_private *i915) 230 { 231 struct intel_gt *gt; 232 int ret, i, j; 233 234 GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 235 236 ret = lmem_restore(i915, 0); 237 GEM_WARN_ON(ret); 238 239 /* 240 * As we didn't flush the kernel context before suspend, we cannot 241 * guarantee that the context image is complete. So let's just reset 242 * it and start again. 243 */ 244 for_each_gt(gt, i915, i) 245 if (intel_gt_resume(gt)) 246 goto err_wedged; 247 248 ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU); 249 GEM_WARN_ON(ret); 250 251 return; 252 253 err_wedged: 254 for_each_gt(gt, i915, j) { 255 if (!intel_gt_is_wedged(gt)) { 256 dev_err(i915->drm.dev, 257 "Failed to re-initialize GPU[%u], declaring it wedged!\n", 258 j); 259 intel_gt_set_wedged(gt); 260 } 261 262 if (j == i) 263 break; 264 } 265 } 266