1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_pm.h" 8 #include "gem/i915_gem_ttm_pm.h" 9 #include "gt/intel_gt.h" 10 #include "gt/intel_gt_pm.h" 11 #include "gt/intel_gt_requests.h" 12 13 #include "i915_driver.h" 14 #include "i915_drv.h" 15 16 #if IS_ENABLED(CONFIG_X86) 17 #include <asm/smp.h> 18 #else 19 #define wbinvd_on_all_cpus() \ 20 pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__) 21 #endif 22 23 void i915_gem_suspend(struct drm_i915_private *i915) 24 { 25 struct intel_gt *gt; 26 unsigned int i; 27 28 GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 29 30 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0); 31 /* 32 * On rare occasions, we've observed the fence completion triggers 33 * free_engines asynchronously via rcu_call. Ensure those are done. 34 * This path is only called on suspend, so it's an acceptable cost. 35 */ 36 rcu_barrier(); 37 38 flush_workqueue(i915->wq); 39 40 /* 41 * We have to flush all the executing contexts to main memory so 42 * that they can saved in the hibernation image. To ensure the last 43 * context image is coherent, we have to switch away from it. That 44 * leaves the i915->kernel_context still active when 45 * we actually suspend, and its image in memory may not match the GPU 46 * state. Fortunately, the kernel_context is disposable and we do 47 * not rely on its state. 48 */ 49 for_each_gt(gt, i915, i) 50 intel_gt_suspend_prepare(gt); 51 52 i915_gem_drain_freed_objects(i915); 53 } 54 55 static int lmem_restore(struct drm_i915_private *i915, u32 flags) 56 { 57 struct intel_memory_region *mr; 58 int ret = 0, id; 59 60 for_each_memory_region(mr, i915, id) { 61 if (mr->type == INTEL_MEMORY_LOCAL) { 62 ret = i915_ttm_restore_region(mr, flags); 63 if (ret) 64 break; 65 } 66 } 67 68 return ret; 69 } 70 71 static int lmem_suspend(struct drm_i915_private *i915, u32 flags) 72 { 73 struct intel_memory_region *mr; 74 int ret = 0, id; 75 76 for_each_memory_region(mr, i915, id) { 77 if (mr->type == INTEL_MEMORY_LOCAL) { 78 ret = i915_ttm_backup_region(mr, flags); 79 if (ret) 80 break; 81 } 82 } 83 84 return ret; 85 } 86 87 static void lmem_recover(struct drm_i915_private *i915) 88 { 89 struct intel_memory_region *mr; 90 int id; 91 92 for_each_memory_region(mr, i915, id) 93 if (mr->type == INTEL_MEMORY_LOCAL) 94 i915_ttm_recover_region(mr); 95 } 96 97 int i915_gem_backup_suspend(struct drm_i915_private *i915) 98 { 99 int ret; 100 101 /* Opportunistically try to evict unpinned objects */ 102 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU); 103 if (ret) 104 goto out_recover; 105 106 i915_gem_suspend(i915); 107 108 /* 109 * More objects may have become unpinned as requests were 110 * retired. Now try to evict again. The gt may be wedged here 111 * in which case we automatically fall back to memcpy. 112 * We allow also backing up pinned objects that have not been 113 * marked for early recover, and that may contain, for example, 114 * page-tables for the migrate context. 115 */ 116 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU | 117 I915_TTM_BACKUP_PINNED); 118 if (ret) 119 goto out_recover; 120 121 /* 122 * Remaining objects are backed up using memcpy once we've stopped 123 * using the migrate context. 124 */ 125 ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED); 126 if (ret) 127 goto out_recover; 128 129 return 0; 130 131 out_recover: 132 lmem_recover(i915); 133 134 return ret; 135 } 136 137 void i915_gem_suspend_late(struct drm_i915_private *i915) 138 { 139 struct drm_i915_gem_object *obj; 140 struct list_head *phases[] = { 141 &i915->mm.shrink_list, 142 &i915->mm.purge_list, 143 NULL 144 }, **phase; 145 struct intel_gt *gt; 146 unsigned long flags; 147 unsigned int i; 148 bool flush = false; 149 150 /* 151 * Neither the BIOS, ourselves or any other kernel 152 * expects the system to be in execlists mode on startup, 153 * so we need to reset the GPU back to legacy mode. And the only 154 * known way to disable logical contexts is through a GPU reset. 155 * 156 * So in order to leave the system in a known default configuration, 157 * always reset the GPU upon unload and suspend. Afterwards we then 158 * clean up the GEM state tracking, flushing off the requests and 159 * leaving the system in a known idle state. 160 * 161 * Note that is of the upmost importance that the GPU is idle and 162 * all stray writes are flushed *before* we dismantle the backing 163 * storage for the pinned objects. 164 * 165 * However, since we are uncertain that resetting the GPU on older 166 * machines is a good idea, we don't - just in case it leaves the 167 * machine in an unusable condition. 168 */ 169 170 /* Like i915_gem_suspend, flush tasks staged from fence triggers */ 171 rcu_barrier(); 172 173 for_each_gt(gt, i915, i) 174 intel_gt_suspend_late(gt); 175 176 spin_lock_irqsave(&i915->mm.obj_lock, flags); 177 for (phase = phases; *phase; phase++) { 178 list_for_each_entry(obj, *phase, mm.link) { 179 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 180 flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0; 181 __start_cpu_write(obj); /* presume auto-hibernate */ 182 } 183 } 184 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 185 if (flush) 186 wbinvd_on_all_cpus(); 187 } 188 189 int i915_gem_freeze(struct drm_i915_private *i915) 190 { 191 /* Discard all purgeable objects, let userspace recover those as 192 * required after resuming. 193 */ 194 i915_gem_shrink_all(i915); 195 196 return 0; 197 } 198 199 int i915_gem_freeze_late(struct drm_i915_private *i915) 200 { 201 struct drm_i915_gem_object *obj; 202 intel_wakeref_t wakeref; 203 204 /* 205 * Called just before we write the hibernation image. 206 * 207 * We need to update the domain tracking to reflect that the CPU 208 * will be accessing all the pages to create and restore from the 209 * hibernation, and so upon restoration those pages will be in the 210 * CPU domain. 211 * 212 * To make sure the hibernation image contains the latest state, 213 * we update that state just before writing out the image. 214 * 215 * To try and reduce the hibernation image, we manually shrink 216 * the objects as well, see i915_gem_freeze() 217 */ 218 219 with_intel_runtime_pm(&i915->runtime_pm, wakeref) 220 i915_gem_shrink(NULL, i915, -1UL, NULL, ~0); 221 i915_gem_drain_freed_objects(i915); 222 223 wbinvd_on_all_cpus(); 224 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) 225 __start_cpu_write(obj); 226 227 return 0; 228 } 229 230 void i915_gem_resume(struct drm_i915_private *i915) 231 { 232 struct intel_gt *gt; 233 int ret, i, j; 234 235 GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 236 237 ret = lmem_restore(i915, 0); 238 GEM_WARN_ON(ret); 239 240 /* 241 * As we didn't flush the kernel context before suspend, we cannot 242 * guarantee that the context image is complete. So let's just reset 243 * it and start again. 244 */ 245 for_each_gt(gt, i915, i) 246 if (intel_gt_resume(gt)) 247 goto err_wedged; 248 249 ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU); 250 GEM_WARN_ON(ret); 251 252 return; 253 254 err_wedged: 255 for_each_gt(gt, i915, j) { 256 if (!intel_gt_is_wedged(gt)) { 257 dev_err(i915->drm.dev, 258 "Failed to re-initialize GPU[%u], declaring it wedged!\n", 259 j); 260 intel_gt_set_wedged(gt); 261 } 262 263 if (j == i) 264 break; 265 } 266 } 267