1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_pm.h" 8 #include "gem/i915_gem_ttm_pm.h" 9 #include "gt/intel_gt.h" 10 #include "gt/intel_gt_pm.h" 11 #include "gt/intel_gt_requests.h" 12 13 #include "i915_driver.h" 14 #include "i915_drv.h" 15 16 #if defined(CONFIG_X86) 17 #include <asm/smp.h> 18 #else 19 #define wbinvd_on_all_cpus() \ 20 pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__) 21 #endif 22 23 void i915_gem_suspend(struct drm_i915_private *i915) 24 { 25 struct intel_gt *gt; 26 unsigned int i; 27 28 GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 29 30 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0); 31 flush_workqueue(i915->wq); 32 33 /* 34 * We have to flush all the executing contexts to main memory so 35 * that they can saved in the hibernation image. To ensure the last 36 * context image is coherent, we have to switch away from it. That 37 * leaves the i915->kernel_context still active when 38 * we actually suspend, and its image in memory may not match the GPU 39 * state. Fortunately, the kernel_context is disposable and we do 40 * not rely on its state. 41 */ 42 for_each_gt(gt, i915, i) 43 intel_gt_suspend_prepare(gt); 44 45 i915_gem_drain_freed_objects(i915); 46 } 47 48 static int lmem_restore(struct drm_i915_private *i915, u32 flags) 49 { 50 struct intel_memory_region *mr; 51 int ret = 0, id; 52 53 for_each_memory_region(mr, i915, id) { 54 if (mr->type == INTEL_MEMORY_LOCAL) { 55 ret = i915_ttm_restore_region(mr, flags); 56 if (ret) 57 break; 58 } 59 } 60 61 return ret; 62 } 63 64 static int lmem_suspend(struct drm_i915_private *i915, u32 flags) 65 { 66 struct intel_memory_region *mr; 67 int ret = 0, id; 68 69 for_each_memory_region(mr, i915, id) { 70 if (mr->type == INTEL_MEMORY_LOCAL) { 71 ret = i915_ttm_backup_region(mr, flags); 72 if (ret) 73 break; 74 } 75 } 76 77 return ret; 78 } 79 80 static void lmem_recover(struct drm_i915_private *i915) 81 { 82 struct intel_memory_region *mr; 83 int id; 84 85 for_each_memory_region(mr, i915, id) 86 if (mr->type == INTEL_MEMORY_LOCAL) 87 i915_ttm_recover_region(mr); 88 } 89 90 int i915_gem_backup_suspend(struct drm_i915_private *i915) 91 { 92 int ret; 93 94 /* Opportunistically try to evict unpinned objects */ 95 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU); 96 if (ret) 97 goto out_recover; 98 99 i915_gem_suspend(i915); 100 101 /* 102 * More objects may have become unpinned as requests were 103 * retired. Now try to evict again. The gt may be wedged here 104 * in which case we automatically fall back to memcpy. 105 * We allow also backing up pinned objects that have not been 106 * marked for early recover, and that may contain, for example, 107 * page-tables for the migrate context. 108 */ 109 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU | 110 I915_TTM_BACKUP_PINNED); 111 if (ret) 112 goto out_recover; 113 114 /* 115 * Remaining objects are backed up using memcpy once we've stopped 116 * using the migrate context. 117 */ 118 ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED); 119 if (ret) 120 goto out_recover; 121 122 return 0; 123 124 out_recover: 125 lmem_recover(i915); 126 127 return ret; 128 } 129 130 void i915_gem_suspend_late(struct drm_i915_private *i915) 131 { 132 struct drm_i915_gem_object *obj; 133 struct list_head *phases[] = { 134 &i915->mm.shrink_list, 135 &i915->mm.purge_list, 136 NULL 137 }, **phase; 138 struct intel_gt *gt; 139 unsigned long flags; 140 unsigned int i; 141 bool flush = false; 142 143 /* 144 * Neither the BIOS, ourselves or any other kernel 145 * expects the system to be in execlists mode on startup, 146 * so we need to reset the GPU back to legacy mode. And the only 147 * known way to disable logical contexts is through a GPU reset. 148 * 149 * So in order to leave the system in a known default configuration, 150 * always reset the GPU upon unload and suspend. Afterwards we then 151 * clean up the GEM state tracking, flushing off the requests and 152 * leaving the system in a known idle state. 153 * 154 * Note that is of the upmost importance that the GPU is idle and 155 * all stray writes are flushed *before* we dismantle the backing 156 * storage for the pinned objects. 157 * 158 * However, since we are uncertain that resetting the GPU on older 159 * machines is a good idea, we don't - just in case it leaves the 160 * machine in an unusable condition. 161 */ 162 163 for_each_gt(gt, i915, i) 164 intel_gt_suspend_late(gt); 165 166 spin_lock_irqsave(&i915->mm.obj_lock, flags); 167 for (phase = phases; *phase; phase++) { 168 list_for_each_entry(obj, *phase, mm.link) { 169 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 170 flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0; 171 __start_cpu_write(obj); /* presume auto-hibernate */ 172 } 173 } 174 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 175 if (flush) 176 wbinvd_on_all_cpus(); 177 } 178 179 int i915_gem_freeze(struct drm_i915_private *i915) 180 { 181 /* Discard all purgeable objects, let userspace recover those as 182 * required after resuming. 183 */ 184 i915_gem_shrink_all(i915); 185 186 return 0; 187 } 188 189 int i915_gem_freeze_late(struct drm_i915_private *i915) 190 { 191 struct drm_i915_gem_object *obj; 192 intel_wakeref_t wakeref; 193 194 /* 195 * Called just before we write the hibernation image. 196 * 197 * We need to update the domain tracking to reflect that the CPU 198 * will be accessing all the pages to create and restore from the 199 * hibernation, and so upon restoration those pages will be in the 200 * CPU domain. 201 * 202 * To make sure the hibernation image contains the latest state, 203 * we update that state just before writing out the image. 204 * 205 * To try and reduce the hibernation image, we manually shrink 206 * the objects as well, see i915_gem_freeze() 207 */ 208 209 with_intel_runtime_pm(&i915->runtime_pm, wakeref) 210 i915_gem_shrink(NULL, i915, -1UL, NULL, ~0); 211 i915_gem_drain_freed_objects(i915); 212 213 wbinvd_on_all_cpus(); 214 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) 215 __start_cpu_write(obj); 216 217 return 0; 218 } 219 220 void i915_gem_resume(struct drm_i915_private *i915) 221 { 222 struct intel_gt *gt; 223 int ret, i, j; 224 225 GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 226 227 ret = lmem_restore(i915, 0); 228 GEM_WARN_ON(ret); 229 230 /* 231 * As we didn't flush the kernel context before suspend, we cannot 232 * guarantee that the context image is complete. So let's just reset 233 * it and start again. 234 */ 235 for_each_gt(gt, i915, i) 236 if (intel_gt_resume(gt)) 237 goto err_wedged; 238 239 ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU); 240 GEM_WARN_ON(ret); 241 242 return; 243 244 err_wedged: 245 for_each_gt(gt, i915, j) { 246 if (!intel_gt_is_wedged(gt)) { 247 dev_err(i915->drm.dev, 248 "Failed to re-initialize GPU[%u], declaring it wedged!\n", 249 j); 250 intel_gt_set_wedged(gt); 251 } 252 253 if (j == i) 254 break; 255 } 256 } 257