1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/drm_print.h> 7 #include <drm/ttm/ttm_placement.h> 8 #include <drm/ttm/ttm_tt.h> 9 10 #include "i915_drv.h" 11 #include "intel_memory_region.h" 12 #include "intel_region_ttm.h" 13 14 #include "gem/i915_gem_region.h" 15 #include "gem/i915_gem_ttm.h" 16 #include "gem/i915_gem_ttm_move.h" 17 #include "gem/i915_gem_ttm_pm.h" 18 19 /** 20 * i915_ttm_backup_free - Free any backup attached to this object 21 * @obj: The object whose backup is to be freed. 22 */ 23 void i915_ttm_backup_free(struct drm_i915_gem_object *obj) 24 { 25 if (obj->ttm.backup) { 26 i915_gem_object_put(obj->ttm.backup); 27 obj->ttm.backup = NULL; 28 } 29 } 30 31 /** 32 * struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore 33 * @base: The i915_gem_apply_to_region we derive from. 34 * @allow_gpu: Whether using the gpu blitter is allowed. 35 * @backup_pinned: On backup, backup also pinned objects. 36 */ 37 struct i915_gem_ttm_pm_apply { 38 struct i915_gem_apply_to_region base; 39 bool allow_gpu : 1; 40 bool backup_pinned : 1; 41 }; 42 43 static int i915_ttm_backup(struct i915_gem_apply_to_region *apply, 44 struct drm_i915_gem_object *obj) 45 { 46 struct i915_gem_ttm_pm_apply *pm_apply = 47 container_of(apply, typeof(*pm_apply), base); 48 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 49 struct ttm_buffer_object *backup_bo; 50 struct drm_i915_private *i915 = 51 container_of(bo->bdev, typeof(*i915), bdev); 52 struct drm_i915_gem_object *backup; 53 struct ttm_operation_ctx ctx = {}; 54 unsigned int flags; 55 int err = 0; 56 57 if (!i915_ttm_cpu_maps_iomem(bo->resource) || obj->ttm.backup) 58 return 0; 59 60 if (pm_apply->allow_gpu && i915_gem_object_evictable(obj)) 61 return ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx); 62 63 if (!pm_apply->backup_pinned || 64 (pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_PM_EARLY))) 65 return 0; 66 67 if (obj->flags & I915_BO_ALLOC_PM_VOLATILE) 68 return 0; 69 70 /* 71 * It seems that we might have some framebuffers still pinned at this 72 * stage, but for such objects we might also need to deal with the CCS 73 * aux state. Make sure we force the save/restore of the CCS state, 74 * otherwise we might observe display corruption, when returning from 75 * suspend. 76 */ 77 flags = 0; 78 if (i915_gem_object_needs_ccs_pages(obj)) { 79 WARN_ON_ONCE(!i915_gem_object_is_framebuffer(obj)); 80 WARN_ON_ONCE(!pm_apply->allow_gpu); 81 82 flags = I915_BO_ALLOC_CCS_AUX; 83 } 84 backup = i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM], 85 obj->base.size, 0, flags); 86 if (IS_ERR(backup)) 87 return PTR_ERR(backup); 88 89 err = i915_gem_object_lock(backup, apply->ww); 90 if (err) 91 goto out_no_lock; 92 93 backup_bo = i915_gem_to_ttm(backup); 94 err = ttm_bo_populate(backup_bo, &ctx); 95 if (err) 96 goto out_no_populate; 97 98 err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false); 99 if (err) { 100 drm_err(&i915->drm, 101 "Unable to copy from device to system memory, err:%pe\n", 102 ERR_PTR(err)); 103 goto out_no_populate; 104 } 105 ttm_bo_wait_ctx(backup_bo, &ctx); 106 107 obj->ttm.backup = backup; 108 return 0; 109 110 out_no_populate: 111 i915_gem_ww_unlock_single(backup); 112 out_no_lock: 113 i915_gem_object_put(backup); 114 115 return err; 116 } 117 118 static int i915_ttm_recover(struct i915_gem_apply_to_region *apply, 119 struct drm_i915_gem_object *obj) 120 { 121 i915_ttm_backup_free(obj); 122 return 0; 123 } 124 125 /** 126 * i915_ttm_recover_region - Free the backup of all objects of a region 127 * @mr: The memory region 128 * 129 * Checks all objects of a region if there is backup attached and if so 130 * frees that backup. Typically this is called to recover after a partially 131 * performed backup. 132 */ 133 void i915_ttm_recover_region(struct intel_memory_region *mr) 134 { 135 static const struct i915_gem_apply_to_region_ops recover_ops = { 136 .process_obj = i915_ttm_recover, 137 }; 138 struct i915_gem_apply_to_region apply = {.ops = &recover_ops}; 139 int ret; 140 141 ret = i915_gem_process_region(mr, &apply); 142 GEM_WARN_ON(ret); 143 } 144 145 /** 146 * i915_ttm_backup_region - Back up all objects of a region to smem. 147 * @mr: The memory region 148 * @flags: TTM backup flags 149 * 150 * Loops over all objects of a region and either evicts them if they are 151 * evictable or backs them up using a backup object if they are pinned. 152 * 153 * Return: Zero on success. Negative error code on error. 154 */ 155 int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags) 156 { 157 static const struct i915_gem_apply_to_region_ops backup_ops = { 158 .process_obj = i915_ttm_backup, 159 }; 160 struct i915_gem_ttm_pm_apply pm_apply = { 161 .base = {.ops = &backup_ops}, 162 .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU, 163 .backup_pinned = flags & I915_TTM_BACKUP_PINNED, 164 }; 165 166 return i915_gem_process_region(mr, &pm_apply.base); 167 } 168 169 static int i915_ttm_restore(struct i915_gem_apply_to_region *apply, 170 struct drm_i915_gem_object *obj) 171 { 172 struct i915_gem_ttm_pm_apply *pm_apply = 173 container_of(apply, typeof(*pm_apply), base); 174 struct drm_i915_gem_object *backup = obj->ttm.backup; 175 struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(backup); 176 struct ttm_operation_ctx ctx = {}; 177 int err; 178 179 if (!backup) 180 return 0; 181 182 if (!pm_apply->allow_gpu && !(obj->flags & I915_BO_ALLOC_PM_EARLY)) 183 return 0; 184 185 err = i915_gem_object_lock(backup, apply->ww); 186 if (err) 187 return err; 188 189 /* Content may have been swapped. */ 190 if (!backup_bo->resource) 191 err = ttm_bo_validate(backup_bo, i915_ttm_sys_placement(), &ctx); 192 if (!err) 193 err = ttm_bo_populate(backup_bo, &ctx); 194 if (!err) { 195 err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu, 196 false); 197 GEM_WARN_ON(err); 198 ttm_bo_wait_ctx(backup_bo, &ctx); 199 200 obj->ttm.backup = NULL; 201 err = 0; 202 } 203 204 i915_gem_ww_unlock_single(backup); 205 206 if (!err) 207 i915_gem_object_put(backup); 208 209 return err; 210 } 211 212 /** 213 * i915_ttm_restore_region - Restore backed-up objects of a region from smem. 214 * @mr: The memory region 215 * @flags: TTM backup flags 216 * 217 * Loops over all objects of a region and if they are backed-up, restores 218 * them from smem. 219 * 220 * Return: Zero on success. Negative error code on error. 221 */ 222 int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags) 223 { 224 static const struct i915_gem_apply_to_region_ops restore_ops = { 225 .process_obj = i915_ttm_restore, 226 }; 227 struct i915_gem_ttm_pm_apply pm_apply = { 228 .base = {.ops = &restore_ops}, 229 .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU, 230 }; 231 232 return i915_gem_process_region(mr, &pm_apply.base); 233 } 234