1 // SPDX-License-Identifier: GPL-2.0 AND MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <kunit/test.h> 7 8 #include "xe_bo_evict.h" 9 #include "xe_pci.h" 10 11 static int ccs_test_migrate(struct xe_gt *gt, struct xe_bo *bo, 12 bool clear, u64 get_val, u64 assign_val, 13 struct kunit *test) 14 { 15 struct dma_fence *fence; 16 struct ttm_tt *ttm; 17 struct page *page; 18 pgoff_t ccs_page; 19 long timeout; 20 u64 *cpu_map; 21 int ret; 22 u32 offset; 23 24 /* Move bo to VRAM if not already there. */ 25 ret = xe_bo_validate(bo, NULL, false); 26 if (ret) { 27 KUNIT_FAIL(test, "Failed to validate bo.\n"); 28 return ret; 29 } 30 31 /* Optionally clear bo *and* CCS data in VRAM. */ 32 if (clear) { 33 fence = xe_migrate_clear(gt->migrate, bo, bo->ttm.resource, 0); 34 if (IS_ERR(fence)) { 35 KUNIT_FAIL(test, "Failed to submit bo clear.\n"); 36 return PTR_ERR(fence); 37 } 38 dma_fence_put(fence); 39 } 40 41 /* Evict to system. CCS data should be copied. */ 42 ret = xe_bo_evict(bo, true); 43 if (ret) { 44 KUNIT_FAIL(test, "Failed to evict bo.\n"); 45 return ret; 46 } 47 48 /* Sync all migration blits */ 49 timeout = dma_resv_wait_timeout(bo->ttm.base.resv, 50 DMA_RESV_USAGE_KERNEL, 51 true, 52 5 * HZ); 53 if (timeout <= 0) { 54 KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); 55 return -ETIME; 56 } 57 58 /* 59 * Bo with CCS data is now in system memory. Verify backing store 60 * and data integrity. Then assign for the next testing round while 61 * we still have a CPU map. 62 */ 63 ttm = bo->ttm.ttm; 64 if (!ttm || !ttm_tt_is_populated(ttm)) { 65 KUNIT_FAIL(test, "Bo was not in expected placement.\n"); 66 return -EINVAL; 67 } 68 69 ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT; 70 if (ccs_page >= ttm->num_pages) { 71 KUNIT_FAIL(test, "No TTM CCS pages present.\n"); 72 return -EINVAL; 73 } 74 75 page = ttm->pages[ccs_page]; 76 cpu_map = kmap_local_page(page); 77 78 /* Check first CCS value */ 79 if (cpu_map[0] != get_val) { 80 KUNIT_FAIL(test, 81 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 82 (unsigned long long)get_val, 83 (unsigned long long)cpu_map[0]); 84 ret = -EINVAL; 85 } 86 87 /* Check last CCS value, or at least last value in page. */ 88 offset = xe_device_ccs_bytes(gt->xe, bo->size); 89 offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1; 90 if (cpu_map[offset] != get_val) { 91 KUNIT_FAIL(test, 92 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 93 (unsigned long long)get_val, 94 (unsigned long long)cpu_map[offset]); 95 ret = -EINVAL; 96 } 97 98 cpu_map[0] = assign_val; 99 cpu_map[offset] = assign_val; 100 kunmap_local(cpu_map); 101 102 return ret; 103 } 104 105 static void ccs_test_run_gt(struct xe_device *xe, struct xe_gt *gt, 106 struct kunit *test) 107 { 108 struct xe_bo *bo; 109 u32 vram_bit; 110 int ret; 111 112 /* TODO: Sanity check */ 113 vram_bit = XE_BO_CREATE_VRAM0_BIT << gt->info.vram_id; 114 kunit_info(test, "Testing gt id %u vram id %u\n", gt->info.id, 115 gt->info.vram_id); 116 117 bo = xe_bo_create_locked(xe, NULL, NULL, SZ_1M, ttm_bo_type_device, 118 vram_bit); 119 if (IS_ERR(bo)) { 120 KUNIT_FAIL(test, "Failed to create bo.\n"); 121 return; 122 } 123 124 kunit_info(test, "Verifying that CCS data is cleared on creation.\n"); 125 ret = ccs_test_migrate(gt, bo, false, 0ULL, 0xdeadbeefdeadbeefULL, 126 test); 127 if (ret) 128 goto out_unlock; 129 130 kunit_info(test, "Verifying that CCS data survives migration.\n"); 131 ret = ccs_test_migrate(gt, bo, false, 0xdeadbeefdeadbeefULL, 132 0xdeadbeefdeadbeefULL, test); 133 if (ret) 134 goto out_unlock; 135 136 kunit_info(test, "Verifying that CCS data can be properly cleared.\n"); 137 ret = ccs_test_migrate(gt, bo, true, 0ULL, 0ULL, test); 138 139 out_unlock: 140 xe_bo_unlock_no_vm(bo); 141 xe_bo_put(bo); 142 } 143 144 static int ccs_test_run_device(struct xe_device *xe) 145 { 146 struct kunit *test = xe_cur_kunit(); 147 struct xe_gt *gt; 148 int id; 149 150 if (!xe_device_has_flat_ccs(xe)) { 151 kunit_info(test, "Skipping non-flat-ccs device.\n"); 152 return 0; 153 } 154 155 for_each_gt(gt, xe, id) 156 ccs_test_run_gt(xe, gt, test); 157 158 return 0; 159 } 160 161 void xe_ccs_migrate_kunit(struct kunit *test) 162 { 163 xe_call_for_each_device(ccs_test_run_device); 164 } 165 EXPORT_SYMBOL(xe_ccs_migrate_kunit); 166 167 static int evict_test_run_gt(struct xe_device *xe, struct xe_gt *gt, struct kunit *test) 168 { 169 struct xe_bo *bo, *external; 170 unsigned int bo_flags = XE_BO_CREATE_USER_BIT | 171 XE_BO_CREATE_VRAM_IF_DGFX(gt); 172 struct xe_vm *vm = xe_migrate_get_vm(xe->gt[0].migrate); 173 struct ww_acquire_ctx ww; 174 int err, i; 175 176 kunit_info(test, "Testing device %s gt id %u vram id %u\n", 177 dev_name(xe->drm.dev), gt->info.id, gt->info.vram_id); 178 179 for (i = 0; i < 2; ++i) { 180 xe_vm_lock(vm, &ww, 0, false); 181 bo = xe_bo_create(xe, NULL, vm, 0x10000, ttm_bo_type_device, 182 bo_flags); 183 xe_vm_unlock(vm, &ww); 184 if (IS_ERR(bo)) { 185 KUNIT_FAIL(test, "bo create err=%pe\n", bo); 186 break; 187 } 188 189 external = xe_bo_create(xe, NULL, NULL, 0x10000, 190 ttm_bo_type_device, bo_flags); 191 if (IS_ERR(external)) { 192 KUNIT_FAIL(test, "external bo create err=%pe\n", external); 193 goto cleanup_bo; 194 } 195 196 xe_bo_lock(external, &ww, 0, false); 197 err = xe_bo_pin_external(external); 198 xe_bo_unlock(external, &ww); 199 if (err) { 200 KUNIT_FAIL(test, "external bo pin err=%pe\n", 201 ERR_PTR(err)); 202 goto cleanup_external; 203 } 204 205 err = xe_bo_evict_all(xe); 206 if (err) { 207 KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err)); 208 goto cleanup_all; 209 } 210 211 err = xe_bo_restore_kernel(xe); 212 if (err) { 213 KUNIT_FAIL(test, "restore kernel err=%pe\n", 214 ERR_PTR(err)); 215 goto cleanup_all; 216 } 217 218 err = xe_bo_restore_user(xe); 219 if (err) { 220 KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); 221 goto cleanup_all; 222 } 223 224 if (!xe_bo_is_vram(external)) { 225 KUNIT_FAIL(test, "external bo is not vram\n"); 226 err = -EPROTO; 227 goto cleanup_all; 228 } 229 230 if (xe_bo_is_vram(bo)) { 231 KUNIT_FAIL(test, "bo is vram\n"); 232 err = -EPROTO; 233 goto cleanup_all; 234 } 235 236 if (i) { 237 down_read(&vm->lock); 238 xe_vm_lock(vm, &ww, 0, false); 239 err = xe_bo_validate(bo, bo->vm, false); 240 xe_vm_unlock(vm, &ww); 241 up_read(&vm->lock); 242 if (err) { 243 KUNIT_FAIL(test, "bo valid err=%pe\n", 244 ERR_PTR(err)); 245 goto cleanup_all; 246 } 247 xe_bo_lock(external, &ww, 0, false); 248 err = xe_bo_validate(external, NULL, false); 249 xe_bo_unlock(external, &ww); 250 if (err) { 251 KUNIT_FAIL(test, "external bo valid err=%pe\n", 252 ERR_PTR(err)); 253 goto cleanup_all; 254 } 255 } 256 257 xe_bo_lock(external, &ww, 0, false); 258 xe_bo_unpin_external(external); 259 xe_bo_unlock(external, &ww); 260 261 xe_bo_put(external); 262 xe_bo_put(bo); 263 continue; 264 265 cleanup_all: 266 xe_bo_lock(external, &ww, 0, false); 267 xe_bo_unpin_external(external); 268 xe_bo_unlock(external, &ww); 269 cleanup_external: 270 xe_bo_put(external); 271 cleanup_bo: 272 xe_bo_put(bo); 273 break; 274 } 275 276 xe_vm_put(vm); 277 278 return 0; 279 } 280 281 static int evict_test_run_device(struct xe_device *xe) 282 { 283 struct kunit *test = xe_cur_kunit(); 284 struct xe_gt *gt; 285 int id; 286 287 if (!IS_DGFX(xe)) { 288 kunit_info(test, "Skipping non-discrete device %s.\n", 289 dev_name(xe->drm.dev)); 290 return 0; 291 } 292 293 for_each_gt(gt, xe, id) 294 evict_test_run_gt(xe, gt, test); 295 296 return 0; 297 } 298 299 void xe_bo_evict_kunit(struct kunit *test) 300 { 301 xe_call_for_each_device(evict_test_run_device); 302 } 303 EXPORT_SYMBOL(xe_bo_evict_kunit); 304