1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_query.h" 7 8 #include <linux/nospec.h> 9 #include <linux/sched/clock.h> 10 11 #include <drm/ttm/ttm_placement.h> 12 #include <drm/xe_drm.h> 13 14 #include "regs/xe_engine_regs.h" 15 #include "xe_bo.h" 16 #include "xe_device.h" 17 #include "xe_exec_queue.h" 18 #include "xe_ggtt.h" 19 #include "xe_gt.h" 20 #include "xe_guc_hwconfig.h" 21 #include "xe_macros.h" 22 #include "xe_mmio.h" 23 #include "xe_ttm_vram_mgr.h" 24 25 static const u16 xe_to_user_engine_class[] = { 26 [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, 27 [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY, 28 [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE, 29 [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE, 30 [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, 31 }; 32 33 static const enum xe_engine_class user_to_xe_engine_class[] = { 34 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 35 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 36 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 37 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 38 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 39 }; 40 41 static size_t calc_hw_engine_info_size(struct xe_device *xe) 42 { 43 struct xe_hw_engine *hwe; 44 enum xe_hw_engine_id id; 45 struct xe_gt *gt; 46 u8 gt_id; 47 int i = 0; 48 49 for_each_gt(gt, xe, gt_id) 50 for_each_hw_engine(hwe, gt, id) { 51 if (xe_hw_engine_is_reserved(hwe)) 52 continue; 53 i++; 54 } 55 56 return sizeof(struct drm_xe_query_engines) + 57 i * sizeof(struct drm_xe_engine); 58 } 59 60 typedef u64 (*__ktime_func_t)(void); 61 static __ktime_func_t __clock_id_to_func(clockid_t clk_id) 62 { 63 /* 64 * Use logic same as the perf subsystem to allow user to select the 65 * reference clock id to be used for timestamps. 66 */ 67 switch (clk_id) { 68 case CLOCK_MONOTONIC: 69 return &ktime_get_ns; 70 case CLOCK_MONOTONIC_RAW: 71 return &ktime_get_raw_ns; 72 case CLOCK_REALTIME: 73 return &ktime_get_real_ns; 74 case CLOCK_BOOTTIME: 75 return &ktime_get_boottime_ns; 76 case CLOCK_TAI: 77 return &ktime_get_clocktai_ns; 78 default: 79 return NULL; 80 } 81 } 82 83 static void 84 __read_timestamps(struct xe_gt *gt, 85 struct xe_reg lower_reg, 86 struct xe_reg upper_reg, 87 u64 *engine_ts, 88 u64 *cpu_ts, 89 u64 *cpu_delta, 90 __ktime_func_t cpu_clock) 91 { 92 u32 upper, lower, old_upper, loop = 0; 93 94 upper = xe_mmio_read32(gt, upper_reg); 95 do { 96 *cpu_delta = local_clock(); 97 *cpu_ts = cpu_clock(); 98 lower = xe_mmio_read32(gt, lower_reg); 99 *cpu_delta = local_clock() - *cpu_delta; 100 old_upper = upper; 101 upper = xe_mmio_read32(gt, upper_reg); 102 } while (upper != old_upper && loop++ < 2); 103 104 *engine_ts = (u64)upper << 32 | lower; 105 } 106 107 static int 108 query_engine_cycles(struct xe_device *xe, 109 struct drm_xe_device_query *query) 110 { 111 struct drm_xe_query_engine_cycles __user *query_ptr; 112 struct drm_xe_engine_class_instance *eci; 113 struct drm_xe_query_engine_cycles resp; 114 size_t size = sizeof(resp); 115 __ktime_func_t cpu_clock; 116 struct xe_hw_engine *hwe; 117 struct xe_gt *gt; 118 119 if (query->size == 0) { 120 query->size = size; 121 return 0; 122 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 123 return -EINVAL; 124 } 125 126 query_ptr = u64_to_user_ptr(query->data); 127 if (copy_from_user(&resp, query_ptr, size)) 128 return -EFAULT; 129 130 cpu_clock = __clock_id_to_func(resp.clockid); 131 if (!cpu_clock) 132 return -EINVAL; 133 134 eci = &resp.eci; 135 if (eci->gt_id > XE_MAX_GT_PER_TILE) 136 return -EINVAL; 137 138 gt = xe_device_get_gt(xe, eci->gt_id); 139 if (!gt) 140 return -EINVAL; 141 142 if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) 143 return -EINVAL; 144 145 hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class], 146 eci->engine_instance, true); 147 if (!hwe) 148 return -EINVAL; 149 150 xe_device_mem_access_get(xe); 151 xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 152 153 __read_timestamps(gt, 154 RING_TIMESTAMP(hwe->mmio_base), 155 RING_TIMESTAMP_UDW(hwe->mmio_base), 156 &resp.engine_cycles, 157 &resp.cpu_timestamp, 158 &resp.cpu_delta, 159 cpu_clock); 160 161 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 162 xe_device_mem_access_put(xe); 163 resp.width = 36; 164 165 /* Only write to the output fields of user query */ 166 if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp)) 167 return -EFAULT; 168 169 if (put_user(resp.cpu_delta, &query_ptr->cpu_delta)) 170 return -EFAULT; 171 172 if (put_user(resp.engine_cycles, &query_ptr->engine_cycles)) 173 return -EFAULT; 174 175 if (put_user(resp.width, &query_ptr->width)) 176 return -EFAULT; 177 178 return 0; 179 } 180 181 static int query_engines(struct xe_device *xe, 182 struct drm_xe_device_query *query) 183 { 184 size_t size = calc_hw_engine_info_size(xe); 185 struct drm_xe_query_engines __user *query_ptr = 186 u64_to_user_ptr(query->data); 187 struct drm_xe_query_engines *engines; 188 struct xe_hw_engine *hwe; 189 enum xe_hw_engine_id id; 190 struct xe_gt *gt; 191 u8 gt_id; 192 int i = 0; 193 194 if (query->size == 0) { 195 query->size = size; 196 return 0; 197 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 198 return -EINVAL; 199 } 200 201 engines = kmalloc(size, GFP_KERNEL); 202 if (!engines) 203 return -ENOMEM; 204 205 for_each_gt(gt, xe, gt_id) 206 for_each_hw_engine(hwe, gt, id) { 207 if (xe_hw_engine_is_reserved(hwe)) 208 continue; 209 210 engines->engines[i].instance.engine_class = 211 xe_to_user_engine_class[hwe->class]; 212 engines->engines[i].instance.engine_instance = 213 hwe->logical_instance; 214 engines->engines[i].instance.gt_id = gt->info.id; 215 engines->engines[i].instance.pad = 0; 216 memset(engines->engines[i].reserved, 0, 217 sizeof(engines->engines[i].reserved)); 218 219 i++; 220 } 221 222 engines->pad = 0; 223 engines->num_engines = i; 224 225 if (copy_to_user(query_ptr, engines, size)) { 226 kfree(engines); 227 return -EFAULT; 228 } 229 kfree(engines); 230 231 return 0; 232 } 233 234 static size_t calc_mem_regions_size(struct xe_device *xe) 235 { 236 u32 num_managers = 1; 237 int i; 238 239 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) 240 if (ttm_manager_type(&xe->ttm, i)) 241 num_managers++; 242 243 return offsetof(struct drm_xe_query_mem_regions, mem_regions[num_managers]); 244 } 245 246 static int query_mem_regions(struct xe_device *xe, 247 struct drm_xe_device_query *query) 248 { 249 size_t size = calc_mem_regions_size(xe); 250 struct drm_xe_query_mem_regions *mem_regions; 251 struct drm_xe_query_mem_regions __user *query_ptr = 252 u64_to_user_ptr(query->data); 253 struct ttm_resource_manager *man; 254 int ret, i; 255 256 if (query->size == 0) { 257 query->size = size; 258 return 0; 259 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 260 return -EINVAL; 261 } 262 263 mem_regions = kzalloc(size, GFP_KERNEL); 264 if (XE_IOCTL_DBG(xe, !mem_regions)) 265 return -ENOMEM; 266 267 man = ttm_manager_type(&xe->ttm, XE_PL_TT); 268 mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; 269 /* 270 * The instance needs to be a unique number that represents the index 271 * in the placement mask used at xe_gem_create_ioctl() for the 272 * xe_bo_create() placement. 273 */ 274 mem_regions->mem_regions[0].instance = 0; 275 mem_regions->mem_regions[0].min_page_size = PAGE_SIZE; 276 mem_regions->mem_regions[0].total_size = man->size << PAGE_SHIFT; 277 if (perfmon_capable()) 278 mem_regions->mem_regions[0].used = ttm_resource_manager_usage(man); 279 mem_regions->num_mem_regions = 1; 280 281 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { 282 man = ttm_manager_type(&xe->ttm, i); 283 if (man) { 284 mem_regions->mem_regions[mem_regions->num_mem_regions].mem_class = 285 DRM_XE_MEM_REGION_CLASS_VRAM; 286 mem_regions->mem_regions[mem_regions->num_mem_regions].instance = 287 mem_regions->num_mem_regions; 288 mem_regions->mem_regions[mem_regions->num_mem_regions].min_page_size = 289 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? 290 SZ_64K : PAGE_SIZE; 291 mem_regions->mem_regions[mem_regions->num_mem_regions].total_size = 292 man->size; 293 294 if (perfmon_capable()) { 295 xe_ttm_vram_get_used(man, 296 &mem_regions->mem_regions 297 [mem_regions->num_mem_regions].used, 298 &mem_regions->mem_regions 299 [mem_regions->num_mem_regions].cpu_visible_used); 300 } 301 302 mem_regions->mem_regions[mem_regions->num_mem_regions].cpu_visible_size = 303 xe_ttm_vram_get_cpu_visible_size(man); 304 mem_regions->num_mem_regions++; 305 } 306 } 307 308 if (!copy_to_user(query_ptr, mem_regions, size)) 309 ret = 0; 310 else 311 ret = -ENOSPC; 312 313 kfree(mem_regions); 314 return ret; 315 } 316 317 static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) 318 { 319 const u32 num_params = DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; 320 size_t size = 321 sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); 322 struct drm_xe_query_config __user *query_ptr = 323 u64_to_user_ptr(query->data); 324 struct drm_xe_query_config *config; 325 326 if (query->size == 0) { 327 query->size = size; 328 return 0; 329 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 330 return -EINVAL; 331 } 332 333 config = kzalloc(size, GFP_KERNEL); 334 if (!config) 335 return -ENOMEM; 336 337 config->num_params = num_params; 338 config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = 339 xe->info.devid | (xe->info.revid << 16); 340 if (xe_device_get_root_tile(xe)->mem.vram.usable_size) 341 config->info[DRM_XE_QUERY_CONFIG_FLAGS] = 342 DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; 343 config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = 344 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; 345 config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; 346 config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = 347 xe_exec_queue_device_get_max_priority(xe); 348 349 if (copy_to_user(query_ptr, config, size)) { 350 kfree(config); 351 return -EFAULT; 352 } 353 kfree(config); 354 355 return 0; 356 } 357 358 static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query) 359 { 360 struct xe_gt *gt; 361 size_t size = sizeof(struct drm_xe_query_gt_list) + 362 xe->info.gt_count * sizeof(struct drm_xe_gt); 363 struct drm_xe_query_gt_list __user *query_ptr = 364 u64_to_user_ptr(query->data); 365 struct drm_xe_query_gt_list *gt_list; 366 u8 id; 367 368 if (query->size == 0) { 369 query->size = size; 370 return 0; 371 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 372 return -EINVAL; 373 } 374 375 gt_list = kzalloc(size, GFP_KERNEL); 376 if (!gt_list) 377 return -ENOMEM; 378 379 gt_list->num_gt = xe->info.gt_count; 380 381 for_each_gt(gt, xe, id) { 382 if (xe_gt_is_media_type(gt)) 383 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA; 384 else 385 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN; 386 gt_list->gt_list[id].tile_id = gt_to_tile(gt)->id; 387 gt_list->gt_list[id].gt_id = gt->info.id; 388 gt_list->gt_list[id].reference_clock = gt->info.reference_clock; 389 /* 390 * The mem_regions indexes in the mask below need to 391 * directly identify the struct 392 * drm_xe_query_mem_regions' instance constructed at 393 * query_mem_regions() 394 * 395 * For our current platforms: 396 * Bit 0 -> System Memory 397 * Bit 1 -> VRAM0 on Tile0 398 * Bit 2 -> VRAM1 on Tile1 399 * However the uAPI is generic and it's userspace's 400 * responsibility to check the mem_class, without any 401 * assumption. 402 */ 403 if (!IS_DGFX(xe)) 404 gt_list->gt_list[id].near_mem_regions = 0x1; 405 else 406 gt_list->gt_list[id].near_mem_regions = 407 BIT(gt_to_tile(gt)->id) << 1; 408 gt_list->gt_list[id].far_mem_regions = xe->info.mem_region_mask ^ 409 gt_list->gt_list[id].near_mem_regions; 410 } 411 412 if (copy_to_user(query_ptr, gt_list, size)) { 413 kfree(gt_list); 414 return -EFAULT; 415 } 416 kfree(gt_list); 417 418 return 0; 419 } 420 421 static int query_hwconfig(struct xe_device *xe, 422 struct drm_xe_device_query *query) 423 { 424 struct xe_gt *gt = xe_root_mmio_gt(xe); 425 size_t size = xe_guc_hwconfig_size(>->uc.guc); 426 void __user *query_ptr = u64_to_user_ptr(query->data); 427 void *hwconfig; 428 429 if (query->size == 0) { 430 query->size = size; 431 return 0; 432 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 433 return -EINVAL; 434 } 435 436 hwconfig = kzalloc(size, GFP_KERNEL); 437 if (!hwconfig) 438 return -ENOMEM; 439 440 xe_device_mem_access_get(xe); 441 xe_guc_hwconfig_copy(>->uc.guc, hwconfig); 442 xe_device_mem_access_put(xe); 443 444 if (copy_to_user(query_ptr, hwconfig, size)) { 445 kfree(hwconfig); 446 return -EFAULT; 447 } 448 kfree(hwconfig); 449 450 return 0; 451 } 452 453 static size_t calc_topo_query_size(struct xe_device *xe) 454 { 455 return xe->info.gt_count * 456 (3 * sizeof(struct drm_xe_query_topology_mask) + 457 sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) + 458 sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) + 459 sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss)); 460 } 461 462 static void __user *copy_mask(void __user *ptr, 463 struct drm_xe_query_topology_mask *topo, 464 void *mask, size_t mask_size) 465 { 466 topo->num_bytes = mask_size; 467 468 if (copy_to_user(ptr, topo, sizeof(*topo))) 469 return ERR_PTR(-EFAULT); 470 ptr += sizeof(topo); 471 472 if (copy_to_user(ptr, mask, mask_size)) 473 return ERR_PTR(-EFAULT); 474 ptr += mask_size; 475 476 return ptr; 477 } 478 479 static int query_gt_topology(struct xe_device *xe, 480 struct drm_xe_device_query *query) 481 { 482 void __user *query_ptr = u64_to_user_ptr(query->data); 483 size_t size = calc_topo_query_size(xe); 484 struct drm_xe_query_topology_mask topo; 485 struct xe_gt *gt; 486 int id; 487 488 if (query->size == 0) { 489 query->size = size; 490 return 0; 491 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 492 return -EINVAL; 493 } 494 495 for_each_gt(gt, xe, id) { 496 topo.gt_id = id; 497 498 topo.type = DRM_XE_TOPO_DSS_GEOMETRY; 499 query_ptr = copy_mask(query_ptr, &topo, 500 gt->fuse_topo.g_dss_mask, 501 sizeof(gt->fuse_topo.g_dss_mask)); 502 if (IS_ERR(query_ptr)) 503 return PTR_ERR(query_ptr); 504 505 topo.type = DRM_XE_TOPO_DSS_COMPUTE; 506 query_ptr = copy_mask(query_ptr, &topo, 507 gt->fuse_topo.c_dss_mask, 508 sizeof(gt->fuse_topo.c_dss_mask)); 509 if (IS_ERR(query_ptr)) 510 return PTR_ERR(query_ptr); 511 512 topo.type = DRM_XE_TOPO_EU_PER_DSS; 513 query_ptr = copy_mask(query_ptr, &topo, 514 gt->fuse_topo.eu_mask_per_dss, 515 sizeof(gt->fuse_topo.eu_mask_per_dss)); 516 if (IS_ERR(query_ptr)) 517 return PTR_ERR(query_ptr); 518 } 519 520 return 0; 521 } 522 523 static int (* const xe_query_funcs[])(struct xe_device *xe, 524 struct drm_xe_device_query *query) = { 525 query_engines, 526 query_mem_regions, 527 query_config, 528 query_gt_list, 529 query_hwconfig, 530 query_gt_topology, 531 query_engine_cycles, 532 }; 533 534 int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 535 { 536 struct xe_device *xe = to_xe_device(dev); 537 struct drm_xe_device_query *query = data; 538 u32 idx; 539 540 if (XE_IOCTL_DBG(xe, query->extensions) || 541 XE_IOCTL_DBG(xe, query->reserved[0] || query->reserved[1])) 542 return -EINVAL; 543 544 if (XE_IOCTL_DBG(xe, query->query >= ARRAY_SIZE(xe_query_funcs))) 545 return -EINVAL; 546 547 idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs)); 548 if (XE_IOCTL_DBG(xe, !xe_query_funcs[idx])) 549 return -EINVAL; 550 551 return xe_query_funcs[idx](xe, query); 552 } 553