1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_query.h" 7 8 #include <linux/nospec.h> 9 #include <linux/sched/clock.h> 10 11 #include <drm/ttm/ttm_placement.h> 12 #include <drm/xe_drm.h> 13 14 #include "regs/xe_engine_regs.h" 15 #include "regs/xe_gt_regs.h" 16 #include "xe_bo.h" 17 #include "xe_device.h" 18 #include "xe_exec_queue.h" 19 #include "xe_ggtt.h" 20 #include "xe_gt.h" 21 #include "xe_guc_hwconfig.h" 22 #include "xe_macros.h" 23 #include "xe_mmio.h" 24 #include "xe_ttm_vram_mgr.h" 25 26 static const u16 xe_to_user_engine_class[] = { 27 [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, 28 [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY, 29 [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE, 30 [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE, 31 [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, 32 }; 33 34 static const enum xe_engine_class user_to_xe_engine_class[] = { 35 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 36 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 37 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 38 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 39 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 40 }; 41 42 static size_t calc_hw_engine_info_size(struct xe_device *xe) 43 { 44 struct xe_hw_engine *hwe; 45 enum xe_hw_engine_id id; 46 struct xe_gt *gt; 47 u8 gt_id; 48 int i = 0; 49 50 for_each_gt(gt, xe, gt_id) 51 for_each_hw_engine(hwe, gt, id) { 52 if (xe_hw_engine_is_reserved(hwe)) 53 continue; 54 i++; 55 } 56 57 return sizeof(struct drm_xe_query_engines) + 58 i * sizeof(struct drm_xe_engine); 59 } 60 61 typedef u64 (*__ktime_func_t)(void); 62 static __ktime_func_t __clock_id_to_func(clockid_t clk_id) 63 { 64 /* 65 * Use logic same as the perf subsystem to allow user to select the 66 * reference clock id to be used for timestamps. 67 */ 68 switch (clk_id) { 69 case CLOCK_MONOTONIC: 70 return &ktime_get_ns; 71 case CLOCK_MONOTONIC_RAW: 72 return &ktime_get_raw_ns; 73 case CLOCK_REALTIME: 74 return &ktime_get_real_ns; 75 case CLOCK_BOOTTIME: 76 return &ktime_get_boottime_ns; 77 case CLOCK_TAI: 78 return &ktime_get_clocktai_ns; 79 default: 80 return NULL; 81 } 82 } 83 84 static void 85 __read_timestamps(struct xe_gt *gt, 86 struct xe_reg lower_reg, 87 struct xe_reg upper_reg, 88 u64 *engine_ts, 89 u64 *cpu_ts, 90 u64 *cpu_delta, 91 __ktime_func_t cpu_clock) 92 { 93 u32 upper, lower, old_upper, loop = 0; 94 95 upper = xe_mmio_read32(gt, upper_reg); 96 do { 97 *cpu_delta = local_clock(); 98 *cpu_ts = cpu_clock(); 99 lower = xe_mmio_read32(gt, lower_reg); 100 *cpu_delta = local_clock() - *cpu_delta; 101 old_upper = upper; 102 upper = xe_mmio_read32(gt, upper_reg); 103 } while (upper != old_upper && loop++ < 2); 104 105 *engine_ts = (u64)upper << 32 | lower; 106 } 107 108 static int 109 query_engine_cycles(struct xe_device *xe, 110 struct drm_xe_device_query *query) 111 { 112 struct drm_xe_query_engine_cycles __user *query_ptr; 113 struct drm_xe_engine_class_instance *eci; 114 struct drm_xe_query_engine_cycles resp; 115 size_t size = sizeof(resp); 116 __ktime_func_t cpu_clock; 117 struct xe_hw_engine *hwe; 118 struct xe_gt *gt; 119 120 if (query->size == 0) { 121 query->size = size; 122 return 0; 123 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 124 return -EINVAL; 125 } 126 127 query_ptr = u64_to_user_ptr(query->data); 128 if (copy_from_user(&resp, query_ptr, size)) 129 return -EFAULT; 130 131 cpu_clock = __clock_id_to_func(resp.clockid); 132 if (!cpu_clock) 133 return -EINVAL; 134 135 eci = &resp.eci; 136 if (eci->gt_id >= XE_MAX_GT_PER_TILE) 137 return -EINVAL; 138 139 gt = xe_device_get_gt(xe, eci->gt_id); 140 if (!gt) 141 return -EINVAL; 142 143 if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) 144 return -EINVAL; 145 146 hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class], 147 eci->engine_instance, true); 148 if (!hwe) 149 return -EINVAL; 150 151 if (xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)) 152 return -EIO; 153 154 __read_timestamps(gt, 155 RING_TIMESTAMP(hwe->mmio_base), 156 RING_TIMESTAMP_UDW(hwe->mmio_base), 157 &resp.engine_cycles, 158 &resp.cpu_timestamp, 159 &resp.cpu_delta, 160 cpu_clock); 161 162 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 163 resp.width = 36; 164 165 /* Only write to the output fields of user query */ 166 if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp)) 167 return -EFAULT; 168 169 if (put_user(resp.cpu_delta, &query_ptr->cpu_delta)) 170 return -EFAULT; 171 172 if (put_user(resp.engine_cycles, &query_ptr->engine_cycles)) 173 return -EFAULT; 174 175 if (put_user(resp.width, &query_ptr->width)) 176 return -EFAULT; 177 178 return 0; 179 } 180 181 static int query_engines(struct xe_device *xe, 182 struct drm_xe_device_query *query) 183 { 184 size_t size = calc_hw_engine_info_size(xe); 185 struct drm_xe_query_engines __user *query_ptr = 186 u64_to_user_ptr(query->data); 187 struct drm_xe_query_engines *engines; 188 struct xe_hw_engine *hwe; 189 enum xe_hw_engine_id id; 190 struct xe_gt *gt; 191 u8 gt_id; 192 int i = 0; 193 194 if (query->size == 0) { 195 query->size = size; 196 return 0; 197 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 198 return -EINVAL; 199 } 200 201 engines = kzalloc(size, GFP_KERNEL); 202 if (!engines) 203 return -ENOMEM; 204 205 for_each_gt(gt, xe, gt_id) 206 for_each_hw_engine(hwe, gt, id) { 207 if (xe_hw_engine_is_reserved(hwe)) 208 continue; 209 210 engines->engines[i].instance.engine_class = 211 xe_to_user_engine_class[hwe->class]; 212 engines->engines[i].instance.engine_instance = 213 hwe->logical_instance; 214 engines->engines[i].instance.gt_id = gt->info.id; 215 216 i++; 217 } 218 219 engines->num_engines = i; 220 221 if (copy_to_user(query_ptr, engines, size)) { 222 kfree(engines); 223 return -EFAULT; 224 } 225 kfree(engines); 226 227 return 0; 228 } 229 230 static size_t calc_mem_regions_size(struct xe_device *xe) 231 { 232 u32 num_managers = 1; 233 int i; 234 235 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) 236 if (ttm_manager_type(&xe->ttm, i)) 237 num_managers++; 238 239 return offsetof(struct drm_xe_query_mem_regions, mem_regions[num_managers]); 240 } 241 242 static int query_mem_regions(struct xe_device *xe, 243 struct drm_xe_device_query *query) 244 { 245 size_t size = calc_mem_regions_size(xe); 246 struct drm_xe_query_mem_regions *mem_regions; 247 struct drm_xe_query_mem_regions __user *query_ptr = 248 u64_to_user_ptr(query->data); 249 struct ttm_resource_manager *man; 250 int ret, i; 251 252 if (query->size == 0) { 253 query->size = size; 254 return 0; 255 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 256 return -EINVAL; 257 } 258 259 mem_regions = kzalloc(size, GFP_KERNEL); 260 if (XE_IOCTL_DBG(xe, !mem_regions)) 261 return -ENOMEM; 262 263 man = ttm_manager_type(&xe->ttm, XE_PL_TT); 264 mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; 265 /* 266 * The instance needs to be a unique number that represents the index 267 * in the placement mask used at xe_gem_create_ioctl() for the 268 * xe_bo_create() placement. 269 */ 270 mem_regions->mem_regions[0].instance = 0; 271 mem_regions->mem_regions[0].min_page_size = PAGE_SIZE; 272 mem_regions->mem_regions[0].total_size = man->size << PAGE_SHIFT; 273 if (perfmon_capable()) 274 mem_regions->mem_regions[0].used = ttm_resource_manager_usage(man); 275 mem_regions->num_mem_regions = 1; 276 277 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { 278 man = ttm_manager_type(&xe->ttm, i); 279 if (man) { 280 mem_regions->mem_regions[mem_regions->num_mem_regions].mem_class = 281 DRM_XE_MEM_REGION_CLASS_VRAM; 282 mem_regions->mem_regions[mem_regions->num_mem_regions].instance = 283 mem_regions->num_mem_regions; 284 mem_regions->mem_regions[mem_regions->num_mem_regions].min_page_size = 285 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? 286 SZ_64K : PAGE_SIZE; 287 mem_regions->mem_regions[mem_regions->num_mem_regions].total_size = 288 man->size; 289 290 if (perfmon_capable()) { 291 xe_ttm_vram_get_used(man, 292 &mem_regions->mem_regions 293 [mem_regions->num_mem_regions].used, 294 &mem_regions->mem_regions 295 [mem_regions->num_mem_regions].cpu_visible_used); 296 } 297 298 mem_regions->mem_regions[mem_regions->num_mem_regions].cpu_visible_size = 299 xe_ttm_vram_get_cpu_visible_size(man); 300 mem_regions->num_mem_regions++; 301 } 302 } 303 304 if (!copy_to_user(query_ptr, mem_regions, size)) 305 ret = 0; 306 else 307 ret = -ENOSPC; 308 309 kfree(mem_regions); 310 return ret; 311 } 312 313 static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) 314 { 315 const u32 num_params = DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; 316 size_t size = 317 sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); 318 struct drm_xe_query_config __user *query_ptr = 319 u64_to_user_ptr(query->data); 320 struct drm_xe_query_config *config; 321 322 if (query->size == 0) { 323 query->size = size; 324 return 0; 325 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 326 return -EINVAL; 327 } 328 329 config = kzalloc(size, GFP_KERNEL); 330 if (!config) 331 return -ENOMEM; 332 333 config->num_params = num_params; 334 config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = 335 xe->info.devid | (xe->info.revid << 16); 336 if (xe_device_get_root_tile(xe)->mem.vram.usable_size) 337 config->info[DRM_XE_QUERY_CONFIG_FLAGS] = 338 DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; 339 config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = 340 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; 341 config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; 342 config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = 343 xe_exec_queue_device_get_max_priority(xe); 344 345 if (copy_to_user(query_ptr, config, size)) { 346 kfree(config); 347 return -EFAULT; 348 } 349 kfree(config); 350 351 return 0; 352 } 353 354 static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query) 355 { 356 struct xe_gt *gt; 357 size_t size = sizeof(struct drm_xe_query_gt_list) + 358 xe->info.gt_count * sizeof(struct drm_xe_gt); 359 struct drm_xe_query_gt_list __user *query_ptr = 360 u64_to_user_ptr(query->data); 361 struct drm_xe_query_gt_list *gt_list; 362 u8 id; 363 364 if (query->size == 0) { 365 query->size = size; 366 return 0; 367 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 368 return -EINVAL; 369 } 370 371 gt_list = kzalloc(size, GFP_KERNEL); 372 if (!gt_list) 373 return -ENOMEM; 374 375 gt_list->num_gt = xe->info.gt_count; 376 377 for_each_gt(gt, xe, id) { 378 if (xe_gt_is_media_type(gt)) 379 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA; 380 else 381 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN; 382 gt_list->gt_list[id].tile_id = gt_to_tile(gt)->id; 383 gt_list->gt_list[id].gt_id = gt->info.id; 384 gt_list->gt_list[id].reference_clock = gt->info.reference_clock; 385 /* 386 * The mem_regions indexes in the mask below need to 387 * directly identify the struct 388 * drm_xe_query_mem_regions' instance constructed at 389 * query_mem_regions() 390 * 391 * For our current platforms: 392 * Bit 0 -> System Memory 393 * Bit 1 -> VRAM0 on Tile0 394 * Bit 2 -> VRAM1 on Tile1 395 * However the uAPI is generic and it's userspace's 396 * responsibility to check the mem_class, without any 397 * assumption. 398 */ 399 if (!IS_DGFX(xe)) 400 gt_list->gt_list[id].near_mem_regions = 0x1; 401 else 402 gt_list->gt_list[id].near_mem_regions = 403 BIT(gt_to_tile(gt)->id) << 1; 404 gt_list->gt_list[id].far_mem_regions = xe->info.mem_region_mask ^ 405 gt_list->gt_list[id].near_mem_regions; 406 407 gt_list->gt_list[id].ip_ver_major = 408 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid); 409 gt_list->gt_list[id].ip_ver_minor = 410 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid); 411 gt_list->gt_list[id].ip_ver_rev = 412 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid); 413 } 414 415 if (copy_to_user(query_ptr, gt_list, size)) { 416 kfree(gt_list); 417 return -EFAULT; 418 } 419 kfree(gt_list); 420 421 return 0; 422 } 423 424 static int query_hwconfig(struct xe_device *xe, 425 struct drm_xe_device_query *query) 426 { 427 struct xe_gt *gt = xe_root_mmio_gt(xe); 428 size_t size = xe_guc_hwconfig_size(>->uc.guc); 429 void __user *query_ptr = u64_to_user_ptr(query->data); 430 void *hwconfig; 431 432 if (query->size == 0) { 433 query->size = size; 434 return 0; 435 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 436 return -EINVAL; 437 } 438 439 hwconfig = kzalloc(size, GFP_KERNEL); 440 if (!hwconfig) 441 return -ENOMEM; 442 443 xe_guc_hwconfig_copy(>->uc.guc, hwconfig); 444 445 if (copy_to_user(query_ptr, hwconfig, size)) { 446 kfree(hwconfig); 447 return -EFAULT; 448 } 449 kfree(hwconfig); 450 451 return 0; 452 } 453 454 static size_t calc_topo_query_size(struct xe_device *xe) 455 { 456 return xe->info.gt_count * 457 (3 * sizeof(struct drm_xe_query_topology_mask) + 458 sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) + 459 sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) + 460 sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss)); 461 } 462 463 static int copy_mask(void __user **ptr, 464 struct drm_xe_query_topology_mask *topo, 465 void *mask, size_t mask_size) 466 { 467 topo->num_bytes = mask_size; 468 469 if (copy_to_user(*ptr, topo, sizeof(*topo))) 470 return -EFAULT; 471 *ptr += sizeof(topo); 472 473 if (copy_to_user(*ptr, mask, mask_size)) 474 return -EFAULT; 475 *ptr += mask_size; 476 477 return 0; 478 } 479 480 static int query_gt_topology(struct xe_device *xe, 481 struct drm_xe_device_query *query) 482 { 483 void __user *query_ptr = u64_to_user_ptr(query->data); 484 size_t size = calc_topo_query_size(xe); 485 struct drm_xe_query_topology_mask topo; 486 struct xe_gt *gt; 487 int id; 488 489 if (query->size == 0) { 490 query->size = size; 491 return 0; 492 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 493 return -EINVAL; 494 } 495 496 for_each_gt(gt, xe, id) { 497 int err; 498 499 topo.gt_id = id; 500 501 topo.type = DRM_XE_TOPO_DSS_GEOMETRY; 502 err = copy_mask(&query_ptr, &topo, gt->fuse_topo.g_dss_mask, 503 sizeof(gt->fuse_topo.g_dss_mask)); 504 if (err) 505 return err; 506 507 topo.type = DRM_XE_TOPO_DSS_COMPUTE; 508 err = copy_mask(&query_ptr, &topo, gt->fuse_topo.c_dss_mask, 509 sizeof(gt->fuse_topo.c_dss_mask)); 510 if (err) 511 return err; 512 513 topo.type = DRM_XE_TOPO_EU_PER_DSS; 514 err = copy_mask(&query_ptr, &topo, 515 gt->fuse_topo.eu_mask_per_dss, 516 sizeof(gt->fuse_topo.eu_mask_per_dss)); 517 if (err) 518 return err; 519 } 520 521 return 0; 522 } 523 524 static int 525 query_uc_fw_version(struct xe_device *xe, struct drm_xe_device_query *query) 526 { 527 struct drm_xe_query_uc_fw_version __user *query_ptr = u64_to_user_ptr(query->data); 528 size_t size = sizeof(struct drm_xe_query_uc_fw_version); 529 struct drm_xe_query_uc_fw_version resp; 530 struct xe_uc_fw_version *version = NULL; 531 532 if (query->size == 0) { 533 query->size = size; 534 return 0; 535 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 536 return -EINVAL; 537 } 538 539 if (copy_from_user(&resp, query_ptr, size)) 540 return -EFAULT; 541 542 if (XE_IOCTL_DBG(xe, resp.pad || resp.pad2 || resp.reserved)) 543 return -EINVAL; 544 545 switch (resp.uc_type) { 546 case XE_QUERY_UC_TYPE_GUC_SUBMISSION: { 547 struct xe_guc *guc = &xe->tiles[0].primary_gt->uc.guc; 548 549 version = &guc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]; 550 break; 551 } 552 case XE_QUERY_UC_TYPE_HUC: { 553 struct xe_gt *media_gt = NULL; 554 struct xe_huc *huc; 555 556 if (MEDIA_VER(xe) >= 13) { 557 struct xe_tile *tile; 558 u8 gt_id; 559 560 for_each_tile(tile, xe, gt_id) { 561 if (tile->media_gt) { 562 media_gt = tile->media_gt; 563 break; 564 } 565 } 566 } else { 567 media_gt = xe->tiles[0].primary_gt; 568 } 569 570 if (!media_gt) 571 break; 572 573 huc = &media_gt->uc.huc; 574 if (huc->fw.status == XE_UC_FIRMWARE_RUNNING) 575 version = &huc->fw.versions.found[XE_UC_FW_VER_RELEASE]; 576 break; 577 } 578 default: 579 return -EINVAL; 580 } 581 582 if (version) { 583 resp.branch_ver = 0; 584 resp.major_ver = version->major; 585 resp.minor_ver = version->minor; 586 resp.patch_ver = version->patch; 587 } else { 588 return -ENODEV; 589 } 590 591 if (copy_to_user(query_ptr, &resp, size)) 592 return -EFAULT; 593 594 return 0; 595 } 596 597 static int (* const xe_query_funcs[])(struct xe_device *xe, 598 struct drm_xe_device_query *query) = { 599 query_engines, 600 query_mem_regions, 601 query_config, 602 query_gt_list, 603 query_hwconfig, 604 query_gt_topology, 605 query_engine_cycles, 606 query_uc_fw_version, 607 }; 608 609 int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 610 { 611 struct xe_device *xe = to_xe_device(dev); 612 struct drm_xe_device_query *query = data; 613 u32 idx; 614 615 if (XE_IOCTL_DBG(xe, query->extensions) || 616 XE_IOCTL_DBG(xe, query->reserved[0] || query->reserved[1])) 617 return -EINVAL; 618 619 if (XE_IOCTL_DBG(xe, query->query >= ARRAY_SIZE(xe_query_funcs))) 620 return -EINVAL; 621 622 idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs)); 623 if (XE_IOCTL_DBG(xe, !xe_query_funcs[idx])) 624 return -EINVAL; 625 626 return xe_query_funcs[idx](xe, query); 627 } 628