1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_query.h" 7 8 #include <linux/nospec.h> 9 #include <linux/sched/clock.h> 10 11 #include <drm/ttm/ttm_placement.h> 12 #include <drm/xe_drm.h> 13 14 #include "regs/xe_engine_regs.h" 15 #include "xe_bo.h" 16 #include "xe_device.h" 17 #include "xe_exec_queue.h" 18 #include "xe_ggtt.h" 19 #include "xe_gt.h" 20 #include "xe_guc_hwconfig.h" 21 #include "xe_macros.h" 22 #include "xe_mmio.h" 23 #include "xe_ttm_vram_mgr.h" 24 25 static const u16 xe_to_user_engine_class[] = { 26 [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, 27 [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY, 28 [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE, 29 [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE, 30 [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, 31 }; 32 33 static const enum xe_engine_class user_to_xe_engine_class[] = { 34 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 35 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 36 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 37 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 38 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 39 }; 40 41 static size_t calc_hw_engine_info_size(struct xe_device *xe) 42 { 43 struct xe_hw_engine *hwe; 44 enum xe_hw_engine_id id; 45 struct xe_gt *gt; 46 u8 gt_id; 47 int i = 0; 48 49 for_each_gt(gt, xe, gt_id) 50 for_each_hw_engine(hwe, gt, id) { 51 if (xe_hw_engine_is_reserved(hwe)) 52 continue; 53 i++; 54 } 55 56 return i * sizeof(struct drm_xe_engine_class_instance); 57 } 58 59 typedef u64 (*__ktime_func_t)(void); 60 static __ktime_func_t __clock_id_to_func(clockid_t clk_id) 61 { 62 /* 63 * Use logic same as the perf subsystem to allow user to select the 64 * reference clock id to be used for timestamps. 65 */ 66 switch (clk_id) { 67 case CLOCK_MONOTONIC: 68 return &ktime_get_ns; 69 case CLOCK_MONOTONIC_RAW: 70 return &ktime_get_raw_ns; 71 case CLOCK_REALTIME: 72 return &ktime_get_real_ns; 73 case CLOCK_BOOTTIME: 74 return &ktime_get_boottime_ns; 75 case CLOCK_TAI: 76 return &ktime_get_clocktai_ns; 77 default: 78 return NULL; 79 } 80 } 81 82 static void 83 __read_timestamps(struct xe_gt *gt, 84 struct xe_reg lower_reg, 85 struct xe_reg upper_reg, 86 u64 *engine_ts, 87 u64 *cpu_ts, 88 u64 *cpu_delta, 89 __ktime_func_t cpu_clock) 90 { 91 u32 upper, lower, old_upper, loop = 0; 92 93 upper = xe_mmio_read32(gt, upper_reg); 94 do { 95 *cpu_delta = local_clock(); 96 *cpu_ts = cpu_clock(); 97 lower = xe_mmio_read32(gt, lower_reg); 98 *cpu_delta = local_clock() - *cpu_delta; 99 old_upper = upper; 100 upper = xe_mmio_read32(gt, upper_reg); 101 } while (upper != old_upper && loop++ < 2); 102 103 *engine_ts = (u64)upper << 32 | lower; 104 } 105 106 static int 107 query_engine_cycles(struct xe_device *xe, 108 struct drm_xe_device_query *query) 109 { 110 struct drm_xe_query_engine_cycles __user *query_ptr; 111 struct drm_xe_engine_class_instance *eci; 112 struct drm_xe_query_engine_cycles resp; 113 size_t size = sizeof(resp); 114 __ktime_func_t cpu_clock; 115 struct xe_hw_engine *hwe; 116 struct xe_gt *gt; 117 118 if (query->size == 0) { 119 query->size = size; 120 return 0; 121 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 122 return -EINVAL; 123 } 124 125 query_ptr = u64_to_user_ptr(query->data); 126 if (copy_from_user(&resp, query_ptr, size)) 127 return -EFAULT; 128 129 cpu_clock = __clock_id_to_func(resp.clockid); 130 if (!cpu_clock) 131 return -EINVAL; 132 133 eci = &resp.eci; 134 if (eci->gt_id > XE_MAX_GT_PER_TILE) 135 return -EINVAL; 136 137 gt = xe_device_get_gt(xe, eci->gt_id); 138 if (!gt) 139 return -EINVAL; 140 141 if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) 142 return -EINVAL; 143 144 hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class], 145 eci->engine_instance, true); 146 if (!hwe) 147 return -EINVAL; 148 149 resp.engine_frequency = gt->info.clock_freq; 150 151 xe_device_mem_access_get(xe); 152 xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 153 154 __read_timestamps(gt, 155 RING_TIMESTAMP(hwe->mmio_base), 156 RING_TIMESTAMP_UDW(hwe->mmio_base), 157 &resp.engine_cycles, 158 &resp.cpu_timestamp, 159 &resp.cpu_delta, 160 cpu_clock); 161 162 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 163 xe_device_mem_access_put(xe); 164 resp.width = 36; 165 166 /* Only write to the output fields of user query */ 167 if (put_user(resp.engine_frequency, &query_ptr->engine_frequency)) 168 return -EFAULT; 169 170 if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp)) 171 return -EFAULT; 172 173 if (put_user(resp.cpu_delta, &query_ptr->cpu_delta)) 174 return -EFAULT; 175 176 if (put_user(resp.engine_cycles, &query_ptr->engine_cycles)) 177 return -EFAULT; 178 179 if (put_user(resp.width, &query_ptr->width)) 180 return -EFAULT; 181 182 return 0; 183 } 184 185 static int query_engines(struct xe_device *xe, 186 struct drm_xe_device_query *query) 187 { 188 size_t size = calc_hw_engine_info_size(xe); 189 struct drm_xe_engine_class_instance __user *query_ptr = 190 u64_to_user_ptr(query->data); 191 struct drm_xe_engine_class_instance *hw_engine_info; 192 struct xe_hw_engine *hwe; 193 enum xe_hw_engine_id id; 194 struct xe_gt *gt; 195 u8 gt_id; 196 int i = 0; 197 198 if (query->size == 0) { 199 query->size = size; 200 return 0; 201 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 202 return -EINVAL; 203 } 204 205 hw_engine_info = kmalloc(size, GFP_KERNEL); 206 if (!hw_engine_info) 207 return -ENOMEM; 208 209 for_each_gt(gt, xe, gt_id) 210 for_each_hw_engine(hwe, gt, id) { 211 if (xe_hw_engine_is_reserved(hwe)) 212 continue; 213 214 hw_engine_info[i].engine_class = 215 xe_to_user_engine_class[hwe->class]; 216 hw_engine_info[i].engine_instance = 217 hwe->logical_instance; 218 hw_engine_info[i].gt_id = gt->info.id; 219 hw_engine_info[i].pad = 0; 220 221 i++; 222 } 223 224 if (copy_to_user(query_ptr, hw_engine_info, size)) { 225 kfree(hw_engine_info); 226 return -EFAULT; 227 } 228 kfree(hw_engine_info); 229 230 return 0; 231 } 232 233 static size_t calc_mem_regions_size(struct xe_device *xe) 234 { 235 u32 num_managers = 1; 236 int i; 237 238 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) 239 if (ttm_manager_type(&xe->ttm, i)) 240 num_managers++; 241 242 return offsetof(struct drm_xe_query_mem_regions, regions[num_managers]); 243 } 244 245 static int query_mem_regions(struct xe_device *xe, 246 struct drm_xe_device_query *query) 247 { 248 size_t size = calc_mem_regions_size(xe); 249 struct drm_xe_query_mem_regions *usage; 250 struct drm_xe_query_mem_regions __user *query_ptr = 251 u64_to_user_ptr(query->data); 252 struct ttm_resource_manager *man; 253 int ret, i; 254 255 if (query->size == 0) { 256 query->size = size; 257 return 0; 258 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 259 return -EINVAL; 260 } 261 262 usage = kzalloc(size, GFP_KERNEL); 263 if (XE_IOCTL_DBG(xe, !usage)) 264 return -ENOMEM; 265 266 man = ttm_manager_type(&xe->ttm, XE_PL_TT); 267 usage->regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; 268 usage->regions[0].instance = 0; 269 usage->regions[0].min_page_size = PAGE_SIZE; 270 usage->regions[0].total_size = man->size << PAGE_SHIFT; 271 if (perfmon_capable()) 272 usage->regions[0].used = ttm_resource_manager_usage(man); 273 usage->num_regions = 1; 274 275 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { 276 man = ttm_manager_type(&xe->ttm, i); 277 if (man) { 278 usage->regions[usage->num_regions].mem_class = 279 DRM_XE_MEM_REGION_CLASS_VRAM; 280 usage->regions[usage->num_regions].instance = 281 usage->num_regions; 282 usage->regions[usage->num_regions].min_page_size = 283 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? 284 SZ_64K : PAGE_SIZE; 285 usage->regions[usage->num_regions].total_size = 286 man->size; 287 288 if (perfmon_capable()) { 289 xe_ttm_vram_get_used(man, 290 &usage->regions[usage->num_regions].used, 291 &usage->regions[usage->num_regions].cpu_visible_used); 292 } 293 294 usage->regions[usage->num_regions].cpu_visible_size = 295 xe_ttm_vram_get_cpu_visible_size(man); 296 usage->num_regions++; 297 } 298 } 299 300 if (!copy_to_user(query_ptr, usage, size)) 301 ret = 0; 302 else 303 ret = -ENOSPC; 304 305 kfree(usage); 306 return ret; 307 } 308 309 static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) 310 { 311 const u32 num_params = DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; 312 size_t size = 313 sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); 314 struct drm_xe_query_config __user *query_ptr = 315 u64_to_user_ptr(query->data); 316 struct drm_xe_query_config *config; 317 318 if (query->size == 0) { 319 query->size = size; 320 return 0; 321 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 322 return -EINVAL; 323 } 324 325 config = kzalloc(size, GFP_KERNEL); 326 if (!config) 327 return -ENOMEM; 328 329 config->num_params = num_params; 330 config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = 331 xe->info.devid | (xe->info.revid << 16); 332 if (xe_device_get_root_tile(xe)->mem.vram.usable_size) 333 config->info[DRM_XE_QUERY_CONFIG_FLAGS] = 334 DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; 335 config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = 336 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; 337 config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; 338 config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = 339 xe_exec_queue_device_get_max_priority(xe); 340 341 if (copy_to_user(query_ptr, config, size)) { 342 kfree(config); 343 return -EFAULT; 344 } 345 kfree(config); 346 347 return 0; 348 } 349 350 static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query) 351 { 352 struct xe_gt *gt; 353 size_t size = sizeof(struct drm_xe_query_gt_list) + 354 xe->info.gt_count * sizeof(struct drm_xe_query_gt); 355 struct drm_xe_query_gt_list __user *query_ptr = 356 u64_to_user_ptr(query->data); 357 struct drm_xe_query_gt_list *gt_list; 358 u8 id; 359 360 if (query->size == 0) { 361 query->size = size; 362 return 0; 363 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 364 return -EINVAL; 365 } 366 367 gt_list = kzalloc(size, GFP_KERNEL); 368 if (!gt_list) 369 return -ENOMEM; 370 371 gt_list->num_gt = xe->info.gt_count; 372 for_each_gt(gt, xe, id) { 373 if (xe_gt_is_media_type(gt)) 374 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA; 375 else 376 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN; 377 gt_list->gt_list[id].gt_id = gt->info.id; 378 gt_list->gt_list[id].clock_freq = gt->info.clock_freq; 379 if (!IS_DGFX(xe)) 380 gt_list->gt_list[id].near_mem_regions = 0x1; 381 else 382 gt_list->gt_list[id].near_mem_regions = 383 BIT(gt_to_tile(gt)->id) << 1; 384 gt_list->gt_list[id].far_mem_regions = xe->info.mem_region_mask ^ 385 gt_list->gt_list[id].near_mem_regions; 386 } 387 388 if (copy_to_user(query_ptr, gt_list, size)) { 389 kfree(gt_list); 390 return -EFAULT; 391 } 392 kfree(gt_list); 393 394 return 0; 395 } 396 397 static int query_hwconfig(struct xe_device *xe, 398 struct drm_xe_device_query *query) 399 { 400 struct xe_gt *gt = xe_root_mmio_gt(xe); 401 size_t size = xe_guc_hwconfig_size(>->uc.guc); 402 void __user *query_ptr = u64_to_user_ptr(query->data); 403 void *hwconfig; 404 405 if (query->size == 0) { 406 query->size = size; 407 return 0; 408 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 409 return -EINVAL; 410 } 411 412 hwconfig = kzalloc(size, GFP_KERNEL); 413 if (!hwconfig) 414 return -ENOMEM; 415 416 xe_device_mem_access_get(xe); 417 xe_guc_hwconfig_copy(>->uc.guc, hwconfig); 418 xe_device_mem_access_put(xe); 419 420 if (copy_to_user(query_ptr, hwconfig, size)) { 421 kfree(hwconfig); 422 return -EFAULT; 423 } 424 kfree(hwconfig); 425 426 return 0; 427 } 428 429 static size_t calc_topo_query_size(struct xe_device *xe) 430 { 431 return xe->info.gt_count * 432 (3 * sizeof(struct drm_xe_query_topology_mask) + 433 sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) + 434 sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) + 435 sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss)); 436 } 437 438 static void __user *copy_mask(void __user *ptr, 439 struct drm_xe_query_topology_mask *topo, 440 void *mask, size_t mask_size) 441 { 442 topo->num_bytes = mask_size; 443 444 if (copy_to_user(ptr, topo, sizeof(*topo))) 445 return ERR_PTR(-EFAULT); 446 ptr += sizeof(topo); 447 448 if (copy_to_user(ptr, mask, mask_size)) 449 return ERR_PTR(-EFAULT); 450 ptr += mask_size; 451 452 return ptr; 453 } 454 455 static int query_gt_topology(struct xe_device *xe, 456 struct drm_xe_device_query *query) 457 { 458 void __user *query_ptr = u64_to_user_ptr(query->data); 459 size_t size = calc_topo_query_size(xe); 460 struct drm_xe_query_topology_mask topo; 461 struct xe_gt *gt; 462 int id; 463 464 if (query->size == 0) { 465 query->size = size; 466 return 0; 467 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 468 return -EINVAL; 469 } 470 471 for_each_gt(gt, xe, id) { 472 topo.gt_id = id; 473 474 topo.type = DRM_XE_TOPO_DSS_GEOMETRY; 475 query_ptr = copy_mask(query_ptr, &topo, 476 gt->fuse_topo.g_dss_mask, 477 sizeof(gt->fuse_topo.g_dss_mask)); 478 if (IS_ERR(query_ptr)) 479 return PTR_ERR(query_ptr); 480 481 topo.type = DRM_XE_TOPO_DSS_COMPUTE; 482 query_ptr = copy_mask(query_ptr, &topo, 483 gt->fuse_topo.c_dss_mask, 484 sizeof(gt->fuse_topo.c_dss_mask)); 485 if (IS_ERR(query_ptr)) 486 return PTR_ERR(query_ptr); 487 488 topo.type = DRM_XE_TOPO_EU_PER_DSS; 489 query_ptr = copy_mask(query_ptr, &topo, 490 gt->fuse_topo.eu_mask_per_dss, 491 sizeof(gt->fuse_topo.eu_mask_per_dss)); 492 if (IS_ERR(query_ptr)) 493 return PTR_ERR(query_ptr); 494 } 495 496 return 0; 497 } 498 499 static int (* const xe_query_funcs[])(struct xe_device *xe, 500 struct drm_xe_device_query *query) = { 501 query_engines, 502 query_mem_regions, 503 query_config, 504 query_gt_list, 505 query_hwconfig, 506 query_gt_topology, 507 query_engine_cycles, 508 }; 509 510 int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 511 { 512 struct xe_device *xe = to_xe_device(dev); 513 struct drm_xe_device_query *query = data; 514 u32 idx; 515 516 if (XE_IOCTL_DBG(xe, query->extensions) || 517 XE_IOCTL_DBG(xe, query->reserved[0] || query->reserved[1])) 518 return -EINVAL; 519 520 if (XE_IOCTL_DBG(xe, query->query >= ARRAY_SIZE(xe_query_funcs))) 521 return -EINVAL; 522 523 idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs)); 524 if (XE_IOCTL_DBG(xe, !xe_query_funcs[idx])) 525 return -EINVAL; 526 527 return xe_query_funcs[idx](xe, query); 528 } 529