1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_query.h" 7 8 #include <linux/nospec.h> 9 #include <linux/sched/clock.h> 10 11 #include <drm/ttm/ttm_placement.h> 12 #include <drm/xe_drm.h> 13 14 #include "regs/xe_engine_regs.h" 15 #include "xe_bo.h" 16 #include "xe_device.h" 17 #include "xe_exec_queue.h" 18 #include "xe_ggtt.h" 19 #include "xe_gt.h" 20 #include "xe_guc_hwconfig.h" 21 #include "xe_macros.h" 22 #include "xe_mmio.h" 23 #include "xe_ttm_vram_mgr.h" 24 25 static const u16 xe_to_user_engine_class[] = { 26 [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, 27 [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY, 28 [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE, 29 [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE, 30 [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, 31 }; 32 33 static const enum xe_engine_class user_to_xe_engine_class[] = { 34 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 35 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 36 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 37 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 38 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 39 }; 40 41 static size_t calc_hw_engine_info_size(struct xe_device *xe) 42 { 43 struct xe_hw_engine *hwe; 44 enum xe_hw_engine_id id; 45 struct xe_gt *gt; 46 u8 gt_id; 47 int i = 0; 48 49 for_each_gt(gt, xe, gt_id) 50 for_each_hw_engine(hwe, gt, id) { 51 if (xe_hw_engine_is_reserved(hwe)) 52 continue; 53 i++; 54 } 55 56 return i * sizeof(struct drm_xe_engine_class_instance); 57 } 58 59 typedef u64 (*__ktime_func_t)(void); 60 static __ktime_func_t __clock_id_to_func(clockid_t clk_id) 61 { 62 /* 63 * Use logic same as the perf subsystem to allow user to select the 64 * reference clock id to be used for timestamps. 65 */ 66 switch (clk_id) { 67 case CLOCK_MONOTONIC: 68 return &ktime_get_ns; 69 case CLOCK_MONOTONIC_RAW: 70 return &ktime_get_raw_ns; 71 case CLOCK_REALTIME: 72 return &ktime_get_real_ns; 73 case CLOCK_BOOTTIME: 74 return &ktime_get_boottime_ns; 75 case CLOCK_TAI: 76 return &ktime_get_clocktai_ns; 77 default: 78 return NULL; 79 } 80 } 81 82 static void 83 __read_timestamps(struct xe_gt *gt, 84 struct xe_reg lower_reg, 85 struct xe_reg upper_reg, 86 u64 *engine_ts, 87 u64 *cpu_ts, 88 u64 *cpu_delta, 89 __ktime_func_t cpu_clock) 90 { 91 u32 upper, lower, old_upper, loop = 0; 92 93 upper = xe_mmio_read32(gt, upper_reg); 94 do { 95 *cpu_delta = local_clock(); 96 *cpu_ts = cpu_clock(); 97 lower = xe_mmio_read32(gt, lower_reg); 98 *cpu_delta = local_clock() - *cpu_delta; 99 old_upper = upper; 100 upper = xe_mmio_read32(gt, upper_reg); 101 } while (upper != old_upper && loop++ < 2); 102 103 *engine_ts = (u64)upper << 32 | lower; 104 } 105 106 static int 107 query_engine_cycles(struct xe_device *xe, 108 struct drm_xe_device_query *query) 109 { 110 struct drm_xe_query_engine_cycles __user *query_ptr; 111 struct drm_xe_engine_class_instance *eci; 112 struct drm_xe_query_engine_cycles resp; 113 size_t size = sizeof(resp); 114 __ktime_func_t cpu_clock; 115 struct xe_hw_engine *hwe; 116 struct xe_gt *gt; 117 118 if (query->size == 0) { 119 query->size = size; 120 return 0; 121 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 122 return -EINVAL; 123 } 124 125 query_ptr = u64_to_user_ptr(query->data); 126 if (copy_from_user(&resp, query_ptr, size)) 127 return -EFAULT; 128 129 cpu_clock = __clock_id_to_func(resp.clockid); 130 if (!cpu_clock) 131 return -EINVAL; 132 133 eci = &resp.eci; 134 if (eci->gt_id > XE_MAX_GT_PER_TILE) 135 return -EINVAL; 136 137 gt = xe_device_get_gt(xe, eci->gt_id); 138 if (!gt) 139 return -EINVAL; 140 141 if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) 142 return -EINVAL; 143 144 hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class], 145 eci->engine_instance, true); 146 if (!hwe) 147 return -EINVAL; 148 149 resp.engine_frequency = gt->info.clock_freq; 150 151 xe_device_mem_access_get(xe); 152 xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 153 154 __read_timestamps(gt, 155 RING_TIMESTAMP(hwe->mmio_base), 156 RING_TIMESTAMP_UDW(hwe->mmio_base), 157 &resp.engine_cycles, 158 &resp.cpu_timestamp, 159 &resp.cpu_delta, 160 cpu_clock); 161 162 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 163 xe_device_mem_access_put(xe); 164 resp.width = 36; 165 166 /* Only write to the output fields of user query */ 167 if (put_user(resp.engine_frequency, &query_ptr->engine_frequency)) 168 return -EFAULT; 169 170 if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp)) 171 return -EFAULT; 172 173 if (put_user(resp.cpu_delta, &query_ptr->cpu_delta)) 174 return -EFAULT; 175 176 if (put_user(resp.engine_cycles, &query_ptr->engine_cycles)) 177 return -EFAULT; 178 179 if (put_user(resp.width, &query_ptr->width)) 180 return -EFAULT; 181 182 return 0; 183 } 184 185 static int query_engines(struct xe_device *xe, 186 struct drm_xe_device_query *query) 187 { 188 size_t size = calc_hw_engine_info_size(xe); 189 struct drm_xe_engine_class_instance __user *query_ptr = 190 u64_to_user_ptr(query->data); 191 struct drm_xe_engine_class_instance *hw_engine_info; 192 struct xe_hw_engine *hwe; 193 enum xe_hw_engine_id id; 194 struct xe_gt *gt; 195 u8 gt_id; 196 int i = 0; 197 198 if (query->size == 0) { 199 query->size = size; 200 return 0; 201 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 202 return -EINVAL; 203 } 204 205 hw_engine_info = kmalloc(size, GFP_KERNEL); 206 if (!hw_engine_info) 207 return -ENOMEM; 208 209 for_each_gt(gt, xe, gt_id) 210 for_each_hw_engine(hwe, gt, id) { 211 if (xe_hw_engine_is_reserved(hwe)) 212 continue; 213 214 hw_engine_info[i].engine_class = 215 xe_to_user_engine_class[hwe->class]; 216 hw_engine_info[i].engine_instance = 217 hwe->logical_instance; 218 hw_engine_info[i++].gt_id = gt->info.id; 219 } 220 221 if (copy_to_user(query_ptr, hw_engine_info, size)) { 222 kfree(hw_engine_info); 223 return -EFAULT; 224 } 225 kfree(hw_engine_info); 226 227 return 0; 228 } 229 230 static size_t calc_memory_usage_size(struct xe_device *xe) 231 { 232 u32 num_managers = 1; 233 int i; 234 235 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) 236 if (ttm_manager_type(&xe->ttm, i)) 237 num_managers++; 238 239 return offsetof(struct drm_xe_query_mem_usage, regions[num_managers]); 240 } 241 242 static int query_memory_usage(struct xe_device *xe, 243 struct drm_xe_device_query *query) 244 { 245 size_t size = calc_memory_usage_size(xe); 246 struct drm_xe_query_mem_usage *usage; 247 struct drm_xe_query_mem_usage __user *query_ptr = 248 u64_to_user_ptr(query->data); 249 struct ttm_resource_manager *man; 250 int ret, i; 251 252 if (query->size == 0) { 253 query->size = size; 254 return 0; 255 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 256 return -EINVAL; 257 } 258 259 usage = kzalloc(size, GFP_KERNEL); 260 if (XE_IOCTL_DBG(xe, !usage)) 261 return -ENOMEM; 262 263 man = ttm_manager_type(&xe->ttm, XE_PL_TT); 264 usage->regions[0].mem_class = XE_MEM_REGION_CLASS_SYSMEM; 265 usage->regions[0].instance = 0; 266 usage->regions[0].min_page_size = PAGE_SIZE; 267 usage->regions[0].total_size = man->size << PAGE_SHIFT; 268 if (perfmon_capable()) 269 usage->regions[0].used = ttm_resource_manager_usage(man); 270 usage->num_regions = 1; 271 272 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { 273 man = ttm_manager_type(&xe->ttm, i); 274 if (man) { 275 usage->regions[usage->num_regions].mem_class = 276 XE_MEM_REGION_CLASS_VRAM; 277 usage->regions[usage->num_regions].instance = 278 usage->num_regions; 279 usage->regions[usage->num_regions].min_page_size = 280 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? 281 SZ_64K : PAGE_SIZE; 282 usage->regions[usage->num_regions].total_size = 283 man->size; 284 285 if (perfmon_capable()) { 286 xe_ttm_vram_get_used(man, 287 &usage->regions[usage->num_regions].used, 288 &usage->regions[usage->num_regions].cpu_visible_used); 289 } 290 291 usage->regions[usage->num_regions].cpu_visible_size = 292 xe_ttm_vram_get_cpu_visible_size(man); 293 usage->num_regions++; 294 } 295 } 296 297 if (!copy_to_user(query_ptr, usage, size)) 298 ret = 0; 299 else 300 ret = -ENOSPC; 301 302 kfree(usage); 303 return ret; 304 } 305 306 static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) 307 { 308 u32 num_params = XE_QUERY_CONFIG_NUM_PARAM; 309 size_t size = 310 sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); 311 struct drm_xe_query_config __user *query_ptr = 312 u64_to_user_ptr(query->data); 313 struct drm_xe_query_config *config; 314 315 if (query->size == 0) { 316 query->size = size; 317 return 0; 318 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 319 return -EINVAL; 320 } 321 322 config = kzalloc(size, GFP_KERNEL); 323 if (!config) 324 return -ENOMEM; 325 326 config->num_params = num_params; 327 config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = 328 xe->info.devid | (xe->info.revid << 16); 329 if (xe_device_get_root_tile(xe)->mem.vram.usable_size) 330 config->info[XE_QUERY_CONFIG_FLAGS] = 331 XE_QUERY_CONFIG_FLAGS_HAS_VRAM; 332 config->info[XE_QUERY_CONFIG_MIN_ALIGNMENT] = 333 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; 334 config->info[XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; 335 config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.gt_count; 336 config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT] = 337 hweight_long(xe->info.mem_region_mask); 338 config->info[XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = 339 xe_exec_queue_device_get_max_priority(xe); 340 341 if (copy_to_user(query_ptr, config, size)) { 342 kfree(config); 343 return -EFAULT; 344 } 345 kfree(config); 346 347 return 0; 348 } 349 350 static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query) 351 { 352 struct xe_gt *gt; 353 size_t size = sizeof(struct drm_xe_query_gt_list) + 354 xe->info.gt_count * sizeof(struct drm_xe_query_gt); 355 struct drm_xe_query_gt_list __user *query_ptr = 356 u64_to_user_ptr(query->data); 357 struct drm_xe_query_gt_list *gt_list; 358 u8 id; 359 360 if (query->size == 0) { 361 query->size = size; 362 return 0; 363 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 364 return -EINVAL; 365 } 366 367 gt_list = kzalloc(size, GFP_KERNEL); 368 if (!gt_list) 369 return -ENOMEM; 370 371 gt_list->num_gt = xe->info.gt_count; 372 for_each_gt(gt, xe, id) { 373 if (xe_gt_is_media_type(gt)) 374 gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MEDIA; 375 else if (gt_to_tile(gt)->id > 0) 376 gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_REMOTE; 377 else 378 gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MAIN; 379 gt_list->gt_list[id].gt_id = gt->info.id; 380 gt_list->gt_list[id].clock_freq = gt->info.clock_freq; 381 if (!IS_DGFX(xe)) 382 gt_list->gt_list[id].native_mem_regions = 0x1; 383 else 384 gt_list->gt_list[id].native_mem_regions = 385 BIT(gt_to_tile(gt)->id) << 1; 386 gt_list->gt_list[id].slow_mem_regions = xe->info.mem_region_mask ^ 387 gt_list->gt_list[id].native_mem_regions; 388 } 389 390 if (copy_to_user(query_ptr, gt_list, size)) { 391 kfree(gt_list); 392 return -EFAULT; 393 } 394 kfree(gt_list); 395 396 return 0; 397 } 398 399 static int query_hwconfig(struct xe_device *xe, 400 struct drm_xe_device_query *query) 401 { 402 struct xe_gt *gt = xe_root_mmio_gt(xe); 403 size_t size = xe_guc_hwconfig_size(>->uc.guc); 404 void __user *query_ptr = u64_to_user_ptr(query->data); 405 void *hwconfig; 406 407 if (query->size == 0) { 408 query->size = size; 409 return 0; 410 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 411 return -EINVAL; 412 } 413 414 hwconfig = kzalloc(size, GFP_KERNEL); 415 if (!hwconfig) 416 return -ENOMEM; 417 418 xe_device_mem_access_get(xe); 419 xe_guc_hwconfig_copy(>->uc.guc, hwconfig); 420 xe_device_mem_access_put(xe); 421 422 if (copy_to_user(query_ptr, hwconfig, size)) { 423 kfree(hwconfig); 424 return -EFAULT; 425 } 426 kfree(hwconfig); 427 428 return 0; 429 } 430 431 static size_t calc_topo_query_size(struct xe_device *xe) 432 { 433 return xe->info.gt_count * 434 (3 * sizeof(struct drm_xe_query_topology_mask) + 435 sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) + 436 sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) + 437 sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss)); 438 } 439 440 static void __user *copy_mask(void __user *ptr, 441 struct drm_xe_query_topology_mask *topo, 442 void *mask, size_t mask_size) 443 { 444 topo->num_bytes = mask_size; 445 446 if (copy_to_user(ptr, topo, sizeof(*topo))) 447 return ERR_PTR(-EFAULT); 448 ptr += sizeof(topo); 449 450 if (copy_to_user(ptr, mask, mask_size)) 451 return ERR_PTR(-EFAULT); 452 ptr += mask_size; 453 454 return ptr; 455 } 456 457 static int query_gt_topology(struct xe_device *xe, 458 struct drm_xe_device_query *query) 459 { 460 void __user *query_ptr = u64_to_user_ptr(query->data); 461 size_t size = calc_topo_query_size(xe); 462 struct drm_xe_query_topology_mask topo; 463 struct xe_gt *gt; 464 int id; 465 466 if (query->size == 0) { 467 query->size = size; 468 return 0; 469 } else if (XE_IOCTL_DBG(xe, query->size != size)) { 470 return -EINVAL; 471 } 472 473 for_each_gt(gt, xe, id) { 474 topo.gt_id = id; 475 476 topo.type = XE_TOPO_DSS_GEOMETRY; 477 query_ptr = copy_mask(query_ptr, &topo, 478 gt->fuse_topo.g_dss_mask, 479 sizeof(gt->fuse_topo.g_dss_mask)); 480 if (IS_ERR(query_ptr)) 481 return PTR_ERR(query_ptr); 482 483 topo.type = XE_TOPO_DSS_COMPUTE; 484 query_ptr = copy_mask(query_ptr, &topo, 485 gt->fuse_topo.c_dss_mask, 486 sizeof(gt->fuse_topo.c_dss_mask)); 487 if (IS_ERR(query_ptr)) 488 return PTR_ERR(query_ptr); 489 490 topo.type = XE_TOPO_EU_PER_DSS; 491 query_ptr = copy_mask(query_ptr, &topo, 492 gt->fuse_topo.eu_mask_per_dss, 493 sizeof(gt->fuse_topo.eu_mask_per_dss)); 494 if (IS_ERR(query_ptr)) 495 return PTR_ERR(query_ptr); 496 } 497 498 return 0; 499 } 500 501 static int (* const xe_query_funcs[])(struct xe_device *xe, 502 struct drm_xe_device_query *query) = { 503 query_engines, 504 query_memory_usage, 505 query_config, 506 query_gt_list, 507 query_hwconfig, 508 query_gt_topology, 509 query_engine_cycles, 510 }; 511 512 int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 513 { 514 struct xe_device *xe = to_xe_device(dev); 515 struct drm_xe_device_query *query = data; 516 u32 idx; 517 518 if (XE_IOCTL_DBG(xe, query->extensions) || 519 XE_IOCTL_DBG(xe, query->reserved[0] || query->reserved[1])) 520 return -EINVAL; 521 522 if (XE_IOCTL_DBG(xe, query->query >= ARRAY_SIZE(xe_query_funcs))) 523 return -EINVAL; 524 525 idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs)); 526 if (XE_IOCTL_DBG(xe, !xe_query_funcs[idx])) 527 return -EINVAL; 528 529 return xe_query_funcs[idx](xe, query); 530 } 531