1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_pci.h" 7 8 #include <kunit/static_stub.h> 9 #include <linux/device/driver.h> 10 #include <linux/module.h> 11 #include <linux/pci.h> 12 #include <linux/pm_runtime.h> 13 14 #include <drm/drm_color_mgmt.h> 15 #include <drm/drm_drv.h> 16 #include <drm/intel/pciids.h> 17 18 #include "display/xe_display.h" 19 #include "regs/xe_gt_regs.h" 20 #include "regs/xe_regs.h" 21 #include "xe_configfs.h" 22 #include "xe_device.h" 23 #include "xe_drv.h" 24 #include "xe_gt.h" 25 #include "xe_gt_printk.h" 26 #include "xe_gt_sriov_vf.h" 27 #include "xe_guc.h" 28 #include "xe_mmio.h" 29 #include "xe_module.h" 30 #include "xe_pci_error.h" 31 #include "xe_pci_rebar.h" 32 #include "xe_pci_sriov.h" 33 #include "xe_pci_types.h" 34 #include "xe_pm.h" 35 #include "xe_printk.h" 36 #include "xe_sriov.h" 37 #include "xe_step.h" 38 #include "xe_survivability_mode.h" 39 #include "xe_tile.h" 40 #include "xe_tile_printk.h" 41 42 enum toggle_d3cold { 43 D3COLD_DISABLE, 44 D3COLD_ENABLE, 45 }; 46 47 __diag_push(); 48 __diag_ignore_all("-Woverride-init", "Allow field overrides in table"); 49 50 #define PLATFORM(x) \ 51 .platform = XE_##x, \ 52 .platform_name = #x 53 54 #define NOP(x) x 55 56 static const struct xe_graphics_desc graphics_xelp = { 57 .hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0), 58 .num_geometry_xecore_fuse_regs = 1, 59 }; 60 61 #define XE_HP_FEATURES \ 62 .has_range_tlb_inval = true 63 64 static const struct xe_graphics_desc graphics_xehpg = { 65 .hw_engine_mask = 66 BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | 67 BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | 68 BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), 69 .num_geometry_xecore_fuse_regs = 1, 70 .num_compute_xecore_fuse_regs = 1, 71 72 XE_HP_FEATURES, 73 }; 74 75 static const struct xe_graphics_desc graphics_xehpc = { 76 .hw_engine_mask = 77 BIT(XE_HW_ENGINE_BCS0) | BIT(XE_HW_ENGINE_BCS1) | 78 BIT(XE_HW_ENGINE_BCS2) | BIT(XE_HW_ENGINE_BCS3) | 79 BIT(XE_HW_ENGINE_BCS4) | BIT(XE_HW_ENGINE_BCS5) | 80 BIT(XE_HW_ENGINE_BCS6) | BIT(XE_HW_ENGINE_BCS7) | 81 BIT(XE_HW_ENGINE_BCS8) | 82 BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | 83 BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), 84 85 XE_HP_FEATURES, 86 87 .has_access_counter = 1, 88 .has_asid = 1, 89 .has_atomic_enable_pte_bit = 1, 90 .has_usm = 1, 91 .num_compute_xecore_fuse_regs = 2, 92 }; 93 94 static const struct xe_graphics_desc graphics_xelpg = { 95 .hw_engine_mask = 96 BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | 97 BIT(XE_HW_ENGINE_CCS0), 98 .num_geometry_xecore_fuse_regs = 1, 99 .num_compute_xecore_fuse_regs = 1, 100 101 XE_HP_FEATURES, 102 }; 103 104 #define XE2_GFX_FEATURES \ 105 .has_access_counter = 1, \ 106 .has_asid = 1, \ 107 .has_atomic_enable_pte_bit = 1, \ 108 .has_range_tlb_inval = 1, \ 109 .has_usm = 1, \ 110 .has_64bit_timestamp = 1, \ 111 .hw_engine_mask = \ 112 BIT(XE_HW_ENGINE_RCS0) | \ 113 BIT(XE_HW_ENGINE_BCS8) | BIT(XE_HW_ENGINE_BCS0) | \ 114 GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0) 115 116 static const struct xe_graphics_desc graphics_xe2 = { 117 XE2_GFX_FEATURES, 118 .num_geometry_xecore_fuse_regs = 3, 119 .num_compute_xecore_fuse_regs = 3, 120 }; 121 122 static const struct xe_graphics_desc graphics_xe3p_lpg = { 123 XE2_GFX_FEATURES, 124 .has_indirect_ring_state = 1, 125 .has_uncorrectable_error_reporting = 1, 126 .multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | BIT(XE_ENGINE_CLASS_COMPUTE), 127 .num_geometry_xecore_fuse_regs = 3, 128 .num_compute_xecore_fuse_regs = 3, 129 }; 130 131 static const struct xe_graphics_desc graphics_xe3p_xpc = { 132 XE2_GFX_FEATURES, 133 .has_access_counter = 0, 134 .has_indirect_ring_state = 1, 135 .has_uncorrectable_error_reporting = 1, 136 .hw_engine_mask = 137 GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) | 138 GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0), 139 .multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | 140 BIT(XE_ENGINE_CLASS_COMPUTE), 141 .num_geometry_xecore_fuse_regs = 4, 142 .num_compute_xecore_fuse_regs = 4, 143 }; 144 145 static const struct xe_media_desc media_xem = { 146 .hw_engine_mask = 147 GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | 148 GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0), 149 }; 150 151 static const struct xe_media_desc media_xelpmp = { 152 .hw_engine_mask = 153 GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | 154 GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) | 155 BIT(XE_HW_ENGINE_GSCCS0) 156 }; 157 158 static const struct xe_media_desc media_xe3p_hpm = { 159 .has_uncorrectable_error_reporting = 1, 160 .hw_engine_mask = 161 GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | 162 GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) | 163 BIT(XE_HW_ENGINE_GSCCS0) 164 }; 165 166 /* Pre-GMDID Graphics IPs */ 167 static const struct xe_ip graphics_ip_xelp = { 1200, "Xe_LP", &graphics_xelp }; 168 static const struct xe_ip graphics_ip_xelpp = { 1210, "Xe_LP+", &graphics_xelp }; 169 static const struct xe_ip graphics_ip_xehpg = { 1255, "Xe_HPG", &graphics_xehpg }; 170 static const struct xe_ip graphics_ip_xehpc = { 1260, "Xe_HPC", &graphics_xehpc }; 171 172 /* GMDID-based Graphics IPs */ 173 static const struct xe_ip graphics_ips[] = { 174 { 1270, "Xe_LPG", &graphics_xelpg }, 175 { 1271, "Xe_LPG", &graphics_xelpg }, 176 { 1274, "Xe_LPG+", &graphics_xelpg }, 177 { 2001, "Xe2_HPG", &graphics_xe2 }, 178 { 2002, "Xe2_HPG", &graphics_xe2 }, 179 { 2004, "Xe2_LPG", &graphics_xe2 }, 180 { 3000, "Xe3_LPG", &graphics_xe2 }, 181 { 3001, "Xe3_LPG", &graphics_xe2 }, 182 { 3003, "Xe3_LPG", &graphics_xe2 }, 183 { 3004, "Xe3_LPG", &graphics_xe2 }, 184 { 3005, "Xe3_LPG", &graphics_xe2 }, 185 { 3510, "Xe3p_LPG", &graphics_xe3p_lpg }, 186 { 3511, "Xe3p_XPC", &graphics_xe3p_xpc }, 187 }; 188 189 /* Pre-GMDID Media IPs */ 190 static const struct xe_ip media_ip_xem = { 1200, "Xe_M", &media_xem }; 191 static const struct xe_ip media_ip_xehpm = { 1255, "Xe_HPM", &media_xem }; 192 193 /* GMDID-based Media IPs */ 194 static const struct xe_ip media_ips[] = { 195 { 1300, "Xe_LPM+", &media_xelpmp }, 196 { 1301, "Xe2_HPM", &media_xelpmp }, 197 { 2000, "Xe2_LPM", &media_xelpmp }, 198 { 3000, "Xe3_LPM", &media_xelpmp }, 199 { 3002, "Xe3_LPM", &media_xelpmp }, 200 { 3500, "Xe3p_LPM", &media_xelpmp }, 201 { 3503, "Xe3p_HPM", &media_xe3p_hpm }, 202 }; 203 204 #define MULTI_LRC_MASK \ 205 .multi_lrc_mask = BIT(XE_ENGINE_CLASS_VIDEO_DECODE) | \ 206 BIT(XE_ENGINE_CLASS_VIDEO_ENHANCE) 207 208 static const struct xe_device_desc tgl_desc = { 209 .pre_gmdid_graphics_ip = &graphics_ip_xelp, 210 .pre_gmdid_media_ip = &media_ip_xem, 211 PLATFORM(TIGERLAKE), 212 .dma_mask_size = 39, 213 .has_cached_pt = true, 214 .has_display = true, 215 .has_llc = true, 216 .has_sriov = true, 217 .max_gt_per_tile = 1, 218 MULTI_LRC_MASK, 219 .require_force_probe = true, 220 .va_bits = 48, 221 .vm_max_level = 3, 222 }; 223 224 static const struct xe_device_desc rkl_desc = { 225 .pre_gmdid_graphics_ip = &graphics_ip_xelp, 226 .pre_gmdid_media_ip = &media_ip_xem, 227 PLATFORM(ROCKETLAKE), 228 .dma_mask_size = 39, 229 .has_cached_pt = true, 230 .has_display = true, 231 .has_llc = true, 232 .max_gt_per_tile = 1, 233 MULTI_LRC_MASK, 234 .require_force_probe = true, 235 .va_bits = 48, 236 .vm_max_level = 3, 237 }; 238 239 static const u16 adls_rpls_ids[] = { INTEL_RPLS_IDS(NOP), 0 }; 240 241 static const struct xe_device_desc adl_s_desc = { 242 .pre_gmdid_graphics_ip = &graphics_ip_xelp, 243 .pre_gmdid_media_ip = &media_ip_xem, 244 PLATFORM(ALDERLAKE_S), 245 .dma_mask_size = 39, 246 .has_cached_pt = true, 247 .has_display = true, 248 .has_llc = true, 249 .has_sriov = true, 250 .max_gt_per_tile = 1, 251 MULTI_LRC_MASK, 252 .require_force_probe = true, 253 .subplatforms = (const struct xe_subplatform_desc[]) { 254 { XE_SUBPLATFORM_ALDERLAKE_S_RPLS, "RPLS", adls_rpls_ids }, 255 {}, 256 }, 257 .va_bits = 48, 258 .vm_max_level = 3, 259 }; 260 261 static const u16 adlp_rplu_ids[] = { INTEL_RPLU_IDS(NOP), 0 }; 262 263 static const struct xe_device_desc adl_p_desc = { 264 .pre_gmdid_graphics_ip = &graphics_ip_xelp, 265 .pre_gmdid_media_ip = &media_ip_xem, 266 PLATFORM(ALDERLAKE_P), 267 .dma_mask_size = 39, 268 .has_cached_pt = true, 269 .has_display = true, 270 .has_llc = true, 271 .has_sriov = true, 272 .max_gt_per_tile = 1, 273 MULTI_LRC_MASK, 274 .require_force_probe = true, 275 .subplatforms = (const struct xe_subplatform_desc[]) { 276 { XE_SUBPLATFORM_ALDERLAKE_P_RPLU, "RPLU", adlp_rplu_ids }, 277 {}, 278 }, 279 .va_bits = 48, 280 .vm_max_level = 3, 281 }; 282 283 static const struct xe_device_desc adl_n_desc = { 284 .pre_gmdid_graphics_ip = &graphics_ip_xelp, 285 .pre_gmdid_media_ip = &media_ip_xem, 286 PLATFORM(ALDERLAKE_N), 287 .dma_mask_size = 39, 288 .has_cached_pt = true, 289 .has_display = true, 290 .has_llc = true, 291 .has_sriov = true, 292 .max_gt_per_tile = 1, 293 MULTI_LRC_MASK, 294 .require_force_probe = true, 295 .va_bits = 48, 296 .vm_max_level = 3, 297 }; 298 299 #define DGFX_FEATURES \ 300 .is_dgfx = 1 301 302 static const struct xe_device_desc dg1_desc = { 303 .pre_gmdid_graphics_ip = &graphics_ip_xelpp, 304 .pre_gmdid_media_ip = &media_ip_xem, 305 DGFX_FEATURES, 306 PLATFORM(DG1), 307 .dma_mask_size = 39, 308 .has_display = true, 309 .has_gsc_nvm = 1, 310 .has_heci_gscfi = 1, 311 .max_gt_per_tile = 1, 312 MULTI_LRC_MASK, 313 .require_force_probe = true, 314 .va_bits = 48, 315 .vm_max_level = 3, 316 }; 317 318 static const u16 dg2_g10_ids[] = { INTEL_DG2_G10_IDS(NOP), INTEL_ATS_M150_IDS(NOP), 0 }; 319 static const u16 dg2_g11_ids[] = { INTEL_DG2_G11_IDS(NOP), INTEL_ATS_M75_IDS(NOP), 0 }; 320 static const u16 dg2_g12_ids[] = { INTEL_DG2_G12_IDS(NOP), 0 }; 321 322 #define DG2_FEATURES \ 323 DGFX_FEATURES, \ 324 PLATFORM(DG2), \ 325 .has_flat_ccs = 1, \ 326 .has_gsc_nvm = 1, \ 327 .has_heci_gscfi = 1, \ 328 .subplatforms = (const struct xe_subplatform_desc[]) { \ 329 { XE_SUBPLATFORM_DG2_G10, "G10", dg2_g10_ids }, \ 330 { XE_SUBPLATFORM_DG2_G11, "G11", dg2_g11_ids }, \ 331 { XE_SUBPLATFORM_DG2_G12, "G12", dg2_g12_ids }, \ 332 { } \ 333 }, \ 334 .va_bits = 48, \ 335 .vm_max_level = 3, \ 336 .vram_flags = XE_VRAM_FLAGS_NEED64K 337 338 static const struct xe_device_desc ats_m_desc = { 339 .pre_gmdid_graphics_ip = &graphics_ip_xehpg, 340 .pre_gmdid_media_ip = &media_ip_xehpm, 341 .dma_mask_size = 46, 342 .max_gt_per_tile = 1, 343 MULTI_LRC_MASK, 344 .require_force_probe = true, 345 346 DG2_FEATURES, 347 .has_display = false, 348 .has_sriov = true, 349 }; 350 351 static const struct xe_device_desc dg2_desc = { 352 .pre_gmdid_graphics_ip = &graphics_ip_xehpg, 353 .pre_gmdid_media_ip = &media_ip_xehpm, 354 .dma_mask_size = 46, 355 .max_gt_per_tile = 1, 356 MULTI_LRC_MASK, 357 .require_force_probe = true, 358 359 DG2_FEATURES, 360 .has_display = true, 361 .has_fan_control = true, 362 .has_mbx_power_limits = false, 363 }; 364 365 static const __maybe_unused struct xe_device_desc pvc_desc = { 366 .pre_gmdid_graphics_ip = &graphics_ip_xehpc, 367 DGFX_FEATURES, 368 PLATFORM(PVC), 369 .dma_mask_size = 52, 370 .has_display = false, 371 .has_drm_ras = true, 372 .has_gsc_nvm = 1, 373 .has_heci_gscfi = 1, 374 .max_gt_per_tile = 1, 375 .max_remote_tiles = 1, 376 MULTI_LRC_MASK, 377 .require_force_probe = true, 378 .va_bits = 57, 379 .vm_max_level = 4, 380 .vram_flags = XE_VRAM_FLAGS_NEED64K, 381 .has_mbx_power_limits = false, 382 }; 383 384 static const struct xe_device_desc mtl_desc = { 385 /* .graphics and .media determined via GMD_ID */ 386 .require_force_probe = true, 387 PLATFORM(METEORLAKE), 388 .dma_mask_size = 46, 389 .has_display = true, 390 .has_pxp = true, 391 .max_gt_per_tile = 2, 392 MULTI_LRC_MASK, 393 .va_bits = 48, 394 .vm_max_level = 3, 395 }; 396 397 static const struct xe_device_desc lnl_desc = { 398 PLATFORM(LUNARLAKE), 399 .dma_mask_size = 46, 400 .has_display = true, 401 .has_flat_ccs = 1, 402 .has_pxp = true, 403 .max_gt_per_tile = 2, 404 MULTI_LRC_MASK, 405 .needs_scratch = true, 406 .va_bits = 48, 407 .vm_max_level = 4, 408 }; 409 410 static const u16 bmg_g21_ids[] = { INTEL_BMG_G21_IDS(NOP), 0 }; 411 412 static const struct xe_device_desc bmg_desc = { 413 DGFX_FEATURES, 414 PLATFORM(BATTLEMAGE), 415 .dma_mask_size = 46, 416 .has_display = true, 417 .has_fan_control = true, 418 .has_flat_ccs = 1, 419 .has_mbx_power_limits = true, 420 .has_mbx_thermal_info = true, 421 .has_gsc_nvm = 1, 422 .has_heci_cscfi = 1, 423 .has_i2c = true, 424 .has_late_bind = true, 425 .has_pre_prod_wa = 1, 426 .has_soc_remapper_telem = true, 427 .has_sriov = true, 428 .max_gt_per_tile = 2, 429 MULTI_LRC_MASK, 430 .needs_scratch = true, 431 .subplatforms = (const struct xe_subplatform_desc[]) { 432 { XE_SUBPLATFORM_BATTLEMAGE_G21, "G21", bmg_g21_ids }, 433 { } 434 }, 435 .va_bits = 48, 436 .vm_max_level = 4, 437 }; 438 439 static const struct xe_device_desc ptl_desc = { 440 PLATFORM(PANTHERLAKE), 441 .dma_mask_size = 46, 442 .has_display = true, 443 .has_flat_ccs = 1, 444 .has_sriov = true, 445 .has_pre_prod_wa = 1, 446 .has_pxp = true, 447 .max_gt_per_tile = 2, 448 MULTI_LRC_MASK, 449 .needs_scratch = true, 450 .needs_shared_vf_gt_wq = true, 451 .va_bits = 48, 452 .vm_max_level = 4, 453 }; 454 455 static const struct xe_device_desc nvls_desc = { 456 PLATFORM(NOVALAKE_S), 457 .dma_mask_size = 46, 458 .has_display = true, 459 .has_flat_ccs = 1, 460 .has_pre_prod_wa = 1, 461 .has_sriov = true, 462 .max_gt_per_tile = 2, 463 MULTI_LRC_MASK, 464 .va_bits = 48, 465 .vm_max_level = 4, 466 }; 467 468 static const struct xe_device_desc cri_desc = { 469 DGFX_FEATURES, 470 PLATFORM(CRESCENTISLAND), 471 .dma_mask_size = 52, 472 .has_display = false, 473 .has_drm_ras = true, 474 .has_flat_ccs = false, 475 .has_gsc_nvm = 1, 476 .has_i2c = true, 477 .has_mbx_power_limits = true, 478 .has_mbx_thermal_info = true, 479 .has_mert = true, 480 .has_pre_prod_wa = 1, 481 .has_soc_remapper_sysctrl = true, 482 .has_soc_remapper_telem = true, 483 .has_sriov = true, 484 .has_sysctrl = true, 485 .max_gt_per_tile = 2, 486 MULTI_LRC_MASK, 487 .require_force_probe = true, 488 .va_bits = 57, 489 .vm_max_level = 4, 490 }; 491 492 static const struct xe_device_desc nvlp_desc = { 493 PLATFORM(NOVALAKE_P), 494 .dma_mask_size = 46, 495 .has_cached_pt = true, 496 .has_display = true, 497 .has_flat_ccs = 1, 498 .has_page_reclaim_hw_assist = true, 499 .has_pre_prod_wa = true, 500 .has_sriov = true, 501 .max_gt_per_tile = 2, 502 MULTI_LRC_MASK, 503 .require_force_probe = true, 504 .va_bits = 48, 505 .vm_max_level = 4, 506 }; 507 508 #undef PLATFORM 509 __diag_pop(); 510 511 /* 512 * Make sure any device matches here are from most specific to most 513 * general. For example, since the Quanta match is based on the subsystem 514 * and subvendor IDs, we need it to come before the more general IVB 515 * PCI ID matches, otherwise we'll use the wrong info struct above. 516 */ 517 static const struct pci_device_id pciidlist[] = { 518 INTEL_TGL_IDS(INTEL_VGA_DEVICE, &tgl_desc), 519 INTEL_RKL_IDS(INTEL_VGA_DEVICE, &rkl_desc), 520 INTEL_ADLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), 521 INTEL_ADLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), 522 INTEL_ADLN_IDS(INTEL_VGA_DEVICE, &adl_n_desc), 523 INTEL_RPLU_IDS(INTEL_VGA_DEVICE, &adl_p_desc), 524 INTEL_RPLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), 525 INTEL_RPLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), 526 INTEL_DG1_IDS(INTEL_VGA_DEVICE, &dg1_desc), 527 INTEL_ATS_M_IDS(INTEL_VGA_DEVICE, &ats_m_desc), 528 INTEL_ARL_IDS(INTEL_VGA_DEVICE, &mtl_desc), 529 INTEL_DG2_IDS(INTEL_VGA_DEVICE, &dg2_desc), 530 INTEL_MTL_IDS(INTEL_VGA_DEVICE, &mtl_desc), 531 INTEL_LNL_IDS(INTEL_VGA_DEVICE, &lnl_desc), 532 INTEL_BMG_IDS(INTEL_VGA_DEVICE, &bmg_desc), 533 INTEL_PTL_IDS(INTEL_VGA_DEVICE, &ptl_desc), 534 INTEL_WCL_IDS(INTEL_VGA_DEVICE, &ptl_desc), 535 INTEL_NVLS_IDS(INTEL_VGA_DEVICE, &nvls_desc), 536 INTEL_CRI_IDS(INTEL_PCI_DEVICE, &cri_desc), 537 INTEL_NVLP_IDS(INTEL_VGA_DEVICE, &nvlp_desc), 538 { } 539 }; 540 MODULE_DEVICE_TABLE(pci, pciidlist); 541 542 /* is device_id present in comma separated list of ids */ 543 static bool device_id_in_list(u16 device_id, const char *devices, bool negative) 544 { 545 char *s, *p, *tok; 546 bool ret; 547 548 if (!devices || !*devices) 549 return false; 550 551 /* match everything */ 552 if (negative && strcmp(devices, "!*") == 0) 553 return true; 554 if (!negative && strcmp(devices, "*") == 0) 555 return true; 556 557 s = kstrdup(devices, GFP_KERNEL); 558 if (!s) 559 return false; 560 561 for (p = s, ret = false; (tok = strsep(&p, ",")) != NULL; ) { 562 u16 val; 563 564 if (negative && tok[0] == '!') 565 tok++; 566 else if ((negative && tok[0] != '!') || 567 (!negative && tok[0] == '!')) 568 continue; 569 570 if (kstrtou16(tok, 16, &val) == 0 && val == device_id) { 571 ret = true; 572 break; 573 } 574 } 575 576 kfree(s); 577 578 return ret; 579 } 580 581 static bool id_forced(u16 device_id) 582 { 583 return device_id_in_list(device_id, xe_modparam.force_probe, false); 584 } 585 586 static bool id_blocked(u16 device_id) 587 { 588 return device_id_in_list(device_id, xe_modparam.force_probe, true); 589 } 590 591 static const struct xe_subplatform_desc * 592 find_subplatform(const struct xe_device_desc *desc, u16 devid) 593 { 594 const struct xe_subplatform_desc *sp; 595 const u16 *id; 596 597 for (sp = desc->subplatforms; sp && sp->subplatform; sp++) 598 for (id = sp->pciidlist; *id; id++) 599 if (*id == devid) 600 return sp; 601 602 return NULL; 603 } 604 605 enum xe_gmdid_type { 606 GMDID_GRAPHICS, 607 GMDID_MEDIA 608 }; 609 610 static int read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u32 *revid) 611 { 612 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 613 struct xe_reg gmdid_reg = GMD_ID; 614 u32 val; 615 616 if (IS_SRIOV_VF(xe)) { 617 /* 618 * To get the value of the GMDID register, VFs must obtain it 619 * from the GuC using MMIO communication. 620 * 621 * Note that at this point the GTs are not initialized and only 622 * tile-level access to MMIO registers is possible. To use our 623 * existing GuC communication functions we must create a dummy 624 * GT structure and perform at least basic xe_gt and xe_guc 625 * initialization. 626 */ 627 struct xe_gt *gt __free(kfree) = NULL; 628 int err; 629 630 /* Don't try to read media ver if media GT is not allowed */ 631 if (type == GMDID_MEDIA && !xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { 632 *ver = *revid = 0; 633 return 0; 634 } 635 636 gt = kzalloc(sizeof(*gt), GFP_KERNEL); 637 if (!gt) 638 return -ENOMEM; 639 640 gt->tile = &xe->tiles[0]; 641 if (type == GMDID_MEDIA) { 642 gt->info.id = 1; 643 gt->info.type = XE_GT_TYPE_MEDIA; 644 } else { 645 gt->info.id = 0; 646 gt->info.type = XE_GT_TYPE_MAIN; 647 } 648 649 xe_gt_mmio_init(gt); 650 xe_guc_comm_init_early(>->uc.guc); 651 652 err = xe_gt_sriov_vf_bootstrap(gt); 653 if (err) 654 return err; 655 656 val = xe_gt_sriov_vf_gmdid(gt); 657 } else { 658 /* 659 * GMD_ID is a GT register, but at this point in the driver 660 * init we haven't fully initialized the GT yet so we need to 661 * read the register with the tile's MMIO accessor. That means 662 * we need to apply the GSI offset manually since it won't get 663 * automatically added as it would if we were using a GT mmio 664 * accessor. 665 */ 666 if (type == GMDID_MEDIA) 667 gmdid_reg.addr += MEDIA_GT_GSI_OFFSET; 668 669 val = xe_mmio_read32(mmio, gmdid_reg); 670 } 671 672 *ver = REG_FIELD_GET(GMD_ID_ARCH_MASK, val) * 100 + REG_FIELD_GET(GMD_ID_RELEASE_MASK, val); 673 *revid = REG_FIELD_GET(GMD_ID_REVID, val); 674 675 return 0; 676 } 677 678 static const struct xe_ip *find_graphics_ip(unsigned int verx100) 679 { 680 KUNIT_STATIC_STUB_REDIRECT(find_graphics_ip, verx100); 681 682 for (int i = 0; i < ARRAY_SIZE(graphics_ips); i++) 683 if (graphics_ips[i].verx100 == verx100) 684 return &graphics_ips[i]; 685 return NULL; 686 } 687 688 static const struct xe_ip *find_media_ip(unsigned int verx100) 689 { 690 KUNIT_STATIC_STUB_REDIRECT(find_media_ip, verx100); 691 692 for (int i = 0; i < ARRAY_SIZE(media_ips); i++) 693 if (media_ips[i].verx100 == verx100) 694 return &media_ips[i]; 695 return NULL; 696 } 697 698 /* 699 * Read IP version from hardware and select graphics/media IP descriptors 700 * based on the result. 701 */ 702 static int handle_gmdid(struct xe_device *xe, 703 const struct xe_ip **graphics_ip, 704 const struct xe_ip **media_ip, 705 u32 *graphics_revid, 706 u32 *media_revid) 707 { 708 u32 ver; 709 int ret; 710 711 *graphics_ip = NULL; 712 *media_ip = NULL; 713 714 ret = read_gmdid(xe, GMDID_GRAPHICS, &ver, graphics_revid); 715 if (ret) 716 return ret; 717 718 *graphics_ip = find_graphics_ip(ver); 719 if (!*graphics_ip) { 720 drm_err(&xe->drm, "Hardware reports unknown graphics version %u.%02u\n", 721 ver / 100, ver % 100); 722 } 723 724 ret = read_gmdid(xe, GMDID_MEDIA, &ver, media_revid); 725 if (ret) 726 return ret; 727 728 /* Media may legitimately be fused off / not present */ 729 if (ver == 0) 730 return 0; 731 732 *media_ip = find_media_ip(ver); 733 if (!*media_ip) { 734 drm_err(&xe->drm, "Hardware reports unknown media version %u.%02u\n", 735 ver / 100, ver % 100); 736 } 737 738 return 0; 739 } 740 741 struct xe_probed_info { 742 u16 devid; 743 u8 revid; 744 u8 tile_count; 745 struct xe_step_info step; 746 const struct xe_ip *graphics_ip; 747 const struct xe_ip *media_ip; 748 }; 749 750 /* 751 * Probe from the hardware the info required by xe_info_init_early(). 752 */ 753 static int xe_probe_info_early(struct xe_device *xe, 754 const struct xe_device_desc *desc, 755 struct xe_probed_info *probed_info) 756 { 757 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 758 759 probed_info->devid = pdev->device; 760 probed_info->revid = pdev->revision; 761 762 xe_step_platform_get(desc->platform, probed_info->revid, &probed_info->step); 763 764 return 0; 765 } 766 767 /* 768 * Initialize device info content that only depends on static driver_data 769 * passed to the driver at probe time from PCI ID table. 770 */ 771 static int xe_info_init_early(struct xe_device *xe, 772 const struct xe_device_desc *desc, 773 const struct xe_subplatform_desc *subplatform_desc, 774 struct xe_probed_info *probed_info) 775 { 776 int err; 777 778 xe->info.devid = probed_info->devid; 779 xe->info.revid = probed_info->revid; 780 xe->info.step.platform = probed_info->step.platform; 781 782 xe->info.platform_name = desc->platform_name; 783 xe->info.platform = desc->platform; 784 xe->info.subplatform = subplatform_desc ? 785 subplatform_desc->subplatform : XE_SUBPLATFORM_NONE; 786 787 xe->info.dma_mask_size = desc->dma_mask_size; 788 xe->info.va_bits = desc->va_bits; 789 xe->info.vm_max_level = desc->vm_max_level; 790 xe->info.vram_flags = desc->vram_flags; 791 792 xe->info.is_dgfx = desc->is_dgfx; 793 xe->info.has_cached_pt = desc->has_cached_pt; 794 xe->info.has_drm_ras = desc->has_drm_ras; 795 xe->info.has_fan_control = desc->has_fan_control; 796 /* runtime fusing may force flat_ccs to disabled later */ 797 xe->info.has_flat_ccs = desc->has_flat_ccs; 798 xe->info.has_mbx_power_limits = desc->has_mbx_power_limits; 799 xe->info.has_mbx_thermal_info = desc->has_mbx_thermal_info; 800 xe->info.has_gsc_nvm = desc->has_gsc_nvm; 801 xe->info.has_heci_gscfi = desc->has_heci_gscfi; 802 xe->info.has_heci_cscfi = desc->has_heci_cscfi; 803 xe->info.has_i2c = desc->has_i2c; 804 xe->info.has_late_bind = desc->has_late_bind; 805 xe->info.has_llc = desc->has_llc; 806 xe->info.has_mert = desc->has_mert; 807 xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; 808 xe->info.has_pre_prod_wa = desc->has_pre_prod_wa; 809 xe->info.has_pxp = desc->has_pxp; 810 xe->info.has_soc_remapper_sysctrl = desc->has_soc_remapper_sysctrl; 811 xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem; 812 xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && 813 desc->has_sriov; 814 xe->info.has_sysctrl = desc->has_sysctrl; 815 xe->info.skip_guc_pc = desc->skip_guc_pc; 816 xe->info.skip_pcode = desc->skip_pcode; 817 xe->info.needs_scratch = desc->needs_scratch; 818 xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq; 819 xe->info.multi_lrc_mask = desc->multi_lrc_mask; 820 821 xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) && 822 xe_modparam.probe_display && 823 desc->has_display && 824 !xe_device_is_admin_only(xe); 825 826 xe_assert(xe, desc->max_gt_per_tile > 0); 827 xe_assert(xe, desc->max_gt_per_tile <= XE_MAX_GT_PER_TILE); 828 xe->info.max_gt_per_tile = desc->max_gt_per_tile; 829 830 err = xe_tile_init_early(xe_device_get_root_tile(xe), xe, 0); 831 if (err) 832 return err; 833 834 return 0; 835 } 836 837 static void xe_probe_tile_count(struct xe_device *xe, 838 const struct xe_device_desc *desc, 839 struct xe_probed_info *probed_info) 840 { 841 struct xe_mmio *mmio; 842 u8 tile_count; 843 u32 mtcfg; 844 845 probed_info->tile_count = 1 + desc->max_remote_tiles; 846 847 /* 848 * Probe for tile count only for platforms that support multiple 849 * tiles. 850 */ 851 if (probed_info->tile_count == 1) 852 return; 853 854 mmio = xe_root_tile_mmio(xe); 855 856 /* 857 * Although the per-tile mmio regs are not yet initialized, this 858 * is fine as it's going to the root tile's mmio, that's 859 * guaranteed to be initialized earlier in xe_mmio_probe_early() 860 */ 861 mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR); 862 tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; 863 864 if (tile_count < probed_info->tile_count) { 865 drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", 866 probed_info->tile_count, tile_count); 867 probed_info->tile_count = tile_count; 868 } 869 } 870 871 static struct xe_gt *alloc_primary_gt(struct xe_tile *tile, 872 const struct xe_graphics_desc *graphics_desc, 873 const struct xe_media_desc *media_desc) 874 { 875 struct xe_device *xe = tile_to_xe(tile); 876 struct xe_gt *gt; 877 878 if (!xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev))) { 879 xe_tile_info(tile, "Primary GT disabled via configfs\n"); 880 return NULL; 881 } 882 883 gt = xe_gt_alloc(tile); 884 if (IS_ERR(gt)) 885 return gt; 886 887 gt->info.type = XE_GT_TYPE_MAIN; 888 gt->info.id = tile->id * xe->info.max_gt_per_tile; 889 gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state; 890 gt->info.has_uncorrectable_error_reporting = 891 graphics_desc->has_uncorrectable_error_reporting; 892 gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask; 893 if (!xe_configfs_get_enable_multi_queue(to_pci_dev(xe->drm.dev))) { 894 xe_gt_info(gt, "Multi-queue disabled via configfs\n"); 895 gt->info.multi_queue_engine_class_mask = 0; 896 } 897 gt->info.engine_mask = graphics_desc->hw_engine_mask; 898 gt->info.num_geometry_xecore_fuse_regs = graphics_desc->num_geometry_xecore_fuse_regs; 899 gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs; 900 901 /* 902 * Even if the service copy engines wind up being fused off, their 903 * presence in the IP descriptor indicates that the platform supports 904 * Xe2-style MEM_SET and MEM_COPY functionality. 905 */ 906 if (graphics_desc->hw_engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 907 XE_HW_ENGINE_BCS1)) 908 gt->info.has_xe2_blt_instructions = true; 909 910 /* 911 * Before media version 13, the media IP was part of the primary GT 912 * so we need to add the media engines to the primary GT's engine list. 913 */ 914 if (MEDIA_VER(xe) < 13 && media_desc) 915 gt->info.engine_mask |= media_desc->hw_engine_mask; 916 917 return gt; 918 } 919 920 static struct xe_gt *alloc_media_gt(struct xe_tile *tile, 921 const struct xe_media_desc *media_desc) 922 { 923 struct xe_device *xe = tile_to_xe(tile); 924 struct xe_gt *gt; 925 926 if (!xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { 927 xe_tile_info(tile, "Media GT disabled via configfs\n"); 928 return NULL; 929 } 930 931 if (MEDIA_VER(xe) < 13 || !media_desc) 932 return NULL; 933 934 gt = xe_gt_alloc(tile); 935 if (IS_ERR(gt)) 936 return gt; 937 938 gt->info.type = XE_GT_TYPE_MEDIA; 939 gt->info.id = tile->id * xe->info.max_gt_per_tile + 1; 940 gt->info.has_indirect_ring_state = media_desc->has_indirect_ring_state; 941 gt->info.has_uncorrectable_error_reporting = media_desc->has_uncorrectable_error_reporting; 942 gt->info.engine_mask = media_desc->hw_engine_mask; 943 944 return gt; 945 } 946 947 static int xe_probe_ips(struct xe_device *xe, 948 const struct xe_device_desc *desc, 949 struct xe_probed_info *probed_info) 950 { 951 /* 952 * If this platform supports GMD_ID, we'll detect the proper IP 953 * descriptor to use from hardware registers. 954 * desc->pre_gmdid_graphics_ip will only ever be set at this point for 955 * platforms before GMD_ID. In that case the IP descriptions and 956 * versions are simply derived from that. 957 */ 958 if (desc->pre_gmdid_graphics_ip) { 959 probed_info->graphics_ip = desc->pre_gmdid_graphics_ip; 960 probed_info->media_ip = desc->pre_gmdid_media_ip; 961 xe_step_pre_gmdid_get(xe, &probed_info->step); 962 } else { 963 int err; 964 u32 graphics_revid, media_revid; 965 966 xe_assert(xe, !desc->pre_gmdid_media_ip); 967 968 err = handle_gmdid(xe, &probed_info->graphics_ip, &probed_info->media_ip, 969 &graphics_revid, &media_revid); 970 if (err) 971 return err; 972 973 xe_step_gmdid_get(xe, graphics_revid, media_revid, &probed_info->step); 974 } 975 976 /* 977 * If we couldn't detect the graphics IP, that's considered a fatal 978 * error and we should abort driver load. Failing to detect media 979 * IP is non-fatal; we'll just proceed without enabling media support. 980 */ 981 if (!probed_info->graphics_ip) 982 return -ENODEV; 983 984 return 0; 985 } 986 987 /* 988 * Probe from the hardware the info required by xe_info_init(). 989 */ 990 static int xe_probe_info(struct xe_device *xe, 991 const struct xe_device_desc *desc, 992 struct xe_probed_info *probed_info) 993 { 994 int err; 995 996 xe_probe_tile_count(xe, desc, probed_info); 997 998 err = xe_probe_ips(xe, desc, probed_info); 999 if (err) 1000 return err; 1001 1002 return 0; 1003 } 1004 1005 /* 1006 * Initialize device info content that does require knowledge about 1007 * graphics / media IP version. 1008 * Make sure that GT / tile structures allocated by the driver match the data 1009 * present in device info. 1010 */ 1011 static int xe_info_init(struct xe_device *xe, 1012 const struct xe_device_desc *desc, 1013 struct xe_probed_info *probed_info) 1014 { 1015 const struct xe_ip *graphics_ip; 1016 const struct xe_ip *media_ip; 1017 const struct xe_graphics_desc *graphics_desc; 1018 const struct xe_media_desc *media_desc; 1019 struct xe_tile *tile; 1020 struct xe_gt *gt; 1021 u8 id; 1022 1023 graphics_ip = probed_info->graphics_ip; 1024 media_ip = probed_info->media_ip; 1025 1026 xe->info.tile_count = probed_info->tile_count; 1027 xe->info.step.basedie = probed_info->step.basedie; 1028 xe->info.step.graphics = probed_info->step.graphics; 1029 xe->info.step.media = probed_info->step.media; 1030 1031 xe->info.graphics_verx100 = graphics_ip->verx100; 1032 xe->info.graphics_name = graphics_ip->name; 1033 graphics_desc = graphics_ip->desc; 1034 1035 if (media_ip) { 1036 xe->info.media_verx100 = media_ip->verx100; 1037 xe->info.media_name = media_ip->name; 1038 media_desc = media_ip->desc; 1039 } else { 1040 xe->info.media_name = "none"; 1041 media_desc = NULL; 1042 } 1043 1044 xe->info.has_access_counter = graphics_desc->has_access_counter; 1045 xe->info.has_asid = graphics_desc->has_asid; 1046 xe->info.has_atomic_enable_pte_bit = graphics_desc->has_atomic_enable_pte_bit; 1047 if (xe->info.platform != XE_PVC) 1048 xe->info.has_device_atomics_on_smem = 1; 1049 1050 xe->info.has_range_tlb_inval = graphics_desc->has_range_tlb_inval; 1051 xe->info.has_ctx_tlb_inval = graphics_desc->has_ctx_tlb_inval; 1052 xe->info.has_usm = graphics_desc->has_usm; 1053 xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; 1054 xe->info.has_mem_copy_instr = GRAPHICS_VER(xe) >= 20; 1055 1056 if (IS_SRIOV_VF(xe)) { 1057 xe->info.has_sysctrl = 0; 1058 xe->info.has_soc_remapper_sysctrl = 0; 1059 xe->info.has_soc_remapper_telem = 0; 1060 } 1061 1062 for_each_remote_tile(tile, xe, id) { 1063 int err; 1064 1065 err = xe_tile_init_early(tile, xe, id); 1066 if (err) 1067 return err; 1068 } 1069 1070 /* Allocate any GT and VRAM structures necessary for the platform. */ 1071 for_each_tile(tile, xe, id) { 1072 int err; 1073 1074 err = xe_tile_alloc_vram(tile); 1075 if (err) 1076 return err; 1077 1078 tile->primary_gt = alloc_primary_gt(tile, graphics_desc, media_desc); 1079 if (IS_ERR(tile->primary_gt)) 1080 return PTR_ERR(tile->primary_gt); 1081 1082 /* 1083 * It's not currently possible to probe a device with the 1084 * primary GT disabled. With some work, this may be future in 1085 * the possible for igpu platforms (although probably not for 1086 * dgpu's since access to the primary GT's BCS engines is 1087 * required for VRAM management). 1088 */ 1089 if (!tile->primary_gt) { 1090 drm_err(&xe->drm, "Cannot probe device with without a primary GT\n"); 1091 return -ENODEV; 1092 } 1093 1094 tile->media_gt = alloc_media_gt(tile, media_desc); 1095 if (IS_ERR(tile->media_gt)) 1096 return PTR_ERR(tile->media_gt); 1097 } 1098 1099 /* 1100 * Now that we have tiles and GTs defined, let's loop over valid GTs 1101 * in order to define gt_count. 1102 */ 1103 for_each_gt(gt, xe, id) 1104 xe->info.gt_count++; 1105 1106 return 0; 1107 } 1108 1109 static void xe_pci_remove(struct pci_dev *pdev) 1110 { 1111 struct xe_device *xe = pdev_to_xe_device(pdev); 1112 1113 if (IS_SRIOV_PF(xe)) 1114 xe_pci_sriov_configure(pdev, 0); 1115 1116 if (xe_survivability_mode_is_boot_enabled(xe)) 1117 return; 1118 1119 xe_device_remove(xe); 1120 1121 /* 1122 * Preserve remove-time flush after moving destroy work to module 1123 * lifetime. 1124 */ 1125 xe_destroy_wq_flush(); 1126 xe_pm_fini(xe); 1127 } 1128 1129 /* 1130 * Probe the PCI device, initialize various parts of the driver. 1131 * 1132 * Fault injection is used to test the error paths of some initialization 1133 * functions called either directly from xe_pci_probe() or indirectly for 1134 * example through xe_device_probe(). Those functions use the kernel fault 1135 * injection capabilities infrastructure, see 1136 * Documentation/fault-injection/fault-injection.rst for details. The macro 1137 * ALLOW_ERROR_INJECTION() is used to conditionally skip function execution 1138 * at runtime and use a provided return value. The first requirement for 1139 * error injectable functions is proper handling of the error code by the 1140 * caller for recovery, which is always the case here. The second 1141 * requirement is that no state is changed before the first error return. 1142 * It is not strictly fulfilled for all initialization functions using the 1143 * ALLOW_ERROR_INJECTION() macro but this is acceptable because for those 1144 * error cases at probe time, the error code is simply propagated up by the 1145 * caller. Therefore there is no consequence on those specific callers when 1146 * function error injection skips the whole function. 1147 */ 1148 static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1149 { 1150 struct xe_probed_info probed_info = {}; 1151 const struct xe_device_desc *desc = (const void *)ent->driver_data; 1152 const struct xe_subplatform_desc *subplatform_desc; 1153 struct xe_device *xe; 1154 void *group; 1155 int err; 1156 1157 subplatform_desc = find_subplatform(desc, pdev->device); 1158 1159 xe_configfs_check_device(pdev); 1160 1161 if (desc->require_force_probe && !id_forced(pdev->device)) { 1162 dev_info(&pdev->dev, 1163 "Your graphics device %04x is not officially supported\n" 1164 "by xe driver in this kernel version. To force Xe probe,\n" 1165 "use xe.force_probe='%04x' and i915.force_probe='!%04x'\n" 1166 "module parameters or CONFIG_DRM_XE_FORCE_PROBE='%04x' and\n" 1167 "CONFIG_DRM_I915_FORCE_PROBE='!%04x' configuration options.\n", 1168 pdev->device, pdev->device, pdev->device, 1169 pdev->device, pdev->device); 1170 return -ENODEV; 1171 } 1172 1173 if (id_blocked(pdev->device)) { 1174 dev_info(&pdev->dev, "Probe blocked for device [%04x:%04x].\n", 1175 pdev->vendor, pdev->device); 1176 return -ENODEV; 1177 } 1178 1179 if (xe_display_driver_probe_defer(pdev)) 1180 return -EPROBE_DEFER; 1181 1182 /* Group all devres so xe_pci_error_slot_reset() can release them as a unit. */ 1183 group = devres_open_group(&pdev->dev, NULL, GFP_KERNEL); 1184 if (!group) 1185 return -ENOMEM; 1186 1187 err = pcim_enable_device(pdev); 1188 if (err) 1189 return err; 1190 1191 xe = xe_device_create(pdev); 1192 if (IS_ERR(xe)) 1193 return PTR_ERR(xe); 1194 1195 xe->devres_group = group; 1196 1197 pci_set_drvdata(pdev, &xe->drm); 1198 1199 xe_pm_assert_unbounded_bridge(xe); 1200 1201 pci_set_master(pdev); 1202 1203 err = xe_probe_info_early(xe, desc, &probed_info); 1204 if (err) 1205 return err; 1206 1207 err = xe_info_init_early(xe, desc, subplatform_desc, &probed_info); 1208 if (err) 1209 return err; 1210 1211 xe_pci_rebar_resize(xe); 1212 1213 err = xe_device_probe_early(xe); 1214 /* 1215 * In Boot Survivability mode, no drm card is exposed and driver 1216 * is loaded with bare minimum to allow for firmware to be 1217 * flashed through mei. Return success, if survivability mode 1218 * is enabled due to pcode failure or configfs being set 1219 */ 1220 if (xe_survivability_mode_is_boot_enabled(xe)) 1221 return 0; 1222 1223 if (err) 1224 return err; 1225 1226 err = xe_probe_info(xe, desc, &probed_info); 1227 if (err) 1228 return err; 1229 1230 err = xe_info_init(xe, desc, &probed_info); 1231 if (err) 1232 return err; 1233 1234 err = xe_display_probe(xe); 1235 if (err) 1236 return err; 1237 1238 drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx:%s (%d.%02d) media:%s (%d.%02d) display:%s dma_m_s:%d tc:%d gscfi:%d cscfi:%d", 1239 desc->platform_name, 1240 subplatform_desc ? subplatform_desc->name : "", 1241 xe->info.devid, xe->info.revid, 1242 xe->info.is_dgfx, 1243 xe->info.graphics_name, 1244 xe->info.graphics_verx100 / 100, 1245 xe->info.graphics_verx100 % 100, 1246 xe->info.media_name, 1247 xe->info.media_verx100 / 100, 1248 xe->info.media_verx100 % 100, 1249 str_yes_no(xe->info.probe_display), 1250 xe->info.dma_mask_size, xe->info.tile_count, 1251 xe->info.has_heci_gscfi, xe->info.has_heci_cscfi); 1252 1253 drm_dbg(&xe->drm, "Stepping = (G:%s, M:%s, B:%s)\n", 1254 xe_step_name(xe->info.step.graphics), 1255 xe_step_name(xe->info.step.media), 1256 xe_step_name(xe->info.step.basedie)); 1257 1258 drm_dbg(&xe->drm, "SR-IOV support: %s (mode: %s)\n", 1259 str_yes_no(xe_device_has_sriov(xe)), 1260 xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); 1261 1262 err = xe_pm_probe(xe); 1263 if (err) 1264 return err; 1265 1266 err = xe_device_probe(xe); 1267 if (err) 1268 return err; 1269 1270 err = xe_pm_init(xe); 1271 if (err) 1272 goto err_driver_cleanup; 1273 1274 return 0; 1275 1276 err_driver_cleanup: 1277 xe_pci_remove(pdev); 1278 return err; 1279 } 1280 1281 static void xe_pci_shutdown(struct pci_dev *pdev) 1282 { 1283 xe_device_shutdown(pdev_to_xe_device(pdev)); 1284 } 1285 1286 #ifdef CONFIG_PM_SLEEP 1287 static void d3cold_toggle(struct pci_dev *pdev, enum toggle_d3cold toggle) 1288 { 1289 struct xe_device *xe = pdev_to_xe_device(pdev); 1290 struct pci_dev *root_pdev; 1291 1292 if (!xe->d3cold.capable) 1293 return; 1294 1295 root_pdev = pcie_find_root_port(pdev); 1296 if (!root_pdev) 1297 return; 1298 1299 switch (toggle) { 1300 case D3COLD_DISABLE: 1301 pci_d3cold_disable(root_pdev); 1302 break; 1303 case D3COLD_ENABLE: 1304 pci_d3cold_enable(root_pdev); 1305 break; 1306 } 1307 } 1308 1309 static int xe_pci_suspend(struct device *dev) 1310 { 1311 struct pci_dev *pdev = to_pci_dev(dev); 1312 struct xe_device *xe = pdev_to_xe_device(pdev); 1313 int err; 1314 1315 if (xe_survivability_mode_is_boot_enabled(xe)) 1316 return -EBUSY; 1317 1318 err = xe_pm_suspend(xe); 1319 if (err) 1320 return err; 1321 1322 /* 1323 * Enabling D3Cold is needed for S2Idle/S0ix. 1324 * It is save to allow here since xe_pm_suspend has evicted 1325 * the local memory and the direct complete optimization is disabled. 1326 */ 1327 d3cold_toggle(pdev, D3COLD_ENABLE); 1328 1329 pci_save_state(pdev); 1330 pci_disable_device(pdev); 1331 pci_set_power_state(pdev, PCI_D3cold); 1332 1333 return 0; 1334 } 1335 1336 static int xe_pci_resume(struct device *dev) 1337 { 1338 struct pci_dev *pdev = to_pci_dev(dev); 1339 int err; 1340 1341 /* Give back the D3Cold decision to the runtime P M*/ 1342 d3cold_toggle(pdev, D3COLD_DISABLE); 1343 1344 err = pci_set_power_state(pdev, PCI_D0); 1345 if (err) 1346 return err; 1347 1348 pci_restore_state(pdev); 1349 1350 err = pci_enable_device(pdev); 1351 if (err) 1352 return err; 1353 1354 pci_set_master(pdev); 1355 1356 err = xe_pm_resume(pdev_to_xe_device(pdev)); 1357 if (err) 1358 return err; 1359 1360 return 0; 1361 } 1362 1363 static int xe_pci_runtime_suspend(struct device *dev) 1364 { 1365 struct pci_dev *pdev = to_pci_dev(dev); 1366 struct xe_device *xe = pdev_to_xe_device(pdev); 1367 int err; 1368 1369 /* 1370 * We hold an additional reference to the runtime PM to keep PF in D0 1371 * during VFs lifetime, as our VFs do not implement the PM capability. 1372 * This means we should never be runtime suspending as long as VFs are 1373 * enabled. 1374 */ 1375 xe_assert(xe, !IS_SRIOV_VF(xe)); 1376 xe_assert(xe, !pci_num_vf(pdev)); 1377 1378 err = xe_pm_runtime_suspend(xe); 1379 if (err) 1380 return err; 1381 1382 pci_save_state(pdev); 1383 1384 if (xe->d3cold.allowed) { 1385 d3cold_toggle(pdev, D3COLD_ENABLE); 1386 pci_disable_device(pdev); 1387 pci_ignore_hotplug(pdev); 1388 pci_set_power_state(pdev, PCI_D3cold); 1389 } else { 1390 d3cold_toggle(pdev, D3COLD_DISABLE); 1391 pci_set_power_state(pdev, PCI_D3hot); 1392 } 1393 1394 return 0; 1395 } 1396 1397 static int xe_pci_runtime_resume(struct device *dev) 1398 { 1399 struct pci_dev *pdev = to_pci_dev(dev); 1400 struct xe_device *xe = pdev_to_xe_device(pdev); 1401 int err; 1402 1403 err = pci_set_power_state(pdev, PCI_D0); 1404 if (err) 1405 return err; 1406 1407 pci_restore_state(pdev); 1408 1409 if (xe->d3cold.allowed) { 1410 err = pci_enable_device(pdev); 1411 if (err) 1412 return err; 1413 1414 pci_set_master(pdev); 1415 } 1416 1417 return xe_pm_runtime_resume(xe); 1418 } 1419 1420 static int xe_pci_runtime_idle(struct device *dev) 1421 { 1422 struct pci_dev *pdev = to_pci_dev(dev); 1423 struct xe_device *xe = pdev_to_xe_device(pdev); 1424 1425 xe_pm_d3cold_allowed_toggle(xe); 1426 1427 return 0; 1428 } 1429 1430 static const struct dev_pm_ops xe_pm_ops = { 1431 SET_SYSTEM_SLEEP_PM_OPS(xe_pci_suspend, xe_pci_resume) 1432 SET_RUNTIME_PM_OPS(xe_pci_runtime_suspend, xe_pci_runtime_resume, xe_pci_runtime_idle) 1433 }; 1434 #endif 1435 1436 static struct pci_driver xe_pci_driver = { 1437 .name = DRIVER_NAME, 1438 .id_table = pciidlist, 1439 .probe = xe_pci_probe, 1440 .remove = xe_pci_remove, 1441 .shutdown = xe_pci_shutdown, 1442 .sriov_configure = xe_pci_sriov_configure, 1443 .err_handler = &xe_pci_error_handlers, 1444 #ifdef CONFIG_PM_SLEEP 1445 .driver.pm = &xe_pm_ops, 1446 #endif 1447 }; 1448 1449 /** 1450 * xe_pci_to_pf_device() - Get PF &xe_device. 1451 * @pdev: the VF &pci_dev device 1452 * 1453 * Return: pointer to PF &xe_device, NULL otherwise. 1454 */ 1455 struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev) 1456 { 1457 struct drm_device *drm; 1458 1459 drm = pci_iov_get_pf_drvdata(pdev, &xe_pci_driver); 1460 if (IS_ERR(drm)) 1461 return NULL; 1462 1463 return to_xe_device(drm); 1464 } 1465 1466 int xe_register_pci_driver(void) 1467 { 1468 return pci_register_driver(&xe_pci_driver); 1469 } 1470 1471 void xe_unregister_pci_driver(void) 1472 { 1473 pci_unregister_driver(&xe_pci_driver); 1474 } 1475 1476 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1477 #include "tests/xe_pci.c" 1478 #endif 1479