1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - 2024 Intel Corporation 4 */ 5 6 #include "ivpu_drv.h" 7 #include "ivpu_hw.h" 8 #include "ivpu_hw_btrs.h" 9 #include "ivpu_hw_ip.h" 10 11 #include <asm/msr-index.h> 12 #include <asm/msr.h> 13 #include <linux/dmi.h> 14 #include <linux/fault-inject.h> 15 #include <linux/pm_runtime.h> 16 17 #ifdef CONFIG_FAULT_INJECTION 18 DECLARE_FAULT_ATTR(ivpu_hw_failure); 19 20 static char *ivpu_fail_hw; 21 module_param_named_unsafe(fail_hw, ivpu_fail_hw, charp, 0444); 22 MODULE_PARM_DESC(fail_hw, "<interval>,<probability>,<space>,<times>"); 23 #endif 24 25 #define FW_SHARED_MEM_ALIGNMENT SZ_512K /* VPU MTRR limitation */ 26 27 #define ECC_MCA_SIGNAL_ENABLE_MASK 0xff 28 29 static char *platform_to_str(u32 platform) 30 { 31 switch (platform) { 32 case IVPU_PLATFORM_SILICON: 33 return "SILICON"; 34 case IVPU_PLATFORM_SIMICS: 35 return "SIMICS"; 36 case IVPU_PLATFORM_FPGA: 37 return "FPGA"; 38 case IVPU_PLATFORM_HSLE: 39 return "HSLE"; 40 default: 41 return "Invalid platform"; 42 } 43 } 44 45 static void platform_init(struct ivpu_device *vdev) 46 { 47 int platform = ivpu_hw_btrs_platform_read(vdev); 48 49 ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", platform_to_str(platform), platform); 50 51 switch (platform) { 52 case IVPU_PLATFORM_SILICON: 53 case IVPU_PLATFORM_SIMICS: 54 case IVPU_PLATFORM_FPGA: 55 case IVPU_PLATFORM_HSLE: 56 vdev->platform = platform; 57 break; 58 59 default: 60 ivpu_err(vdev, "Invalid platform type: %d\n", platform); 61 break; 62 } 63 } 64 65 static void wa_init(struct ivpu_device *vdev) 66 { 67 vdev->wa.punit_disabled = false; 68 vdev->wa.clear_runtime_mem = false; 69 70 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 71 vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev); 72 73 if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && 74 ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) 75 vdev->wa.disable_clock_relinquish = true; 76 77 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE) 78 vdev->wa.disable_clock_relinquish = false; 79 80 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_DISABLE) 81 vdev->wa.disable_clock_relinquish = true; 82 83 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 84 vdev->wa.wp0_during_power_up = true; 85 86 if (ivpu_test_mode & IVPU_TEST_MODE_D0I2_DISABLE) 87 vdev->wa.disable_d0i2 = true; 88 89 IVPU_PRINT_WA(punit_disabled); 90 IVPU_PRINT_WA(clear_runtime_mem); 91 IVPU_PRINT_WA(interrupt_clear_with_0); 92 IVPU_PRINT_WA(disable_clock_relinquish); 93 IVPU_PRINT_WA(wp0_during_power_up); 94 IVPU_PRINT_WA(disable_d0i2); 95 } 96 97 static void timeouts_init(struct ivpu_device *vdev) 98 { 99 if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) { 100 vdev->timeout.boot = -1; 101 vdev->timeout.jsm = -1; 102 vdev->timeout.tdr = -1; 103 vdev->timeout.inference = -1; 104 vdev->timeout.autosuspend = -1; 105 vdev->timeout.d0i3_entry_msg = -1; 106 } else if (ivpu_is_fpga(vdev)) { 107 vdev->timeout.boot = 50; 108 vdev->timeout.jsm = 15000; 109 vdev->timeout.tdr = 30000; 110 vdev->timeout.inference = 900000; 111 vdev->timeout.autosuspend = -1; 112 vdev->timeout.d0i3_entry_msg = 500; 113 vdev->timeout.state_dump_msg = 10000; 114 } else if (ivpu_is_simics(vdev)) { 115 vdev->timeout.boot = 50; 116 vdev->timeout.jsm = 500; 117 vdev->timeout.tdr = 10000; 118 vdev->timeout.inference = 300000; 119 vdev->timeout.autosuspend = 100; 120 vdev->timeout.d0i3_entry_msg = 100; 121 vdev->timeout.state_dump_msg = 10; 122 } else { 123 vdev->timeout.boot = 1000; 124 vdev->timeout.jsm = 500; 125 vdev->timeout.tdr = 2000; 126 vdev->timeout.inference = 60000; 127 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 128 vdev->timeout.autosuspend = 10; 129 else 130 vdev->timeout.autosuspend = 100; 131 vdev->timeout.d0i3_entry_msg = 5; 132 vdev->timeout.state_dump_msg = 100; 133 } 134 } 135 136 static void priority_bands_init(struct ivpu_device *vdev) 137 { 138 /* Idle */ 139 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 0; 140 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 50000; 141 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 160000; 142 /* Normal */ 143 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000; 144 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000; 145 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 300000; 146 /* Focus */ 147 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000; 148 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000; 149 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 200000; 150 /* Realtime */ 151 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 0; 152 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 50000; 153 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 200000; 154 } 155 156 int ivpu_hw_range_init(struct ivpu_device *vdev, struct ivpu_addr_range *range, u64 start, u64 size) 157 { 158 u64 end; 159 160 if (!range || check_add_overflow(start, size, &end)) { 161 ivpu_err(vdev, "Invalid range: start 0x%llx size %llu\n", start, size); 162 return -EINVAL; 163 } 164 165 range->start = start; 166 range->end = end; 167 168 return 0; 169 } 170 171 static void memory_ranges_init(struct ivpu_device *vdev) 172 { 173 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 174 ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x84800000, SZ_64M); 175 ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M); 176 ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0xa0000000, 511 * SZ_1M); 177 ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x180000000, SZ_2G); 178 ivpu_hw_range_init(vdev, &vdev->hw->ranges.dma, 0x200000000, SZ_128G); 179 } else { 180 ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x80000000, SZ_64M); 181 ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M); 182 ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x80000000, SZ_2G); 183 ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0x100000000, SZ_256G); 184 vdev->hw->ranges.dma = vdev->hw->ranges.user; 185 } 186 187 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vdev->hw->ranges.global.start, 188 FW_SHARED_MEM_ALIGNMENT)); 189 } 190 191 static int wp_enable(struct ivpu_device *vdev) 192 { 193 return ivpu_hw_btrs_wp_drive(vdev, true); 194 } 195 196 static int wp_disable(struct ivpu_device *vdev) 197 { 198 return ivpu_hw_btrs_wp_drive(vdev, false); 199 } 200 201 int ivpu_hw_power_up(struct ivpu_device *vdev) 202 { 203 int ret; 204 205 if (IVPU_WA(wp0_during_power_up)) { 206 /* WP requests may fail when powering down, so issue WP 0 here */ 207 ret = wp_disable(vdev); 208 if (ret) 209 ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret); 210 } 211 212 ret = ivpu_hw_btrs_d0i3_disable(vdev); 213 if (ret) 214 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret); 215 216 ret = wp_enable(vdev); 217 if (ret) { 218 ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret); 219 return ret; 220 } 221 222 if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) { 223 if (IVPU_WA(disable_clock_relinquish)) 224 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev); 225 ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev); 226 ivpu_hw_btrs_ats_print_lnl(vdev); 227 } 228 229 ret = ivpu_hw_ip_host_ss_configure(vdev); 230 if (ret) { 231 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret); 232 return ret; 233 } 234 235 ivpu_hw_ip_idle_gen_disable(vdev); 236 237 ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev); 238 if (ret) { 239 ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n"); 240 return ret; 241 } 242 243 ret = ivpu_hw_ip_pwr_domain_enable(vdev); 244 if (ret) { 245 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret); 246 return ret; 247 } 248 249 ret = ivpu_hw_ip_host_ss_axi_enable(vdev); 250 if (ret) { 251 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret); 252 return ret; 253 } 254 255 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL) 256 ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev); 257 258 ret = ivpu_hw_ip_top_noc_enable(vdev); 259 if (ret) 260 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret); 261 262 return ret; 263 } 264 265 static void save_d0i3_entry_timestamp(struct ivpu_device *vdev) 266 { 267 vdev->hw->d0i3_entry_host_ts = ktime_get_boottime(); 268 vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev); 269 } 270 271 int ivpu_hw_reset(struct ivpu_device *vdev) 272 { 273 int ret = 0; 274 275 if (ivpu_hw_btrs_ip_reset(vdev)) { 276 ivpu_err(vdev, "Failed to reset NPU IP\n"); 277 ret = -EIO; 278 } 279 280 if (wp_disable(vdev)) { 281 ivpu_err(vdev, "Failed to disable workpoint\n"); 282 ret = -EIO; 283 } 284 285 return ret; 286 } 287 288 int ivpu_hw_power_down(struct ivpu_device *vdev) 289 { 290 int ret = 0; 291 292 save_d0i3_entry_timestamp(vdev); 293 294 if (!ivpu_hw_is_idle(vdev)) 295 ivpu_warn(vdev, "NPU not idle during power down\n"); 296 297 if (ivpu_hw_reset(vdev)) { 298 ivpu_err(vdev, "Failed to reset NPU\n"); 299 ret = -EIO; 300 } 301 302 if (ivpu_hw_btrs_d0i3_enable(vdev)) { 303 ivpu_err(vdev, "Failed to enter D0I3\n"); 304 ret = -EIO; 305 } 306 307 return ret; 308 } 309 310 int ivpu_hw_init(struct ivpu_device *vdev) 311 { 312 ivpu_hw_btrs_info_init(vdev); 313 ivpu_hw_btrs_freq_ratios_init(vdev); 314 priority_bands_init(vdev); 315 memory_ranges_init(vdev); 316 platform_init(vdev); 317 wa_init(vdev); 318 timeouts_init(vdev); 319 atomic_set(&vdev->hw->firewall_irq_counter, 0); 320 321 #ifdef CONFIG_FAULT_INJECTION 322 if (ivpu_fail_hw) 323 setup_fault_attr(&ivpu_hw_failure, ivpu_fail_hw); 324 #endif 325 326 return 0; 327 } 328 329 int ivpu_hw_boot_fw(struct ivpu_device *vdev) 330 { 331 int ret; 332 333 ivpu_hw_ip_snoop_disable(vdev); 334 ivpu_hw_ip_tbu_mmu_enable(vdev); 335 ret = ivpu_hw_ip_soc_cpu_boot(vdev); 336 if (ret) 337 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret); 338 339 return ret; 340 } 341 342 void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable) 343 { 344 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 345 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 346 return; 347 } 348 349 if (enable) 350 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH; 351 else 352 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 353 } 354 355 void ivpu_irq_handlers_init(struct ivpu_device *vdev) 356 { 357 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 358 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx; 359 else 360 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx; 361 362 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 363 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl; 364 else 365 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl; 366 } 367 368 void ivpu_hw_irq_enable(struct ivpu_device *vdev) 369 { 370 ivpu_hw_ip_irq_enable(vdev); 371 ivpu_hw_btrs_irq_enable(vdev); 372 } 373 374 void ivpu_hw_irq_disable(struct ivpu_device *vdev) 375 { 376 ivpu_hw_btrs_irq_disable(vdev); 377 ivpu_hw_ip_irq_disable(vdev); 378 } 379 380 irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr) 381 { 382 struct ivpu_device *vdev = ptr; 383 bool ip_handled, btrs_handled; 384 385 ivpu_hw_btrs_global_int_disable(vdev); 386 387 btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq); 388 if (!ivpu_hw_is_idle((vdev)) || !btrs_handled) 389 ip_handled = ivpu_hw_ip_irq_handler(vdev, irq); 390 else 391 ip_handled = false; 392 393 /* Re-enable global interrupts to re-trigger MSI for pending interrupts */ 394 ivpu_hw_btrs_global_int_enable(vdev); 395 396 if (!ip_handled && !btrs_handled) 397 return IRQ_NONE; 398 399 pm_runtime_mark_last_busy(vdev->drm.dev); 400 return IRQ_HANDLED; 401 } 402 403 bool ivpu_hw_uses_ecc_mca_signal(struct ivpu_device *vdev) 404 { 405 unsigned long long msr_integrity_caps; 406 int ret; 407 408 if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX) 409 return false; 410 411 ret = rdmsrq_safe(MSR_INTEGRITY_CAPS, &msr_integrity_caps); 412 if (ret) { 413 ivpu_warn(vdev, "Error reading MSR_INTEGRITY_CAPS: %d", ret); 414 return false; 415 } 416 417 ivpu_dbg(vdev, MISC, "MSR_INTEGRITY_CAPS: 0x%llx\n", msr_integrity_caps); 418 419 return msr_integrity_caps & ECC_MCA_SIGNAL_ENABLE_MASK; 420 } 421