1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - 2024 Intel Corporation 4 */ 5 6 #include "ivpu_drv.h" 7 #include "ivpu_hw.h" 8 #include "ivpu_hw_btrs.h" 9 #include "ivpu_hw_ip.h" 10 11 #include <linux/dmi.h> 12 #include <linux/fault-inject.h> 13 #include <linux/pm_runtime.h> 14 15 #ifdef CONFIG_FAULT_INJECTION 16 DECLARE_FAULT_ATTR(ivpu_hw_failure); 17 18 static char *ivpu_fail_hw; 19 module_param_named_unsafe(fail_hw, ivpu_fail_hw, charp, 0444); 20 MODULE_PARM_DESC(fail_hw, "<interval>,<probability>,<space>,<times>"); 21 #endif 22 23 #define FW_SHARED_MEM_ALIGNMENT SZ_512K /* VPU MTRR limitation */ 24 25 static char *platform_to_str(u32 platform) 26 { 27 switch (platform) { 28 case IVPU_PLATFORM_SILICON: 29 return "SILICON"; 30 case IVPU_PLATFORM_SIMICS: 31 return "SIMICS"; 32 case IVPU_PLATFORM_FPGA: 33 return "FPGA"; 34 case IVPU_PLATFORM_HSLE: 35 return "HSLE"; 36 default: 37 return "Invalid platform"; 38 } 39 } 40 41 static void platform_init(struct ivpu_device *vdev) 42 { 43 int platform = ivpu_hw_btrs_platform_read(vdev); 44 45 ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", platform_to_str(platform), platform); 46 47 switch (platform) { 48 case IVPU_PLATFORM_SILICON: 49 case IVPU_PLATFORM_SIMICS: 50 case IVPU_PLATFORM_FPGA: 51 case IVPU_PLATFORM_HSLE: 52 vdev->platform = platform; 53 break; 54 55 default: 56 ivpu_err(vdev, "Invalid platform type: %d\n", platform); 57 break; 58 } 59 } 60 61 static void wa_init(struct ivpu_device *vdev) 62 { 63 vdev->wa.punit_disabled = false; 64 vdev->wa.clear_runtime_mem = false; 65 66 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 67 vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev); 68 69 if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && 70 ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) 71 vdev->wa.disable_clock_relinquish = true; 72 73 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE) 74 vdev->wa.disable_clock_relinquish = false; 75 76 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_DISABLE) 77 vdev->wa.disable_clock_relinquish = true; 78 79 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 80 vdev->wa.wp0_during_power_up = true; 81 82 if (ivpu_test_mode & IVPU_TEST_MODE_D0I2_DISABLE) 83 vdev->wa.disable_d0i2 = true; 84 85 IVPU_PRINT_WA(punit_disabled); 86 IVPU_PRINT_WA(clear_runtime_mem); 87 IVPU_PRINT_WA(interrupt_clear_with_0); 88 IVPU_PRINT_WA(disable_clock_relinquish); 89 IVPU_PRINT_WA(wp0_during_power_up); 90 IVPU_PRINT_WA(disable_d0i2); 91 } 92 93 static void timeouts_init(struct ivpu_device *vdev) 94 { 95 if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) { 96 vdev->timeout.boot = -1; 97 vdev->timeout.jsm = -1; 98 vdev->timeout.tdr = -1; 99 vdev->timeout.inference = -1; 100 vdev->timeout.autosuspend = -1; 101 vdev->timeout.d0i3_entry_msg = -1; 102 } else if (ivpu_is_fpga(vdev)) { 103 vdev->timeout.boot = 50; 104 vdev->timeout.jsm = 15000; 105 vdev->timeout.tdr = 30000; 106 vdev->timeout.inference = 900000; 107 vdev->timeout.autosuspend = -1; 108 vdev->timeout.d0i3_entry_msg = 500; 109 vdev->timeout.state_dump_msg = 10000; 110 } else if (ivpu_is_simics(vdev)) { 111 vdev->timeout.boot = 50; 112 vdev->timeout.jsm = 500; 113 vdev->timeout.tdr = 10000; 114 vdev->timeout.inference = 300000; 115 vdev->timeout.autosuspend = 100; 116 vdev->timeout.d0i3_entry_msg = 100; 117 vdev->timeout.state_dump_msg = 10; 118 } else { 119 vdev->timeout.boot = 1000; 120 vdev->timeout.jsm = 500; 121 vdev->timeout.tdr = 2000; 122 vdev->timeout.inference = 60000; 123 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 124 vdev->timeout.autosuspend = 10; 125 else 126 vdev->timeout.autosuspend = 100; 127 vdev->timeout.d0i3_entry_msg = 5; 128 vdev->timeout.state_dump_msg = 100; 129 } 130 } 131 132 static void priority_bands_init(struct ivpu_device *vdev) 133 { 134 /* Idle */ 135 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 0; 136 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 50000; 137 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 160000; 138 /* Normal */ 139 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000; 140 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000; 141 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 300000; 142 /* Focus */ 143 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000; 144 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000; 145 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 200000; 146 /* Realtime */ 147 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 0; 148 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 50000; 149 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 200000; 150 } 151 152 int ivpu_hw_range_init(struct ivpu_device *vdev, struct ivpu_addr_range *range, u64 start, u64 size) 153 { 154 u64 end; 155 156 if (!range || check_add_overflow(start, size, &end)) { 157 ivpu_err(vdev, "Invalid range: start 0x%llx size %llu\n", start, size); 158 return -EINVAL; 159 } 160 161 range->start = start; 162 range->end = end; 163 164 return 0; 165 } 166 167 static void memory_ranges_init(struct ivpu_device *vdev) 168 { 169 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 170 ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x84800000, SZ_64M); 171 ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M); 172 ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0xa0000000, 511 * SZ_1M); 173 ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x180000000, SZ_2G); 174 ivpu_hw_range_init(vdev, &vdev->hw->ranges.dma, 0x200000000, SZ_128G); 175 } else { 176 ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x80000000, SZ_64M); 177 ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M); 178 ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x80000000, SZ_2G); 179 ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0x100000000, SZ_256G); 180 vdev->hw->ranges.dma = vdev->hw->ranges.user; 181 } 182 183 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vdev->hw->ranges.global.start, 184 FW_SHARED_MEM_ALIGNMENT)); 185 } 186 187 static int wp_enable(struct ivpu_device *vdev) 188 { 189 return ivpu_hw_btrs_wp_drive(vdev, true); 190 } 191 192 static int wp_disable(struct ivpu_device *vdev) 193 { 194 return ivpu_hw_btrs_wp_drive(vdev, false); 195 } 196 197 int ivpu_hw_power_up(struct ivpu_device *vdev) 198 { 199 int ret; 200 201 if (IVPU_WA(wp0_during_power_up)) { 202 /* WP requests may fail when powering down, so issue WP 0 here */ 203 ret = wp_disable(vdev); 204 if (ret) 205 ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret); 206 } 207 208 ret = ivpu_hw_btrs_d0i3_disable(vdev); 209 if (ret) 210 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret); 211 212 ret = wp_enable(vdev); 213 if (ret) { 214 ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret); 215 return ret; 216 } 217 218 if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) { 219 if (IVPU_WA(disable_clock_relinquish)) 220 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev); 221 ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev); 222 ivpu_hw_btrs_ats_print_lnl(vdev); 223 } 224 225 ret = ivpu_hw_ip_host_ss_configure(vdev); 226 if (ret) { 227 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret); 228 return ret; 229 } 230 231 ivpu_hw_ip_idle_gen_disable(vdev); 232 233 ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev); 234 if (ret) { 235 ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n"); 236 return ret; 237 } 238 239 ret = ivpu_hw_ip_pwr_domain_enable(vdev); 240 if (ret) { 241 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret); 242 return ret; 243 } 244 245 ret = ivpu_hw_ip_host_ss_axi_enable(vdev); 246 if (ret) { 247 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret); 248 return ret; 249 } 250 251 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL) 252 ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev); 253 254 ret = ivpu_hw_ip_top_noc_enable(vdev); 255 if (ret) 256 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret); 257 258 return ret; 259 } 260 261 static void save_d0i3_entry_timestamp(struct ivpu_device *vdev) 262 { 263 vdev->hw->d0i3_entry_host_ts = ktime_get_boottime(); 264 vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev); 265 } 266 267 int ivpu_hw_reset(struct ivpu_device *vdev) 268 { 269 int ret = 0; 270 271 if (ivpu_hw_btrs_ip_reset(vdev)) { 272 ivpu_err(vdev, "Failed to reset NPU IP\n"); 273 ret = -EIO; 274 } 275 276 if (wp_disable(vdev)) { 277 ivpu_err(vdev, "Failed to disable workpoint\n"); 278 ret = -EIO; 279 } 280 281 return ret; 282 } 283 284 int ivpu_hw_power_down(struct ivpu_device *vdev) 285 { 286 int ret = 0; 287 288 save_d0i3_entry_timestamp(vdev); 289 290 if (!ivpu_hw_is_idle(vdev)) 291 ivpu_warn(vdev, "NPU not idle during power down\n"); 292 293 if (ivpu_hw_reset(vdev)) { 294 ivpu_err(vdev, "Failed to reset NPU\n"); 295 ret = -EIO; 296 } 297 298 if (ivpu_hw_btrs_d0i3_enable(vdev)) { 299 ivpu_err(vdev, "Failed to enter D0I3\n"); 300 ret = -EIO; 301 } 302 303 return ret; 304 } 305 306 int ivpu_hw_init(struct ivpu_device *vdev) 307 { 308 ivpu_hw_btrs_info_init(vdev); 309 ivpu_hw_btrs_freq_ratios_init(vdev); 310 priority_bands_init(vdev); 311 memory_ranges_init(vdev); 312 platform_init(vdev); 313 wa_init(vdev); 314 timeouts_init(vdev); 315 atomic_set(&vdev->hw->firewall_irq_counter, 0); 316 317 #ifdef CONFIG_FAULT_INJECTION 318 if (ivpu_fail_hw) 319 setup_fault_attr(&ivpu_hw_failure, ivpu_fail_hw); 320 #endif 321 322 return 0; 323 } 324 325 int ivpu_hw_boot_fw(struct ivpu_device *vdev) 326 { 327 int ret; 328 329 ivpu_hw_ip_snoop_disable(vdev); 330 ivpu_hw_ip_tbu_mmu_enable(vdev); 331 ret = ivpu_hw_ip_soc_cpu_boot(vdev); 332 if (ret) 333 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret); 334 335 return ret; 336 } 337 338 void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable) 339 { 340 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 341 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 342 return; 343 } 344 345 if (enable) 346 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH; 347 else 348 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 349 } 350 351 void ivpu_irq_handlers_init(struct ivpu_device *vdev) 352 { 353 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 354 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx; 355 else 356 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx; 357 358 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 359 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl; 360 else 361 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl; 362 } 363 364 void ivpu_hw_irq_enable(struct ivpu_device *vdev) 365 { 366 ivpu_hw_ip_irq_enable(vdev); 367 ivpu_hw_btrs_irq_enable(vdev); 368 } 369 370 void ivpu_hw_irq_disable(struct ivpu_device *vdev) 371 { 372 ivpu_hw_btrs_irq_disable(vdev); 373 ivpu_hw_ip_irq_disable(vdev); 374 } 375 376 irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr) 377 { 378 struct ivpu_device *vdev = ptr; 379 bool ip_handled, btrs_handled; 380 381 ivpu_hw_btrs_global_int_disable(vdev); 382 383 btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq); 384 if (!ivpu_hw_is_idle((vdev)) || !btrs_handled) 385 ip_handled = ivpu_hw_ip_irq_handler(vdev, irq); 386 else 387 ip_handled = false; 388 389 /* Re-enable global interrupts to re-trigger MSI for pending interrupts */ 390 ivpu_hw_btrs_global_int_enable(vdev); 391 392 if (!ip_handled && !btrs_handled) 393 return IRQ_NONE; 394 395 pm_runtime_mark_last_busy(vdev->drm.dev); 396 return IRQ_HANDLED; 397 } 398