1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2024 Intel Corporation 4 */ 5 6 #include "ivpu_drv.h" 7 #include "ivpu_hw.h" 8 #include "ivpu_hw_btrs.h" 9 #include "ivpu_hw_btrs_lnl_reg.h" 10 #include "ivpu_hw_btrs_mtl_reg.h" 11 #include "ivpu_hw_reg_io.h" 12 #include "ivpu_pm.h" 13 14 #define BTRS_MTL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR)) | \ 15 (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR))) 16 17 #define BTRS_LNL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR)) | \ 18 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR)) | \ 19 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR)) | \ 20 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR)) | \ 21 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR)) | \ 22 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR))) 23 24 #define BTRS_MTL_ALL_IRQ_MASK (BTRS_MTL_IRQ_MASK | (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, \ 25 FREQ_CHANGE))) 26 27 #define BTRS_IRQ_DISABLE_MASK ((u32)-1) 28 29 #define BTRS_LNL_ALL_IRQ_MASK ((u32)-1) 30 31 #define BTRS_MTL_WP_CONFIG_1_TILE_5_3_RATIO WP_CONFIG(MTL_CONFIG_1_TILE, MTL_PLL_RATIO_5_3) 32 #define BTRS_MTL_WP_CONFIG_1_TILE_4_3_RATIO WP_CONFIG(MTL_CONFIG_1_TILE, MTL_PLL_RATIO_4_3) 33 #define BTRS_MTL_WP_CONFIG_2_TILE_5_3_RATIO WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_5_3) 34 #define BTRS_MTL_WP_CONFIG_2_TILE_4_3_RATIO WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_4_3) 35 #define BTRS_MTL_WP_CONFIG_0_TILE_PLL_OFF WP_CONFIG(0, 0) 36 37 #define PLL_CDYN_DEFAULT 0x80 38 #define PLL_EPP_DEFAULT 0x80 39 #define PLL_CONFIG_DEFAULT 0x0 40 #define PLL_SIMULATION_FREQ 10000000 41 #define PLL_REF_CLK_FREQ 50000000 42 #define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC) 43 #define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC) 44 #define TIMEOUT_US (150 * USEC_PER_MSEC) 45 46 /* Work point configuration values */ 47 #define WP_CONFIG(tile, ratio) (((tile) << 8) | (ratio)) 48 #define MTL_CONFIG_1_TILE 0x01 49 #define MTL_CONFIG_2_TILE 0x02 50 #define MTL_PLL_RATIO_5_3 0x01 51 #define MTL_PLL_RATIO_4_3 0x02 52 #define BTRS_MTL_TILE_FUSE_ENABLE_BOTH 0x0 53 #define BTRS_MTL_TILE_SKU_BOTH 0x3630 54 55 #define BTRS_LNL_TILE_MAX_NUM 6 56 #define BTRS_LNL_TILE_MAX_MASK 0x3f 57 58 #define WEIGHTS_DEFAULT 0xf711f711u 59 #define WEIGHTS_ATS_DEFAULT 0x0000f711u 60 61 #define DCT_REQ 0x2 62 #define DCT_ENABLE 0x1 63 #define DCT_DISABLE 0x0 64 65 int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev) 66 { 67 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, BTRS_MTL_ALL_IRQ_MASK); 68 if (REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) == BTRS_MTL_ALL_IRQ_MASK) { 69 /* Writing 1s does not clear the interrupt status register */ 70 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0); 71 return true; 72 } 73 74 return false; 75 } 76 77 static void freq_ratios_init_mtl(struct ivpu_device *vdev) 78 { 79 struct ivpu_hw_info *hw = vdev->hw; 80 u32 fmin_fuse, fmax_fuse; 81 82 fmin_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMIN_FUSE); 83 hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, MIN_RATIO, fmin_fuse); 84 hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, PN_RATIO, fmin_fuse); 85 86 fmax_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMAX_FUSE); 87 hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMAX_FUSE, MAX_RATIO, fmax_fuse); 88 } 89 90 static void freq_ratios_init_lnl(struct ivpu_device *vdev) 91 { 92 struct ivpu_hw_info *hw = vdev->hw; 93 u32 fmin_fuse, fmax_fuse; 94 95 fmin_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMIN_FUSE); 96 hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, MIN_RATIO, fmin_fuse); 97 hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, PN_RATIO, fmin_fuse); 98 99 fmax_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMAX_FUSE); 100 hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMAX_FUSE, MAX_RATIO, fmax_fuse); 101 } 102 103 void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev) 104 { 105 struct ivpu_hw_info *hw = vdev->hw; 106 107 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 108 freq_ratios_init_mtl(vdev); 109 else 110 freq_ratios_init_lnl(vdev); 111 112 hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 113 hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 114 hw->pll.pn_ratio = clamp_t(u8, hw->pll.pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 115 } 116 117 static bool tile_disable_check(u32 config) 118 { 119 /* Allowed values: 0 or one bit from range 0-5 (6 tiles) */ 120 if (config == 0) 121 return true; 122 123 if (config > BIT(BTRS_LNL_TILE_MAX_NUM - 1)) 124 return false; 125 126 if ((config & (config - 1)) == 0) 127 return true; 128 129 return false; 130 } 131 132 static int read_tile_config_fuse(struct ivpu_device *vdev, u32 *tile_fuse_config) 133 { 134 u32 fuse; 135 u32 config; 136 137 fuse = REGB_RD32(VPU_HW_BTRS_LNL_TILE_FUSE); 138 if (!REG_TEST_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, VALID, fuse)) { 139 ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse); 140 return -EIO; 141 } 142 143 config = REG_GET_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, CONFIG, fuse); 144 if (!tile_disable_check(config)) 145 ivpu_warn(vdev, "More than 1 tile disabled, tile fuse config mask: 0x%x\n", config); 146 147 ivpu_dbg(vdev, MISC, "Tile disable config mask: 0x%x\n", config); 148 149 *tile_fuse_config = config; 150 return 0; 151 } 152 153 static int info_init_mtl(struct ivpu_device *vdev) 154 { 155 struct ivpu_hw_info *hw = vdev->hw; 156 157 hw->tile_fuse = BTRS_MTL_TILE_FUSE_ENABLE_BOTH; 158 hw->sku = BTRS_MTL_TILE_SKU_BOTH; 159 hw->config = BTRS_MTL_WP_CONFIG_2_TILE_4_3_RATIO; 160 161 return 0; 162 } 163 164 static int info_init_lnl(struct ivpu_device *vdev) 165 { 166 struct ivpu_hw_info *hw = vdev->hw; 167 u32 tile_fuse_config; 168 int ret; 169 170 ret = read_tile_config_fuse(vdev, &tile_fuse_config); 171 if (ret) 172 return ret; 173 174 hw->tile_fuse = tile_fuse_config; 175 hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 176 177 return 0; 178 } 179 180 int ivpu_hw_btrs_info_init(struct ivpu_device *vdev) 181 { 182 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 183 return info_init_mtl(vdev); 184 else 185 return info_init_lnl(vdev); 186 } 187 188 static int wp_request_sync(struct ivpu_device *vdev) 189 { 190 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 191 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); 192 else 193 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); 194 } 195 196 static int wait_for_status_ready(struct ivpu_device *vdev, bool enable) 197 { 198 u32 exp_val = enable ? 0x1 : 0x0; 199 200 if (IVPU_WA(punit_disabled)) 201 return 0; 202 203 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 204 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US); 205 else 206 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US); 207 } 208 209 struct wp_request { 210 u16 min; 211 u16 max; 212 u16 target; 213 u16 cfg; 214 u16 epp; 215 u16 cdyn; 216 }; 217 218 static void wp_request_mtl(struct ivpu_device *vdev, struct wp_request *wp) 219 { 220 u32 val; 221 222 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0); 223 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val); 224 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val); 225 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, val); 226 227 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1); 228 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val); 229 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, EPP, PLL_EPP_DEFAULT, val); 230 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, val); 231 232 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2); 233 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val); 234 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, val); 235 236 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_CMD); 237 val = REG_SET_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, val); 238 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_CMD, val); 239 } 240 241 static void wp_request_lnl(struct ivpu_device *vdev, struct wp_request *wp) 242 { 243 u32 val; 244 245 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0); 246 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val); 247 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val); 248 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, val); 249 250 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1); 251 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val); 252 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, EPP, wp->epp, val); 253 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, val); 254 255 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2); 256 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val); 257 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CDYN, wp->cdyn, val); 258 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, val); 259 260 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_CMD); 261 val = REG_SET_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, val); 262 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_CMD, val); 263 } 264 265 static void wp_request(struct ivpu_device *vdev, struct wp_request *wp) 266 { 267 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 268 wp_request_mtl(vdev, wp); 269 else 270 wp_request_lnl(vdev, wp); 271 } 272 273 static int wp_request_send(struct ivpu_device *vdev, struct wp_request *wp) 274 { 275 int ret; 276 277 ret = wp_request_sync(vdev); 278 if (ret) { 279 ivpu_err(vdev, "Failed to sync before workpoint request: %d\n", ret); 280 return ret; 281 } 282 283 wp_request(vdev, wp); 284 285 ret = wp_request_sync(vdev); 286 if (ret) 287 ivpu_err(vdev, "Failed to sync after workpoint request: %d\n", ret); 288 289 return ret; 290 } 291 292 static void prepare_wp_request(struct ivpu_device *vdev, struct wp_request *wp, bool enable) 293 { 294 struct ivpu_hw_info *hw = vdev->hw; 295 296 wp->min = hw->pll.min_ratio; 297 wp->max = hw->pll.max_ratio; 298 299 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 300 wp->target = enable ? hw->pll.pn_ratio : 0; 301 wp->cfg = enable ? hw->config : 0; 302 wp->cdyn = 0; 303 wp->epp = 0; 304 } else { 305 wp->target = hw->pll.pn_ratio; 306 wp->cfg = enable ? PLL_CONFIG_DEFAULT : 0; 307 wp->cdyn = enable ? PLL_CDYN_DEFAULT : 0; 308 wp->epp = enable ? PLL_EPP_DEFAULT : 0; 309 } 310 } 311 312 static int wait_for_pll_lock(struct ivpu_device *vdev, bool enable) 313 { 314 u32 exp_val = enable ? 0x1 : 0x0; 315 316 if (ivpu_hw_btrs_gen(vdev) != IVPU_HW_BTRS_MTL) 317 return 0; 318 319 if (IVPU_WA(punit_disabled)) 320 return 0; 321 322 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_PLL_STATUS, LOCK, exp_val, PLL_TIMEOUT_US); 323 } 324 325 int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable) 326 { 327 struct wp_request wp; 328 int ret; 329 330 if (IVPU_WA(punit_disabled)) { 331 ivpu_dbg(vdev, PM, "Skipping workpoint request\n"); 332 return 0; 333 } 334 335 prepare_wp_request(vdev, &wp, enable); 336 337 ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, config: 0x%x, epp: 0x%x, cdyn: 0x%x\n", 338 PLL_RATIO_TO_FREQ(wp.target), wp.cfg, wp.epp, wp.cdyn); 339 340 ret = wp_request_send(vdev, &wp); 341 if (ret) { 342 ivpu_err(vdev, "Failed to send workpoint request: %d\n", ret); 343 return ret; 344 } 345 346 ret = wait_for_pll_lock(vdev, enable); 347 if (ret) { 348 ivpu_err(vdev, "Timed out waiting for PLL lock\n"); 349 return ret; 350 } 351 352 ret = wait_for_status_ready(vdev, enable); 353 if (ret) { 354 ivpu_err(vdev, "Timed out waiting for NPU ready status\n"); 355 return ret; 356 } 357 358 return 0; 359 } 360 361 static int d0i3_drive_mtl(struct ivpu_device *vdev, bool enable) 362 { 363 int ret; 364 u32 val; 365 366 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 367 if (ret) { 368 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); 369 return ret; 370 } 371 372 val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL); 373 if (enable) 374 val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val); 375 else 376 val = REG_CLR_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val); 377 REGB_WR32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, val); 378 379 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 380 if (ret) 381 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); 382 383 return ret; 384 } 385 386 static int d0i3_drive_lnl(struct ivpu_device *vdev, bool enable) 387 { 388 int ret; 389 u32 val; 390 391 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 392 if (ret) { 393 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); 394 return ret; 395 } 396 397 val = REGB_RD32(VPU_HW_BTRS_LNL_D0I3_CONTROL); 398 if (enable) 399 val = REG_SET_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val); 400 else 401 val = REG_CLR_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val); 402 REGB_WR32(VPU_HW_BTRS_LNL_D0I3_CONTROL, val); 403 404 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 405 if (ret) { 406 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); 407 return ret; 408 } 409 410 return 0; 411 } 412 413 static int d0i3_drive(struct ivpu_device *vdev, bool enable) 414 { 415 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 416 return d0i3_drive_mtl(vdev, enable); 417 else 418 return d0i3_drive_lnl(vdev, enable); 419 } 420 421 int ivpu_hw_btrs_d0i3_enable(struct ivpu_device *vdev) 422 { 423 int ret; 424 425 if (IVPU_WA(punit_disabled)) 426 return 0; 427 428 ret = d0i3_drive(vdev, true); 429 if (ret) 430 ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret); 431 432 udelay(5); /* VPU requires 5 us to complete the transition */ 433 434 return ret; 435 } 436 437 int ivpu_hw_btrs_d0i3_disable(struct ivpu_device *vdev) 438 { 439 int ret; 440 441 if (IVPU_WA(punit_disabled)) 442 return 0; 443 444 ret = d0i3_drive(vdev, false); 445 if (ret) 446 ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret); 447 448 return ret; 449 } 450 451 int ivpu_hw_btrs_wait_for_clock_res_own_ack(struct ivpu_device *vdev) 452 { 453 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 454 return 0; 455 456 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US); 457 } 458 459 void ivpu_hw_btrs_set_port_arbitration_weights_lnl(struct ivpu_device *vdev) 460 { 461 REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT); 462 REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT); 463 } 464 465 static int ip_reset_mtl(struct ivpu_device *vdev) 466 { 467 int ret; 468 u32 val; 469 470 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US); 471 if (ret) { 472 ivpu_err(vdev, "Timed out waiting for TRIGGER bit\n"); 473 return ret; 474 } 475 476 val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_IP_RESET); 477 val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, val); 478 REGB_WR32(VPU_HW_BTRS_MTL_VPU_IP_RESET, val); 479 480 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US); 481 if (ret) 482 ivpu_err(vdev, "Timed out waiting for RESET completion\n"); 483 484 return ret; 485 } 486 487 static int ip_reset_lnl(struct ivpu_device *vdev) 488 { 489 int ret; 490 u32 val; 491 492 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev); 493 494 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US); 495 if (ret) { 496 ivpu_err(vdev, "Wait for *_TRIGGER timed out\n"); 497 return ret; 498 } 499 500 val = REGB_RD32(VPU_HW_BTRS_LNL_IP_RESET); 501 val = REG_SET_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, val); 502 REGB_WR32(VPU_HW_BTRS_LNL_IP_RESET, val); 503 504 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US); 505 if (ret) 506 ivpu_err(vdev, "Timed out waiting for RESET completion\n"); 507 508 return ret; 509 } 510 511 int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev) 512 { 513 if (IVPU_WA(punit_disabled)) 514 return 0; 515 516 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 517 return ip_reset_mtl(vdev); 518 else 519 return ip_reset_lnl(vdev); 520 } 521 522 void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev) 523 { 524 u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 525 526 if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT) 527 val = REG_CLR_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val); 528 else 529 val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val); 530 531 REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val); 532 } 533 534 void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev) 535 { 536 ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n", 537 REGB_RD32(VPU_HW_BTRS_LNL_HM_ATS) ? "Enable" : "Disable"); 538 } 539 540 void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev) 541 { 542 u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 543 544 val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, DISABLE_CLK_RELINQUISH, val); 545 REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val); 546 } 547 548 bool ivpu_hw_btrs_is_idle(struct ivpu_device *vdev) 549 { 550 u32 val; 551 552 if (IVPU_WA(punit_disabled)) 553 return true; 554 555 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 556 val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_STATUS); 557 558 return REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, val) && 559 REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, val); 560 } else { 561 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 562 563 return REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, val) && 564 REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, val); 565 } 566 } 567 568 int ivpu_hw_btrs_wait_for_idle(struct ivpu_device *vdev) 569 { 570 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 571 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); 572 else 573 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); 574 } 575 576 /* Handler for IRQs from Buttress core (irqB) */ 577 bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq) 578 { 579 u32 status = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK; 580 bool schedule_recovery = false; 581 582 if (!status) 583 return false; 584 585 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, FREQ_CHANGE, status)) 586 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", 587 REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL)); 588 589 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, status)) { 590 ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0)); 591 REGB_WR32(VPU_HW_BTRS_MTL_ATS_ERR_CLEAR, 0x1); 592 schedule_recovery = true; 593 } 594 595 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, status)) { 596 u32 ufi_log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG); 597 598 ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx", 599 ufi_log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, ufi_log), 600 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, ufi_log), 601 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, ufi_log)); 602 REGB_WR32(VPU_HW_BTRS_MTL_UFI_ERR_CLEAR, 0x1); 603 schedule_recovery = true; 604 } 605 606 /* This must be done after interrupts are cleared at the source. */ 607 if (IVPU_WA(interrupt_clear_with_0)) 608 /* 609 * Writing 1 triggers an interrupt, so we can't perform read update write. 610 * Clear local interrupt status by writing 0 to all bits. 611 */ 612 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0); 613 else 614 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, status); 615 616 if (schedule_recovery) 617 ivpu_pm_trigger_recovery(vdev, "Buttress IRQ"); 618 619 return true; 620 } 621 622 /* Handler for IRQs from Buttress core (irqB) */ 623 bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq) 624 { 625 u32 status = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; 626 bool schedule_recovery = false; 627 628 if (!status) 629 return false; 630 631 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status)) { 632 ivpu_dbg(vdev, IRQ, "Survivability IRQ\n"); 633 if (!kfifo_put(&vdev->hw->irq.fifo, IVPU_HW_IRQ_SRC_DCT)) 634 ivpu_err_ratelimited(vdev, "IRQ FIFO full\n"); 635 } 636 637 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status)) 638 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ)); 639 640 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, status)) { 641 ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", 642 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1), 643 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2)); 644 REGB_WR32(VPU_HW_BTRS_LNL_ATS_ERR_CLEAR, 0x1); 645 schedule_recovery = true; 646 } 647 648 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, status)) { 649 ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG)); 650 REGB_WR32(VPU_HW_BTRS_LNL_CFI0_ERR_CLEAR, 0x1); 651 schedule_recovery = true; 652 } 653 654 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, status)) { 655 ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG)); 656 REGB_WR32(VPU_HW_BTRS_LNL_CFI1_ERR_CLEAR, 0x1); 657 schedule_recovery = true; 658 } 659 660 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, status)) { 661 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x", 662 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW), 663 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH)); 664 REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_CLEAR, 0x1); 665 schedule_recovery = true; 666 } 667 668 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, status)) { 669 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x", 670 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW), 671 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH)); 672 REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR, 0x1); 673 schedule_recovery = true; 674 } 675 676 /* This must be done after interrupts are cleared at the source. */ 677 REGB_WR32(VPU_HW_BTRS_LNL_INTERRUPT_STAT, status); 678 679 if (schedule_recovery) 680 ivpu_pm_trigger_recovery(vdev, "Buttress IRQ"); 681 682 return true; 683 } 684 685 int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable) 686 { 687 u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW); 688 u32 cmd = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, CMD, val); 689 u32 param1 = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, PARAM1, val); 690 691 if (cmd != DCT_REQ) { 692 ivpu_err_ratelimited(vdev, "Unsupported PCODE command: 0x%x\n", cmd); 693 return -EBADR; 694 } 695 696 switch (param1) { 697 case DCT_ENABLE: 698 *enable = true; 699 return 0; 700 case DCT_DISABLE: 701 *enable = false; 702 return 0; 703 default: 704 ivpu_err_ratelimited(vdev, "Invalid PARAM1 value: %u\n", param1); 705 return -EINVAL; 706 } 707 } 708 709 void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 active_percent) 710 { 711 u32 val = 0; 712 u32 cmd = enable ? DCT_ENABLE : DCT_DISABLE; 713 714 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, CMD, DCT_REQ, val); 715 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM1, cmd, val); 716 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM2, active_percent, val); 717 718 REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, val); 719 } 720 721 static u32 pll_ratio_to_freq_mtl(u32 ratio, u32 config) 722 { 723 u32 pll_clock = PLL_REF_CLK_FREQ * ratio; 724 u32 cpu_clock; 725 726 if ((config & 0xff) == MTL_PLL_RATIO_4_3) 727 cpu_clock = pll_clock * 2 / 4; 728 else 729 cpu_clock = pll_clock * 2 / 5; 730 731 return cpu_clock; 732 } 733 734 u32 ivpu_hw_btrs_ratio_to_freq(struct ivpu_device *vdev, u32 ratio) 735 { 736 struct ivpu_hw_info *hw = vdev->hw; 737 738 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 739 return pll_ratio_to_freq_mtl(ratio, hw->config); 740 else 741 return PLL_RATIO_TO_FREQ(ratio); 742 } 743 744 static u32 pll_freq_get_mtl(struct ivpu_device *vdev) 745 { 746 u32 pll_curr_ratio; 747 748 pll_curr_ratio = REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL); 749 pll_curr_ratio &= VPU_HW_BTRS_MTL_CURRENT_PLL_RATIO_MASK; 750 751 if (!ivpu_is_silicon(vdev)) 752 return PLL_SIMULATION_FREQ; 753 754 return pll_ratio_to_freq_mtl(pll_curr_ratio, vdev->hw->config); 755 } 756 757 static u32 pll_freq_get_lnl(struct ivpu_device *vdev) 758 { 759 u32 pll_curr_ratio; 760 761 pll_curr_ratio = REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ); 762 pll_curr_ratio &= VPU_HW_BTRS_LNL_PLL_FREQ_RATIO_MASK; 763 764 return PLL_RATIO_TO_FREQ(pll_curr_ratio); 765 } 766 767 u32 ivpu_hw_btrs_pll_freq_get(struct ivpu_device *vdev) 768 { 769 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 770 return pll_freq_get_mtl(vdev); 771 else 772 return pll_freq_get_lnl(vdev); 773 } 774 775 u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev) 776 { 777 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 778 return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_OFFSET); 779 else 780 return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_OFFSET); 781 } 782 783 u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev) 784 { 785 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 786 return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_SIZE); 787 else 788 return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_SIZE); 789 } 790 791 u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev) 792 { 793 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 794 return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_ENABLE); 795 else 796 return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_ENABLE); 797 } 798 799 void ivpu_hw_btrs_global_int_disable(struct ivpu_device *vdev) 800 { 801 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 802 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1); 803 else 804 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1); 805 } 806 807 void ivpu_hw_btrs_global_int_enable(struct ivpu_device *vdev) 808 { 809 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 810 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0); 811 else 812 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0); 813 } 814 815 void ivpu_hw_btrs_irq_enable(struct ivpu_device *vdev) 816 { 817 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 818 REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, (u32)(~BTRS_MTL_IRQ_MASK)); 819 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0); 820 } else { 821 REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, (u32)(~BTRS_LNL_IRQ_MASK)); 822 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0); 823 } 824 } 825 826 void ivpu_hw_btrs_irq_disable(struct ivpu_device *vdev) 827 { 828 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 829 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1); 830 REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK); 831 } else { 832 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1); 833 REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK); 834 } 835 } 836 837 static void diagnose_failure_mtl(struct ivpu_device *vdev) 838 { 839 u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK; 840 841 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, reg)) 842 ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0)); 843 844 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, reg)) { 845 u32 log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG); 846 847 ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx", 848 log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, log), 849 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, log), 850 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, log)); 851 } 852 } 853 854 static void diagnose_failure_lnl(struct ivpu_device *vdev) 855 { 856 u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; 857 858 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) { 859 ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", 860 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1), 861 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2)); 862 } 863 864 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, reg)) 865 ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG)); 866 867 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, reg)) 868 ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG)); 869 870 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, reg)) 871 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n", 872 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW), 873 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH)); 874 875 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, reg)) 876 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n", 877 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW), 878 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH)); 879 880 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, reg)) 881 ivpu_err(vdev, "Survivability IRQ\n"); 882 } 883 884 void ivpu_hw_btrs_diagnose_failure(struct ivpu_device *vdev) 885 { 886 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 887 return diagnose_failure_mtl(vdev); 888 else 889 return diagnose_failure_lnl(vdev); 890 } 891