1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2024 Intel Corporation 4 */ 5 6 #include "ivpu_drv.h" 7 #include "ivpu_hw.h" 8 #include "ivpu_hw_btrs.h" 9 #include "ivpu_hw_btrs_lnl_reg.h" 10 #include "ivpu_hw_btrs_mtl_reg.h" 11 #include "ivpu_hw_reg_io.h" 12 #include "ivpu_pm.h" 13 14 #define BTRS_MTL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR)) | \ 15 (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR))) 16 17 #define BTRS_LNL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR)) | \ 18 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR)) | \ 19 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR)) | \ 20 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR)) | \ 21 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR)) | \ 22 (REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR))) 23 24 #define BTRS_MTL_ALL_IRQ_MASK (BTRS_MTL_IRQ_MASK | (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, \ 25 FREQ_CHANGE))) 26 27 #define BTRS_IRQ_DISABLE_MASK ((u32)-1) 28 29 #define BTRS_LNL_ALL_IRQ_MASK ((u32)-1) 30 31 #define BTRS_MTL_WP_CONFIG_1_TILE_5_3_RATIO WP_CONFIG(MTL_CONFIG_1_TILE, MTL_PLL_RATIO_5_3) 32 #define BTRS_MTL_WP_CONFIG_1_TILE_4_3_RATIO WP_CONFIG(MTL_CONFIG_1_TILE, MTL_PLL_RATIO_4_3) 33 #define BTRS_MTL_WP_CONFIG_2_TILE_5_3_RATIO WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_5_3) 34 #define BTRS_MTL_WP_CONFIG_2_TILE_4_3_RATIO WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_4_3) 35 #define BTRS_MTL_WP_CONFIG_0_TILE_PLL_OFF WP_CONFIG(0, 0) 36 37 #define PLL_CDYN_DEFAULT 0x80 38 #define PLL_EPP_DEFAULT 0x80 39 #define PLL_CONFIG_DEFAULT 0x0 40 #define PLL_SIMULATION_FREQ 10000000 41 #define PLL_REF_CLK_FREQ 50000000 42 #define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC) 43 #define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC) 44 #define TIMEOUT_US (150 * USEC_PER_MSEC) 45 46 /* Work point configuration values */ 47 #define WP_CONFIG(tile, ratio) (((tile) << 8) | (ratio)) 48 #define MTL_CONFIG_1_TILE 0x01 49 #define MTL_CONFIG_2_TILE 0x02 50 #define MTL_PLL_RATIO_5_3 0x01 51 #define MTL_PLL_RATIO_4_3 0x02 52 #define BTRS_MTL_TILE_FUSE_ENABLE_BOTH 0x0 53 #define BTRS_MTL_TILE_SKU_BOTH 0x3630 54 55 #define BTRS_LNL_TILE_MAX_NUM 6 56 #define BTRS_LNL_TILE_MAX_MASK 0x3f 57 58 #define WEIGHTS_DEFAULT 0xf711f711u 59 #define WEIGHTS_ATS_DEFAULT 0x0000f711u 60 61 #define DCT_REQ 0x2 62 #define DCT_ENABLE 0x1 63 #define DCT_DISABLE 0x0 64 65 int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev) 66 { 67 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, BTRS_MTL_ALL_IRQ_MASK); 68 if (REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) == BTRS_MTL_ALL_IRQ_MASK) { 69 /* Writing 1s does not clear the interrupt status register */ 70 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0); 71 return true; 72 } 73 74 return false; 75 } 76 77 static void freq_ratios_init_mtl(struct ivpu_device *vdev) 78 { 79 struct ivpu_hw_info *hw = vdev->hw; 80 u32 fmin_fuse, fmax_fuse; 81 82 fmin_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMIN_FUSE); 83 hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, MIN_RATIO, fmin_fuse); 84 hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, PN_RATIO, fmin_fuse); 85 86 fmax_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMAX_FUSE); 87 hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMAX_FUSE, MAX_RATIO, fmax_fuse); 88 } 89 90 static void freq_ratios_init_lnl(struct ivpu_device *vdev) 91 { 92 struct ivpu_hw_info *hw = vdev->hw; 93 u32 fmin_fuse, fmax_fuse; 94 95 fmin_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMIN_FUSE); 96 hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, MIN_RATIO, fmin_fuse); 97 hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, PN_RATIO, fmin_fuse); 98 99 fmax_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMAX_FUSE); 100 hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMAX_FUSE, MAX_RATIO, fmax_fuse); 101 } 102 103 void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev) 104 { 105 struct ivpu_hw_info *hw = vdev->hw; 106 107 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 108 freq_ratios_init_mtl(vdev); 109 else 110 freq_ratios_init_lnl(vdev); 111 112 hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 113 hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 114 hw->pll.pn_ratio = clamp_t(u8, hw->pll.pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 115 } 116 117 static bool tile_disable_check(u32 config) 118 { 119 /* Allowed values: 0 or one bit from range 0-5 (6 tiles) */ 120 if (config == 0) 121 return true; 122 123 if (config > BIT(BTRS_LNL_TILE_MAX_NUM - 1)) 124 return false; 125 126 if ((config & (config - 1)) == 0) 127 return true; 128 129 return false; 130 } 131 132 static int read_tile_config_fuse(struct ivpu_device *vdev, u32 *tile_fuse_config) 133 { 134 u32 fuse; 135 u32 config; 136 137 fuse = REGB_RD32(VPU_HW_BTRS_LNL_TILE_FUSE); 138 if (!REG_TEST_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, VALID, fuse)) { 139 ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse); 140 return -EIO; 141 } 142 143 config = REG_GET_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, CONFIG, fuse); 144 if (!tile_disable_check(config)) 145 ivpu_warn(vdev, "More than 1 tile disabled, tile fuse config mask: 0x%x\n", config); 146 147 ivpu_dbg(vdev, MISC, "Tile disable config mask: 0x%x\n", config); 148 149 *tile_fuse_config = config; 150 return 0; 151 } 152 153 static int info_init_mtl(struct ivpu_device *vdev) 154 { 155 struct ivpu_hw_info *hw = vdev->hw; 156 157 hw->tile_fuse = BTRS_MTL_TILE_FUSE_ENABLE_BOTH; 158 hw->sku = BTRS_MTL_TILE_SKU_BOTH; 159 hw->config = BTRS_MTL_WP_CONFIG_2_TILE_4_3_RATIO; 160 161 return 0; 162 } 163 164 static int info_init_lnl(struct ivpu_device *vdev) 165 { 166 struct ivpu_hw_info *hw = vdev->hw; 167 u32 tile_fuse_config; 168 int ret; 169 170 ret = read_tile_config_fuse(vdev, &tile_fuse_config); 171 if (ret) 172 return ret; 173 174 hw->tile_fuse = tile_fuse_config; 175 hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 176 177 return 0; 178 } 179 180 int ivpu_hw_btrs_info_init(struct ivpu_device *vdev) 181 { 182 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 183 return info_init_mtl(vdev); 184 else 185 return info_init_lnl(vdev); 186 } 187 188 static int wp_request_sync(struct ivpu_device *vdev) 189 { 190 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 191 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); 192 else 193 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); 194 } 195 196 static int wait_for_status_ready(struct ivpu_device *vdev, bool enable) 197 { 198 u32 exp_val = enable ? 0x1 : 0x0; 199 200 if (IVPU_WA(punit_disabled)) 201 return 0; 202 203 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 204 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US); 205 else 206 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US); 207 } 208 209 struct wp_request { 210 u16 min; 211 u16 max; 212 u16 target; 213 u16 cfg; 214 u16 epp; 215 u16 cdyn; 216 }; 217 218 static void wp_request_mtl(struct ivpu_device *vdev, struct wp_request *wp) 219 { 220 u32 val; 221 222 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0); 223 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val); 224 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val); 225 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, val); 226 227 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1); 228 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val); 229 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, EPP, PLL_EPP_DEFAULT, val); 230 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, val); 231 232 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2); 233 val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val); 234 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, val); 235 236 val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_CMD); 237 val = REG_SET_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, val); 238 REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_CMD, val); 239 } 240 241 static void wp_request_lnl(struct ivpu_device *vdev, struct wp_request *wp) 242 { 243 u32 val; 244 245 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0); 246 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val); 247 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val); 248 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, val); 249 250 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1); 251 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val); 252 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, EPP, wp->epp, val); 253 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, val); 254 255 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2); 256 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val); 257 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CDYN, wp->cdyn, val); 258 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, val); 259 260 val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_CMD); 261 val = REG_SET_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, val); 262 REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_CMD, val); 263 } 264 265 static void wp_request(struct ivpu_device *vdev, struct wp_request *wp) 266 { 267 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 268 wp_request_mtl(vdev, wp); 269 else 270 wp_request_lnl(vdev, wp); 271 } 272 273 static int wp_request_send(struct ivpu_device *vdev, struct wp_request *wp) 274 { 275 int ret; 276 277 ret = wp_request_sync(vdev); 278 if (ret) { 279 ivpu_err(vdev, "Failed to sync before workpoint request: %d\n", ret); 280 return ret; 281 } 282 283 wp_request(vdev, wp); 284 285 ret = wp_request_sync(vdev); 286 if (ret) 287 ivpu_err(vdev, "Failed to sync after workpoint request: %d\n", ret); 288 289 return ret; 290 } 291 292 static void prepare_wp_request(struct ivpu_device *vdev, struct wp_request *wp, bool enable) 293 { 294 struct ivpu_hw_info *hw = vdev->hw; 295 296 wp->min = hw->pll.min_ratio; 297 wp->max = hw->pll.max_ratio; 298 299 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 300 wp->target = enable ? hw->pll.pn_ratio : 0; 301 wp->cfg = enable ? hw->config : 0; 302 wp->cdyn = 0; 303 wp->epp = 0; 304 } else { 305 wp->target = hw->pll.pn_ratio; 306 wp->cfg = enable ? PLL_CONFIG_DEFAULT : 0; 307 wp->cdyn = enable ? PLL_CDYN_DEFAULT : 0; 308 wp->epp = enable ? PLL_EPP_DEFAULT : 0; 309 } 310 } 311 312 static int wait_for_pll_lock(struct ivpu_device *vdev, bool enable) 313 { 314 u32 exp_val = enable ? 0x1 : 0x0; 315 316 if (ivpu_hw_btrs_gen(vdev) != IVPU_HW_BTRS_MTL) 317 return 0; 318 319 if (IVPU_WA(punit_disabled)) 320 return 0; 321 322 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_PLL_STATUS, LOCK, exp_val, PLL_TIMEOUT_US); 323 } 324 325 int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable) 326 { 327 struct wp_request wp; 328 int ret; 329 330 if (IVPU_WA(punit_disabled)) { 331 ivpu_dbg(vdev, PM, "Skipping workpoint request\n"); 332 return 0; 333 } 334 335 prepare_wp_request(vdev, &wp, enable); 336 337 ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, config: 0x%x, epp: 0x%x, cdyn: 0x%x\n", 338 PLL_RATIO_TO_FREQ(wp.target), wp.cfg, wp.epp, wp.cdyn); 339 340 ret = wp_request_send(vdev, &wp); 341 if (ret) { 342 ivpu_err(vdev, "Failed to send workpoint request: %d\n", ret); 343 return ret; 344 } 345 346 ret = wait_for_pll_lock(vdev, enable); 347 if (ret) { 348 ivpu_err(vdev, "Timed out waiting for PLL lock\n"); 349 return ret; 350 } 351 352 ret = wait_for_status_ready(vdev, enable); 353 if (ret) { 354 ivpu_err(vdev, "Timed out waiting for NPU ready status\n"); 355 return ret; 356 } 357 358 return 0; 359 } 360 361 static int d0i3_drive_mtl(struct ivpu_device *vdev, bool enable) 362 { 363 int ret; 364 u32 val; 365 366 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 367 if (ret) { 368 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); 369 return ret; 370 } 371 372 val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL); 373 if (enable) 374 val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val); 375 else 376 val = REG_CLR_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val); 377 REGB_WR32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, val); 378 379 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 380 if (ret) 381 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); 382 383 return ret; 384 } 385 386 static int d0i3_drive_lnl(struct ivpu_device *vdev, bool enable) 387 { 388 int ret; 389 u32 val; 390 391 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 392 if (ret) { 393 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); 394 return ret; 395 } 396 397 val = REGB_RD32(VPU_HW_BTRS_LNL_D0I3_CONTROL); 398 if (enable) 399 val = REG_SET_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val); 400 else 401 val = REG_CLR_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val); 402 REGB_WR32(VPU_HW_BTRS_LNL_D0I3_CONTROL, val); 403 404 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 405 if (ret) { 406 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); 407 return ret; 408 } 409 410 return 0; 411 } 412 413 static int d0i3_drive(struct ivpu_device *vdev, bool enable) 414 { 415 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 416 return d0i3_drive_mtl(vdev, enable); 417 else 418 return d0i3_drive_lnl(vdev, enable); 419 } 420 421 int ivpu_hw_btrs_d0i3_enable(struct ivpu_device *vdev) 422 { 423 int ret; 424 425 if (IVPU_WA(punit_disabled)) 426 return 0; 427 428 ret = d0i3_drive(vdev, true); 429 if (ret) 430 ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret); 431 432 udelay(5); /* VPU requires 5 us to complete the transition */ 433 434 return ret; 435 } 436 437 int ivpu_hw_btrs_d0i3_disable(struct ivpu_device *vdev) 438 { 439 int ret; 440 441 if (IVPU_WA(punit_disabled)) 442 return 0; 443 444 ret = d0i3_drive(vdev, false); 445 if (ret) 446 ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret); 447 448 return ret; 449 } 450 451 int ivpu_hw_btrs_wait_for_clock_res_own_ack(struct ivpu_device *vdev) 452 { 453 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 454 return 0; 455 456 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US); 457 } 458 459 void ivpu_hw_btrs_set_port_arbitration_weights_lnl(struct ivpu_device *vdev) 460 { 461 REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT); 462 REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT); 463 } 464 465 static int ip_reset_mtl(struct ivpu_device *vdev) 466 { 467 int ret; 468 u32 val; 469 470 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US); 471 if (ret) { 472 ivpu_err(vdev, "Timed out waiting for TRIGGER bit\n"); 473 return ret; 474 } 475 476 val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_IP_RESET); 477 val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, val); 478 REGB_WR32(VPU_HW_BTRS_MTL_VPU_IP_RESET, val); 479 480 ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US); 481 if (ret) 482 ivpu_err(vdev, "Timed out waiting for RESET completion\n"); 483 484 return ret; 485 } 486 487 static int ip_reset_lnl(struct ivpu_device *vdev) 488 { 489 int ret; 490 u32 val; 491 492 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev); 493 494 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US); 495 if (ret) { 496 ivpu_err(vdev, "Wait for *_TRIGGER timed out\n"); 497 return ret; 498 } 499 500 val = REGB_RD32(VPU_HW_BTRS_LNL_IP_RESET); 501 val = REG_SET_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, val); 502 REGB_WR32(VPU_HW_BTRS_LNL_IP_RESET, val); 503 504 ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US); 505 if (ret) 506 ivpu_err(vdev, "Timed out waiting for RESET completion\n"); 507 508 return ret; 509 } 510 511 int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev) 512 { 513 if (IVPU_WA(punit_disabled)) 514 return 0; 515 516 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 517 return ip_reset_mtl(vdev); 518 else 519 return ip_reset_lnl(vdev); 520 } 521 522 void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev) 523 { 524 u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 525 526 if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT) 527 val = REG_CLR_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val); 528 else 529 val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val); 530 531 REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val); 532 } 533 534 void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev) 535 { 536 ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n", 537 REGB_RD32(VPU_HW_BTRS_LNL_HM_ATS) ? "Enable" : "Disable"); 538 } 539 540 void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev) 541 { 542 u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 543 544 val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, DISABLE_CLK_RELINQUISH, val); 545 REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val); 546 } 547 548 bool ivpu_hw_btrs_is_idle(struct ivpu_device *vdev) 549 { 550 u32 val; 551 552 if (IVPU_WA(punit_disabled)) 553 return true; 554 555 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 556 val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_STATUS); 557 558 return REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, val) && 559 REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, val); 560 } else { 561 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 562 563 return REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, val) && 564 REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, val); 565 } 566 } 567 568 int ivpu_hw_btrs_wait_for_idle(struct ivpu_device *vdev) 569 { 570 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 571 return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); 572 else 573 return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); 574 } 575 576 /* Handler for IRQs from Buttress core (irqB) */ 577 bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq) 578 { 579 u32 status = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK; 580 bool schedule_recovery = false; 581 582 if (!status) 583 return false; 584 585 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, FREQ_CHANGE, status)) 586 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", 587 REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL)); 588 589 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, status)) { 590 ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0)); 591 REGB_WR32(VPU_HW_BTRS_MTL_ATS_ERR_CLEAR, 0x1); 592 schedule_recovery = true; 593 } 594 595 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, status)) { 596 u32 ufi_log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG); 597 598 ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx", 599 ufi_log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, ufi_log), 600 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, ufi_log), 601 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, ufi_log)); 602 REGB_WR32(VPU_HW_BTRS_MTL_UFI_ERR_CLEAR, 0x1); 603 schedule_recovery = true; 604 } 605 606 /* This must be done after interrupts are cleared at the source. */ 607 if (IVPU_WA(interrupt_clear_with_0)) 608 /* 609 * Writing 1 triggers an interrupt, so we can't perform read update write. 610 * Clear local interrupt status by writing 0 to all bits. 611 */ 612 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0); 613 else 614 REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, status); 615 616 if (schedule_recovery) 617 ivpu_pm_trigger_recovery(vdev, "Buttress IRQ"); 618 619 return true; 620 } 621 622 /* Handler for IRQs from Buttress core (irqB) */ 623 bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq) 624 { 625 u32 status = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; 626 bool schedule_recovery = false; 627 628 if (!status) 629 return false; 630 631 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status)) { 632 ivpu_dbg(vdev, IRQ, "Survivability IRQ\n"); 633 queue_work(system_wq, &vdev->irq_dct_work); 634 } 635 636 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status)) 637 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ)); 638 639 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, status)) { 640 ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", 641 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1), 642 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2)); 643 REGB_WR32(VPU_HW_BTRS_LNL_ATS_ERR_CLEAR, 0x1); 644 schedule_recovery = true; 645 } 646 647 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, status)) { 648 ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG)); 649 REGB_WR32(VPU_HW_BTRS_LNL_CFI0_ERR_CLEAR, 0x1); 650 schedule_recovery = true; 651 } 652 653 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, status)) { 654 ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG)); 655 REGB_WR32(VPU_HW_BTRS_LNL_CFI1_ERR_CLEAR, 0x1); 656 schedule_recovery = true; 657 } 658 659 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, status)) { 660 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x", 661 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW), 662 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH)); 663 REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_CLEAR, 0x1); 664 schedule_recovery = true; 665 } 666 667 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, status)) { 668 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x", 669 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW), 670 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH)); 671 REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR, 0x1); 672 schedule_recovery = true; 673 } 674 675 /* This must be done after interrupts are cleared at the source. */ 676 REGB_WR32(VPU_HW_BTRS_LNL_INTERRUPT_STAT, status); 677 678 if (schedule_recovery) 679 ivpu_pm_trigger_recovery(vdev, "Buttress IRQ"); 680 681 return true; 682 } 683 684 int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable) 685 { 686 u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW); 687 u32 cmd = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, CMD, val); 688 u32 param1 = REG_GET_FLD(VPU_HW_BTRS_LNL_PCODE_MAILBOX_SHADOW, PARAM1, val); 689 690 if (cmd != DCT_REQ) { 691 ivpu_err_ratelimited(vdev, "Unsupported PCODE command: 0x%x\n", cmd); 692 return -EBADR; 693 } 694 695 switch (param1) { 696 case DCT_ENABLE: 697 *enable = true; 698 return 0; 699 case DCT_DISABLE: 700 *enable = false; 701 return 0; 702 default: 703 ivpu_err_ratelimited(vdev, "Invalid PARAM1 value: %u\n", param1); 704 return -EINVAL; 705 } 706 } 707 708 void ivpu_hw_btrs_dct_set_status(struct ivpu_device *vdev, bool enable, u32 active_percent) 709 { 710 u32 val = 0; 711 u32 cmd = enable ? DCT_ENABLE : DCT_DISABLE; 712 713 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, CMD, DCT_REQ, val); 714 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM1, cmd, val); 715 val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, PARAM2, active_percent, val); 716 717 REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX_STATUS, val); 718 } 719 720 static u32 pll_ratio_to_freq_mtl(u32 ratio, u32 config) 721 { 722 u32 pll_clock = PLL_REF_CLK_FREQ * ratio; 723 u32 cpu_clock; 724 725 if ((config & 0xff) == MTL_PLL_RATIO_4_3) 726 cpu_clock = pll_clock * 2 / 4; 727 else 728 cpu_clock = pll_clock * 2 / 5; 729 730 return cpu_clock; 731 } 732 733 u32 ivpu_hw_btrs_ratio_to_freq(struct ivpu_device *vdev, u32 ratio) 734 { 735 struct ivpu_hw_info *hw = vdev->hw; 736 737 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 738 return pll_ratio_to_freq_mtl(ratio, hw->config); 739 else 740 return PLL_RATIO_TO_FREQ(ratio); 741 } 742 743 static u32 pll_freq_get_mtl(struct ivpu_device *vdev) 744 { 745 u32 pll_curr_ratio; 746 747 pll_curr_ratio = REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL); 748 pll_curr_ratio &= VPU_HW_BTRS_MTL_CURRENT_PLL_RATIO_MASK; 749 750 if (!ivpu_is_silicon(vdev)) 751 return PLL_SIMULATION_FREQ; 752 753 return pll_ratio_to_freq_mtl(pll_curr_ratio, vdev->hw->config); 754 } 755 756 static u32 pll_freq_get_lnl(struct ivpu_device *vdev) 757 { 758 u32 pll_curr_ratio; 759 760 pll_curr_ratio = REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ); 761 pll_curr_ratio &= VPU_HW_BTRS_LNL_PLL_FREQ_RATIO_MASK; 762 763 return PLL_RATIO_TO_FREQ(pll_curr_ratio); 764 } 765 766 u32 ivpu_hw_btrs_pll_freq_get(struct ivpu_device *vdev) 767 { 768 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 769 return pll_freq_get_mtl(vdev); 770 else 771 return pll_freq_get_lnl(vdev); 772 } 773 774 u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev) 775 { 776 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 777 return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_OFFSET); 778 else 779 return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_OFFSET); 780 } 781 782 u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev) 783 { 784 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 785 return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_SIZE); 786 else 787 return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_SIZE); 788 } 789 790 u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev) 791 { 792 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 793 return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_ENABLE); 794 else 795 return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_ENABLE); 796 } 797 798 void ivpu_hw_btrs_global_int_disable(struct ivpu_device *vdev) 799 { 800 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 801 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1); 802 else 803 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1); 804 } 805 806 void ivpu_hw_btrs_global_int_enable(struct ivpu_device *vdev) 807 { 808 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 809 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0); 810 else 811 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0); 812 } 813 814 void ivpu_hw_btrs_irq_enable(struct ivpu_device *vdev) 815 { 816 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 817 REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, (u32)(~BTRS_MTL_IRQ_MASK)); 818 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0); 819 } else { 820 REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, (u32)(~BTRS_LNL_IRQ_MASK)); 821 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0); 822 } 823 } 824 825 void ivpu_hw_btrs_irq_disable(struct ivpu_device *vdev) 826 { 827 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) { 828 REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1); 829 REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK); 830 } else { 831 REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1); 832 REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK); 833 } 834 } 835 836 static void diagnose_failure_mtl(struct ivpu_device *vdev) 837 { 838 u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK; 839 840 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, reg)) 841 ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0)); 842 843 if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, reg)) { 844 u32 log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG); 845 846 ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx", 847 log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, log), 848 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, log), 849 REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, log)); 850 } 851 } 852 853 static void diagnose_failure_lnl(struct ivpu_device *vdev) 854 { 855 u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; 856 857 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) { 858 ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", 859 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1), 860 REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2)); 861 } 862 863 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, reg)) 864 ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG)); 865 866 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, reg)) 867 ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG)); 868 869 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, reg)) 870 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n", 871 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW), 872 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH)); 873 874 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, reg)) 875 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n", 876 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW), 877 REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH)); 878 879 if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, reg)) 880 ivpu_err(vdev, "Survivability IRQ\n"); 881 } 882 883 void ivpu_hw_btrs_diagnose_failure(struct ivpu_device *vdev) 884 { 885 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) 886 return diagnose_failure_mtl(vdev); 887 else 888 return diagnose_failure_lnl(vdev); 889 } 890 891 int ivpu_hw_btrs_platform_read(struct ivpu_device *vdev) 892 { 893 u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS); 894 895 return REG_GET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PLATFORM, reg); 896 } 897