1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2024 Intel Corporation 4 */ 5 6 #include "ivpu_drv.h" 7 #include "ivpu_fw.h" 8 #include "ivpu_hw.h" 9 #include "ivpu_hw_37xx_reg.h" 10 #include "ivpu_hw_40xx_reg.h" 11 #include "ivpu_hw_btrs.h" 12 #include "ivpu_hw_ip.h" 13 #include "ivpu_hw_reg_io.h" 14 #include "ivpu_mmu.h" 15 #include "ivpu_pm.h" 16 17 #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC) 18 19 #define TIM_SAFE_ENABLE 0xf1d0dead 20 #define TIM_WATCHDOG_RESET_VALUE 0xffffffff 21 22 #define ICB_0_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \ 23 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \ 24 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \ 25 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \ 26 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \ 27 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \ 28 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT))) 29 30 #define ICB_1_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \ 31 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \ 32 (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT))) 33 34 #define ICB_0_1_IRQ_MASK_37XX ((((u64)ICB_1_IRQ_MASK_37XX) << 32) | ICB_0_IRQ_MASK_37XX) 35 36 #define ICB_0_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \ 37 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \ 38 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \ 39 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \ 40 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \ 41 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \ 42 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT))) 43 44 #define ICB_1_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \ 45 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \ 46 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT))) 47 48 #define ICB_0_1_IRQ_MASK_40XX ((((u64)ICB_1_IRQ_MASK_40XX) << 32) | ICB_0_IRQ_MASK_40XX) 49 50 #define ITF_FIREWALL_VIOLATION_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \ 51 (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \ 52 (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \ 53 (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \ 54 (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \ 55 (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \ 56 (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX))) 57 58 #define ITF_FIREWALL_VIOLATION_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \ 59 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \ 60 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \ 61 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \ 62 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \ 63 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \ 64 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX))) 65 66 static int wait_for_ip_bar(struct ivpu_device *vdev) 67 { 68 return REGV_POLL_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, AON, 0, 100); 69 } 70 71 static void host_ss_rst_clr(struct ivpu_device *vdev) 72 { 73 u32 val = 0; 74 75 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, TOP_NOC, val); 76 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, DSS_MAS, val); 77 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, MSS_MAS, val); 78 79 REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_CLR, val); 80 } 81 82 static int host_ss_noc_qreqn_check_37xx(struct ivpu_device *vdev, u32 exp_val) 83 { 84 u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN); 85 86 if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val)) 87 return -EIO; 88 89 return 0; 90 } 91 92 static int host_ss_noc_qreqn_check_40xx(struct ivpu_device *vdev, u32 exp_val) 93 { 94 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); 95 96 if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val)) 97 return -EIO; 98 99 return 0; 100 } 101 102 static int host_ss_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val) 103 { 104 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 105 return host_ss_noc_qreqn_check_37xx(vdev, exp_val); 106 else 107 return host_ss_noc_qreqn_check_40xx(vdev, exp_val); 108 } 109 110 static int host_ss_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val) 111 { 112 u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QACCEPTN); 113 114 if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val)) 115 return -EIO; 116 117 return 0; 118 } 119 120 static int host_ss_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val) 121 { 122 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN); 123 124 if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val)) 125 return -EIO; 126 127 return 0; 128 } 129 130 static int host_ss_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) 131 { 132 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 133 return host_ss_noc_qacceptn_check_37xx(vdev, exp_val); 134 else 135 return host_ss_noc_qacceptn_check_40xx(vdev, exp_val); 136 } 137 138 static int host_ss_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val) 139 { 140 u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QDENY); 141 142 if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val)) 143 return -EIO; 144 145 return 0; 146 } 147 148 static int host_ss_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val) 149 { 150 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY); 151 152 if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val)) 153 return -EIO; 154 155 return 0; 156 } 157 158 static int host_ss_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) 159 { 160 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 161 return host_ss_noc_qdeny_check_37xx(vdev, exp_val); 162 else 163 return host_ss_noc_qdeny_check_40xx(vdev, exp_val); 164 } 165 166 static int top_noc_qrenqn_check_37xx(struct ivpu_device *vdev, u32 exp_val) 167 { 168 u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN); 169 170 if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) || 171 !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val)) 172 return -EIO; 173 174 return 0; 175 } 176 177 static int top_noc_qrenqn_check_40xx(struct ivpu_device *vdev, u32 exp_val) 178 { 179 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); 180 181 if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) || 182 !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val)) 183 return -EIO; 184 185 return 0; 186 } 187 188 static int top_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val) 189 { 190 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 191 return top_noc_qrenqn_check_37xx(vdev, exp_val); 192 else 193 return top_noc_qrenqn_check_40xx(vdev, exp_val); 194 } 195 196 int ivpu_hw_ip_host_ss_configure(struct ivpu_device *vdev) 197 { 198 int ret; 199 200 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 201 ret = wait_for_ip_bar(vdev); 202 if (ret) { 203 ivpu_err(vdev, "Timed out waiting for NPU IP bar\n"); 204 return ret; 205 } 206 host_ss_rst_clr(vdev); 207 } 208 209 ret = host_ss_noc_qreqn_check(vdev, 0x0); 210 if (ret) { 211 ivpu_err(vdev, "Failed qreqn check: %d\n", ret); 212 return ret; 213 } 214 215 ret = host_ss_noc_qacceptn_check(vdev, 0x0); 216 if (ret) { 217 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); 218 return ret; 219 } 220 221 ret = host_ss_noc_qdeny_check(vdev, 0x0); 222 if (ret) 223 ivpu_err(vdev, "Failed qdeny check %d\n", ret); 224 225 return ret; 226 } 227 228 static void idle_gen_drive_37xx(struct ivpu_device *vdev, bool enable) 229 { 230 u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN); 231 232 if (enable) 233 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val); 234 else 235 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val); 236 237 REGV_WR32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, val); 238 } 239 240 static void idle_gen_drive_40xx(struct ivpu_device *vdev, bool enable) 241 { 242 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN); 243 244 if (enable) 245 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); 246 else 247 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); 248 249 REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val); 250 } 251 252 void ivpu_hw_ip_idle_gen_enable(struct ivpu_device *vdev) 253 { 254 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 255 idle_gen_drive_37xx(vdev, true); 256 else 257 idle_gen_drive_40xx(vdev, true); 258 } 259 260 void ivpu_hw_ip_idle_gen_disable(struct ivpu_device *vdev) 261 { 262 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 263 idle_gen_drive_37xx(vdev, false); 264 else 265 idle_gen_drive_40xx(vdev, false); 266 } 267 268 static void 269 pwr_island_delay_set_50xx(struct ivpu_device *vdev, u32 post, u32 post1, u32 post2, u32 status) 270 { 271 u32 val; 272 273 val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY); 274 val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST_DLY, post, val); 275 val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST1_DLY, post1, val); 276 val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST2_DLY, post2, val); 277 REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, val); 278 279 val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY); 280 val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, STATUS_DLY, status, val); 281 REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, val); 282 } 283 284 static void pwr_island_trickle_drive_37xx(struct ivpu_device *vdev, bool enable) 285 { 286 u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0); 287 288 if (enable) 289 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val); 290 else 291 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val); 292 293 REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val); 294 } 295 296 static void pwr_island_trickle_drive_40xx(struct ivpu_device *vdev, bool enable) 297 { 298 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0); 299 300 if (enable) 301 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); 302 else 303 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); 304 305 REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val); 306 } 307 308 static void pwr_island_drive_37xx(struct ivpu_device *vdev, bool enable) 309 { 310 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0); 311 312 if (enable) 313 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); 314 else 315 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); 316 317 REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val); 318 } 319 320 static void pwr_island_drive_40xx(struct ivpu_device *vdev, bool enable) 321 { 322 u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0); 323 324 if (enable) 325 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val); 326 else 327 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val); 328 329 REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val); 330 } 331 332 static void pwr_island_enable(struct ivpu_device *vdev) 333 { 334 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 335 pwr_island_trickle_drive_37xx(vdev, true); 336 ndelay(500); 337 pwr_island_drive_37xx(vdev, true); 338 } else { 339 pwr_island_trickle_drive_40xx(vdev, true); 340 ndelay(500); 341 pwr_island_drive_40xx(vdev, true); 342 } 343 } 344 345 static int wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val) 346 { 347 if (IVPU_WA(punit_disabled)) 348 return 0; 349 350 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 351 return REGV_POLL_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0, MSS_CPU, exp_val, 352 PWR_ISLAND_STATUS_TIMEOUT_US); 353 else 354 return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, exp_val, 355 PWR_ISLAND_STATUS_TIMEOUT_US); 356 } 357 358 static void pwr_island_isolation_drive_37xx(struct ivpu_device *vdev, bool enable) 359 { 360 u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0); 361 362 if (enable) 363 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val); 364 else 365 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val); 366 367 REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, val); 368 } 369 370 static void pwr_island_isolation_drive_40xx(struct ivpu_device *vdev, bool enable) 371 { 372 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0); 373 374 if (enable) 375 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); 376 else 377 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); 378 379 REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val); 380 } 381 382 static void pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable) 383 { 384 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 385 pwr_island_isolation_drive_37xx(vdev, enable); 386 else 387 pwr_island_isolation_drive_40xx(vdev, enable); 388 } 389 390 static void pwr_island_isolation_disable(struct ivpu_device *vdev) 391 { 392 pwr_island_isolation_drive(vdev, false); 393 } 394 395 static void host_ss_clk_drive_37xx(struct ivpu_device *vdev, bool enable) 396 { 397 u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_CLK_SET); 398 399 if (enable) { 400 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val); 401 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val); 402 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val); 403 } else { 404 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val); 405 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val); 406 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val); 407 } 408 409 REGV_WR32(VPU_37XX_HOST_SS_CPR_CLK_SET, val); 410 } 411 412 static void host_ss_clk_drive_40xx(struct ivpu_device *vdev, bool enable) 413 { 414 u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN); 415 416 if (enable) { 417 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); 418 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); 419 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); 420 } else { 421 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); 422 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); 423 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); 424 } 425 426 REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val); 427 } 428 429 static void host_ss_clk_drive(struct ivpu_device *vdev, bool enable) 430 { 431 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 432 host_ss_clk_drive_37xx(vdev, enable); 433 else 434 host_ss_clk_drive_40xx(vdev, enable); 435 } 436 437 static void host_ss_clk_enable(struct ivpu_device *vdev) 438 { 439 host_ss_clk_drive(vdev, true); 440 } 441 442 static void host_ss_rst_drive_37xx(struct ivpu_device *vdev, bool enable) 443 { 444 u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_RST_SET); 445 446 if (enable) { 447 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val); 448 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val); 449 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val); 450 } else { 451 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val); 452 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val); 453 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val); 454 } 455 456 REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_SET, val); 457 } 458 459 static void host_ss_rst_drive_40xx(struct ivpu_device *vdev, bool enable) 460 { 461 u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN); 462 463 if (enable) { 464 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); 465 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); 466 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); 467 } else { 468 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); 469 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); 470 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); 471 } 472 473 REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val); 474 } 475 476 static void host_ss_rst_drive(struct ivpu_device *vdev, bool enable) 477 { 478 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 479 host_ss_rst_drive_37xx(vdev, enable); 480 else 481 host_ss_rst_drive_40xx(vdev, enable); 482 } 483 484 static void host_ss_rst_enable(struct ivpu_device *vdev) 485 { 486 host_ss_rst_drive(vdev, true); 487 } 488 489 static void host_ss_noc_qreqn_top_socmmio_drive_37xx(struct ivpu_device *vdev, bool enable) 490 { 491 u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN); 492 493 if (enable) 494 val = REG_SET_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); 495 else 496 val = REG_CLR_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); 497 REGV_WR32(VPU_37XX_HOST_SS_NOC_QREQN, val); 498 } 499 500 static void host_ss_noc_qreqn_top_socmmio_drive_40xx(struct ivpu_device *vdev, bool enable) 501 { 502 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); 503 504 if (enable) 505 val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); 506 else 507 val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); 508 REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val); 509 } 510 511 static void host_ss_noc_qreqn_top_socmmio_drive(struct ivpu_device *vdev, bool enable) 512 { 513 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 514 host_ss_noc_qreqn_top_socmmio_drive_37xx(vdev, enable); 515 else 516 host_ss_noc_qreqn_top_socmmio_drive_40xx(vdev, enable); 517 } 518 519 static int host_ss_axi_drive(struct ivpu_device *vdev, bool enable) 520 { 521 int ret; 522 523 host_ss_noc_qreqn_top_socmmio_drive(vdev, enable); 524 525 ret = host_ss_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); 526 if (ret) { 527 ivpu_err(vdev, "Failed HOST SS NOC QACCEPTN check: %d\n", ret); 528 return ret; 529 } 530 531 ret = host_ss_noc_qdeny_check(vdev, 0x0); 532 if (ret) 533 ivpu_err(vdev, "Failed HOST SS NOC QDENY check: %d\n", ret); 534 535 return ret; 536 } 537 538 static void top_noc_qreqn_drive_40xx(struct ivpu_device *vdev, bool enable) 539 { 540 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); 541 542 if (enable) { 543 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); 544 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); 545 } else { 546 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); 547 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); 548 } 549 550 REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val); 551 } 552 553 static void top_noc_qreqn_drive_37xx(struct ivpu_device *vdev, bool enable) 554 { 555 u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN); 556 557 if (enable) { 558 val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val); 559 val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); 560 } else { 561 val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val); 562 val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); 563 } 564 565 REGV_WR32(VPU_37XX_TOP_NOC_QREQN, val); 566 } 567 568 static void top_noc_qreqn_drive(struct ivpu_device *vdev, bool enable) 569 { 570 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 571 top_noc_qreqn_drive_37xx(vdev, enable); 572 else 573 top_noc_qreqn_drive_40xx(vdev, enable); 574 } 575 576 int ivpu_hw_ip_host_ss_axi_enable(struct ivpu_device *vdev) 577 { 578 return host_ss_axi_drive(vdev, true); 579 } 580 581 static int top_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val) 582 { 583 u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QACCEPTN); 584 585 if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) || 586 !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val)) 587 return -EIO; 588 589 return 0; 590 } 591 592 static int top_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val) 593 { 594 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN); 595 596 if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) || 597 !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val)) 598 return -EIO; 599 600 return 0; 601 } 602 603 static int top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) 604 { 605 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 606 return top_noc_qacceptn_check_37xx(vdev, exp_val); 607 else 608 return top_noc_qacceptn_check_40xx(vdev, exp_val); 609 } 610 611 static int top_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val) 612 { 613 u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QDENY); 614 615 if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) || 616 !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val)) 617 return -EIO; 618 619 return 0; 620 } 621 622 static int top_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val) 623 { 624 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY); 625 626 if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) || 627 !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val)) 628 return -EIO; 629 630 return 0; 631 } 632 633 static int top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) 634 { 635 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 636 return top_noc_qdeny_check_37xx(vdev, exp_val); 637 else 638 return top_noc_qdeny_check_40xx(vdev, exp_val); 639 } 640 641 static int top_noc_drive(struct ivpu_device *vdev, bool enable) 642 { 643 int ret; 644 645 top_noc_qreqn_drive(vdev, enable); 646 647 ret = top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); 648 if (ret) { 649 ivpu_err(vdev, "Failed TOP NOC QACCEPTN check: %d\n", ret); 650 return ret; 651 } 652 653 ret = top_noc_qdeny_check(vdev, 0x0); 654 if (ret) 655 ivpu_err(vdev, "Failed TOP NOC QDENY check: %d\n", ret); 656 657 return ret; 658 } 659 660 int ivpu_hw_ip_top_noc_enable(struct ivpu_device *vdev) 661 { 662 return top_noc_drive(vdev, true); 663 } 664 665 static void dpu_active_drive_37xx(struct ivpu_device *vdev, bool enable) 666 { 667 u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE); 668 669 if (enable) 670 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val); 671 else 672 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val); 673 674 REGV_WR32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, val); 675 } 676 677 static void pwr_island_delay_set(struct ivpu_device *vdev) 678 { 679 bool high = vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_HIGH; 680 u32 post, post1, post2, status; 681 682 if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX) 683 return; 684 685 switch (ivpu_device_id(vdev)) { 686 case PCI_DEVICE_ID_WCL: 687 case PCI_DEVICE_ID_PTL_P: 688 post = high ? 18 : 0; 689 post1 = 0; 690 post2 = 0; 691 status = high ? 46 : 3; 692 break; 693 694 case PCI_DEVICE_ID_NVL: 695 post = high ? 198 : 17; 696 post1 = 0; 697 post2 = high ? 198 : 17; 698 status = 0; 699 break; 700 701 default: 702 dump_stack(); 703 ivpu_err(vdev, "Unknown device ID\n"); 704 return; 705 } 706 707 pwr_island_delay_set_50xx(vdev, post, post1, post2, status); 708 } 709 710 int ivpu_hw_ip_pwr_domain_enable(struct ivpu_device *vdev) 711 { 712 int ret; 713 714 pwr_island_delay_set(vdev); 715 pwr_island_enable(vdev); 716 717 ret = wait_for_pwr_island_status(vdev, 0x1); 718 if (ret) { 719 ivpu_err(vdev, "Timed out waiting for power island status\n"); 720 return ret; 721 } 722 723 ret = top_noc_qreqn_check(vdev, 0x0); 724 if (ret) { 725 ivpu_err(vdev, "Failed TOP NOC QREQN check %d\n", ret); 726 return ret; 727 } 728 729 host_ss_clk_enable(vdev); 730 pwr_island_isolation_disable(vdev); 731 host_ss_rst_enable(vdev); 732 733 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 734 dpu_active_drive_37xx(vdev, true); 735 736 return ret; 737 } 738 739 u64 ivpu_hw_ip_read_perf_timer_counter(struct ivpu_device *vdev) 740 { 741 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 742 return REGV_RD64(VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT); 743 else 744 return REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT); 745 } 746 747 static void ivpu_hw_ip_snoop_disable_37xx(struct ivpu_device *vdev) 748 { 749 u32 val = REGV_RD32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES); 750 751 val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, NOSNOOP_OVERRIDE_EN, val); 752 val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AW_NOSNOOP_OVERRIDE, val); 753 754 if (ivpu_is_force_snoop_enabled(vdev)) 755 val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val); 756 else 757 val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val); 758 759 REGV_WR32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, val); 760 } 761 762 static void ivpu_hw_ip_snoop_disable_40xx(struct ivpu_device *vdev) 763 { 764 u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES); 765 766 val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val); 767 val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val); 768 769 if (ivpu_is_force_snoop_enabled(vdev)) 770 val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val); 771 else 772 val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val); 773 774 REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val); 775 } 776 777 void ivpu_hw_ip_snoop_disable(struct ivpu_device *vdev) 778 { 779 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 780 return ivpu_hw_ip_snoop_disable_37xx(vdev); 781 else 782 return ivpu_hw_ip_snoop_disable_40xx(vdev); 783 } 784 785 static void ivpu_hw_ip_tbu_mmu_enable_37xx(struct ivpu_device *vdev) 786 { 787 u32 val = REGV_RD32(VPU_37XX_HOST_IF_TBU_MMUSSIDV); 788 789 val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); 790 val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); 791 val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); 792 val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); 793 794 REGV_WR32(VPU_37XX_HOST_IF_TBU_MMUSSIDV, val); 795 } 796 797 static void ivpu_hw_ip_tbu_mmu_enable_40xx(struct ivpu_device *vdev) 798 { 799 u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV); 800 801 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); 802 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); 803 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val); 804 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val); 805 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); 806 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); 807 808 REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val); 809 } 810 811 void ivpu_hw_ip_tbu_mmu_enable(struct ivpu_device *vdev) 812 { 813 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 814 return ivpu_hw_ip_tbu_mmu_enable_37xx(vdev); 815 else 816 return ivpu_hw_ip_tbu_mmu_enable_40xx(vdev); 817 } 818 819 static int soc_cpu_boot_37xx(struct ivpu_device *vdev) 820 { 821 u32 val; 822 823 val = REGV_RD32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC); 824 val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTRUN0, val); 825 826 val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTVEC, val); 827 REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val); 828 829 val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val); 830 REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val); 831 832 val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val); 833 REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val); 834 835 val = vdev->fw->entry_point >> 9; 836 REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val); 837 838 val = REG_SET_FLD(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, DONE, val); 839 REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val); 840 841 ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n", 842 vdev->fw->entry_point == vdev->fw->cold_boot_entry_point ? "cold boot" : "resume"); 843 844 return 0; 845 } 846 847 static int cpu_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val) 848 { 849 u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN); 850 851 if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val)) 852 return -EIO; 853 854 return 0; 855 } 856 857 static int cpu_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val) 858 { 859 u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY); 860 861 if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val)) 862 return -EIO; 863 864 return 0; 865 } 866 867 static void cpu_noc_top_mmio_drive_40xx(struct ivpu_device *vdev, bool enable) 868 { 869 u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN); 870 871 if (enable) 872 val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); 873 else 874 val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); 875 REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val); 876 } 877 878 static int soc_cpu_drive_40xx(struct ivpu_device *vdev, bool enable) 879 { 880 int ret; 881 882 cpu_noc_top_mmio_drive_40xx(vdev, enable); 883 884 ret = cpu_noc_qacceptn_check_40xx(vdev, enable ? 0x1 : 0x0); 885 if (ret) { 886 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); 887 return ret; 888 } 889 890 ret = cpu_noc_qdeny_check_40xx(vdev, 0x0); 891 if (ret) 892 ivpu_err(vdev, "Failed qdeny check: %d\n", ret); 893 894 return ret; 895 } 896 897 static int soc_cpu_enable(struct ivpu_device *vdev) 898 { 899 if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_60XX) 900 return 0; 901 902 return soc_cpu_drive_40xx(vdev, true); 903 } 904 905 static int soc_cpu_boot_40xx(struct ivpu_device *vdev) 906 { 907 int ret; 908 u32 val; 909 u64 val64; 910 911 ret = soc_cpu_enable(vdev); 912 if (ret) { 913 ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret); 914 return ret; 915 } 916 917 val64 = vdev->fw->entry_point; 918 val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1; 919 REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64); 920 921 val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO); 922 val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val); 923 REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val); 924 925 ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n", 926 ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume"); 927 928 return 0; 929 } 930 931 int ivpu_hw_ip_soc_cpu_boot(struct ivpu_device *vdev) 932 { 933 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 934 return soc_cpu_boot_37xx(vdev); 935 else 936 return soc_cpu_boot_40xx(vdev); 937 } 938 939 static void wdt_disable_37xx(struct ivpu_device *vdev) 940 { 941 u32 val; 942 943 /* Enable writing and set non-zero WDT value */ 944 REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); 945 REGV_WR32(VPU_37XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE); 946 947 /* Enable writing and disable watchdog timer */ 948 REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); 949 REGV_WR32(VPU_37XX_CPU_SS_TIM_WDOG_EN, 0); 950 951 /* Now clear the timeout interrupt */ 952 val = REGV_RD32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG); 953 val = REG_CLR_FLD(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val); 954 REGV_WR32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, val); 955 } 956 957 static void wdt_disable_40xx(struct ivpu_device *vdev) 958 { 959 u32 val; 960 961 REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); 962 REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE); 963 964 REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); 965 REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0); 966 967 val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG); 968 val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val); 969 REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val); 970 } 971 972 void ivpu_hw_ip_wdt_disable(struct ivpu_device *vdev) 973 { 974 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 975 return wdt_disable_37xx(vdev); 976 else 977 return wdt_disable_40xx(vdev); 978 } 979 980 static u32 ipc_rx_count_get_37xx(struct ivpu_device *vdev) 981 { 982 u32 count = readl(vdev->regv + VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT); 983 984 return REG_GET_FLD(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count); 985 } 986 987 static u32 ipc_rx_count_get_40xx(struct ivpu_device *vdev) 988 { 989 u32 count = readl(vdev->regv + VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT); 990 991 return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count); 992 } 993 994 u32 ivpu_hw_ip_ipc_rx_count_get(struct ivpu_device *vdev) 995 { 996 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 997 return ipc_rx_count_get_37xx(vdev); 998 else 999 return ipc_rx_count_get_40xx(vdev); 1000 } 1001 1002 void ivpu_hw_ip_irq_enable(struct ivpu_device *vdev) 1003 { 1004 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 1005 REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_37XX); 1006 REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_37XX); 1007 } else { 1008 REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_40XX); 1009 REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_40XX); 1010 } 1011 } 1012 1013 void ivpu_hw_ip_irq_disable(struct ivpu_device *vdev) 1014 { 1015 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) { 1016 REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, 0x0ull); 1017 REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, 0x0); 1018 } else { 1019 REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull); 1020 REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul); 1021 } 1022 } 1023 1024 static void diagnose_failure_37xx(struct ivpu_device *vdev) 1025 { 1026 u32 reg = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX; 1027 1028 if (ipc_rx_count_get_37xx(vdev)) 1029 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ"); 1030 1031 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg)) 1032 ivpu_err(vdev, "WDT MSS timeout detected\n"); 1033 1034 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg)) 1035 ivpu_err(vdev, "WDT NCE timeout detected\n"); 1036 1037 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg)) 1038 ivpu_err(vdev, "NOC Firewall irq detected\n"); 1039 } 1040 1041 static void diagnose_failure_40xx(struct ivpu_device *vdev) 1042 { 1043 u32 reg = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX; 1044 1045 if (ipc_rx_count_get_40xx(vdev)) 1046 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ"); 1047 1048 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg)) 1049 ivpu_err(vdev, "WDT MSS timeout detected\n"); 1050 1051 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg)) 1052 ivpu_err(vdev, "WDT NCE timeout detected\n"); 1053 1054 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg)) 1055 ivpu_err(vdev, "NOC Firewall irq detected\n"); 1056 } 1057 1058 void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev) 1059 { 1060 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 1061 diagnose_failure_37xx(vdev); 1062 else 1063 diagnose_failure_40xx(vdev); 1064 } 1065 1066 void ivpu_hw_ip_irq_clear(struct ivpu_device *vdev) 1067 { 1068 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 1069 REGV_WR64(VPU_37XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_37XX); 1070 else 1071 REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_40XX); 1072 } 1073 1074 static void irq_wdt_nce_handler(struct ivpu_device *vdev) 1075 { 1076 ivpu_pm_trigger_recovery(vdev, "WDT NCE IRQ"); 1077 } 1078 1079 static void irq_wdt_mss_handler(struct ivpu_device *vdev) 1080 { 1081 ivpu_hw_ip_wdt_disable(vdev); 1082 ivpu_pm_trigger_recovery(vdev, "WDT MSS IRQ"); 1083 } 1084 1085 static void irq_noc_firewall_handler(struct ivpu_device *vdev) 1086 { 1087 atomic_inc(&vdev->hw->firewall_irq_counter); 1088 1089 ivpu_dbg(vdev, IRQ, "NOC Firewall interrupt detected, counter %d\n", 1090 atomic_read(&vdev->hw->firewall_irq_counter)); 1091 } 1092 1093 /* Handler for IRQs from NPU core */ 1094 bool ivpu_hw_ip_irq_handler_37xx(struct ivpu_device *vdev, int irq) 1095 { 1096 u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX; 1097 1098 if (!status) 1099 return false; 1100 1101 REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status); 1102 1103 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status)) 1104 ivpu_mmu_irq_evtq_handler(vdev); 1105 1106 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status)) 1107 ivpu_ipc_irq_handler(vdev); 1108 1109 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status)) 1110 ivpu_dbg(vdev, IRQ, "MMU sync complete\n"); 1111 1112 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status)) 1113 ivpu_mmu_irq_gerr_handler(vdev); 1114 1115 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status)) 1116 irq_wdt_mss_handler(vdev); 1117 1118 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status)) 1119 irq_wdt_nce_handler(vdev); 1120 1121 if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status)) 1122 irq_noc_firewall_handler(vdev); 1123 1124 return true; 1125 } 1126 1127 /* Handler for IRQs from NPU core */ 1128 bool ivpu_hw_ip_irq_handler_40xx(struct ivpu_device *vdev, int irq) 1129 { 1130 u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX; 1131 1132 if (!status) 1133 return false; 1134 1135 REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status); 1136 1137 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status)) 1138 ivpu_mmu_irq_evtq_handler(vdev); 1139 1140 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status)) 1141 ivpu_ipc_irq_handler(vdev); 1142 1143 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status)) 1144 ivpu_dbg(vdev, IRQ, "MMU sync complete\n"); 1145 1146 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status)) 1147 ivpu_mmu_irq_gerr_handler(vdev); 1148 1149 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status)) 1150 irq_wdt_mss_handler(vdev); 1151 1152 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status)) 1153 irq_wdt_nce_handler(vdev); 1154 1155 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status)) 1156 irq_noc_firewall_handler(vdev); 1157 1158 return true; 1159 } 1160 1161 static void db_set_37xx(struct ivpu_device *vdev, u32 db_id) 1162 { 1163 u32 reg_stride = VPU_37XX_CPU_SS_DOORBELL_1 - VPU_37XX_CPU_SS_DOORBELL_0; 1164 u32 val = REG_FLD(VPU_37XX_CPU_SS_DOORBELL_0, SET); 1165 1166 REGV_WR32I(VPU_37XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val); 1167 } 1168 1169 static void db_set_40xx(struct ivpu_device *vdev, u32 db_id) 1170 { 1171 u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0; 1172 u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET); 1173 1174 REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val); 1175 } 1176 1177 void ivpu_hw_ip_db_set(struct ivpu_device *vdev, u32 db_id) 1178 { 1179 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 1180 db_set_37xx(vdev, db_id); 1181 else 1182 db_set_40xx(vdev, db_id); 1183 } 1184 1185 u32 ivpu_hw_ip_ipc_rx_addr_get(struct ivpu_device *vdev) 1186 { 1187 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 1188 return REGV_RD32(VPU_37XX_HOST_SS_TIM_IPC_FIFO_ATM); 1189 else 1190 return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM); 1191 } 1192 1193 void ivpu_hw_ip_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr) 1194 { 1195 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) 1196 REGV_WR32(VPU_37XX_CPU_SS_TIM_IPC_FIFO, vpu_addr); 1197 else 1198 REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr); 1199 } 1200