1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2020-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include "gaudi2P.h" 9 #include "gaudi2_masks.h" 10 #include "../include/gaudi2/gaudi2_special_blocks.h" 11 #include "../include/hw_ip/mmu/mmu_general.h" 12 #include "../include/hw_ip/mmu/mmu_v2_0.h" 13 #include "../include/gaudi2/gaudi2_packets.h" 14 #include "../include/gaudi2/gaudi2_reg_map.h" 15 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h" 16 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h" 17 18 #include <linux/module.h> 19 #include <linux/pci.h> 20 #include <linux/hwmon.h> 21 #include <linux/iommu.h> 22 23 #define GAUDI2_DMA_POOL_BLK_SIZE SZ_256 /* 256 bytes */ 24 25 #define GAUDI2_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ 26 27 #define GAUDI2_RESET_POLL_TIMEOUT_USEC 500000 /* 500ms */ 28 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC 25000 /* 25s */ 29 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC 25000 /* 25s */ 30 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC 3000000 /* 3s */ 31 #define GAUDI2_RESET_POLL_CNT 3 32 #define GAUDI2_RESET_WAIT_MSEC 1 /* 1ms */ 33 #define GAUDI2_CPU_RESET_WAIT_MSEC 100 /* 100ms */ 34 #define GAUDI2_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 35 #define GAUDI2_CB_POOL_CB_CNT 512 36 #define GAUDI2_CB_POOL_CB_SIZE SZ_128K /* 128KB */ 37 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 38 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC 25000000 /* 25s */ 39 #define GAUDI2_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */ 41 42 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT 3 43 44 /* 45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs 46 * and the code relies on that value (for array size etc..) we define another value 47 * for MAX faulty TPCs which reflects the cluster binning requirements 48 */ 49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS 1 50 #define MAX_FAULTY_XBARS 1 51 #define MAX_FAULTY_EDMAS 1 52 #define MAX_FAULTY_DECODERS 1 53 54 #define GAUDI2_TPC_FULL_MASK 0x1FFFFFF 55 #define GAUDI2_HIF_HMMU_FULL_MASK 0xFFFF 56 #define GAUDI2_DECODER_FULL_MASK 0x3FF 57 58 #define GAUDI2_NA_EVENT_CAUSE 0xFF 59 #define GAUDI2_NUM_OF_QM_ERR_CAUSE 18 60 #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE 25 61 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE 3 62 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE 14 63 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE 3 64 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE 2 65 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE 22 66 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE 31 67 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE 25 68 #define GAUDI2_NUM_OF_MME_ERR_CAUSE 16 69 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE 7 70 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE 8 71 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE 19 72 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE 9 73 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE 3 74 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE 3 75 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE 2 76 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE 2 77 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE 2 78 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE 5 79 80 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 10) 81 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 200) 82 #define GAUDI2_ARB_WDT_TIMEOUT (0x1000000) 83 84 #define GAUDI2_VDEC_TIMEOUT_USEC 10000 /* 10ms */ 85 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC (GAUDI2_VDEC_TIMEOUT_USEC * 100) 86 87 #define KDMA_TIMEOUT_USEC USEC_PER_SEC 88 89 #define IS_DMA_IDLE(dma_core_sts0) \ 90 (!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK))) 91 92 #define IS_DMA_HALTED(dma_core_sts1) \ 93 ((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK)) 94 95 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) 96 97 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK)) 98 99 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \ 100 ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \ 101 (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \ 102 (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK))) 103 104 #define PCIE_DEC_EN_MASK 0x300 105 #define DEC_WORK_STATE_IDLE 0 106 #define DEC_WORK_STATE_PEND 3 107 #define IS_DEC_IDLE(dec_swreg15) \ 108 (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \ 109 ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_PEND) 110 111 /* HBM MMU address scrambling parameters */ 112 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE SZ_8M 113 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT 26 114 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT 0 115 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK DRAM_VA_HINT_MASK 116 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR 16 117 #define MMU_RANGE_INV_VA_LSB_SHIFT 12 118 #define MMU_RANGE_INV_VA_MSB_SHIFT 44 119 #define MMU_RANGE_INV_EN_SHIFT 0 120 #define MMU_RANGE_INV_ASID_EN_SHIFT 1 121 #define MMU_RANGE_INV_ASID_SHIFT 2 122 123 /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has 124 * a 2 entries FIFO, and hence it is not enabled for it. 125 */ 126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0) 127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0) 128 129 #define GAUDI2_MAX_STRING_LEN 64 130 131 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \ 132 GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1) 133 134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) 135 136 /* RAZWI initiator coordinates */ 137 #define RAZWI_GET_AXUSER_XY(x) \ 138 ((x & 0xF8001FF0) >> 4) 139 140 #define RAZWI_GET_AXUSER_LOW_XY(x) \ 141 ((x & 0x00001FF0) >> 4) 142 143 #define RAZWI_INITIATOR_AXUER_L_X_SHIFT 0 144 #define RAZWI_INITIATOR_AXUER_L_X_MASK 0x1F 145 #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT 5 146 #define RAZWI_INITIATOR_AXUER_L_Y_MASK 0xF 147 148 #define RAZWI_INITIATOR_AXUER_H_X_SHIFT 23 149 #define RAZWI_INITIATOR_AXUER_H_X_MASK 0x1F 150 151 #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \ 152 ((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \ 153 (((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT)) 154 155 #define RAZWI_INITIATOR_ID_X_HIGH(x) \ 156 (((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT) 157 158 #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \ 159 (RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh)) 160 161 #define PSOC_RAZWI_ENG_STR_SIZE 128 162 #define PSOC_RAZWI_MAX_ENG_PER_RTR 5 163 164 /* HW scrambles only bits 0-25 */ 165 #define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26) 166 167 struct gaudi2_razwi_info { 168 u32 axuser_xy; 169 u32 rtr_ctrl; 170 u16 eng_id; 171 char *eng_name; 172 }; 173 174 static struct gaudi2_razwi_info common_razwi_info[] = { 175 {RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE, 176 GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"}, 177 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 178 GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"}, 179 {RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE, 180 GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"}, 181 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 182 GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"}, 183 {RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE, 184 GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"}, 185 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 186 GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"}, 187 {RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE, 188 GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"}, 189 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 190 GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"}, 191 {RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE, 192 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"}, 193 {RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE, 194 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"}, 195 {RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE, 196 GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"}, 197 {RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE, 198 GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"}, 199 {RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE, 200 GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"}, 201 {RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE, 202 GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"}, 203 {RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE, 204 GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"}, 205 {RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE, 206 GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"}, 207 {RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE, 208 GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"}, 209 {RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE, 210 GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"}, 211 {RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE, 212 GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"}, 213 {RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE, 214 GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"}, 215 {RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE, 216 GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"}, 217 {RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE, 218 GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"}, 219 {RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE, 220 GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"}, 221 {RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE, 222 GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"}, 223 {RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE, 224 GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"}, 225 {RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE, 226 GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"}, 227 {RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE, 228 GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"}, 229 {RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE, 230 GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"}, 231 {RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE, 232 GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"}, 233 {RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE, 234 GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"}, 235 {RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE, 236 GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"}, 237 {RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE, 238 GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"}, 239 {RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE, 240 GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"}, 241 {RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE, 242 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"}, 243 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 244 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"}, 245 {RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE, 246 GAUDI2_ENGINE_ID_NIC0_0, "NIC0"}, 247 {RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE, 248 GAUDI2_ENGINE_ID_NIC0_1, "NIC1"}, 249 {RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE, 250 GAUDI2_ENGINE_ID_NIC1_0, "NIC2"}, 251 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 252 GAUDI2_ENGINE_ID_NIC1_1, "NIC3"}, 253 {RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE, 254 GAUDI2_ENGINE_ID_NIC2_0, "NIC4"}, 255 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 256 GAUDI2_ENGINE_ID_NIC2_1, "NIC5"}, 257 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 258 GAUDI2_ENGINE_ID_NIC3_0, "NIC6"}, 259 {RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE, 260 GAUDI2_ENGINE_ID_NIC3_1, "NIC7"}, 261 {RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE, 262 GAUDI2_ENGINE_ID_NIC4_0, "NIC8"}, 263 {RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE, 264 GAUDI2_ENGINE_ID_NIC4_1, "NIC9"}, 265 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 266 GAUDI2_ENGINE_ID_NIC5_0, "NIC10"}, 267 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 268 GAUDI2_ENGINE_ID_NIC5_1, "NIC11"}, 269 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 270 GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"}, 271 {RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE, 272 GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"}, 273 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 274 GAUDI2_ENGINE_ID_SIZE, "PMMU"}, 275 {RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE, 276 GAUDI2_ENGINE_ID_SIZE, "PCIE"}, 277 {RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE, 278 GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"}, 279 {RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE, 280 GAUDI2_ENGINE_ID_KDMA, "KDMA"}, 281 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE, 282 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"}, 283 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE, 284 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"}, 285 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE, 286 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"}, 287 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE, 288 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"}, 289 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 290 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"}, 291 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 292 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"}, 293 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 294 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"}, 295 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 296 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"}, 297 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 298 GAUDI2_ENGINE_ID_SIZE, "HMMU0"}, 299 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 300 GAUDI2_ENGINE_ID_SIZE, "HMMU1"}, 301 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 302 GAUDI2_ENGINE_ID_SIZE, "HMMU2"}, 303 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 304 GAUDI2_ENGINE_ID_SIZE, "HMMU3"}, 305 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 306 GAUDI2_ENGINE_ID_SIZE, "HMMU4"}, 307 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 308 GAUDI2_ENGINE_ID_SIZE, "HMMU5"}, 309 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 310 GAUDI2_ENGINE_ID_SIZE, "HMMU6"}, 311 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 312 GAUDI2_ENGINE_ID_SIZE, "HMMU7"}, 313 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 314 GAUDI2_ENGINE_ID_SIZE, "HMMU8"}, 315 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 316 GAUDI2_ENGINE_ID_SIZE, "HMMU9"}, 317 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 318 GAUDI2_ENGINE_ID_SIZE, "HMMU10"}, 319 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 320 GAUDI2_ENGINE_ID_SIZE, "HMMU11"}, 321 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 322 GAUDI2_ENGINE_ID_SIZE, "HMMU12"}, 323 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 324 GAUDI2_ENGINE_ID_SIZE, "HMMU13"}, 325 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 326 GAUDI2_ENGINE_ID_SIZE, "HMMU14"}, 327 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 328 GAUDI2_ENGINE_ID_SIZE, "HMMU15"}, 329 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 330 GAUDI2_ENGINE_ID_ROT_0, "ROT0"}, 331 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 332 GAUDI2_ENGINE_ID_ROT_1, "ROT1"}, 333 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 334 GAUDI2_ENGINE_ID_PSOC, "CPU"}, 335 {RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE, 336 GAUDI2_ENGINE_ID_PSOC, "PSOC"} 337 }; 338 339 static struct gaudi2_razwi_info mme_razwi_info[] = { 340 /* MME X high coordinate is N/A, hence using only low coordinates */ 341 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 342 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"}, 343 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 344 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"}, 345 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 346 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"}, 347 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 348 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"}, 349 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 350 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"}, 351 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 352 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"}, 353 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 354 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"}, 355 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 356 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"}, 357 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 358 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"}, 359 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 360 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"}, 361 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 362 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"}, 363 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 364 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"}, 365 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 366 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"}, 367 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 368 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"}, 369 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 370 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"}, 371 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 372 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"}, 373 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 374 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"}, 375 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 376 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"}, 377 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 378 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"}, 379 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 380 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"}, 381 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 382 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"}, 383 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 384 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"}, 385 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 386 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"}, 387 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 388 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"}, 389 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 390 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"}, 391 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 392 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"}, 393 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 394 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"}, 395 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 396 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"}, 397 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 398 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"}, 399 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 400 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"}, 401 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 402 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"}, 403 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 404 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"}, 405 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 406 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"}, 407 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 408 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"}, 409 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 410 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"}, 411 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 412 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"} 413 }; 414 415 enum hl_pmmu_fatal_cause { 416 LATENCY_RD_OUT_FIFO_OVERRUN, 417 LATENCY_WR_OUT_FIFO_OVERRUN, 418 }; 419 420 enum hl_pcie_drain_ind_cause { 421 LBW_AXI_DRAIN_IND, 422 HBW_AXI_DRAIN_IND 423 }; 424 425 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = { 426 [HBM_ID0] = 0xFFFC, 427 [HBM_ID1] = 0xFFCF, 428 [HBM_ID2] = 0xF7F7, 429 [HBM_ID3] = 0x7F7F, 430 [HBM_ID4] = 0xFCFF, 431 [HBM_ID5] = 0xCFFF, 432 }; 433 434 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = { 435 [0] = HBM_ID0, 436 [1] = HBM_ID1, 437 [2] = HBM_ID4, 438 [3] = HBM_ID5, 439 }; 440 441 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = { 442 [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0, 443 [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2, 444 [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1, 445 [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3, 446 [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2, 447 [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4, 448 [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3, 449 [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5, 450 }; 451 452 static const int gaudi2_qman_async_event_id[] = { 453 [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM, 454 [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM, 455 [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM, 456 [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM, 457 [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM, 458 [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM, 459 [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM, 460 [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM, 461 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM, 462 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM, 463 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM, 464 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM, 465 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM, 466 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM, 467 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM, 468 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM, 469 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM, 470 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM, 471 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM, 472 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM, 473 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM, 474 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM, 475 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM, 476 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM, 477 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM, 478 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM, 479 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM, 480 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM, 481 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM, 482 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM, 483 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM, 484 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM, 485 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM, 486 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM, 487 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM, 488 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM, 489 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM, 490 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM, 491 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM, 492 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM, 493 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM, 494 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM, 495 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM, 496 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM, 497 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM, 498 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM, 499 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM, 500 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM, 501 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM, 502 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM, 503 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM, 504 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM, 505 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM, 506 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM, 507 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM, 508 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM, 509 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM, 510 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM, 511 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM, 512 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM, 513 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM, 514 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM, 515 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM, 516 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM, 517 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM, 518 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM, 519 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM, 520 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM, 521 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM, 522 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM, 523 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM, 524 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM, 525 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM, 526 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM, 527 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM, 528 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM, 529 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM, 530 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM, 531 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM, 532 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM, 533 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM, 534 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM, 535 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM, 536 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM, 537 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM, 538 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM, 539 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM, 540 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM, 541 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM, 542 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM, 543 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM, 544 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM, 545 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM, 546 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM, 547 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM, 548 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM, 549 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM, 550 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM, 551 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM, 552 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM, 553 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM, 554 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM, 555 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM, 556 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM, 557 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM, 558 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM, 559 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM, 560 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM, 561 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM, 562 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM, 563 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM, 564 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM, 565 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM, 566 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM, 567 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM, 568 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM, 569 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM, 570 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM, 571 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM, 572 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM, 573 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM, 574 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM, 575 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM, 576 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM, 577 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM, 578 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM, 579 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM, 580 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM, 581 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM, 582 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM, 583 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM, 584 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM, 585 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM, 586 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM, 587 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM, 588 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM, 589 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM, 590 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM, 591 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM, 592 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM, 593 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM, 594 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM, 595 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM, 596 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM, 597 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM, 598 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM, 599 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM, 600 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM, 601 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM, 602 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM, 603 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM, 604 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM, 605 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM, 606 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM, 607 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM, 608 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM, 609 [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0, 610 [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0, 611 [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0, 612 [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0, 613 [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1, 614 [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1, 615 [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1, 616 [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1, 617 [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0, 618 [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0, 619 [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0, 620 [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0, 621 [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1, 622 [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1, 623 [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1, 624 [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1, 625 [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0, 626 [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0, 627 [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0, 628 [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0, 629 [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1, 630 [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1, 631 [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1, 632 [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1, 633 [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0, 634 [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0, 635 [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0, 636 [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0, 637 [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1, 638 [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1, 639 [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1, 640 [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1, 641 [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0, 642 [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0, 643 [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0, 644 [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0, 645 [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1, 646 [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1, 647 [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1, 648 [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1, 649 [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0, 650 [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0, 651 [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0, 652 [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0, 653 [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1, 654 [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1, 655 [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1, 656 [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1, 657 [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0, 658 [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0, 659 [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0, 660 [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0, 661 [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1, 662 [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1, 663 [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1, 664 [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1, 665 [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0, 666 [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0, 667 [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0, 668 [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0, 669 [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1, 670 [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1, 671 [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1, 672 [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1, 673 [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0, 674 [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0, 675 [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0, 676 [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0, 677 [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1, 678 [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1, 679 [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1, 680 [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1, 681 [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0, 682 [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0, 683 [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0, 684 [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0, 685 [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1, 686 [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1, 687 [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1, 688 [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1, 689 [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0, 690 [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0, 691 [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0, 692 [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0, 693 [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1, 694 [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1, 695 [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1, 696 [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1, 697 [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0, 698 [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0, 699 [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0, 700 [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0, 701 [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1, 702 [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1, 703 [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1, 704 [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1, 705 [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 706 [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 707 [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 708 [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 709 [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 710 [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 711 [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 712 [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM 713 }; 714 715 static const int gaudi2_dma_core_async_event_id[] = { 716 [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE, 717 [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE, 718 [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE, 719 [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE, 720 [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE, 721 [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE, 722 [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE, 723 [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE, 724 [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE, 725 [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE, 726 [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE, 727 }; 728 729 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = { 730 "qman sei intr", 731 "arc sei intr" 732 }; 733 734 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = { 735 "AXI_TERMINATOR WR", 736 "AXI_TERMINATOR RD", 737 "AXI SPLIT SEI Status" 738 }; 739 740 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = { 741 "cbu_bresp_sei_intr_cause", 742 "cbu_rresp_sei_intr_cause", 743 "lbu_bresp_sei_intr_cause", 744 "lbu_rresp_sei_intr_cause", 745 "cbu_axi_split_intr_cause", 746 "lbu_axi_split_intr_cause", 747 "arc_ip_excptn_sei_intr_cause", 748 "dmi_bresp_sei_intr_cause", 749 "aux2apb_err_sei_intr_cause", 750 "cfg_lbw_wr_terminated_intr_cause", 751 "cfg_lbw_rd_terminated_intr_cause", 752 "cfg_dccm_wr_terminated_intr_cause", 753 "cfg_dccm_rd_terminated_intr_cause", 754 "cfg_hbw_rd_terminated_intr_cause" 755 }; 756 757 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = { 758 "msix_vcd_hbw_sei", 759 "msix_l2c_hbw_sei", 760 "msix_nrm_hbw_sei", 761 "msix_abnrm_hbw_sei", 762 "msix_vcd_lbw_sei", 763 "msix_l2c_lbw_sei", 764 "msix_nrm_lbw_sei", 765 "msix_abnrm_lbw_sei", 766 "apb_vcd_lbw_sei", 767 "apb_l2c_lbw_sei", 768 "apb_nrm_lbw_sei", 769 "apb_abnrm_lbw_sei", 770 "dec_sei", 771 "dec_apb_sei", 772 "trc_apb_sei", 773 "lbw_mstr_if_sei", 774 "axi_split_bresp_err_sei", 775 "hbw_axi_wr_viol_sei", 776 "hbw_axi_rd_viol_sei", 777 "lbw_axi_wr_viol_sei", 778 "lbw_axi_rd_viol_sei", 779 "vcd_spi", 780 "l2c_spi", 781 "nrm_spi", 782 "abnrm_spi", 783 }; 784 785 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = { 786 "PQ AXI HBW error", 787 "CQ AXI HBW error", 788 "CP AXI HBW error", 789 "CP error due to undefined OPCODE", 790 "CP encountered STOP OPCODE", 791 "CP AXI LBW error", 792 "CP WRREG32 or WRBULK returned error", 793 "N/A", 794 "FENCE 0 inc over max value and clipped", 795 "FENCE 1 inc over max value and clipped", 796 "FENCE 2 inc over max value and clipped", 797 "FENCE 3 inc over max value and clipped", 798 "FENCE 0 dec under min value and clipped", 799 "FENCE 1 dec under min value and clipped", 800 "FENCE 2 dec under min value and clipped", 801 "FENCE 3 dec under min value and clipped", 802 "CPDMA Up overflow", 803 "PQC L2H error" 804 }; 805 806 static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = { 807 "RSVD0", 808 "CQ AXI HBW error", 809 "CP AXI HBW error", 810 "CP error due to undefined OPCODE", 811 "CP encountered STOP OPCODE", 812 "CP AXI LBW error", 813 "CP WRREG32 or WRBULK returned error", 814 "N/A", 815 "FENCE 0 inc over max value and clipped", 816 "FENCE 1 inc over max value and clipped", 817 "FENCE 2 inc over max value and clipped", 818 "FENCE 3 inc over max value and clipped", 819 "FENCE 0 dec under min value and clipped", 820 "FENCE 1 dec under min value and clipped", 821 "FENCE 2 dec under min value and clipped", 822 "FENCE 3 dec under min value and clipped", 823 "CPDMA Up overflow", 824 "RSVD17", 825 "CQ_WR_IFIFO_CI_ERR", 826 "CQ_WR_CTL_CI_ERR", 827 "ARC_CQF_RD_ERR", 828 "ARC_CQ_WR_IFIFO_CI_ERR", 829 "ARC_CQ_WR_CTL_CI_ERR", 830 "ARC_AXI_ERR", 831 "CP_SWITCH_WDT_ERR" 832 }; 833 834 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = { 835 "Choice push while full error", 836 "Choice Q watchdog error", 837 "MSG AXI LBW returned with error" 838 }; 839 840 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = { 841 "qm_axi_err", 842 "qm_trace_fence_events", 843 "qm_sw_err", 844 "qm_cp_sw_stop", 845 "lbw_mstr_rresp_err", 846 "lbw_mstr_bresp_err", 847 "lbw_msg_slverr", 848 "hbw_msg_slverr", 849 "wbc_slverr", 850 "hbw_mstr_rresp_err", 851 "hbw_mstr_bresp_err", 852 "sb_resp_intr", 853 "mrsb_resp_intr", 854 "core_dw_status_0", 855 "core_dw_status_1", 856 "core_dw_status_2", 857 "core_dw_status_3", 858 "core_dw_status_4", 859 "core_dw_status_5", 860 "core_dw_status_6", 861 "core_dw_status_7", 862 "async_arc2cpu_sei_intr", 863 }; 864 865 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = { 866 "tpc_address_exceed_slm", 867 "tpc_div_by_0", 868 "tpc_spu_mac_overflow", 869 "tpc_spu_addsub_overflow", 870 "tpc_spu_abs_overflow", 871 "tpc_spu_fma_fp_dst_nan", 872 "tpc_spu_fma_fp_dst_inf", 873 "tpc_spu_convert_fp_dst_nan", 874 "tpc_spu_convert_fp_dst_inf", 875 "tpc_spu_fp_dst_denorm", 876 "tpc_vpu_mac_overflow", 877 "tpc_vpu_addsub_overflow", 878 "tpc_vpu_abs_overflow", 879 "tpc_vpu_convert_fp_dst_nan", 880 "tpc_vpu_convert_fp_dst_inf", 881 "tpc_vpu_fma_fp_dst_nan", 882 "tpc_vpu_fma_fp_dst_inf", 883 "tpc_vpu_fp_dst_denorm", 884 "tpc_assertions", 885 "tpc_illegal_instruction", 886 "tpc_pc_wrap_around", 887 "tpc_qm_sw_err", 888 "tpc_hbw_rresp_err", 889 "tpc_hbw_bresp_err", 890 "tpc_lbw_rresp_err", 891 "tpc_lbw_bresp_err", 892 "st_unlock_already_locked", 893 "invalid_lock_access", 894 "LD_L protection violation", 895 "ST_L protection violation", 896 "D$ L0CS mismatch", 897 }; 898 899 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = { 900 "agu_resp_intr", 901 "qman_axi_err", 902 "wap sei (wbc axi err)", 903 "arc sei", 904 "cfg access error", 905 "qm_sw_err", 906 "sbte_dbg_intr_0", 907 "sbte_dbg_intr_1", 908 "sbte_dbg_intr_2", 909 "sbte_dbg_intr_3", 910 "sbte_dbg_intr_4", 911 "sbte_prtn_intr_0", 912 "sbte_prtn_intr_1", 913 "sbte_prtn_intr_2", 914 "sbte_prtn_intr_3", 915 "sbte_prtn_intr_4", 916 }; 917 918 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = { 919 "WBC ERR RESP_0", 920 "WBC ERR RESP_1", 921 "AP SOURCE POS INF", 922 "AP SOURCE NEG INF", 923 "AP SOURCE NAN", 924 "AP RESULT POS INF", 925 "AP RESULT NEG INF", 926 }; 927 928 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 929 "HBW Read returned with error RRESP", 930 "HBW write returned with error BRESP", 931 "LBW write returned with error BRESP", 932 "descriptor_fifo_overflow", 933 "KDMA SB LBW Read returned with error", 934 "KDMA WBC LBW Write returned with error", 935 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 936 "WRONG CFG FOR COMMIT IN LIN DMA" 937 }; 938 939 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 940 "HBW/LBW Read returned with error RRESP", 941 "HBW/LBW write returned with error BRESP", 942 "LBW write returned with error BRESP", 943 "descriptor_fifo_overflow", 944 "KDMA SB LBW Read returned with error", 945 "KDMA WBC LBW Write returned with error", 946 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 947 "WRONG CFG FOR COMMIT IN LIN DMA" 948 }; 949 950 struct gaudi2_sm_sei_cause_data { 951 const char *cause_name; 952 const char *log_name; 953 }; 954 955 static const struct gaudi2_sm_sei_cause_data 956 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = { 957 {"calculated SO value overflow/underflow", "SOB ID"}, 958 {"payload address of monitor is not aligned to 4B", "monitor addr"}, 959 {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"}, 960 }; 961 962 static const char * const 963 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = { 964 "LATENCY_RD_OUT_FIFO_OVERRUN", 965 "LATENCY_WR_OUT_FIFO_OVERRUN", 966 }; 967 968 static const char * const 969 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = { 970 "LATENCY_RD_OUT_FIFO_OVERRUN", 971 "LATENCY_WR_OUT_FIFO_OVERRUN", 972 }; 973 974 static const char * const 975 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = { 976 "AXI drain HBW", 977 "AXI drain LBW", 978 }; 979 980 static const char * const 981 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = { 982 "HBW error response", 983 "LBW error response", 984 "TLP is blocked by RR" 985 }; 986 987 static const int gaudi2_queue_id_to_engine_id[] = { 988 [GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_ENGINE_ID_PDMA_0, 989 [GAUDI2_QUEUE_ID_PDMA_1_0...GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_ENGINE_ID_PDMA_1, 990 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = 991 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, 992 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = 993 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, 994 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = 995 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, 996 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = 997 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, 998 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = 999 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, 1000 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = 1001 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, 1002 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = 1003 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, 1004 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = 1005 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, 1006 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = 1007 GAUDI2_DCORE0_ENGINE_ID_MME, 1008 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = 1009 GAUDI2_DCORE1_ENGINE_ID_MME, 1010 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = 1011 GAUDI2_DCORE2_ENGINE_ID_MME, 1012 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = 1013 GAUDI2_DCORE3_ENGINE_ID_MME, 1014 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0...GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = 1015 GAUDI2_DCORE0_ENGINE_ID_TPC_0, 1016 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0...GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = 1017 GAUDI2_DCORE0_ENGINE_ID_TPC_1, 1018 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0...GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = 1019 GAUDI2_DCORE0_ENGINE_ID_TPC_2, 1020 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0...GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = 1021 GAUDI2_DCORE0_ENGINE_ID_TPC_3, 1022 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0...GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = 1023 GAUDI2_DCORE0_ENGINE_ID_TPC_4, 1024 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0...GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = 1025 GAUDI2_DCORE0_ENGINE_ID_TPC_5, 1026 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0...GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = 1027 GAUDI2_DCORE0_ENGINE_ID_TPC_6, 1028 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0...GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = 1029 GAUDI2_DCORE1_ENGINE_ID_TPC_0, 1030 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0...GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = 1031 GAUDI2_DCORE1_ENGINE_ID_TPC_1, 1032 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0...GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = 1033 GAUDI2_DCORE1_ENGINE_ID_TPC_2, 1034 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0...GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = 1035 GAUDI2_DCORE1_ENGINE_ID_TPC_3, 1036 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0...GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = 1037 GAUDI2_DCORE1_ENGINE_ID_TPC_4, 1038 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0...GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = 1039 GAUDI2_DCORE1_ENGINE_ID_TPC_5, 1040 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0...GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = 1041 GAUDI2_DCORE2_ENGINE_ID_TPC_0, 1042 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0...GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = 1043 GAUDI2_DCORE2_ENGINE_ID_TPC_1, 1044 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0...GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = 1045 GAUDI2_DCORE2_ENGINE_ID_TPC_2, 1046 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0...GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = 1047 GAUDI2_DCORE2_ENGINE_ID_TPC_3, 1048 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0...GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = 1049 GAUDI2_DCORE2_ENGINE_ID_TPC_4, 1050 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0...GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = 1051 GAUDI2_DCORE2_ENGINE_ID_TPC_5, 1052 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0...GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = 1053 GAUDI2_DCORE3_ENGINE_ID_TPC_0, 1054 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0...GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = 1055 GAUDI2_DCORE3_ENGINE_ID_TPC_1, 1056 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0...GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = 1057 GAUDI2_DCORE3_ENGINE_ID_TPC_2, 1058 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0...GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = 1059 GAUDI2_DCORE3_ENGINE_ID_TPC_3, 1060 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0...GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = 1061 GAUDI2_DCORE3_ENGINE_ID_TPC_4, 1062 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0...GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = 1063 GAUDI2_DCORE3_ENGINE_ID_TPC_5, 1064 [GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_ENGINE_ID_NIC0_0, 1065 [GAUDI2_QUEUE_ID_NIC_1_0...GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_ENGINE_ID_NIC0_1, 1066 [GAUDI2_QUEUE_ID_NIC_2_0...GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_ENGINE_ID_NIC1_0, 1067 [GAUDI2_QUEUE_ID_NIC_3_0...GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_ENGINE_ID_NIC1_1, 1068 [GAUDI2_QUEUE_ID_NIC_4_0...GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_ENGINE_ID_NIC2_0, 1069 [GAUDI2_QUEUE_ID_NIC_5_0...GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_ENGINE_ID_NIC2_1, 1070 [GAUDI2_QUEUE_ID_NIC_6_0...GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_ENGINE_ID_NIC3_0, 1071 [GAUDI2_QUEUE_ID_NIC_7_0...GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_ENGINE_ID_NIC3_1, 1072 [GAUDI2_QUEUE_ID_NIC_8_0...GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_ENGINE_ID_NIC4_0, 1073 [GAUDI2_QUEUE_ID_NIC_9_0...GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_ENGINE_ID_NIC4_1, 1074 [GAUDI2_QUEUE_ID_NIC_10_0...GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_ENGINE_ID_NIC5_0, 1075 [GAUDI2_QUEUE_ID_NIC_11_0...GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_ENGINE_ID_NIC5_1, 1076 [GAUDI2_QUEUE_ID_NIC_12_0...GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_ENGINE_ID_NIC6_0, 1077 [GAUDI2_QUEUE_ID_NIC_13_0...GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_ENGINE_ID_NIC6_1, 1078 [GAUDI2_QUEUE_ID_NIC_14_0...GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_ENGINE_ID_NIC7_0, 1079 [GAUDI2_QUEUE_ID_NIC_15_0...GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_ENGINE_ID_NIC7_1, 1080 [GAUDI2_QUEUE_ID_NIC_16_0...GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_ENGINE_ID_NIC8_0, 1081 [GAUDI2_QUEUE_ID_NIC_17_0...GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_ENGINE_ID_NIC8_1, 1082 [GAUDI2_QUEUE_ID_NIC_18_0...GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_ENGINE_ID_NIC9_0, 1083 [GAUDI2_QUEUE_ID_NIC_19_0...GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_ENGINE_ID_NIC9_1, 1084 [GAUDI2_QUEUE_ID_NIC_20_0...GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_ENGINE_ID_NIC10_0, 1085 [GAUDI2_QUEUE_ID_NIC_21_0...GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_ENGINE_ID_NIC10_1, 1086 [GAUDI2_QUEUE_ID_NIC_22_0...GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_ENGINE_ID_NIC11_0, 1087 [GAUDI2_QUEUE_ID_NIC_23_0...GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_ENGINE_ID_NIC11_1, 1088 [GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_ENGINE_ID_ROT_0, 1089 [GAUDI2_QUEUE_ID_ROT_1_0...GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_ENGINE_ID_ROT_1, 1090 }; 1091 1092 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = { 1093 [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE, 1094 [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE, 1095 [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE, 1096 [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE, 1097 [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE, 1098 [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE, 1099 [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE, 1100 [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE, 1101 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE, 1102 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE, 1103 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE, 1104 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE, 1105 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE, 1106 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE, 1107 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE, 1108 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE, 1109 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE, 1110 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE, 1111 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE, 1112 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE, 1113 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE, 1114 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE, 1115 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE, 1116 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE, 1117 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE, 1118 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE, 1119 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE, 1120 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE, 1121 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE, 1122 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE, 1123 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE, 1124 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE, 1125 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE, 1126 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE, 1127 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE, 1128 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE, 1129 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE, 1130 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE, 1131 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE, 1132 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE, 1133 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE, 1134 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE, 1135 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE, 1136 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE, 1137 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE, 1138 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE, 1139 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE, 1140 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE, 1141 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE, 1142 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE, 1143 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE, 1144 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE, 1145 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE, 1146 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE, 1147 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE, 1148 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE, 1149 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE, 1150 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE, 1151 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE, 1152 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE, 1153 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE, 1154 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE, 1155 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE, 1156 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE, 1157 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE, 1158 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE, 1159 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE, 1160 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE, 1161 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE, 1162 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE, 1163 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE, 1164 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE, 1165 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE, 1166 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE, 1167 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE, 1168 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE, 1169 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE, 1170 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE, 1171 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE, 1172 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE, 1173 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE, 1174 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE, 1175 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE, 1176 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE, 1177 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE, 1178 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE, 1179 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE, 1180 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE, 1181 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE, 1182 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE, 1183 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE, 1184 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE, 1185 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE, 1186 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE, 1187 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE, 1188 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE, 1189 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE, 1190 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE, 1191 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE, 1192 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE, 1193 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE, 1194 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE, 1195 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE, 1196 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE, 1197 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE, 1198 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE, 1199 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE, 1200 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE, 1201 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE, 1202 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE, 1203 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE, 1204 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE, 1205 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE, 1206 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE, 1207 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE, 1208 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE, 1209 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE, 1210 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE, 1211 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE, 1212 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE, 1213 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE, 1214 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE, 1215 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE, 1216 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE, 1217 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE, 1218 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE, 1219 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE, 1220 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE, 1221 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE, 1222 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE, 1223 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE, 1224 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE, 1225 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE, 1226 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE, 1227 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE, 1228 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE, 1229 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE, 1230 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE, 1231 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE, 1232 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE, 1233 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE, 1234 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE, 1235 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE, 1236 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE, 1237 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE, 1238 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE, 1239 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE, 1240 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE, 1241 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE, 1242 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE, 1243 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE, 1244 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE, 1245 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE, 1246 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE, 1247 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE, 1248 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE, 1249 [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE, 1250 [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE, 1251 [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE, 1252 [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE, 1253 [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE, 1254 [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE, 1255 [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE, 1256 [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE, 1257 [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE, 1258 [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE, 1259 [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE, 1260 [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE, 1261 [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE, 1262 [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE, 1263 [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE, 1264 [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE, 1265 [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE, 1266 [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE, 1267 [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE, 1268 [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE, 1269 [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE, 1270 [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE, 1271 [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE, 1272 [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE, 1273 [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE, 1274 [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE, 1275 [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE, 1276 [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE, 1277 [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE, 1278 [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE, 1279 [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE, 1280 [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE, 1281 [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE, 1282 [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE, 1283 [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE, 1284 [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE, 1285 [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE, 1286 [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE, 1287 [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE, 1288 [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE, 1289 [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE, 1290 [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE, 1291 [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE, 1292 [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE, 1293 [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE, 1294 [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE, 1295 [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE, 1296 [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE, 1297 [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE, 1298 [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE, 1299 [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE, 1300 [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE, 1301 [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE, 1302 [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE, 1303 [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE, 1304 [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE, 1305 [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE, 1306 [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE, 1307 [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE, 1308 [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE, 1309 [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE, 1310 [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE, 1311 [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE, 1312 [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE, 1313 [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE, 1314 [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE, 1315 [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE, 1316 [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE, 1317 [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE, 1318 [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE, 1319 [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE, 1320 [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE, 1321 [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE, 1322 [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE, 1323 [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE, 1324 [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE, 1325 [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE, 1326 [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE, 1327 [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE, 1328 [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE, 1329 [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE, 1330 [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE, 1331 [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE, 1332 [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE, 1333 [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE, 1334 [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE, 1335 [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE, 1336 [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE, 1337 [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE, 1338 [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE, 1339 [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE, 1340 [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE, 1341 [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE, 1342 [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE, 1343 [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE, 1344 [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE, 1345 [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE, 1346 [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE, 1347 [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE, 1348 [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE, 1349 [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE, 1350 [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE, 1351 [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE, 1352 [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE 1353 }; 1354 1355 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = { 1356 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE, 1357 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE, 1358 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE, 1359 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE, 1360 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE, 1361 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE, 1362 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE, 1363 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE, 1364 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE, 1365 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE, 1366 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE, 1367 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE, 1368 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE, 1369 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE, 1370 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE, 1371 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE, 1372 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE, 1373 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE, 1374 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE, 1375 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE, 1376 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE, 1377 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE, 1378 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE, 1379 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE, 1380 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE, 1381 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE, 1382 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE, 1383 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE, 1384 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE, 1385 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE, 1386 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE, 1387 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE, 1388 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE, 1389 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE, 1390 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE, 1391 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE, 1392 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE, 1393 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE, 1394 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE, 1395 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE, 1396 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE, 1397 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE, 1398 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE, 1399 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE, 1400 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE, 1401 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE, 1402 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE, 1403 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE, 1404 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE, 1405 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE, 1406 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE, 1407 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE, 1408 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE, 1409 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE, 1410 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE, 1411 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE, 1412 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE, 1413 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE, 1414 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE, 1415 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE, 1416 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE, 1417 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE, 1418 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE, 1419 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE, 1420 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE, 1421 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE, 1422 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE, 1423 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE, 1424 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE, 1425 }; 1426 1427 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = { 1428 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE, 1429 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE, 1430 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE, 1431 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE, 1432 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE, 1433 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE, 1434 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE, 1435 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE, 1436 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE, 1437 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE, 1438 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE, 1439 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE, 1440 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE, 1441 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE, 1442 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE, 1443 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE, 1444 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE, 1445 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE, 1446 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE, 1447 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE, 1448 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE, 1449 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE, 1450 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE, 1451 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE, 1452 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE, 1453 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE, 1454 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE, 1455 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE, 1456 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE, 1457 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE, 1458 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE, 1459 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE, 1460 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE, 1461 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE, 1462 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE, 1463 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE, 1464 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE, 1465 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE, 1466 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE, 1467 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE, 1468 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE, 1469 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE, 1470 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE, 1471 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE, 1472 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE, 1473 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE, 1474 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE, 1475 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE, 1476 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE, 1477 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE, 1478 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE, 1479 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE, 1480 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE, 1481 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE, 1482 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE, 1483 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE, 1484 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE, 1485 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE, 1486 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE, 1487 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE, 1488 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE, 1489 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE, 1490 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE, 1491 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE, 1492 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE, 1493 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE, 1494 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE, 1495 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE, 1496 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE, 1497 }; 1498 1499 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = { 1500 [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE, 1501 [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE, 1502 [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE, 1503 [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE, 1504 }; 1505 1506 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = { 1507 [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0, 1508 [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0, 1509 [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0, 1510 [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0, 1511 [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1, 1512 [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1, 1513 [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1, 1514 [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1, 1515 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0, 1516 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0, 1517 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0, 1518 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0, 1519 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1, 1520 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1, 1521 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1, 1522 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1, 1523 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0, 1524 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0, 1525 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0, 1526 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0, 1527 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0, 1528 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0, 1529 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0, 1530 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0, 1531 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1, 1532 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1, 1533 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1, 1534 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1, 1535 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2, 1536 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2, 1537 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2, 1538 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2, 1539 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3, 1540 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3, 1541 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3, 1542 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3, 1543 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4, 1544 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4, 1545 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4, 1546 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4, 1547 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5, 1548 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5, 1549 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5, 1550 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5, 1551 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24, 1552 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24, 1553 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24, 1554 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24, 1555 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2, 1556 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2, 1557 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2, 1558 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2, 1559 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3, 1560 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3, 1561 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3, 1562 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3, 1563 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4, 1564 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4, 1565 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4, 1566 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4, 1567 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6, 1568 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6, 1569 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6, 1570 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6, 1571 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7, 1572 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7, 1573 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7, 1574 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7, 1575 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8, 1576 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8, 1577 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8, 1578 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8, 1579 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9, 1580 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9, 1581 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9, 1582 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9, 1583 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10, 1584 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10, 1585 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10, 1586 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10, 1587 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11, 1588 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11, 1589 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11, 1590 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11, 1591 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4, 1592 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4, 1593 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4, 1594 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4, 1595 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5, 1596 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5, 1597 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5, 1598 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5, 1599 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1, 1600 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1, 1601 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1, 1602 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1, 1603 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12, 1604 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12, 1605 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12, 1606 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12, 1607 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13, 1608 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13, 1609 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13, 1610 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13, 1611 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14, 1612 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14, 1613 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14, 1614 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14, 1615 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15, 1616 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15, 1617 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15, 1618 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15, 1619 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16, 1620 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16, 1621 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16, 1622 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16, 1623 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17, 1624 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17, 1625 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17, 1626 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17, 1627 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6, 1628 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6, 1629 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6, 1630 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6, 1631 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7, 1632 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7, 1633 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7, 1634 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7, 1635 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5, 1636 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5, 1637 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5, 1638 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5, 1639 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18, 1640 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18, 1641 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18, 1642 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18, 1643 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19, 1644 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19, 1645 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19, 1646 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19, 1647 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20, 1648 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20, 1649 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20, 1650 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20, 1651 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21, 1652 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21, 1653 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21, 1654 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21, 1655 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22, 1656 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22, 1657 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22, 1658 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22, 1659 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23, 1660 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23, 1661 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23, 1662 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23, 1663 [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0, 1664 [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0, 1665 [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0, 1666 [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0, 1667 [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1, 1668 [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1, 1669 [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1, 1670 [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1, 1671 [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2, 1672 [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2, 1673 [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2, 1674 [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2, 1675 [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3, 1676 [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3, 1677 [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3, 1678 [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3, 1679 [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4, 1680 [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4, 1681 [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4, 1682 [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4, 1683 [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5, 1684 [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5, 1685 [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5, 1686 [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5, 1687 [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6, 1688 [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6, 1689 [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6, 1690 [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6, 1691 [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7, 1692 [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7, 1693 [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7, 1694 [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7, 1695 [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8, 1696 [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8, 1697 [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8, 1698 [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8, 1699 [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9, 1700 [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9, 1701 [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9, 1702 [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9, 1703 [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10, 1704 [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10, 1705 [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10, 1706 [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10, 1707 [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11, 1708 [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11, 1709 [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11, 1710 [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11, 1711 [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12, 1712 [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12, 1713 [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12, 1714 [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12, 1715 [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13, 1716 [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13, 1717 [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13, 1718 [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13, 1719 [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14, 1720 [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14, 1721 [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14, 1722 [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14, 1723 [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15, 1724 [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15, 1725 [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15, 1726 [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15, 1727 [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16, 1728 [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16, 1729 [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16, 1730 [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16, 1731 [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17, 1732 [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17, 1733 [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17, 1734 [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17, 1735 [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18, 1736 [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18, 1737 [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18, 1738 [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18, 1739 [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19, 1740 [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19, 1741 [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19, 1742 [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19, 1743 [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20, 1744 [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20, 1745 [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20, 1746 [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20, 1747 [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21, 1748 [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21, 1749 [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21, 1750 [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21, 1751 [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22, 1752 [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22, 1753 [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22, 1754 [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22, 1755 [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23, 1756 [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23, 1757 [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23, 1758 [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23, 1759 [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0, 1760 [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0, 1761 [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0, 1762 [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0, 1763 [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1, 1764 [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1, 1765 [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1, 1766 [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1 1767 }; 1768 1769 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = { 1770 [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE, 1771 [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE, 1772 [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE, 1773 [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE, 1774 [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE, 1775 [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE, 1776 [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE, 1777 [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE, 1778 [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE, 1779 [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE, 1780 [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE 1781 }; 1782 1783 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = { 1784 [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE, 1785 [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE, 1786 [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE, 1787 [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE 1788 }; 1789 1790 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = { 1791 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE, 1792 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE, 1793 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE, 1794 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE, 1795 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE, 1796 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE, 1797 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE, 1798 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE, 1799 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE, 1800 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE, 1801 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE, 1802 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE, 1803 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE, 1804 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE, 1805 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE, 1806 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE, 1807 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE, 1808 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE, 1809 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE, 1810 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE, 1811 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE, 1812 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE, 1813 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE, 1814 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE, 1815 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE, 1816 }; 1817 1818 static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = { 1819 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE, 1820 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE, 1821 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE, 1822 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE, 1823 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE, 1824 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE, 1825 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE, 1826 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE, 1827 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE, 1828 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE, 1829 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE, 1830 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE, 1831 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE, 1832 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE, 1833 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE, 1834 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE, 1835 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE, 1836 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE, 1837 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE, 1838 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE, 1839 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE, 1840 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE, 1841 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE, 1842 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE, 1843 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE, 1844 }; 1845 1846 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = { 1847 [ROTATOR_ID_0] = mmROT0_BASE, 1848 [ROTATOR_ID_1] = mmROT1_BASE 1849 }; 1850 1851 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = { 1852 [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0, 1853 [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0, 1854 [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0, 1855 [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0, 1856 [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0, 1857 [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0, 1858 [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0, 1859 [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0, 1860 [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0, 1861 [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0, 1862 [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0, 1863 [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0, 1864 [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0, 1865 [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0, 1866 [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0, 1867 [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0, 1868 [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0, 1869 [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0, 1870 [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0, 1871 [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0, 1872 [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0, 1873 [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0, 1874 [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0, 1875 [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0, 1876 [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0, 1877 }; 1878 1879 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = { 1880 [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0, 1881 [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0, 1882 }; 1883 1884 static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = { 1885 [GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0, 1886 [GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1, 1887 [GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2, 1888 [GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3, 1889 [GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4, 1890 [GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5, 1891 [GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0, 1892 [GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1, 1893 [GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2, 1894 [GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3, 1895 [GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4, 1896 [GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5, 1897 [GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0, 1898 [GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1, 1899 [GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2, 1900 [GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3, 1901 [GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4, 1902 [GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5, 1903 [GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0, 1904 [GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1, 1905 [GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2, 1906 [GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3, 1907 [GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4, 1908 [GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5, 1909 /* the PCI TPC is placed last (mapped liked HW) */ 1910 [GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6, 1911 }; 1912 1913 static const u32 gaudi2_mme_engine_id_to_mme_id[] = { 1914 [GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0, 1915 [GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1, 1916 [GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2, 1917 [GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3, 1918 }; 1919 1920 static const u32 gaudi2_edma_engine_id_to_edma_id[] = { 1921 [GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0, 1922 [GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1, 1923 [GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0, 1924 [GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1, 1925 [GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2, 1926 [GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3, 1927 [GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4, 1928 [GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5, 1929 [GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6, 1930 [GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7, 1931 [GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA, 1932 }; 1933 1934 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 1935 GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 1936 GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0, 1937 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 1938 GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0, 1939 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 1940 GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0, 1941 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0, 1942 GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0, 1943 }; 1944 1945 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = { 1946 "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal", 1947 "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal", 1948 "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal", 1949 "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal", 1950 "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal", 1951 "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal", 1952 "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal", 1953 "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal", 1954 "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal", 1955 "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal" 1956 }; 1957 1958 enum rtr_id { 1959 DCORE0_RTR0, 1960 DCORE0_RTR1, 1961 DCORE0_RTR2, 1962 DCORE0_RTR3, 1963 DCORE0_RTR4, 1964 DCORE0_RTR5, 1965 DCORE0_RTR6, 1966 DCORE0_RTR7, 1967 DCORE1_RTR0, 1968 DCORE1_RTR1, 1969 DCORE1_RTR2, 1970 DCORE1_RTR3, 1971 DCORE1_RTR4, 1972 DCORE1_RTR5, 1973 DCORE1_RTR6, 1974 DCORE1_RTR7, 1975 DCORE2_RTR0, 1976 DCORE2_RTR1, 1977 DCORE2_RTR2, 1978 DCORE2_RTR3, 1979 DCORE2_RTR4, 1980 DCORE2_RTR5, 1981 DCORE2_RTR6, 1982 DCORE2_RTR7, 1983 DCORE3_RTR0, 1984 DCORE3_RTR1, 1985 DCORE3_RTR2, 1986 DCORE3_RTR3, 1987 DCORE3_RTR4, 1988 DCORE3_RTR5, 1989 DCORE3_RTR6, 1990 DCORE3_RTR7, 1991 }; 1992 1993 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 1994 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3, 1995 DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4, 1996 DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, 1997 DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, 1998 DCORE0_RTR0 1999 }; 2000 2001 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 2002 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, 2003 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, 2004 DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0, 2005 DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7, 2006 DCORE0_RTR0 2007 }; 2008 2009 static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = { 2010 DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0, 2011 DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0 2012 }; 2013 2014 static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = { 2015 DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1, 2016 DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0 2017 }; 2018 2019 static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2020 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2021 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2022 }; 2023 2024 static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2025 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2026 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2027 }; 2028 2029 static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 2030 mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2031 mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2032 mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2033 mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2034 mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2035 mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2036 mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2037 mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE 2038 }; 2039 2040 static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = { 2041 DCORE0_RTR0, DCORE0_RTR0 2042 }; 2043 2044 static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = { 2045 DCORE0_RTR2, DCORE0_RTR2 2046 }; 2047 2048 static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = { 2049 DCORE2_RTR0, DCORE3_RTR7 2050 }; 2051 2052 static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = { 2053 DCORE2_RTR2, DCORE3_RTR5 2054 }; 2055 2056 struct mme_initiators_rtr_id { 2057 u32 wap0; 2058 u32 wap1; 2059 u32 write; 2060 u32 read; 2061 u32 sbte0; 2062 u32 sbte1; 2063 u32 sbte2; 2064 u32 sbte3; 2065 u32 sbte4; 2066 }; 2067 2068 enum mme_initiators { 2069 MME_WAP0 = 0, 2070 MME_WAP1, 2071 MME_WRITE, 2072 MME_READ, 2073 MME_SBTE0, 2074 MME_SBTE1, 2075 MME_SBTE2, 2076 MME_SBTE3, 2077 MME_SBTE4, 2078 MME_INITIATORS_MAX 2079 }; 2080 2081 static const struct mme_initiators_rtr_id 2082 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = { 2083 { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7, 2084 .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6}, 2085 { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8, 2086 .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8}, 2087 { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23, 2088 .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23}, 2089 { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30, 2090 .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28}, 2091 }; 2092 2093 enum razwi_event_sources { 2094 RAZWI_TPC, 2095 RAZWI_MME, 2096 RAZWI_EDMA, 2097 RAZWI_PDMA, 2098 RAZWI_NIC, 2099 RAZWI_DEC, 2100 RAZWI_ROT, 2101 RAZWI_ARC_FARM 2102 }; 2103 2104 struct hbm_mc_error_causes { 2105 u32 mask; 2106 char cause[50]; 2107 }; 2108 2109 static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS; 2110 2111 /* Special blocks iterator is currently used to configure security protection bits, 2112 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)- 2113 * must be skipped. Following configurations are commonly used for both PB config 2114 * and global error reading, since currently they both share the same settings. 2115 * Once it changes, we must remember to use separate configurations for either one. 2116 */ 2117 static int gaudi2_iterator_skip_block_types[] = { 2118 GAUDI2_BLOCK_TYPE_PLL, 2119 GAUDI2_BLOCK_TYPE_EU_BIST, 2120 GAUDI2_BLOCK_TYPE_HBM, 2121 GAUDI2_BLOCK_TYPE_XFT 2122 }; 2123 2124 static struct range gaudi2_iterator_skip_block_ranges[] = { 2125 /* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */ 2126 {mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE}, 2127 {mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE}, 2128 /* Skip all CPU blocks except for CPU_IF */ 2129 {mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE}, 2130 {mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE} 2131 }; 2132 2133 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = { 2134 {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"}, 2135 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"}, 2136 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"}, 2137 {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"}, 2138 {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"}, 2139 }; 2140 2141 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = { 2142 [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even", 2143 [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd", 2144 [HBM_SEI_READ_ERR] = "SEI read data error", 2145 [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error", 2146 [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted", 2147 [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail", 2148 [HBM_SEI_DFI] = "SEI DFI error", 2149 [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read", 2150 [HBM_SEI_BIST_FAIL] = "SEI BIST fail" 2151 }; 2152 2153 struct mmu_spi_sei_cause { 2154 char cause[50]; 2155 int clear_bit; 2156 }; 2157 2158 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = { 2159 {"page fault", 1}, /* INTERRUPT_CLR[1] */ 2160 {"page access", 1}, /* INTERRUPT_CLR[1] */ 2161 {"bypass ddr", 2}, /* INTERRUPT_CLR[2] */ 2162 {"multi hit", 2}, /* INTERRUPT_CLR[2] */ 2163 {"mmu rei0", -1}, /* no clear register bit */ 2164 {"mmu rei1", -1}, /* no clear register bit */ 2165 {"stlb rei0", -1}, /* no clear register bit */ 2166 {"stlb rei1", -1}, /* no clear register bit */ 2167 {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */ 2168 {"rr privileged read hit", 2}, /* INTERRUPT_CLR[2] */ 2169 {"rr secure write hit", 2}, /* INTERRUPT_CLR[2] */ 2170 {"rr secure read hit", 2}, /* INTERRUPT_CLR[2] */ 2171 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2172 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2173 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2174 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2175 {"slave error", 16}, /* INTERRUPT_CLR[16] */ 2176 {"dec error", 17}, /* INTERRUPT_CLR[17] */ 2177 {"burst fifo full", 2} /* INTERRUPT_CLR[2] */ 2178 }; 2179 2180 struct gaudi2_cache_invld_params { 2181 u64 start_va; 2182 u64 end_va; 2183 u32 inv_start_val; 2184 u32 flags; 2185 bool range_invalidation; 2186 }; 2187 2188 struct gaudi2_tpc_idle_data { 2189 struct engines_data *e; 2190 unsigned long *mask; 2191 bool *is_idle; 2192 const char *tpc_fmt; 2193 }; 2194 2195 struct gaudi2_tpc_mmu_data { 2196 u32 rw_asid; 2197 }; 2198 2199 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0}; 2200 2201 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val); 2202 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id); 2203 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id); 2204 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2205 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2206 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val); 2207 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, 2208 bool is_memset); 2209 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2210 struct engines_data *e); 2211 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2212 struct engines_data *e); 2213 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2214 struct engines_data *e); 2215 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr); 2216 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr); 2217 2218 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) 2219 { 2220 2221 } 2222 2223 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) 2224 { 2225 return sizeof(struct packet_msg_short); 2226 } 2227 2228 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) 2229 { 2230 return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence); 2231 } 2232 2233 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) 2234 { 2235 struct asic_fixed_properties *prop = &hdev->asic_prop; 2236 int dcore, inst, tpc_seq; 2237 u32 offset; 2238 2239 /* init the return code */ 2240 ctx->rc = 0; 2241 2242 for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) { 2243 for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) { 2244 tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 2245 2246 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) 2247 continue; 2248 2249 offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst); 2250 2251 ctx->fn(hdev, dcore, inst, offset, ctx); 2252 if (ctx->rc) { 2253 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", 2254 dcore, inst); 2255 return; 2256 } 2257 } 2258 } 2259 2260 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) 2261 return; 2262 2263 /* special check for PCI TPC (DCORE0_TPC6) */ 2264 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); 2265 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); 2266 if (ctx->rc) 2267 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); 2268 } 2269 2270 static bool gaudi2_host_phys_addr_valid(u64 addr) 2271 { 2272 if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1)) 2273 return true; 2274 2275 return false; 2276 } 2277 2278 static int set_number_of_functional_hbms(struct hl_device *hdev) 2279 { 2280 struct asic_fixed_properties *prop = &hdev->asic_prop; 2281 u8 faulty_hbms = hweight64(hdev->dram_binning); 2282 2283 /* check if all HBMs should be used */ 2284 if (!faulty_hbms) { 2285 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); 2286 prop->num_functional_hbms = GAUDI2_HBM_NUM; 2287 return 0; 2288 } 2289 2290 /* 2291 * check for error condition in which number of binning 2292 * candidates is higher than the maximum supported by the 2293 * driver (in which case binning mask shall be ignored and driver will 2294 * set the default) 2295 */ 2296 if (faulty_hbms > MAX_FAULTY_HBMS) { 2297 dev_err(hdev->dev, 2298 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", 2299 MAX_FAULTY_HBMS, hdev->dram_binning); 2300 return -EINVAL; 2301 } 2302 2303 /* 2304 * by default, number of functional HBMs in Gaudi2 is always 2305 * GAUDI2_HBM_NUM - 1. 2306 */ 2307 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; 2308 return 0; 2309 } 2310 2311 static int gaudi2_set_dram_properties(struct hl_device *hdev) 2312 { 2313 struct asic_fixed_properties *prop = &hdev->asic_prop; 2314 u32 basic_hbm_page_size; 2315 int rc; 2316 2317 rc = set_number_of_functional_hbms(hdev); 2318 if (rc) 2319 return -EINVAL; 2320 2321 /* 2322 * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround 2323 * in which we are using x16 bigger page size to be able to populate the entire 2324 * HBM mappings in the TLB 2325 */ 2326 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; 2327 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; 2328 prop->device_mem_alloc_default_page_size = prop->dram_page_size; 2329 prop->dram_size = prop->num_functional_hbms * SZ_16G; 2330 prop->dram_base_address = DRAM_PHYS_BASE; 2331 prop->dram_end_address = prop->dram_base_address + prop->dram_size; 2332 prop->dram_supports_virtual_memory = true; 2333 2334 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; 2335 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; 2336 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; 2337 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; 2338 2339 /* since DRAM page size differs from DMMU page size we need to allocate 2340 * DRAM memory in units of dram_page size and mapping this memory in 2341 * units of DMMU page size. we overcome this size mismatch using a 2342 * scrambling routine which takes a DRAM page and converts it to a DMMU 2343 * page. 2344 * We therefore: 2345 * 1. partition the virtual address space to DRAM-page (whole) pages. 2346 * (suppose we get n such pages) 2347 * 2. limit the amount of virtual address space we got from 1 above to 2348 * a multiple of 64M as we don't want the scrambled address to cross 2349 * the DRAM virtual address space. 2350 * ( m = (n * DRAM_page_size) / DMMU_page_size). 2351 * 3. determine the and address accordingly 2352 * end_addr = start_addr + m * 48M 2353 * 2354 * the DRAM address MSBs (63:48) are not part of the roundup calculation 2355 */ 2356 prop->dmmu.start_addr = prop->dram_base_address + 2357 (prop->dram_page_size * 2358 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); 2359 2360 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * 2361 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); 2362 2363 return 0; 2364 } 2365 2366 static int gaudi2_set_fixed_properties(struct hl_device *hdev) 2367 { 2368 struct asic_fixed_properties *prop = &hdev->asic_prop; 2369 struct hw_queue_properties *q_props; 2370 u32 num_sync_stream_queues = 0; 2371 int i; 2372 2373 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; 2374 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), 2375 GFP_KERNEL); 2376 2377 if (!prop->hw_queues_props) 2378 return -ENOMEM; 2379 2380 q_props = prop->hw_queues_props; 2381 2382 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2383 q_props[i].type = QUEUE_TYPE_HW; 2384 q_props[i].driver_only = 0; 2385 2386 if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) { 2387 q_props[i].supports_sync_stream = 0; 2388 } else { 2389 q_props[i].supports_sync_stream = 1; 2390 num_sync_stream_queues++; 2391 } 2392 2393 q_props[i].cb_alloc_flags = CB_ALLOC_USER; 2394 } 2395 2396 q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU; 2397 q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1; 2398 q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL; 2399 2400 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; 2401 prop->cfg_base_address = CFG_BASE; 2402 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; 2403 prop->host_base_address = HOST_PHYS_BASE_0; 2404 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; 2405 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; 2406 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; 2407 prop->user_dec_intr_count = NUMBER_OF_DEC; 2408 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; 2409 prop->completion_mode = HL_COMPLETION_MODE_CS; 2410 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; 2411 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; 2412 2413 prop->sram_base_address = SRAM_BASE_ADDR; 2414 prop->sram_size = SRAM_SIZE; 2415 prop->sram_end_address = prop->sram_base_address + prop->sram_size; 2416 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; 2417 2418 prop->hints_range_reservation = true; 2419 2420 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; 2421 2422 if (hdev->pldm) 2423 prop->mmu_pgt_size = 0x800000; /* 8MB */ 2424 else 2425 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE; 2426 2427 prop->mmu_pte_size = HL_PTE_SIZE; 2428 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; 2429 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; 2430 2431 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; 2432 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; 2433 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; 2434 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; 2435 prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT; 2436 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; 2437 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; 2438 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; 2439 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; 2440 prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK; 2441 prop->dmmu.page_size = PAGE_SIZE_1GB; 2442 prop->dmmu.num_hops = MMU_ARCH_6_HOPS; 2443 prop->dmmu.last_mask = LAST_MASK; 2444 prop->dmmu.host_resident = 1; 2445 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; 2446 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2447 2448 /* 2449 * this is done in order to be able to validate FW descriptor (i.e. validating that 2450 * the addresses and allocated space for FW image does not cross memory bounds). 2451 * for this reason we set the DRAM size to the minimum possible and later it will 2452 * be modified according to what reported in the cpucp info packet 2453 */ 2454 prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G; 2455 2456 hdev->pmmu_huge_range = true; 2457 prop->pmmu.host_resident = 1; 2458 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; 2459 prop->pmmu.last_mask = LAST_MASK; 2460 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; 2461 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2462 2463 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; 2464 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; 2465 prop->hints_host_hpage_reserved_va_range.start_addr = 2466 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START; 2467 prop->hints_host_hpage_reserved_va_range.end_addr = 2468 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END; 2469 2470 if (PAGE_SIZE == SZ_64K) { 2471 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; 2472 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; 2473 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; 2474 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; 2475 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; 2476 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; 2477 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; 2478 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; 2479 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; 2480 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; 2481 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; 2482 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; 2483 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2484 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2485 prop->pmmu.page_size = PAGE_SIZE_64KB; 2486 2487 /* shifts and masks are the same in PMMU and HPMMU */ 2488 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2489 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; 2490 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2491 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2492 } else { 2493 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; 2494 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; 2495 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; 2496 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; 2497 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; 2498 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; 2499 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; 2500 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; 2501 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; 2502 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; 2503 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; 2504 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; 2505 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2506 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2507 prop->pmmu.page_size = PAGE_SIZE_4KB; 2508 2509 /* shifts and masks are the same in PMMU and HPMMU */ 2510 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2511 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 2512 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2513 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2514 } 2515 2516 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; 2517 prop->num_engine_cores = CPU_ID_MAX; 2518 prop->cfg_size = CFG_SIZE; 2519 prop->max_asid = MAX_ASID; 2520 prop->num_of_events = GAUDI2_EVENT_SIZE; 2521 2522 prop->supports_engine_modes = true; 2523 2524 prop->dc_power_default = DC_POWER_DEFAULT; 2525 2526 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; 2527 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; 2528 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; 2529 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 2530 2531 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); 2532 2533 prop->mme_master_slave_mode = 1; 2534 2535 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + 2536 (num_sync_stream_queues * HL_RSVD_SOBS); 2537 2538 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + 2539 (num_sync_stream_queues * HL_RSVD_MONS); 2540 2541 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; 2542 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; 2543 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; 2544 2545 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; 2546 2547 prop->fw_cpu_boot_dev_sts0_valid = false; 2548 prop->fw_cpu_boot_dev_sts1_valid = false; 2549 prop->hard_reset_done_by_fw = false; 2550 prop->gic_interrupts_enable = true; 2551 2552 prop->server_type = HL_SERVER_TYPE_UNKNOWN; 2553 2554 prop->max_dec = NUMBER_OF_DEC; 2555 2556 prop->clk_pll_index = HL_GAUDI2_MME_PLL; 2557 2558 prop->dma_mask = 64; 2559 2560 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; 2561 2562 return 0; 2563 } 2564 2565 static int gaudi2_pci_bars_map(struct hl_device *hdev) 2566 { 2567 static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"}; 2568 bool is_wc[3] = {false, false, true}; 2569 int rc; 2570 2571 rc = hl_pci_bars_map(hdev, name, is_wc); 2572 if (rc) 2573 return rc; 2574 2575 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); 2576 2577 return 0; 2578 } 2579 2580 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) 2581 { 2582 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2583 struct hl_inbound_pci_region pci_region; 2584 u64 old_addr = addr; 2585 int rc; 2586 2587 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) 2588 return old_addr; 2589 2590 if (hdev->asic_prop.iatu_done_by_fw) 2591 return U64_MAX; 2592 2593 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2594 pci_region.mode = PCI_BAR_MATCH_MODE; 2595 pci_region.bar = DRAM_BAR_ID; 2596 pci_region.addr = addr; 2597 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region); 2598 if (rc) 2599 return U64_MAX; 2600 2601 if (gaudi2) { 2602 old_addr = gaudi2->dram_bar_cur_addr; 2603 gaudi2->dram_bar_cur_addr = addr; 2604 } 2605 2606 return old_addr; 2607 } 2608 2609 static int gaudi2_init_iatu(struct hl_device *hdev) 2610 { 2611 struct hl_inbound_pci_region inbound_region; 2612 struct hl_outbound_pci_region outbound_region; 2613 u32 bar_addr_low, bar_addr_high; 2614 int rc; 2615 2616 if (hdev->asic_prop.iatu_done_by_fw) 2617 return 0; 2618 2619 /* Temporary inbound Region 0 - Bar 0 - Point to CFG 2620 * We must map this region in BAR match mode in order to 2621 * fetch BAR physical base address 2622 */ 2623 inbound_region.mode = PCI_BAR_MATCH_MODE; 2624 inbound_region.bar = SRAM_CFG_BAR_ID; 2625 /* Base address must be aligned to Bar size which is 256 MB */ 2626 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; 2627 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2628 if (rc) 2629 return rc; 2630 2631 /* Fetch physical BAR address */ 2632 bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF); 2633 bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF; 2634 2635 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; 2636 2637 /* Inbound Region 0 - Bar 0 - Point to CFG */ 2638 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2639 inbound_region.bar = SRAM_CFG_BAR_ID; 2640 inbound_region.offset_in_bar = 0; 2641 inbound_region.addr = STM_FLASH_BASE_ADDR; 2642 inbound_region.size = CFG_REGION_SIZE; 2643 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2644 if (rc) 2645 return rc; 2646 2647 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ 2648 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2649 inbound_region.bar = SRAM_CFG_BAR_ID; 2650 inbound_region.offset_in_bar = CFG_REGION_SIZE; 2651 inbound_region.addr = BAR0_RSRVD_BASE_ADDR; 2652 inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE; 2653 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 2654 if (rc) 2655 return rc; 2656 2657 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2658 inbound_region.mode = PCI_BAR_MATCH_MODE; 2659 inbound_region.bar = DRAM_BAR_ID; 2660 inbound_region.addr = DRAM_PHYS_BASE; 2661 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region); 2662 if (rc) 2663 return rc; 2664 2665 /* Outbound Region 0 - Point to Host */ 2666 outbound_region.addr = HOST_PHYS_BASE_0; 2667 outbound_region.size = HOST_PHYS_SIZE_0; 2668 rc = hl_pci_set_outbound_region(hdev, &outbound_region); 2669 2670 return rc; 2671 } 2672 2673 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) 2674 { 2675 return RREG32(mmHW_STATE); 2676 } 2677 2678 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) 2679 { 2680 struct asic_fixed_properties *prop = &hdev->asic_prop; 2681 2682 /* 2683 * check for error condition in which number of binning candidates 2684 * is higher than the maximum supported by the driver 2685 */ 2686 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { 2687 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", 2688 MAX_CLUSTER_BINNING_FAULTY_TPCS, 2689 hdev->tpc_binning); 2690 return -EINVAL; 2691 } 2692 2693 prop->tpc_binning_mask = hdev->tpc_binning; 2694 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; 2695 2696 return 0; 2697 } 2698 2699 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) 2700 { 2701 struct asic_fixed_properties *prop = &hdev->asic_prop; 2702 struct hw_queue_properties *q_props = prop->hw_queues_props; 2703 u64 tpc_binning_mask; 2704 u8 subst_idx = 0; 2705 int i, rc; 2706 2707 rc = gaudi2_tpc_binning_init_prop(hdev); 2708 if (rc) 2709 return rc; 2710 2711 tpc_binning_mask = prop->tpc_binning_mask; 2712 2713 for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) { 2714 u8 subst_seq, binned, qid_base; 2715 2716 if (tpc_binning_mask == 0) 2717 break; 2718 2719 if (subst_idx == 0) { 2720 subst_seq = TPC_ID_DCORE0_TPC6; 2721 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 2722 } else { 2723 subst_seq = TPC_ID_DCORE3_TPC5; 2724 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0; 2725 } 2726 2727 2728 /* clear bit from mask */ 2729 binned = __ffs(tpc_binning_mask); 2730 /* 2731 * Coverity complains about possible out-of-bound access in 2732 * clear_bit 2733 */ 2734 if (binned >= TPC_ID_SIZE) { 2735 dev_err(hdev->dev, 2736 "Invalid binned TPC (binning mask: %llx)\n", 2737 tpc_binning_mask); 2738 return -EINVAL; 2739 } 2740 clear_bit(binned, (unsigned long *)&tpc_binning_mask); 2741 2742 /* also clear replacing TPC bit from enabled mask */ 2743 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); 2744 2745 /* bin substite TPC's Qs */ 2746 q_props[qid_base].binned = 1; 2747 q_props[qid_base + 1].binned = 1; 2748 q_props[qid_base + 2].binned = 1; 2749 q_props[qid_base + 3].binned = 1; 2750 2751 subst_idx++; 2752 } 2753 2754 return 0; 2755 } 2756 2757 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) 2758 { 2759 struct asic_fixed_properties *prop = &hdev->asic_prop; 2760 u8 num_faulty; 2761 2762 num_faulty = hweight32(hdev->decoder_binning); 2763 2764 /* 2765 * check for error condition in which number of binning candidates 2766 * is higher than the maximum supported by the driver 2767 */ 2768 if (num_faulty > MAX_FAULTY_DECODERS) { 2769 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n", 2770 hdev->decoder_binning); 2771 return -EINVAL; 2772 } 2773 2774 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); 2775 2776 if (prop->decoder_binning_mask) 2777 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); 2778 else 2779 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; 2780 2781 return 0; 2782 } 2783 2784 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) 2785 { 2786 struct asic_fixed_properties *prop = &hdev->asic_prop; 2787 2788 /* check if we should override default binning */ 2789 if (!hdev->dram_binning) { 2790 prop->dram_binning_mask = 0; 2791 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; 2792 return; 2793 } 2794 2795 /* set DRAM binning constraints */ 2796 prop->faulty_dram_cluster_map |= hdev->dram_binning; 2797 prop->dram_binning_mask = hdev->dram_binning; 2798 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); 2799 } 2800 2801 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) 2802 { 2803 struct asic_fixed_properties *prop = &hdev->asic_prop; 2804 struct hw_queue_properties *q_props; 2805 u8 seq, num_faulty; 2806 2807 num_faulty = hweight32(hdev->edma_binning); 2808 2809 /* 2810 * check for error condition in which number of binning candidates 2811 * is higher than the maximum supported by the driver 2812 */ 2813 if (num_faulty > MAX_FAULTY_EDMAS) { 2814 dev_err(hdev->dev, 2815 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", 2816 hdev->edma_binning); 2817 return -EINVAL; 2818 } 2819 2820 if (!hdev->edma_binning) { 2821 prop->edma_binning_mask = 0; 2822 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; 2823 return 0; 2824 } 2825 2826 seq = __ffs((unsigned long)hdev->edma_binning); 2827 2828 /* set binning constraints */ 2829 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); 2830 prop->edma_binning_mask = hdev->edma_binning; 2831 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); 2832 2833 /* bin substitute EDMA's queue */ 2834 q_props = prop->hw_queues_props; 2835 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1; 2836 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1; 2837 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1; 2838 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1; 2839 2840 return 0; 2841 } 2842 2843 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) 2844 { 2845 struct asic_fixed_properties *prop = &hdev->asic_prop; 2846 u8 num_faulty, seq; 2847 2848 /* check if we should override default binning */ 2849 if (!xbar_edge_iso_mask) { 2850 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; 2851 return 0; 2852 } 2853 2854 /* 2855 * note that it can be set to value other than 0 only after cpucp packet (i.e. 2856 * only the FW can set a redundancy value). for user it'll always be 0. 2857 */ 2858 num_faulty = hweight32(xbar_edge_iso_mask); 2859 2860 /* 2861 * check for error condition in which number of binning candidates 2862 * is higher than the maximum supported by the driver 2863 */ 2864 if (num_faulty > MAX_FAULTY_XBARS) { 2865 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", 2866 MAX_FAULTY_XBARS); 2867 return -EINVAL; 2868 } 2869 2870 seq = __ffs((unsigned long)xbar_edge_iso_mask); 2871 2872 /* set binning constraints */ 2873 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); 2874 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; 2875 2876 return 0; 2877 } 2878 2879 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) 2880 { 2881 int rc; 2882 2883 /* 2884 * mark all clusters as good, each component will "fail" cluster 2885 * based on eFuse/user values. 2886 * If more than single cluster is faulty- the chip is unusable 2887 */ 2888 hdev->asic_prop.faulty_dram_cluster_map = 0; 2889 2890 gaudi2_set_dram_binning_masks(hdev); 2891 2892 rc = gaudi2_set_edma_binning_masks(hdev); 2893 if (rc) 2894 return rc; 2895 2896 rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask); 2897 if (rc) 2898 return rc; 2899 2900 2901 /* always initially set to full mask */ 2902 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; 2903 2904 return 0; 2905 } 2906 2907 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) 2908 { 2909 struct asic_fixed_properties *prop = &hdev->asic_prop; 2910 int rc; 2911 2912 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); 2913 if (rc) 2914 return rc; 2915 2916 /* if we have DRAM binning reported by FW we should perform cluster config */ 2917 if (prop->faulty_dram_cluster_map) { 2918 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); 2919 2920 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; 2921 } 2922 2923 return 0; 2924 } 2925 2926 static int gaudi2_set_binning_masks(struct hl_device *hdev) 2927 { 2928 int rc; 2929 2930 rc = gaudi2_set_cluster_binning_masks(hdev); 2931 if (rc) 2932 return rc; 2933 2934 rc = gaudi2_set_tpc_binning_masks(hdev); 2935 if (rc) 2936 return rc; 2937 2938 rc = gaudi2_set_dec_binning_masks(hdev); 2939 if (rc) 2940 return rc; 2941 2942 return 0; 2943 } 2944 2945 static int gaudi2_cpucp_info_get(struct hl_device *hdev) 2946 { 2947 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2948 struct asic_fixed_properties *prop = &hdev->asic_prop; 2949 long max_power; 2950 u64 dram_size; 2951 int rc; 2952 2953 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 2954 return 0; 2955 2956 /* No point of asking this information again when not doing hard reset, as the device 2957 * CPU hasn't been reset 2958 */ 2959 if (hdev->reset_info.in_compute_reset) 2960 return 0; 2961 2962 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 2963 mmCPU_BOOT_ERR1); 2964 if (rc) 2965 return rc; 2966 2967 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); 2968 if (dram_size) { 2969 /* we can have wither 5 or 6 HBMs. other values are invalid */ 2970 2971 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && 2972 (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) { 2973 dev_err(hdev->dev, 2974 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n", 2975 dram_size, prop->dram_size); 2976 dram_size = prop->dram_size; 2977 } 2978 2979 prop->dram_size = dram_size; 2980 prop->dram_end_address = prop->dram_base_address + dram_size; 2981 } 2982 2983 if (!strlen(prop->cpucp_info.card_name)) 2984 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, 2985 CARD_NAME_MAX_LEN); 2986 2987 /* Overwrite binning masks with the actual binning values from F/W */ 2988 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; 2989 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; 2990 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); 2991 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); 2992 2993 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", 2994 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, 2995 hdev->decoder_binning); 2996 2997 /* 2998 * at this point the DRAM parameters need to be updated according to data obtained 2999 * from the FW 3000 */ 3001 rc = hdev->asic_funcs->set_dram_properties(hdev); 3002 if (rc) 3003 return rc; 3004 3005 rc = hdev->asic_funcs->set_binning_masks(hdev); 3006 if (rc) 3007 return rc; 3008 3009 max_power = hl_fw_get_max_power(hdev); 3010 if (max_power < 0) 3011 return max_power; 3012 3013 prop->max_power_default = (u64) max_power; 3014 3015 return 0; 3016 } 3017 3018 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) 3019 { 3020 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3021 u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS]; 3022 int rc; 3023 3024 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 3025 return 0; 3026 3027 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr); 3028 if (rc) 3029 return rc; 3030 3031 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; 3032 3033 return 0; 3034 } 3035 3036 static int gaudi2_early_init(struct hl_device *hdev) 3037 { 3038 struct asic_fixed_properties *prop = &hdev->asic_prop; 3039 struct pci_dev *pdev = hdev->pdev; 3040 resource_size_t pci_bar_size; 3041 int rc; 3042 3043 rc = gaudi2_set_fixed_properties(hdev); 3044 if (rc) 3045 return rc; 3046 3047 /* Check BAR sizes */ 3048 pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID); 3049 3050 if (pci_bar_size != CFG_BAR_SIZE) { 3051 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3052 SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 3053 rc = -ENODEV; 3054 goto free_queue_props; 3055 } 3056 3057 pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID); 3058 if (pci_bar_size != MSIX_BAR_SIZE) { 3059 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3060 MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE); 3061 rc = -ENODEV; 3062 goto free_queue_props; 3063 } 3064 3065 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); 3066 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); 3067 3068 /* 3069 * Only in pldm driver config iATU 3070 */ 3071 if (hdev->pldm) 3072 hdev->asic_prop.iatu_done_by_fw = false; 3073 else 3074 hdev->asic_prop.iatu_done_by_fw = true; 3075 3076 rc = hl_pci_init(hdev); 3077 if (rc) 3078 goto free_queue_props; 3079 3080 /* Before continuing in the initialization, we need to read the preboot 3081 * version to determine whether we run with a security-enabled firmware 3082 */ 3083 rc = hl_fw_read_preboot_status(hdev); 3084 if (rc) { 3085 if (hdev->reset_on_preboot_fail) 3086 /* we are already on failure flow, so don't check if hw_fini fails. */ 3087 hdev->asic_funcs->hw_fini(hdev, true, false); 3088 goto pci_fini; 3089 } 3090 3091 if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 3092 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 3093 rc = hdev->asic_funcs->hw_fini(hdev, true, false); 3094 if (rc) { 3095 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); 3096 goto pci_fini; 3097 } 3098 } 3099 3100 return 0; 3101 3102 pci_fini: 3103 hl_pci_fini(hdev); 3104 free_queue_props: 3105 kfree(hdev->asic_prop.hw_queues_props); 3106 return rc; 3107 } 3108 3109 static int gaudi2_early_fini(struct hl_device *hdev) 3110 { 3111 kfree(hdev->asic_prop.hw_queues_props); 3112 hl_pci_fini(hdev); 3113 3114 return 0; 3115 } 3116 3117 static bool gaudi2_is_arc_nic_owned(u64 arc_id) 3118 { 3119 switch (arc_id) { 3120 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 3121 return true; 3122 default: 3123 return false; 3124 } 3125 } 3126 3127 static bool gaudi2_is_arc_tpc_owned(u64 arc_id) 3128 { 3129 switch (arc_id) { 3130 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 3131 return true; 3132 default: 3133 return false; 3134 } 3135 } 3136 3137 static void gaudi2_init_arcs(struct hl_device *hdev) 3138 { 3139 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3140 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3141 u64 arc_id; 3142 u32 i; 3143 3144 for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) { 3145 if (gaudi2_is_arc_enabled(hdev, i)) 3146 continue; 3147 3148 gaudi2_set_arc_id_cap(hdev, i); 3149 } 3150 3151 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 3152 if (!gaudi2_is_queue_enabled(hdev, i)) 3153 continue; 3154 3155 arc_id = gaudi2_queue_id_to_arc_id[i]; 3156 if (gaudi2_is_arc_enabled(hdev, arc_id)) 3157 continue; 3158 3159 if (gaudi2_is_arc_nic_owned(arc_id) && 3160 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) 3161 continue; 3162 3163 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & 3164 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) 3165 continue; 3166 3167 gaudi2_set_arc_id_cap(hdev, arc_id); 3168 } 3169 3170 /* Fetch ARC scratchpad address */ 3171 hdev->asic_prop.engine_core_interrupt_reg_addr = 3172 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); 3173 } 3174 3175 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) 3176 { 3177 u32 reg_base, reg_val; 3178 int rc; 3179 3180 switch (cpu_id) { 3181 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3: 3182 /* Each ARC scheduler has 2 consecutive DCCM blocks */ 3183 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3184 ARC_DCCM_BLOCK_SIZE * 2, true); 3185 if (rc) 3186 return rc; 3187 break; 3188 case CPU_ID_SCHED_ARC4: 3189 case CPU_ID_SCHED_ARC5: 3190 case CPU_ID_MME_QMAN_ARC0: 3191 case CPU_ID_MME_QMAN_ARC1: 3192 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 3193 3194 /* Scrub lower DCCM block */ 3195 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3196 ARC_DCCM_BLOCK_SIZE, true); 3197 if (rc) 3198 return rc; 3199 3200 /* Switch to upper DCCM block */ 3201 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1); 3202 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3203 3204 /* Scrub upper DCCM block */ 3205 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3206 ARC_DCCM_BLOCK_SIZE, true); 3207 if (rc) 3208 return rc; 3209 3210 /* Switch to lower DCCM block */ 3211 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0); 3212 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3213 break; 3214 default: 3215 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3216 ARC_DCCM_BLOCK_SIZE, true); 3217 if (rc) 3218 return rc; 3219 } 3220 3221 return 0; 3222 } 3223 3224 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) 3225 { 3226 u16 arc_id; 3227 int rc; 3228 3229 for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) { 3230 if (!gaudi2_is_arc_enabled(hdev, arc_id)) 3231 continue; 3232 3233 rc = gaudi2_scrub_arc_dccm(hdev, arc_id); 3234 if (rc) 3235 return rc; 3236 } 3237 3238 return 0; 3239 } 3240 3241 static int gaudi2_late_init(struct hl_device *hdev) 3242 { 3243 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3244 int rc; 3245 3246 hdev->asic_prop.supports_advanced_cpucp_rc = true; 3247 3248 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 3249 gaudi2->virt_msix_db_dma_addr); 3250 if (rc) { 3251 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); 3252 return rc; 3253 } 3254 3255 rc = gaudi2_fetch_psoc_frequency(hdev); 3256 if (rc) { 3257 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); 3258 goto disable_pci_access; 3259 } 3260 3261 gaudi2_init_arcs(hdev); 3262 3263 rc = gaudi2_scrub_arcs_dccm(hdev); 3264 if (rc) { 3265 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 3266 goto disable_pci_access; 3267 } 3268 3269 gaudi2_init_security(hdev); 3270 3271 return 0; 3272 3273 disable_pci_access: 3274 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 3275 3276 return rc; 3277 } 3278 3279 static void gaudi2_late_fini(struct hl_device *hdev) 3280 { 3281 hl_hwmon_release_resources(hdev); 3282 } 3283 3284 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) 3285 { 3286 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3287 3288 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3289 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3290 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3291 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3292 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3293 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3294 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3295 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3296 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3297 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3298 } 3299 3300 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) 3301 { 3302 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3303 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3304 u32 block_size, umr_start_idx, num_umr_blocks; 3305 int i; 3306 3307 for (i = 0 ; i < NUM_ARC_CPUS ; i++) { 3308 if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3) 3309 block_size = ARC_DCCM_BLOCK_SIZE * 2; 3310 else 3311 block_size = ARC_DCCM_BLOCK_SIZE; 3312 3313 blocks[i].address = gaudi2_arc_dccm_bases[i]; 3314 blocks[i].size = block_size; 3315 } 3316 3317 blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE; 3318 blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE; 3319 3320 blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE; 3321 blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE; 3322 3323 blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE; 3324 blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE; 3325 3326 blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE; 3327 blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE; 3328 3329 blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE; 3330 blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE; 3331 3332 blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE; 3333 blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE; 3334 3335 blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE; 3336 blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE; 3337 3338 blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE; 3339 blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE; 3340 3341 umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS; 3342 num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS; 3343 for (i = 0 ; i < num_umr_blocks ; i++) { 3344 u8 nic_id, umr_block_id; 3345 3346 nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS; 3347 umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS; 3348 3349 blocks[umr_start_idx + i].address = 3350 mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE + 3351 (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET + 3352 (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET + 3353 umr_block_id * NIC_UMR_OFFSET; 3354 blocks[umr_start_idx + i].size = HL_BLOCK_SIZE; 3355 } 3356 3357 /* Expose decoder HW configuration block to user */ 3358 gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX); 3359 3360 for (i = 1; i < NUM_OF_DCORES; ++i) { 3361 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; 3362 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; 3363 3364 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = 3365 mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET; 3366 3367 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = 3368 mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET; 3369 } 3370 } 3371 3372 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) 3373 { 3374 dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr; 3375 void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}; 3376 int i, j, rc = 0; 3377 3378 /* The device ARC works with 32-bits addresses, and because there is a single HW register 3379 * that holds the extension bits (49..28), these bits must be identical in all the allocated 3380 * range. 3381 */ 3382 3383 for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) { 3384 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 3385 &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO); 3386 if (!virt_addr_arr[i]) { 3387 rc = -ENOMEM; 3388 goto free_dma_mem_arr; 3389 } 3390 3391 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; 3392 if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr)) 3393 break; 3394 } 3395 3396 if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) { 3397 dev_err(hdev->dev, 3398 "MSB of ARC accessible DMA memory are not identical in all range\n"); 3399 rc = -EFAULT; 3400 goto free_dma_mem_arr; 3401 } 3402 3403 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; 3404 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; 3405 3406 free_dma_mem_arr: 3407 for (j = 0 ; j < i ; j++) 3408 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j], 3409 dma_addr_arr[j]); 3410 3411 return rc; 3412 } 3413 3414 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) 3415 { 3416 struct asic_fixed_properties *prop = &hdev->asic_prop; 3417 struct pci_mem_region *region; 3418 3419 /* CFG */ 3420 region = &hdev->pci_mem_region[PCI_REGION_CFG]; 3421 region->region_base = CFG_BASE; 3422 region->region_size = CFG_SIZE; 3423 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; 3424 region->bar_size = CFG_BAR_SIZE; 3425 region->bar_id = SRAM_CFG_BAR_ID; 3426 region->used = 1; 3427 3428 /* SRAM */ 3429 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 3430 region->region_base = SRAM_BASE_ADDR; 3431 region->region_size = SRAM_SIZE; 3432 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; 3433 region->bar_size = CFG_BAR_SIZE; 3434 region->bar_id = SRAM_CFG_BAR_ID; 3435 region->used = 1; 3436 3437 /* DRAM */ 3438 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 3439 region->region_base = DRAM_PHYS_BASE; 3440 region->region_size = hdev->asic_prop.dram_size; 3441 region->offset_in_bar = 0; 3442 region->bar_size = prop->dram_pci_bar_size; 3443 region->bar_id = DRAM_BAR_ID; 3444 region->used = 1; 3445 } 3446 3447 static void gaudi2_user_interrupt_setup(struct hl_device *hdev) 3448 { 3449 struct asic_fixed_properties *prop = &hdev->asic_prop; 3450 int i, j, k; 3451 3452 /* Initialize TPC interrupt */ 3453 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); 3454 3455 /* Initialize unexpected error interrupt */ 3456 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, 3457 HL_USR_INTERRUPT_UNEXPECTED); 3458 3459 /* Initialize common user CQ interrupt */ 3460 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, 3461 HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ); 3462 3463 /* Initialize common decoder interrupt */ 3464 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, 3465 HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER); 3466 3467 /* User interrupts structure holds both decoder and user interrupts from various engines. 3468 * We first initialize the decoder interrupts and then we add the user interrupts. 3469 * The only limitation is that the last decoder interrupt id must be smaller 3470 * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time. 3471 */ 3472 3473 /* Initialize decoder interrupts, expose only normal interrupts, 3474 * error interrupts to be handled by driver 3475 */ 3476 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM; 3477 i += 2, j++) 3478 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, 3479 HL_USR_INTERRUPT_DECODER); 3480 3481 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) 3482 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); 3483 } 3484 3485 static inline int gaudi2_get_non_zero_random_int(void) 3486 { 3487 int rand = get_random_u32(); 3488 3489 return rand ? rand : 1; 3490 } 3491 3492 static void gaudi2_special_blocks_free(struct hl_device *hdev) 3493 { 3494 struct asic_fixed_properties *prop = &hdev->asic_prop; 3495 struct hl_skip_blocks_cfg *skip_special_blocks_cfg = 3496 &prop->skip_special_blocks_cfg; 3497 3498 kfree(prop->special_blocks); 3499 kfree(skip_special_blocks_cfg->block_types); 3500 kfree(skip_special_blocks_cfg->block_ranges); 3501 } 3502 3503 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) 3504 { 3505 gaudi2_special_blocks_free(hdev); 3506 } 3507 3508 static bool gaudi2_special_block_skip(struct hl_device *hdev, 3509 struct hl_special_blocks_cfg *special_blocks_cfg, 3510 u32 blk_idx, u32 major, u32 minor, u32 sub_minor) 3511 { 3512 return false; 3513 } 3514 3515 static int gaudi2_special_blocks_config(struct hl_device *hdev) 3516 { 3517 struct asic_fixed_properties *prop = &hdev->asic_prop; 3518 int i, rc; 3519 3520 /* Configure Special blocks */ 3521 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; 3522 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); 3523 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, 3524 sizeof(*prop->special_blocks), GFP_KERNEL); 3525 if (!prop->special_blocks) 3526 return -ENOMEM; 3527 3528 for (i = 0 ; i < prop->num_of_special_blocks ; i++) 3529 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], 3530 sizeof(*prop->special_blocks)); 3531 3532 /* Configure when to skip Special blocks */ 3533 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); 3534 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; 3535 3536 if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) { 3537 prop->skip_special_blocks_cfg.block_types = 3538 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types), 3539 sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL); 3540 if (!prop->skip_special_blocks_cfg.block_types) { 3541 rc = -ENOMEM; 3542 goto free_special_blocks; 3543 } 3544 3545 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, 3546 sizeof(gaudi2_iterator_skip_block_types)); 3547 3548 prop->skip_special_blocks_cfg.block_types_len = 3549 ARRAY_SIZE(gaudi2_iterator_skip_block_types); 3550 } 3551 3552 if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) { 3553 prop->skip_special_blocks_cfg.block_ranges = 3554 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges), 3555 sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL); 3556 if (!prop->skip_special_blocks_cfg.block_ranges) { 3557 rc = -ENOMEM; 3558 goto free_skip_special_blocks_types; 3559 } 3560 3561 for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++) 3562 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], 3563 &gaudi2_iterator_skip_block_ranges[i], 3564 sizeof(struct range)); 3565 3566 prop->skip_special_blocks_cfg.block_ranges_len = 3567 ARRAY_SIZE(gaudi2_iterator_skip_block_ranges); 3568 } 3569 3570 return 0; 3571 3572 free_skip_special_blocks_types: 3573 kfree(prop->skip_special_blocks_cfg.block_types); 3574 free_special_blocks: 3575 kfree(prop->special_blocks); 3576 3577 return rc; 3578 } 3579 3580 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) 3581 { 3582 return gaudi2_special_blocks_config(hdev); 3583 } 3584 3585 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) 3586 { 3587 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3588 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3589 int i; 3590 3591 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3592 /* bail-out if this is an allocation failure point */ 3593 if (!msg_info[i].kern_addr) 3594 break; 3595 3596 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr); 3597 msg_info[i].kern_addr = NULL; 3598 } 3599 } 3600 3601 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) 3602 { 3603 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3604 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3605 int i, rc; 3606 3607 /* allocate a message-short buf for each Q we intend to test */ 3608 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3609 msg_info[i].kern_addr = 3610 (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short), 3611 GFP_KERNEL, &msg_info[i].dma_addr); 3612 if (!msg_info[i].kern_addr) { 3613 dev_err(hdev->dev, 3614 "Failed to allocate dma memory for H/W queue %d testing\n", i); 3615 rc = -ENOMEM; 3616 goto err_exit; 3617 } 3618 } 3619 3620 return 0; 3621 3622 err_exit: 3623 gaudi2_test_queues_msgs_free(hdev); 3624 return rc; 3625 } 3626 3627 static int gaudi2_sw_init(struct hl_device *hdev) 3628 { 3629 struct asic_fixed_properties *prop = &hdev->asic_prop; 3630 struct gaudi2_device *gaudi2; 3631 int i, rc; 3632 3633 /* Allocate device structure */ 3634 gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL); 3635 if (!gaudi2) 3636 return -ENOMEM; 3637 3638 for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) { 3639 if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid) 3640 continue; 3641 3642 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { 3643 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", 3644 GAUDI2_EVENT_SIZE); 3645 rc = -EINVAL; 3646 goto free_gaudi2_device; 3647 } 3648 3649 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; 3650 } 3651 3652 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) 3653 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); 3654 3655 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; 3656 3657 hdev->asic_specific = gaudi2; 3658 3659 /* Create DMA pool for small allocations. 3660 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped 3661 * PI/CI registers allocated from this pool have this restriction 3662 */ 3663 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, 3664 GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0); 3665 if (!hdev->dma_pool) { 3666 dev_err(hdev->dev, "failed to create DMA pool\n"); 3667 rc = -ENOMEM; 3668 goto free_gaudi2_device; 3669 } 3670 3671 rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev); 3672 if (rc) 3673 goto free_dma_pool; 3674 3675 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 3676 if (!hdev->cpu_accessible_dma_pool) { 3677 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); 3678 rc = -ENOMEM; 3679 goto free_cpu_dma_mem; 3680 } 3681 3682 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, 3683 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 3684 if (rc) { 3685 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); 3686 rc = -EFAULT; 3687 goto free_cpu_accessible_dma_pool; 3688 } 3689 3690 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, 3691 &gaudi2->virt_msix_db_dma_addr); 3692 if (!gaudi2->virt_msix_db_cpu_addr) { 3693 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); 3694 rc = -ENOMEM; 3695 goto free_cpu_accessible_dma_pool; 3696 } 3697 3698 spin_lock_init(&gaudi2->hw_queues_lock); 3699 3700 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, 3701 &gaudi2->scratchpad_bus_address, 3702 GFP_KERNEL | __GFP_ZERO); 3703 if (!gaudi2->scratchpad_kernel_address) { 3704 rc = -ENOMEM; 3705 goto free_virt_msix_db_mem; 3706 } 3707 3708 gaudi2_user_mapped_blocks_init(hdev); 3709 3710 /* Initialize user interrupts */ 3711 gaudi2_user_interrupt_setup(hdev); 3712 3713 hdev->supports_coresight = true; 3714 hdev->supports_sync_stream = true; 3715 hdev->supports_cb_mapping = true; 3716 hdev->supports_wait_for_multi_cs = false; 3717 3718 prop->supports_compute_reset = true; 3719 3720 /* Event queue sanity check added in FW version 1.11 */ 3721 if (hl_is_fw_sw_ver_below(hdev, 1, 11)) 3722 hdev->event_queue.check_eqe_index = false; 3723 else 3724 hdev->event_queue.check_eqe_index = true; 3725 3726 hdev->asic_funcs->set_pci_memory_regions(hdev); 3727 3728 rc = gaudi2_special_blocks_iterator_config(hdev); 3729 if (rc) 3730 goto free_scratchpad_mem; 3731 3732 rc = gaudi2_test_queues_msgs_alloc(hdev); 3733 if (rc) 3734 goto special_blocks_free; 3735 3736 return 0; 3737 3738 special_blocks_free: 3739 gaudi2_special_blocks_iterator_free(hdev); 3740 free_scratchpad_mem: 3741 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, 3742 gaudi2->scratchpad_bus_address); 3743 free_virt_msix_db_mem: 3744 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3745 free_cpu_accessible_dma_pool: 3746 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3747 free_cpu_dma_mem: 3748 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3749 hdev->cpu_accessible_dma_address); 3750 free_dma_pool: 3751 dma_pool_destroy(hdev->dma_pool); 3752 free_gaudi2_device: 3753 kfree(gaudi2); 3754 return rc; 3755 } 3756 3757 static int gaudi2_sw_fini(struct hl_device *hdev) 3758 { 3759 struct asic_fixed_properties *prop = &hdev->asic_prop; 3760 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3761 3762 gaudi2_test_queues_msgs_free(hdev); 3763 3764 gaudi2_special_blocks_iterator_free(hdev); 3765 3766 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3767 3768 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3769 3770 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3771 hdev->cpu_accessible_dma_address); 3772 3773 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, 3774 gaudi2->scratchpad_bus_address); 3775 3776 dma_pool_destroy(hdev->dma_pool); 3777 3778 kfree(gaudi2); 3779 3780 return 0; 3781 } 3782 3783 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) 3784 { 3785 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP | 3786 QM_GLBL_CFG1_CQF_STOP | 3787 QM_GLBL_CFG1_CP_STOP); 3788 3789 /* stop also the ARC */ 3790 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP); 3791 } 3792 3793 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) 3794 { 3795 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH | 3796 QM_GLBL_CFG1_CQF_FLUSH | 3797 QM_GLBL_CFG1_CP_FLUSH); 3798 } 3799 3800 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) 3801 { 3802 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH); 3803 } 3804 3805 /** 3806 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters 3807 * 3808 * @hdev: pointer to the habanalabs device structure 3809 * @queue_id: queue to clear fence counters to 3810 * @skip_fence: if true set maximum fence value to all fence counters to avoid 3811 * getting stuck on any fence value. otherwise set all fence 3812 * counters to 0 (standard clear of fence counters) 3813 */ 3814 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, 3815 bool skip_fence) 3816 { 3817 u32 size, reg_base; 3818 u32 addr, val; 3819 3820 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3821 3822 addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET; 3823 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; 3824 3825 /* 3826 * in case we want to make sure that QM that is stuck on a fence will 3827 * be released we should set the fence counter to a higher value that 3828 * the value the QM waiting for. to comply with any fence counter of 3829 * any value we set maximum fence value to all counters 3830 */ 3831 val = skip_fence ? U32_MAX : 0; 3832 gaudi2_memset_device_lbw(hdev, addr, size, val); 3833 } 3834 3835 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) 3836 { 3837 u32 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3838 3839 gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true); 3840 gaudi2_flush_qman_common(hdev, reg_base); 3841 gaudi2_flush_qman_arc_common(hdev, reg_base); 3842 } 3843 3844 static void gaudi2_stop_dma_qmans(struct hl_device *hdev) 3845 { 3846 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3847 int dcore, inst; 3848 3849 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3850 goto stop_edma_qmans; 3851 3852 /* Stop CPs of PDMA QMANs */ 3853 gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE); 3854 gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE); 3855 3856 stop_edma_qmans: 3857 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3858 return; 3859 3860 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3861 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3862 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3863 u32 qm_base; 3864 3865 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3866 continue; 3867 3868 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 3869 inst * DCORE_EDMA_OFFSET; 3870 3871 /* Stop CPs of EDMA QMANs */ 3872 gaudi2_stop_qman_common(hdev, qm_base); 3873 } 3874 } 3875 } 3876 3877 static void gaudi2_stop_mme_qmans(struct hl_device *hdev) 3878 { 3879 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3880 u32 offset, i; 3881 3882 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 3883 3884 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 3885 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) 3886 continue; 3887 3888 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 3889 } 3890 } 3891 3892 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) 3893 { 3894 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3895 u32 reg_base; 3896 int i; 3897 3898 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 3899 return; 3900 3901 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 3902 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 3903 continue; 3904 3905 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 3906 gaudi2_stop_qman_common(hdev, reg_base); 3907 } 3908 } 3909 3910 static void gaudi2_stop_rot_qmans(struct hl_device *hdev) 3911 { 3912 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3913 u32 reg_base; 3914 int i; 3915 3916 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 3917 return; 3918 3919 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 3920 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 3921 continue; 3922 3923 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 3924 gaudi2_stop_qman_common(hdev, reg_base); 3925 } 3926 } 3927 3928 static void gaudi2_stop_nic_qmans(struct hl_device *hdev) 3929 { 3930 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3931 u32 reg_base, queue_id; 3932 int i; 3933 3934 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 3935 return; 3936 3937 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 3938 3939 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 3940 if (!(hdev->nic_ports_mask & BIT(i))) 3941 continue; 3942 3943 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3944 gaudi2_stop_qman_common(hdev, reg_base); 3945 } 3946 } 3947 3948 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) 3949 { 3950 u32 reg_val; 3951 3952 reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1); 3953 WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val); 3954 } 3955 3956 static void gaudi2_dma_stall(struct hl_device *hdev) 3957 { 3958 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3959 int dcore, inst; 3960 3961 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3962 goto stall_edma; 3963 3964 gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE); 3965 gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE); 3966 3967 stall_edma: 3968 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3969 return; 3970 3971 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3972 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3973 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3974 u32 core_base; 3975 3976 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3977 continue; 3978 3979 core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET + 3980 inst * DCORE_EDMA_OFFSET; 3981 3982 /* Stall CPs of EDMA QMANs */ 3983 gaudi2_stall_dma_common(hdev, core_base); 3984 } 3985 } 3986 } 3987 3988 static void gaudi2_mme_stall(struct hl_device *hdev) 3989 { 3990 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3991 u32 offset, i; 3992 3993 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; 3994 3995 for (i = 0 ; i < NUM_OF_DCORES ; i++) 3996 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 3997 WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1); 3998 } 3999 4000 static void gaudi2_tpc_stall(struct hl_device *hdev) 4001 { 4002 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4003 u32 reg_base; 4004 int i; 4005 4006 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4007 return; 4008 4009 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4010 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4011 continue; 4012 4013 reg_base = gaudi2_tpc_cfg_blocks_bases[i]; 4014 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1); 4015 } 4016 } 4017 4018 static void gaudi2_rotator_stall(struct hl_device *hdev) 4019 { 4020 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4021 u32 reg_val; 4022 int i; 4023 4024 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4025 return; 4026 4027 reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) | 4028 FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) | 4029 FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1); 4030 4031 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4032 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4033 continue; 4034 4035 WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val); 4036 } 4037 } 4038 4039 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) 4040 { 4041 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0); 4042 } 4043 4044 static void gaudi2_disable_dma_qmans(struct hl_device *hdev) 4045 { 4046 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4047 int dcore, inst; 4048 4049 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4050 goto stop_edma_qmans; 4051 4052 gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE); 4053 gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE); 4054 4055 stop_edma_qmans: 4056 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4057 return; 4058 4059 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4060 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4061 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4062 u32 qm_base; 4063 4064 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4065 continue; 4066 4067 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 4068 inst * DCORE_EDMA_OFFSET; 4069 4070 /* Disable CPs of EDMA QMANs */ 4071 gaudi2_disable_qman_common(hdev, qm_base); 4072 } 4073 } 4074 } 4075 4076 static void gaudi2_disable_mme_qmans(struct hl_device *hdev) 4077 { 4078 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4079 u32 offset, i; 4080 4081 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 4082 4083 for (i = 0 ; i < NUM_OF_DCORES ; i++) 4084 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 4085 gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 4086 } 4087 4088 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) 4089 { 4090 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4091 u32 reg_base; 4092 int i; 4093 4094 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4095 return; 4096 4097 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4098 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4099 continue; 4100 4101 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 4102 gaudi2_disable_qman_common(hdev, reg_base); 4103 } 4104 } 4105 4106 static void gaudi2_disable_rot_qmans(struct hl_device *hdev) 4107 { 4108 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4109 u32 reg_base; 4110 int i; 4111 4112 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4113 return; 4114 4115 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4116 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4117 continue; 4118 4119 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 4120 gaudi2_disable_qman_common(hdev, reg_base); 4121 } 4122 } 4123 4124 static void gaudi2_disable_nic_qmans(struct hl_device *hdev) 4125 { 4126 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4127 u32 reg_base, queue_id; 4128 int i; 4129 4130 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4131 return; 4132 4133 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4134 4135 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4136 if (!(hdev->nic_ports_mask & BIT(i))) 4137 continue; 4138 4139 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4140 gaudi2_disable_qman_common(hdev, reg_base); 4141 } 4142 } 4143 4144 static void gaudi2_enable_timestamp(struct hl_device *hdev) 4145 { 4146 /* Disable the timestamp counter */ 4147 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4148 4149 /* Zero the lower/upper parts of the 64-bit counter */ 4150 WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0); 4151 WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0); 4152 4153 /* Enable the counter */ 4154 WREG32(mmPSOC_TIMESTAMP_BASE, 1); 4155 } 4156 4157 static void gaudi2_disable_timestamp(struct hl_device *hdev) 4158 { 4159 /* Disable the timestamp counter */ 4160 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4161 } 4162 4163 static const char *gaudi2_irq_name(u16 irq_number) 4164 { 4165 switch (irq_number) { 4166 case GAUDI2_IRQ_NUM_EVENT_QUEUE: 4167 return "gaudi2 cpu eq"; 4168 case GAUDI2_IRQ_NUM_COMPLETION: 4169 return "gaudi2 completion"; 4170 case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM: 4171 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; 4172 case GAUDI2_IRQ_NUM_TPC_ASSERT: 4173 return "gaudi2 tpc assert"; 4174 case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR: 4175 return "gaudi2 unexpected error"; 4176 case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST: 4177 return "gaudi2 user completion"; 4178 case GAUDI2_IRQ_NUM_EQ_ERROR: 4179 return "gaudi2 eq error"; 4180 default: 4181 return "invalid"; 4182 } 4183 } 4184 4185 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) 4186 { 4187 int i, irq, relative_idx; 4188 struct hl_dec *dec; 4189 4190 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) { 4191 irq = pci_irq_vector(hdev->pdev, i); 4192 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4193 4194 dec = hdev->dec + relative_idx / 2; 4195 4196 /* We pass different structures depending on the irq handler. For the abnormal 4197 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4198 * user_interrupt entry 4199 */ 4200 free_irq(irq, ((relative_idx % 2) ? 4201 (void *) dec : 4202 (void *) &hdev->user_interrupt[dec->core_id])); 4203 } 4204 } 4205 4206 static int gaudi2_dec_enable_msix(struct hl_device *hdev) 4207 { 4208 int rc, i, irq_init_cnt, irq, relative_idx; 4209 struct hl_dec *dec; 4210 4211 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0; 4212 i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM; 4213 i++, irq_init_cnt++) { 4214 4215 irq = pci_irq_vector(hdev->pdev, i); 4216 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4217 4218 /* We pass different structures depending on the irq handler. For the abnormal 4219 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4220 * user_interrupt entry 4221 * 4222 * TODO: change the dec abnrm to threaded irq 4223 */ 4224 4225 dec = hdev->dec + relative_idx / 2; 4226 if (relative_idx % 2) { 4227 rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0, 4228 gaudi2_irq_name(i), (void *) dec); 4229 } else { 4230 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4231 (void *) &hdev->user_interrupt[dec->core_id]); 4232 } 4233 4234 if (rc) { 4235 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4236 goto free_dec_irqs; 4237 } 4238 } 4239 4240 return 0; 4241 4242 free_dec_irqs: 4243 gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt)); 4244 return rc; 4245 } 4246 4247 static int gaudi2_enable_msix(struct hl_device *hdev) 4248 { 4249 struct asic_fixed_properties *prop = &hdev->asic_prop; 4250 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4251 int rc, irq, i, j, user_irq_init_cnt; 4252 struct hl_cq *cq; 4253 4254 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) 4255 return 0; 4256 4257 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, 4258 PCI_IRQ_MSIX); 4259 if (rc < 0) { 4260 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", 4261 GAUDI2_MSIX_ENTRIES, rc); 4262 return rc; 4263 } 4264 4265 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4266 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4267 rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq); 4268 if (rc) { 4269 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4270 goto free_irq_vectors; 4271 } 4272 4273 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4274 rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE), 4275 &hdev->event_queue); 4276 if (rc) { 4277 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4278 goto free_completion_irq; 4279 } 4280 4281 rc = gaudi2_dec_enable_msix(hdev); 4282 if (rc) { 4283 dev_err(hdev->dev, "Failed to enable decoder IRQ"); 4284 goto free_event_irq; 4285 } 4286 4287 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4288 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4289 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), 4290 &hdev->tpc_interrupt); 4291 if (rc) { 4292 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4293 goto free_dec_irq; 4294 } 4295 4296 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4297 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4298 gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR), 4299 &hdev->unexpected_error_interrupt); 4300 if (rc) { 4301 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4302 goto free_tpc_irq; 4303 } 4304 4305 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; 4306 user_irq_init_cnt < prop->user_interrupt_count; 4307 i++, j++, user_irq_init_cnt++) { 4308 4309 irq = pci_irq_vector(hdev->pdev, i); 4310 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4311 &hdev->user_interrupt[j]); 4312 if (rc) { 4313 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4314 goto free_user_irq; 4315 } 4316 } 4317 4318 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4319 rc = request_threaded_irq(irq, NULL, hl_irq_eq_error_interrupt_thread_handler, 4320 IRQF_ONESHOT, gaudi2_irq_name(GAUDI2_IRQ_NUM_EQ_ERROR), 4321 hdev); 4322 if (rc) { 4323 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4324 goto free_user_irq; 4325 } 4326 4327 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; 4328 4329 return 0; 4330 4331 free_user_irq: 4332 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; 4333 i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) { 4334 4335 irq = pci_irq_vector(hdev->pdev, i); 4336 free_irq(irq, &hdev->user_interrupt[j]); 4337 } 4338 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4339 free_irq(irq, &hdev->unexpected_error_interrupt); 4340 free_tpc_irq: 4341 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4342 free_irq(irq, &hdev->tpc_interrupt); 4343 free_dec_irq: 4344 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1); 4345 free_event_irq: 4346 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4347 free_irq(irq, cq); 4348 4349 free_completion_irq: 4350 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4351 free_irq(irq, cq); 4352 4353 free_irq_vectors: 4354 pci_free_irq_vectors(hdev->pdev); 4355 4356 return rc; 4357 } 4358 4359 static void gaudi2_sync_irqs(struct hl_device *hdev) 4360 { 4361 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4362 int i, j; 4363 int irq; 4364 4365 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4366 return; 4367 4368 /* Wait for all pending IRQs to be finished */ 4369 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); 4370 4371 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) { 4372 irq = pci_irq_vector(hdev->pdev, i); 4373 synchronize_irq(irq); 4374 } 4375 4376 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); 4377 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); 4378 4379 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; 4380 i++, j++) { 4381 irq = pci_irq_vector(hdev->pdev, i); 4382 synchronize_irq(irq); 4383 } 4384 4385 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); 4386 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); 4387 } 4388 4389 static void gaudi2_disable_msix(struct hl_device *hdev) 4390 { 4391 struct asic_fixed_properties *prop = &hdev->asic_prop; 4392 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4393 struct hl_cq *cq; 4394 int irq, i, j, k; 4395 4396 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4397 return; 4398 4399 gaudi2_sync_irqs(hdev); 4400 4401 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4402 free_irq(irq, &hdev->event_queue); 4403 4404 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); 4405 4406 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4407 free_irq(irq, &hdev->tpc_interrupt); 4408 4409 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4410 free_irq(irq, &hdev->unexpected_error_interrupt); 4411 4412 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; 4413 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { 4414 4415 irq = pci_irq_vector(hdev->pdev, i); 4416 free_irq(irq, &hdev->user_interrupt[j]); 4417 } 4418 4419 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4420 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4421 free_irq(irq, cq); 4422 4423 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4424 free_irq(irq, hdev); 4425 4426 pci_free_irq_vectors(hdev->pdev); 4427 4428 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; 4429 } 4430 4431 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) 4432 { 4433 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4434 u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4435 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4436 int rc; 4437 4438 if (hdev->pldm) 4439 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4440 else 4441 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4442 4443 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4444 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 4445 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4446 continue; 4447 4448 offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET; 4449 4450 WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0); 4451 4452 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4453 4454 /* Wait till all traffic from decoder stops 4455 * before apply core reset. 4456 */ 4457 rc = hl_poll_timeout( 4458 hdev, 4459 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4460 graceful, 4461 (graceful & graceful_pend_mask), 4462 100, 4463 timeout_usec); 4464 if (rc) 4465 dev_err(hdev->dev, 4466 "Failed to stop traffic from DCORE%d Decoder %d\n", 4467 dcore_id, dec_id); 4468 } 4469 } 4470 4471 static void gaudi2_stop_pcie_dec(struct hl_device *hdev) 4472 { 4473 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4474 u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4475 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4476 int rc; 4477 4478 if (hdev->pldm) 4479 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4480 else 4481 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4482 4483 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4484 dec_bit = PCIE_DEC_SHIFT + dec_id; 4485 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4486 continue; 4487 4488 offset = dec_id * PCIE_VDEC_OFFSET; 4489 4490 WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0); 4491 4492 WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4493 4494 /* Wait till all traffic from decoder stops 4495 * before apply core reset. 4496 */ 4497 rc = hl_poll_timeout( 4498 hdev, 4499 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4500 graceful, 4501 (graceful & graceful_pend_mask), 4502 100, 4503 timeout_usec); 4504 if (rc) 4505 dev_err(hdev->dev, 4506 "Failed to stop traffic from PCIe Decoder %d\n", 4507 dec_id); 4508 } 4509 } 4510 4511 static void gaudi2_stop_dec(struct hl_device *hdev) 4512 { 4513 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4514 int dcore_id; 4515 4516 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) 4517 return; 4518 4519 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 4520 gaudi2_stop_dcore_dec(hdev, dcore_id); 4521 4522 gaudi2_stop_pcie_dec(hdev); 4523 } 4524 4525 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4526 { 4527 u32 reg_base, reg_val; 4528 4529 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4530 if (run_mode == HL_ENGINE_CORE_RUN) 4531 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); 4532 else 4533 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); 4534 4535 WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val); 4536 } 4537 4538 static void gaudi2_halt_arcs(struct hl_device *hdev) 4539 { 4540 u16 arc_id; 4541 4542 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) { 4543 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4544 gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT); 4545 } 4546 } 4547 4548 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4549 { 4550 int rc; 4551 u32 reg_base, val, ack_mask, timeout_usec = 100000; 4552 4553 if (hdev->pldm) 4554 timeout_usec *= 100; 4555 4556 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4557 if (run_mode == HL_ENGINE_CORE_RUN) 4558 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK; 4559 else 4560 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK; 4561 4562 rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET, 4563 val, ((val & ack_mask) == ack_mask), 4564 1000, timeout_usec); 4565 4566 if (!rc) { 4567 /* Clear */ 4568 val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0); 4569 WREG32(reg_base + ARC_HALT_REQ_OFFSET, val); 4570 } 4571 4572 return rc; 4573 } 4574 4575 static void gaudi2_reset_arcs(struct hl_device *hdev) 4576 { 4577 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4578 u16 arc_id; 4579 4580 if (!gaudi2) 4581 return; 4582 4583 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) 4584 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4585 gaudi2_clr_arc_id_cap(hdev, arc_id); 4586 } 4587 4588 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) 4589 { 4590 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4591 u32 queue_id; 4592 int i; 4593 4594 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4595 return; 4596 4597 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4598 4599 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4600 if (!(hdev->nic_ports_mask & BIT(i))) 4601 continue; 4602 4603 gaudi2_qman_manual_flush_common(hdev, queue_id); 4604 } 4605 } 4606 4607 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, 4608 u32 num_cores, u32 core_command) 4609 { 4610 int i, rc; 4611 4612 for (i = 0 ; i < num_cores ; i++) { 4613 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) 4614 gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command); 4615 } 4616 4617 for (i = 0 ; i < num_cores ; i++) { 4618 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) { 4619 rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command); 4620 4621 if (rc) { 4622 dev_err(hdev->dev, "failed to %s arc: %d\n", 4623 (core_command == HL_ENGINE_CORE_HALT) ? 4624 "HALT" : "RUN", core_ids[i]); 4625 return -1; 4626 } 4627 } 4628 } 4629 4630 return 0; 4631 } 4632 4633 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4634 { 4635 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4636 u32 reg_base, reg_addr, reg_val, tpc_id; 4637 4638 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4639 return 0; 4640 4641 tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id]; 4642 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) 4643 return 0; 4644 4645 reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id]; 4646 reg_addr = reg_base + TPC_CFG_STALL_OFFSET; 4647 reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK, 4648 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4649 WREG32(reg_addr, reg_val); 4650 4651 if (engine_command == HL_ENGINE_RESUME) { 4652 reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id]; 4653 reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET; 4654 RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK); 4655 } 4656 4657 return 0; 4658 } 4659 4660 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4661 { 4662 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4663 u32 reg_base, reg_addr, reg_val, mme_id; 4664 4665 mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id]; 4666 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) 4667 return 0; 4668 4669 reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id]; 4670 reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET; 4671 reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK, 4672 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4673 WREG32(reg_addr, reg_val); 4674 4675 return 0; 4676 } 4677 4678 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4679 { 4680 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4681 u32 reg_base, reg_addr, reg_val, edma_id; 4682 4683 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4684 return 0; 4685 4686 edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id]; 4687 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) 4688 return 0; 4689 4690 reg_base = gaudi2_dma_core_blocks_bases[edma_id]; 4691 reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET; 4692 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 4693 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4694 WREG32(reg_addr, reg_val); 4695 4696 if (engine_command == HL_ENGINE_STALL) { 4697 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) | 4698 FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1); 4699 WREG32(reg_addr, reg_val); 4700 } 4701 4702 return 0; 4703 } 4704 4705 static int gaudi2_set_engine_modes(struct hl_device *hdev, 4706 u32 *engine_ids, u32 num_engines, u32 engine_command) 4707 { 4708 int i, rc; 4709 4710 for (i = 0 ; i < num_engines ; ++i) { 4711 switch (engine_ids[i]) { 4712 case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5: 4713 case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5: 4714 case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5: 4715 case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5: 4716 rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command); 4717 if (rc) 4718 return rc; 4719 4720 break; 4721 case GAUDI2_DCORE0_ENGINE_ID_MME: 4722 case GAUDI2_DCORE1_ENGINE_ID_MME: 4723 case GAUDI2_DCORE2_ENGINE_ID_MME: 4724 case GAUDI2_DCORE3_ENGINE_ID_MME: 4725 rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command); 4726 if (rc) 4727 return rc; 4728 4729 break; 4730 case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1: 4731 case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1: 4732 case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1: 4733 case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1: 4734 rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command); 4735 if (rc) 4736 return rc; 4737 4738 break; 4739 default: 4740 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); 4741 return -EINVAL; 4742 } 4743 } 4744 4745 return 0; 4746 } 4747 4748 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, 4749 u32 num_engines, u32 engine_command) 4750 { 4751 switch (engine_command) { 4752 case HL_ENGINE_CORE_HALT: 4753 case HL_ENGINE_CORE_RUN: 4754 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command); 4755 4756 case HL_ENGINE_STALL: 4757 case HL_ENGINE_RESUME: 4758 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command); 4759 4760 default: 4761 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); 4762 return -EINVAL; 4763 } 4764 } 4765 4766 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 4767 { 4768 u32 wait_timeout_ms; 4769 4770 if (hdev->pldm) 4771 wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC; 4772 else 4773 wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC; 4774 4775 if (fw_reset) 4776 goto skip_engines; 4777 4778 gaudi2_stop_dma_qmans(hdev); 4779 gaudi2_stop_mme_qmans(hdev); 4780 gaudi2_stop_tpc_qmans(hdev); 4781 gaudi2_stop_rot_qmans(hdev); 4782 gaudi2_stop_nic_qmans(hdev); 4783 msleep(wait_timeout_ms); 4784 4785 gaudi2_halt_arcs(hdev); 4786 gaudi2_dma_stall(hdev); 4787 gaudi2_mme_stall(hdev); 4788 gaudi2_tpc_stall(hdev); 4789 gaudi2_rotator_stall(hdev); 4790 4791 msleep(wait_timeout_ms); 4792 4793 gaudi2_stop_dec(hdev); 4794 4795 /* 4796 * in case of soft reset do a manual flush for QMANs (currently called 4797 * only for NIC QMANs 4798 */ 4799 if (!hard_reset) 4800 gaudi2_nic_qmans_manual_flush(hdev); 4801 4802 gaudi2_disable_dma_qmans(hdev); 4803 gaudi2_disable_mme_qmans(hdev); 4804 gaudi2_disable_tpc_qmans(hdev); 4805 gaudi2_disable_rot_qmans(hdev); 4806 gaudi2_disable_nic_qmans(hdev); 4807 gaudi2_disable_timestamp(hdev); 4808 4809 skip_engines: 4810 if (hard_reset) { 4811 gaudi2_disable_msix(hdev); 4812 return; 4813 } 4814 4815 gaudi2_sync_irqs(hdev); 4816 } 4817 4818 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) 4819 { 4820 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 4821 4822 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 4823 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 4824 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 4825 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 4826 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 4827 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; 4828 pre_fw_load->wait_for_preboot_extended_timeout = 4829 GAUDI2_PREBOOT_EXTENDED_REQ_TIMEOUT_USEC; 4830 } 4831 4832 static void gaudi2_init_firmware_loader(struct hl_device *hdev) 4833 { 4834 struct fw_load_mgr *fw_loader = &hdev->fw_loader; 4835 struct dynamic_fw_load_mgr *dynamic_loader; 4836 struct cpu_dyn_regs *dyn_regs; 4837 4838 /* fill common fields */ 4839 fw_loader->fw_comp_loaded = FW_TYPE_NONE; 4840 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; 4841 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; 4842 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; 4843 fw_loader->skip_bmc = false; 4844 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; 4845 fw_loader->dram_bar_id = DRAM_BAR_ID; 4846 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; 4847 4848 /* here we update initial values for few specific dynamic regs (as 4849 * before reading the first descriptor from FW those value has to be 4850 * hard-coded). in later stages of the protocol those values will be 4851 * updated automatically by reading the FW descriptor so data there 4852 * will always be up-to-date 4853 */ 4854 dynamic_loader = &hdev->fw_loader.dynamic_loader; 4855 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 4856 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 4857 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 4858 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; 4859 } 4860 4861 static int gaudi2_init_cpu(struct hl_device *hdev) 4862 { 4863 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4864 int rc; 4865 4866 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 4867 return 0; 4868 4869 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) 4870 return 0; 4871 4872 rc = hl_fw_init_cpu(hdev); 4873 if (rc) 4874 return rc; 4875 4876 gaudi2->hw_cap_initialized |= HW_CAP_CPU; 4877 4878 return 0; 4879 } 4880 4881 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) 4882 { 4883 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 4884 struct asic_fixed_properties *prop = &hdev->asic_prop; 4885 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4886 struct cpu_dyn_regs *dyn_regs; 4887 struct hl_eq *eq; 4888 u32 status; 4889 int err; 4890 4891 if (!hdev->cpu_queues_enable) 4892 return 0; 4893 4894 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 4895 return 0; 4896 4897 eq = &hdev->event_queue; 4898 4899 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4900 4901 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 4902 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 4903 4904 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 4905 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 4906 4907 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); 4908 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); 4909 4910 WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 4911 WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 4912 WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 4913 4914 /* Used for EQ CI */ 4915 WREG32(mmCPU_IF_EQ_RD_OFFS, 0); 4916 4917 WREG32(mmCPU_IF_PF_PQ_PI, 0); 4918 4919 WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP); 4920 4921 /* Let the ARC know we are ready as it is now handling those queues */ 4922 4923 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 4924 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 4925 4926 err = hl_poll_timeout( 4927 hdev, 4928 mmCPU_IF_QUEUE_INIT, 4929 status, 4930 (status == PQ_INIT_STATUS_READY_FOR_HOST), 4931 1000, 4932 cpu_timeout); 4933 4934 if (err) { 4935 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); 4936 return -EIO; 4937 } 4938 4939 /* update FW application security bits */ 4940 if (prop->fw_cpu_boot_dev_sts0_valid) 4941 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 4942 4943 if (prop->fw_cpu_boot_dev_sts1_valid) 4944 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 4945 4946 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; 4947 return 0; 4948 } 4949 4950 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, 4951 u32 queue_id_base) 4952 { 4953 struct hl_hw_queue *q; 4954 u32 pq_id, pq_offset; 4955 4956 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 4957 q = &hdev->kernel_queues[queue_id_base + pq_id]; 4958 pq_offset = pq_id * 4; 4959 4960 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 4961 lower_32_bits(q->bus_address)); 4962 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 4963 upper_32_bits(q->bus_address)); 4964 WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH)); 4965 WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0); 4966 WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0); 4967 } 4968 } 4969 4970 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) 4971 { 4972 u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi; 4973 4974 mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 4975 mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 4976 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4977 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4978 4979 for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) { 4980 cp_offset = cp_id * 4; 4981 4982 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo); 4983 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi); 4984 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo); 4985 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi); 4986 } 4987 4988 /* allow QMANs to accept work from ARC CQF */ 4989 WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1)); 4990 } 4991 4992 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, 4993 u32 queue_id_base) 4994 { 4995 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4996 u32 pq_id, pq_offset, so_base_lo, so_base_hi; 4997 4998 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4999 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5000 5001 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 5002 pq_offset = pq_id * 4; 5003 5004 /* Configure QMAN HBW to scratchpad as it is not needed */ 5005 WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset, 5006 lower_32_bits(gaudi2->scratchpad_bus_address)); 5007 WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset, 5008 upper_32_bits(gaudi2->scratchpad_bus_address)); 5009 WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset, 5010 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry))); 5011 5012 WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0); 5013 WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA); 5014 WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo); 5015 WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi); 5016 } 5017 5018 /* Enable QMAN H/W completion */ 5019 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 5020 } 5021 5022 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) 5023 { 5024 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5025 u32 sp_reg_addr; 5026 5027 switch (queue_id_base) { 5028 case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3: 5029 fallthrough; 5030 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 5031 fallthrough; 5032 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 5033 fallthrough; 5034 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 5035 fallthrough; 5036 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 5037 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 5038 break; 5039 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 5040 fallthrough; 5041 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 5042 fallthrough; 5043 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 5044 fallthrough; 5045 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 5046 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); 5047 break; 5048 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 5049 fallthrough; 5050 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 5051 fallthrough; 5052 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 5053 fallthrough; 5054 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 5055 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); 5056 break; 5057 case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3: 5058 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); 5059 break; 5060 case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3: 5061 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); 5062 break; 5063 default: 5064 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); 5065 return 0; 5066 } 5067 5068 return sp_reg_addr; 5069 } 5070 5071 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, 5072 u32 queue_id_base) 5073 { 5074 u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset; 5075 int map_table_entry; 5076 5077 WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot); 5078 5079 irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base); 5080 WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset)); 5081 WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset)); 5082 5083 map_table_entry = gaudi2_qman_async_event_id[queue_id_base]; 5084 WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET, 5085 gaudi2_irq_map_table[map_table_entry].cpu_id); 5086 5087 WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK); 5088 5089 WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT); 5090 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0); 5091 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0); 5092 5093 /* Enable the QMAN channel. 5094 * PDMA QMAN configuration is different, as we do not allow user to 5095 * access some of the CPs. 5096 * PDMA0: CP2/3 are reserved for the ARC usage. 5097 * PDMA1: CP1/2/3 are reserved for the ARC usage. 5098 */ 5099 if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]) 5100 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE); 5101 else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]) 5102 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE); 5103 else 5104 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE); 5105 } 5106 5107 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, 5108 u32 queue_id_base) 5109 { 5110 u32 pq_id; 5111 5112 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) 5113 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; 5114 5115 gaudi2_init_qman_pq(hdev, reg_base, queue_id_base); 5116 gaudi2_init_qman_cp(hdev, reg_base); 5117 gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base); 5118 gaudi2_init_qman_common(hdev, reg_base, queue_id_base); 5119 } 5120 5121 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, 5122 u32 dma_core_id, bool is_secure) 5123 { 5124 u32 prot, irq_handler_offset; 5125 struct cpu_dyn_regs *dyn_regs; 5126 int map_table_entry; 5127 5128 prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT; 5129 if (is_secure) 5130 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT; 5131 5132 WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot); 5133 5134 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5135 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); 5136 5137 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET, 5138 lower_32_bits(CFG_BASE + irq_handler_offset)); 5139 5140 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET, 5141 upper_32_bits(CFG_BASE + irq_handler_offset)); 5142 5143 map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id]; 5144 WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET, 5145 gaudi2_irq_map_table[map_table_entry].cpu_id); 5146 5147 /* Enable the DMA channel */ 5148 WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT); 5149 } 5150 5151 static void gaudi2_init_kdma(struct hl_device *hdev) 5152 { 5153 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5154 u32 reg_base; 5155 5156 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) 5157 return; 5158 5159 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA]; 5160 5161 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true); 5162 5163 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; 5164 } 5165 5166 static void gaudi2_init_pdma(struct hl_device *hdev) 5167 { 5168 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5169 u32 reg_base; 5170 5171 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) 5172 return; 5173 5174 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0]; 5175 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false); 5176 5177 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]; 5178 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0); 5179 5180 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1]; 5181 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false); 5182 5183 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]; 5184 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0); 5185 5186 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; 5187 } 5188 5189 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) 5190 { 5191 u32 reg_base, base_edma_core_id, base_edma_qman_id; 5192 5193 base_edma_core_id = DMA_CORE_ID_EDMA0 + seq; 5194 base_edma_qman_id = edma_stream_base[seq]; 5195 5196 reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id]; 5197 gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false); 5198 5199 reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id]; 5200 gaudi2_init_qman(hdev, reg_base, base_edma_qman_id); 5201 } 5202 5203 static void gaudi2_init_edma(struct hl_device *hdev) 5204 { 5205 struct asic_fixed_properties *prop = &hdev->asic_prop; 5206 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5207 int dcore, inst; 5208 5209 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) 5210 return; 5211 5212 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 5213 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 5214 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 5215 5216 if (!(prop->edma_enabled_mask & BIT(seq))) 5217 continue; 5218 5219 gaudi2_init_edma_instance(hdev, seq); 5220 5221 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); 5222 } 5223 } 5224 } 5225 5226 /* 5227 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell. 5228 * @hdev: pointer to habanalabs device structure. 5229 * @sob_id: sync object ID. 5230 * @first_mon_id: ID of first monitor out of 3 consecutive monitors. 5231 * @interrupt_id: interrupt ID. 5232 * 5233 * Some initiators cannot have HBW address in their completion address registers, and thus cannot 5234 * write directly to the HBW host memory of the virtual MSI-X doorbell. 5235 * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write. 5236 * 5237 * The mechanism in the sync manager block is composed of a master monitor with 3 messages. 5238 * In addition to the HBW write, the other 2 messages are for preparing the monitor to next 5239 * completion, by decrementing the sync object value and re-arming the monitor. 5240 */ 5241 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, 5242 u32 first_mon_id, u32 interrupt_id) 5243 { 5244 u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config; 5245 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5246 u64 addr; 5247 u8 mask; 5248 5249 /* Reset the SOB value */ 5250 sob_offset = sob_id * sizeof(u32); 5251 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 5252 5253 /* Configure 3 monitors: 5254 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) 5255 * 2. Decrement SOB value by 1. 5256 * 3. Re-arm the master monitor. 5257 */ 5258 5259 first_mon_offset = first_mon_id * sizeof(u32); 5260 5261 /* 2nd monitor: Decrement SOB value by 1 */ 5262 mon_offset = first_mon_offset + sizeof(u32); 5263 5264 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 5265 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5266 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5267 5268 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ 5269 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) | 5270 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1); 5271 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5272 5273 /* 3rd monitor: Re-arm the master monitor */ 5274 mon_offset = first_mon_offset + 2 * sizeof(u32); 5275 5276 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset; 5277 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5278 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5279 5280 sob_group = sob_id / 8; 5281 mask = ~BIT(sob_id & 0x7); 5282 mode = 0; /* comparison mode is "greater than or equal to" */ 5283 arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) | 5284 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) | 5285 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) | 5286 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1); 5287 5288 payload = arm; 5289 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5290 5291 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ 5292 mon_offset = first_mon_offset; 5293 5294 config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */ 5295 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config); 5296 5297 addr = gaudi2->virt_msix_db_dma_addr; 5298 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5299 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5300 5301 payload = interrupt_id; 5302 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5303 5304 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm); 5305 } 5306 5307 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) 5308 { 5309 u32 decoder_id, sob_id, first_mon_id, interrupt_id; 5310 struct asic_fixed_properties *prop = &hdev->asic_prop; 5311 5312 /* Decoder normal/abnormal interrupts */ 5313 for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) { 5314 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) 5315 continue; 5316 5317 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5318 first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id; 5319 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id; 5320 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5321 5322 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5323 first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id; 5324 interrupt_id += 1; 5325 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5326 } 5327 } 5328 5329 static void gaudi2_init_sm(struct hl_device *hdev) 5330 { 5331 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5332 u64 cq_address; 5333 u32 reg_val; 5334 int i; 5335 5336 /* Enable HBW/LBW CQ for completion monitors */ 5337 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5338 reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1); 5339 5340 for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++) 5341 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5342 5343 /* Enable only HBW CQ for KDMA completion monitor */ 5344 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5345 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5346 5347 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ 5348 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); 5349 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); 5350 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION); 5351 5352 for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) { 5353 cq_address = 5354 hdev->completion_queue[i].bus_address; 5355 5356 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i), 5357 lower_32_bits(cq_address)); 5358 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i), 5359 upper_32_bits(cq_address)); 5360 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i), 5361 ilog2(HL_CQ_SIZE_IN_BYTES)); 5362 } 5363 5364 /* Configure kernel ASID and MMU BP*/ 5365 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000); 5366 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0); 5367 5368 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ 5369 gaudi2_prepare_sm_for_virt_msix_db(hdev); 5370 } 5371 5372 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) 5373 { 5374 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5375 u32 reg_val; 5376 int i; 5377 5378 reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0); 5379 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1); 5380 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1); 5381 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1); 5382 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1); 5383 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1); 5384 5385 WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val); 5386 WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF); 5387 5388 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) { 5389 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i); 5390 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); 5391 } 5392 } 5393 5394 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, 5395 bool config_qman_only) 5396 { 5397 u32 queue_id_base, reg_base; 5398 5399 switch (dcore_id) { 5400 case 0: 5401 queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 5402 break; 5403 case 1: 5404 queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 5405 break; 5406 case 2: 5407 queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 5408 break; 5409 case 3: 5410 queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 5411 break; 5412 default: 5413 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); 5414 return; 5415 } 5416 5417 if (!config_qman_only) { 5418 reg_base = gaudi2_mme_acc_blocks_bases[dcore_id]; 5419 gaudi2_init_mme_acc(hdev, reg_base); 5420 } 5421 5422 reg_base = gaudi2_qm_blocks_bases[queue_id_base]; 5423 gaudi2_init_qman(hdev, reg_base, queue_id_base); 5424 } 5425 5426 static void gaudi2_init_mme(struct hl_device *hdev) 5427 { 5428 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5429 int i; 5430 5431 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) 5432 return; 5433 5434 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 5435 gaudi2_init_dcore_mme(hdev, i, false); 5436 5437 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); 5438 } 5439 } 5440 5441 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) 5442 { 5443 /* Mask arithmetic and QM interrupts in TPC */ 5444 WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE); 5445 5446 /* Set 16 cache lines */ 5447 WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET, 5448 2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT); 5449 } 5450 5451 struct gaudi2_tpc_init_cfg_data { 5452 enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES]; 5453 }; 5454 5455 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, 5456 u32 offset, struct iterate_module_ctx *ctx) 5457 { 5458 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5459 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; 5460 u32 queue_id_base; 5461 u8 seq; 5462 5463 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); 5464 5465 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) 5466 /* gets last sequence number */ 5467 seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE; 5468 else 5469 seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 5470 5471 gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset); 5472 gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base); 5473 5474 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); 5475 } 5476 5477 static void gaudi2_init_tpc(struct hl_device *hdev) 5478 { 5479 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5480 struct gaudi2_tpc_init_cfg_data init_cfg_data; 5481 struct iterate_module_ctx tpc_iter; 5482 5483 if (!hdev->asic_prop.tpc_enabled_mask) 5484 return; 5485 5486 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) 5487 return; 5488 5489 init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0; 5490 init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0; 5491 init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0; 5492 init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0; 5493 tpc_iter.fn = &gaudi2_init_tpc_config; 5494 tpc_iter.data = &init_cfg_data; 5495 gaudi2_iterate_tpcs(hdev, &tpc_iter); 5496 } 5497 5498 static void gaudi2_init_rotator(struct hl_device *hdev) 5499 { 5500 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5501 u32 i, reg_base, queue_id; 5502 5503 queue_id = GAUDI2_QUEUE_ID_ROT_0_0; 5504 5505 for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 5506 reg_base = gaudi2_qm_blocks_bases[queue_id]; 5507 gaudi2_init_qman(hdev, reg_base, queue_id); 5508 5509 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); 5510 } 5511 } 5512 5513 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) 5514 { 5515 u32 sob_id; 5516 5517 /* VCMD normal interrupt */ 5518 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5519 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, 5520 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5521 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5522 5523 /* VCMD abnormal interrupt */ 5524 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5525 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, 5526 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5527 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5528 } 5529 5530 static void gaudi2_init_dec(struct hl_device *hdev) 5531 { 5532 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5533 u32 dcore_id, dec_id, dec_bit; 5534 u64 base_addr; 5535 5536 if (!hdev->asic_prop.decoder_enabled_mask) 5537 return; 5538 5539 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) 5540 return; 5541 5542 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 5543 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 5544 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 5545 5546 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5547 continue; 5548 5549 base_addr = mmDCORE0_DEC0_CMD_BASE + 5550 BRDG_CTRL_BLOCK_OFFSET + 5551 dcore_id * DCORE_OFFSET + 5552 dec_id * DCORE_VDEC_OFFSET; 5553 5554 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5555 5556 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5557 } 5558 5559 for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) { 5560 dec_bit = PCIE_DEC_SHIFT + dec_id; 5561 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5562 continue; 5563 5564 base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET + 5565 dec_id * DCORE_VDEC_OFFSET; 5566 5567 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5568 5569 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5570 } 5571 } 5572 5573 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, 5574 u32 stlb_base, u32 asid, u64 phys_addr) 5575 { 5576 u32 status, timeout_usec; 5577 int rc; 5578 5579 if (hdev->pldm || !hdev->pdev) 5580 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5581 else 5582 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 5583 5584 WREG32(stlb_base + STLB_ASID_OFFSET, asid); 5585 WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 5586 WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT); 5587 WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000); 5588 5589 rc = hl_poll_timeout( 5590 hdev, 5591 stlb_base + STLB_BUSY_OFFSET, 5592 status, 5593 !(status & 0x80000000), 5594 1000, 5595 timeout_usec); 5596 5597 if (rc) { 5598 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); 5599 return rc; 5600 } 5601 5602 return 0; 5603 } 5604 5605 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, 5606 u32 start_offset, u32 inv_start_val, 5607 u32 flags) 5608 { 5609 /* clear PMMU mem line cache (only needed in mmu range invalidation) */ 5610 if (flags & MMU_OP_CLEAR_MEMCACHE) 5611 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1); 5612 5613 if (flags & MMU_OP_SKIP_LOW_CACHE_INV) 5614 return; 5615 5616 WREG32(stlb_base + start_offset, inv_start_val); 5617 } 5618 5619 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, 5620 struct gaudi2_cache_invld_params *inv_params) 5621 { 5622 u32 status, timeout_usec, start_offset; 5623 int rc; 5624 5625 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : 5626 GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5627 5628 /* poll PMMU mem line cache (only needed in mmu range invalidation) */ 5629 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { 5630 rc = hl_poll_timeout( 5631 hdev, 5632 mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 5633 status, 5634 status & 0x1, 5635 1000, 5636 timeout_usec); 5637 5638 if (rc) 5639 return rc; 5640 5641 /* Need to manually reset the status to 0 */ 5642 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0); 5643 } 5644 5645 /* Lower cache does not work with cache lines, hence we can skip its 5646 * invalidation upon map and invalidate only upon unmap 5647 */ 5648 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) 5649 return 0; 5650 5651 start_offset = inv_params->range_invalidation ? 5652 STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET; 5653 5654 rc = hl_poll_timeout( 5655 hdev, 5656 stlb_base + start_offset, 5657 status, 5658 !(status & 0x1), 5659 1000, 5660 timeout_usec); 5661 5662 return rc; 5663 } 5664 5665 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) 5666 { 5667 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5668 u32 hw_cap; 5669 5670 hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id); 5671 5672 if (gaudi2->hw_cap_initialized & hw_cap) 5673 return true; 5674 5675 return false; 5676 } 5677 5678 /* this function shall be called only for HMMUs for which capability bit is set */ 5679 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id) 5680 { 5681 u32 offset; 5682 5683 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5684 return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset); 5685 } 5686 5687 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, 5688 struct gaudi2_cache_invld_params *inv_params) 5689 { 5690 u32 start_offset; 5691 5692 if (inv_params->range_invalidation) { 5693 /* Set the addresses range 5694 * Note: that the start address we set in register, is not included in 5695 * the range of the invalidation, by design. 5696 * that's why we need to set lower address than the one we actually 5697 * want to be included in the range invalidation. 5698 */ 5699 u64 start = inv_params->start_va - 1; 5700 5701 start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET; 5702 5703 WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET, 5704 start >> MMU_RANGE_INV_VA_LSB_SHIFT); 5705 5706 WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET, 5707 start >> MMU_RANGE_INV_VA_MSB_SHIFT); 5708 5709 WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET, 5710 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); 5711 5712 WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET, 5713 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); 5714 } else { 5715 start_offset = STLB_INV_ALL_START_OFFSET; 5716 } 5717 5718 gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset, 5719 inv_params->inv_start_val, inv_params->flags); 5720 } 5721 5722 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, 5723 int dcore_id, int hmmu_id, 5724 struct gaudi2_cache_invld_params *inv_params) 5725 { 5726 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5727 5728 gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params); 5729 } 5730 5731 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, 5732 int dcore_id, int hmmu_id, 5733 struct gaudi2_cache_invld_params *inv_params) 5734 { 5735 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5736 5737 return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params); 5738 } 5739 5740 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, 5741 struct gaudi2_cache_invld_params *inv_params) 5742 { 5743 int dcore_id, hmmu_id; 5744 5745 /* first send all invalidation commands */ 5746 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5747 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5748 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5749 continue; 5750 5751 gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params); 5752 } 5753 } 5754 5755 /* next, poll all invalidations status */ 5756 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5757 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5758 int rc; 5759 5760 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5761 continue; 5762 5763 rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id, 5764 inv_params); 5765 if (rc) 5766 return rc; 5767 } 5768 } 5769 5770 return 0; 5771 } 5772 5773 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) 5774 { 5775 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5776 struct gaudi2_cache_invld_params invld_params; 5777 int rc = 0; 5778 5779 if (hdev->reset_info.hard_reset_pending) 5780 return rc; 5781 5782 invld_params.range_invalidation = false; 5783 invld_params.inv_start_val = 1; 5784 5785 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5786 invld_params.flags = flags; 5787 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5788 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5789 &invld_params); 5790 } else if (flags & MMU_OP_PHYS_PACK) { 5791 invld_params.flags = 0; 5792 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5793 } 5794 5795 return rc; 5796 } 5797 5798 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, 5799 u32 flags, u32 asid, u64 va, u64 size) 5800 { 5801 struct gaudi2_cache_invld_params invld_params = {0}; 5802 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5803 u64 start_va, end_va; 5804 u32 inv_start_val; 5805 int rc = 0; 5806 5807 if (hdev->reset_info.hard_reset_pending) 5808 return 0; 5809 5810 inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT | 5811 1 << MMU_RANGE_INV_ASID_EN_SHIFT | 5812 asid << MMU_RANGE_INV_ASID_SHIFT); 5813 start_va = va; 5814 end_va = start_va + size; 5815 5816 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5817 /* As range invalidation does not support zero address we will 5818 * do full invalidation in this case 5819 */ 5820 if (start_va) { 5821 invld_params.range_invalidation = true; 5822 invld_params.start_va = start_va; 5823 invld_params.end_va = end_va; 5824 invld_params.inv_start_val = inv_start_val; 5825 invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE; 5826 } else { 5827 invld_params.range_invalidation = false; 5828 invld_params.inv_start_val = 1; 5829 invld_params.flags = flags; 5830 } 5831 5832 5833 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5834 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5835 &invld_params); 5836 if (rc) 5837 return rc; 5838 5839 } else if (flags & MMU_OP_PHYS_PACK) { 5840 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va); 5841 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va); 5842 invld_params.inv_start_val = inv_start_val; 5843 invld_params.flags = flags; 5844 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5845 } 5846 5847 return rc; 5848 } 5849 5850 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base) 5851 { 5852 struct asic_fixed_properties *prop = &hdev->asic_prop; 5853 u64 hop0_addr; 5854 u32 asid, max_asid = prop->max_asid; 5855 int rc; 5856 5857 /* it takes too much time to init all of the ASIDs on palladium */ 5858 if (hdev->pldm) 5859 max_asid = min((u32) 8, max_asid); 5860 5861 for (asid = 0 ; asid < max_asid ; asid++) { 5862 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; 5863 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr); 5864 if (rc) { 5865 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); 5866 return rc; 5867 } 5868 } 5869 5870 return 0; 5871 } 5872 5873 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base) 5874 { 5875 u32 status, timeout_usec; 5876 int rc; 5877 5878 if (hdev->pldm || !hdev->pdev) 5879 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5880 else 5881 timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5882 5883 WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1); 5884 5885 rc = hl_poll_timeout( 5886 hdev, 5887 stlb_base + STLB_SRAM_INIT_OFFSET, 5888 status, 5889 !status, 5890 1000, 5891 timeout_usec); 5892 5893 if (rc) 5894 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); 5895 5896 rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base); 5897 if (rc) 5898 return rc; 5899 5900 WREG32(mmu_base + MMU_BYPASS_OFFSET, 0); 5901 5902 rc = hl_poll_timeout( 5903 hdev, 5904 stlb_base + STLB_INV_ALL_START_OFFSET, 5905 status, 5906 !status, 5907 1000, 5908 timeout_usec); 5909 5910 if (rc) 5911 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); 5912 5913 WREG32(mmu_base + MMU_ENABLE_OFFSET, 1); 5914 5915 return rc; 5916 } 5917 5918 static int gaudi2_pci_mmu_init(struct hl_device *hdev) 5919 { 5920 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5921 u32 mmu_base, stlb_base; 5922 int rc; 5923 5924 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) 5925 return 0; 5926 5927 mmu_base = mmPMMU_HBW_MMU_BASE; 5928 stlb_base = mmPMMU_HBW_STLB_BASE; 5929 5930 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 5931 (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) | 5932 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) | 5933 (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) | 5934 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) | 5935 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT), 5936 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 5937 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 5938 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 5939 PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 5940 PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 5941 5942 WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0); 5943 5944 if (PAGE_SIZE == SZ_64K) { 5945 /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */ 5946 RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5947 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) | 5948 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) | 5949 FIELD_PREP( 5950 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK, 5951 1), 5952 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK | 5953 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK | 5954 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK); 5955 } 5956 5957 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK); 5958 5959 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); 5960 if (rc) 5961 return rc; 5962 5963 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; 5964 5965 return 0; 5966 } 5967 5968 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, 5969 int hmmu_id) 5970 { 5971 struct asic_fixed_properties *prop = &hdev->asic_prop; 5972 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5973 u32 offset, mmu_base, stlb_base, hw_cap; 5974 u8 dmmu_seq; 5975 int rc; 5976 5977 dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id; 5978 hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq; 5979 5980 /* 5981 * return if DMMU is already initialized or if it's not out of 5982 * isolation (due to cluster binning) 5983 */ 5984 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) 5985 return 0; 5986 5987 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5988 mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset; 5989 stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset; 5990 5991 RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */, 5992 MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK); 5993 5994 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 5995 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) | 5996 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) | 5997 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) | 5998 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) | 5999 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3), 6000 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 6001 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 6002 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 6003 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 6004 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 6005 6006 RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, 6007 STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK); 6008 6009 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK); 6010 6011 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); 6012 if (rc) 6013 return rc; 6014 6015 gaudi2->hw_cap_initialized |= hw_cap; 6016 6017 return 0; 6018 } 6019 6020 static int gaudi2_hbm_mmu_init(struct hl_device *hdev) 6021 { 6022 int rc, dcore_id, hmmu_id; 6023 6024 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 6025 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) { 6026 rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id); 6027 if (rc) 6028 return rc; 6029 } 6030 6031 return 0; 6032 } 6033 6034 static int gaudi2_mmu_init(struct hl_device *hdev) 6035 { 6036 int rc; 6037 6038 rc = gaudi2_pci_mmu_init(hdev); 6039 if (rc) 6040 return rc; 6041 6042 rc = gaudi2_hbm_mmu_init(hdev); 6043 if (rc) 6044 return rc; 6045 6046 return 0; 6047 } 6048 6049 static int gaudi2_hw_init(struct hl_device *hdev) 6050 { 6051 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6052 int rc; 6053 6054 /* Let's mark in the H/W that we have reached this point. We check 6055 * this value in the reset_before_init function to understand whether 6056 * we need to reset the chip before doing H/W init. This register is 6057 * cleared by the H/W upon H/W reset 6058 */ 6059 WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 6060 6061 /* Perform read from the device to make sure device is up */ 6062 RREG32(mmHW_STATE); 6063 6064 /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. 6065 * So we set it here and if anyone tries to move it later to 6066 * a different address, there will be an error 6067 */ 6068 if (hdev->asic_prop.iatu_done_by_fw) 6069 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; 6070 6071 /* 6072 * Before pushing u-boot/linux to device, need to set the hbm bar to 6073 * base address of dram 6074 */ 6075 if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 6076 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); 6077 return -EIO; 6078 } 6079 6080 rc = gaudi2_init_cpu(hdev); 6081 if (rc) { 6082 dev_err(hdev->dev, "failed to initialize CPU\n"); 6083 return rc; 6084 } 6085 6086 gaudi2_init_scrambler_hbm(hdev); 6087 gaudi2_init_kdma(hdev); 6088 6089 rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC); 6090 if (rc) { 6091 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); 6092 return rc; 6093 } 6094 6095 rc = gaudi2->cpucp_info_get(hdev); 6096 if (rc) { 6097 dev_err(hdev->dev, "Failed to get cpucp info\n"); 6098 return rc; 6099 } 6100 6101 rc = gaudi2_mmu_init(hdev); 6102 if (rc) 6103 return rc; 6104 6105 gaudi2_init_pdma(hdev); 6106 gaudi2_init_edma(hdev); 6107 gaudi2_init_sm(hdev); 6108 gaudi2_init_tpc(hdev); 6109 gaudi2_init_mme(hdev); 6110 gaudi2_init_rotator(hdev); 6111 gaudi2_init_dec(hdev); 6112 gaudi2_enable_timestamp(hdev); 6113 6114 rc = gaudi2_coresight_init(hdev); 6115 if (rc) 6116 goto disable_queues; 6117 6118 rc = gaudi2_enable_msix(hdev); 6119 if (rc) 6120 goto disable_queues; 6121 6122 /* Perform read from the device to flush all configuration */ 6123 RREG32(mmHW_STATE); 6124 6125 return 0; 6126 6127 disable_queues: 6128 gaudi2_disable_dma_qmans(hdev); 6129 gaudi2_disable_mme_qmans(hdev); 6130 gaudi2_disable_tpc_qmans(hdev); 6131 gaudi2_disable_rot_qmans(hdev); 6132 gaudi2_disable_nic_qmans(hdev); 6133 6134 gaudi2_disable_timestamp(hdev); 6135 6136 return rc; 6137 } 6138 6139 /** 6140 * gaudi2_send_hard_reset_cmd - common function to handle reset 6141 * 6142 * @hdev: pointer to the habanalabs device structure 6143 * 6144 * This function handles the various possible scenarios for reset. 6145 * It considers if reset is handled by driver\FW and what FW components are loaded 6146 */ 6147 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) 6148 { 6149 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6150 bool heartbeat_reset, preboot_only, cpu_initialized = false; 6151 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6152 u32 cpu_boot_status; 6153 6154 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); 6155 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); 6156 6157 /* 6158 * Handle corner case where failure was at cpu management app load, 6159 * and driver didn't detect any failure while loading the FW, 6160 * then at such scenario driver will send only HALT_MACHINE 6161 * and no one will respond to this request since FW already back to preboot 6162 * and it cannot handle such cmd. 6163 * In this case next time the management app loads it'll check on events register 6164 * which will still have the halt indication, and will reboot the device. 6165 * The solution is to let preboot clear all relevant registers before next boot 6166 * once driver send COMMS_RST_DEV. 6167 */ 6168 cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS); 6169 6170 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && 6171 (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL)) 6172 cpu_initialized = true; 6173 6174 /* 6175 * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways: 6176 * 1. FW reset: FW initiate the reset sequence 6177 * 2. driver reset: FW will start HALT sequence (the preparations for the 6178 * reset but not the reset itself as it is not implemented 6179 * on their part) and LKD will wait to let FW complete the 6180 * sequence before issuing the reset 6181 */ 6182 if (!preboot_only && cpu_initialized) { 6183 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), 6184 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id); 6185 6186 msleep(GAUDI2_CPU_RESET_WAIT_MSEC); 6187 } 6188 6189 /* 6190 * When working with preboot (without Linux/Boot fit) we can 6191 * communicate only using the COMMS commands to issue halt/reset. 6192 * 6193 * For the case in which we are working with Linux/Bootfit this is a hail-mary 6194 * attempt to revive the card in the small chance that the f/w has 6195 * experienced a watchdog event, which caused it to return back to preboot. 6196 * In that case, triggering reset through GIC won't help. We need to 6197 * trigger the reset as if Linux wasn't loaded. 6198 * 6199 * We do it only if the reset cause was HB, because that would be the 6200 * indication of such an event. 6201 * 6202 * In case watchdog hasn't expired but we still got HB, then this won't 6203 * do any damage. 6204 */ 6205 6206 if (heartbeat_reset || preboot_only || !cpu_initialized) { 6207 if (hdev->asic_prop.hard_reset_done_by_fw) 6208 hl_fw_ask_hard_reset_without_linux(hdev); 6209 else 6210 hl_fw_ask_halt_machine_without_linux(hdev); 6211 } 6212 } 6213 6214 /** 6215 * gaudi2_execute_hard_reset - execute hard reset by driver/FW 6216 * 6217 * @hdev: pointer to the habanalabs device structure 6218 * 6219 * This function executes hard reset based on if driver/FW should do the reset 6220 */ 6221 static void gaudi2_execute_hard_reset(struct hl_device *hdev) 6222 { 6223 if (hdev->asic_prop.hard_reset_done_by_fw) { 6224 gaudi2_send_hard_reset_cmd(hdev); 6225 return; 6226 } 6227 6228 /* Set device to handle FLR by H/W as we will put the device 6229 * CPU to halt mode 6230 */ 6231 WREG32(mmPCIE_AUX_FLR_CTRL, 6232 (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); 6233 6234 gaudi2_send_hard_reset_cmd(hdev); 6235 6236 WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1); 6237 } 6238 6239 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us) 6240 { 6241 int i, rc = 0; 6242 u32 reg_val; 6243 6244 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6245 rc = hl_poll_timeout( 6246 hdev, 6247 mmCPU_RST_STATUS_TO_HOST, 6248 reg_val, 6249 reg_val == CPU_RST_STATUS_SOFT_RST_DONE, 6250 1000, 6251 poll_timeout_us); 6252 6253 if (rc) 6254 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", 6255 reg_val); 6256 return rc; 6257 } 6258 6259 /** 6260 * gaudi2_execute_soft_reset - execute soft reset by driver/FW 6261 * 6262 * @hdev: pointer to the habanalabs device structure 6263 * @driver_performs_reset: true if driver should perform reset instead of f/w. 6264 * @poll_timeout_us: time to wait for response from f/w. 6265 * 6266 * This function executes soft reset based on if driver/FW should do the reset 6267 */ 6268 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, 6269 u32 poll_timeout_us) 6270 { 6271 int rc; 6272 6273 if (!driver_performs_reset) { 6274 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) { 6275 /* set SP to indicate reset request sent to FW */ 6276 WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA); 6277 6278 WREG32(mmGIC_HOST_SOFT_RST_IRQ_POLL_REG, 6279 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id); 6280 6281 /* wait for f/w response */ 6282 rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us); 6283 } else { 6284 rc = hl_fw_send_soft_reset(hdev); 6285 } 6286 return rc; 6287 } 6288 6289 /* Block access to engines, QMANs and SM during reset, these 6290 * RRs will be reconfigured after soft reset. 6291 * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset. 6292 */ 6293 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, 6294 mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE); 6295 6296 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, 6297 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE, 6298 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE); 6299 6300 WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1); 6301 return 0; 6302 } 6303 6304 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) 6305 { 6306 int i, rc = 0; 6307 u32 reg_val; 6308 6309 /* We poll the BTM done indication multiple times after reset due to 6310 * a HW errata 'GAUDI2_0300' 6311 */ 6312 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6313 rc = hl_poll_timeout( 6314 hdev, 6315 mmPSOC_GLOBAL_CONF_BTM_FSM, 6316 reg_val, 6317 reg_val == 0, 6318 1000, 6319 poll_timeout_us); 6320 6321 if (rc) 6322 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); 6323 } 6324 6325 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 6326 { 6327 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6328 u32 poll_timeout_us, reset_sleep_ms; 6329 bool driver_performs_reset = false; 6330 int rc; 6331 6332 if (hdev->pldm) { 6333 reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC : 6334 GAUDI2_PLDM_SRESET_TIMEOUT_MSEC; 6335 poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC; 6336 } else { 6337 reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC; 6338 poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC; 6339 } 6340 6341 if (fw_reset) 6342 goto skip_reset; 6343 6344 gaudi2_reset_arcs(hdev); 6345 6346 if (hard_reset) { 6347 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; 6348 gaudi2_execute_hard_reset(hdev); 6349 } else { 6350 /* 6351 * As we have to support also work with preboot only (which does not supports 6352 * soft reset) we have to make sure that security is disabled before letting driver 6353 * do the reset. user shall control the BFE flags to avoid asking soft reset in 6354 * secured device with preboot only. 6355 */ 6356 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && 6357 !hdev->asic_prop.fw_security_enabled); 6358 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us); 6359 if (rc) 6360 return rc; 6361 } 6362 6363 skip_reset: 6364 if (driver_performs_reset || hard_reset) { 6365 /* 6366 * Instead of waiting for BTM indication we should wait for preboot ready: 6367 * Consider the below scenario: 6368 * 1. FW update is being triggered 6369 * - setting the dirty bit 6370 * 2. hard reset will be triggered due to the dirty bit 6371 * 3. FW initiates the reset: 6372 * - dirty bit cleared 6373 * - BTM indication cleared 6374 * - preboot ready indication cleared 6375 * 4. during hard reset: 6376 * - BTM indication will be set 6377 * - BIST test performed and another reset triggered 6378 * 5. only after this reset the preboot will set the preboot ready 6379 * 6380 * when polling on BTM indication alone we can lose sync with FW while trying to 6381 * communicate with FW that is during reset. 6382 * to overcome this we will always wait to preboot ready indication 6383 */ 6384 6385 /* without this sleep reset will not work */ 6386 msleep(reset_sleep_ms); 6387 6388 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) 6389 hl_fw_wait_preboot_ready(hdev); 6390 else 6391 gaudi2_poll_btm_indication(hdev, poll_timeout_us); 6392 } 6393 6394 if (!gaudi2) 6395 return 0; 6396 6397 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); 6398 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); 6399 6400 /* 6401 * Clear NIC capability mask in order for driver to re-configure 6402 * NIC QMANs. NIC ports will not be re-configured during soft 6403 * reset as we call gaudi2_nic_init only during hard reset 6404 */ 6405 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); 6406 6407 if (hard_reset) { 6408 gaudi2->hw_cap_initialized &= 6409 ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK | 6410 HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q | 6411 HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK | 6412 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA | 6413 HW_CAP_MME_MASK | HW_CAP_ROT_MASK); 6414 6415 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); 6416 } else { 6417 gaudi2->hw_cap_initialized &= 6418 ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET | 6419 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK | 6420 HW_CAP_ROT_MASK); 6421 } 6422 return 0; 6423 } 6424 6425 static int gaudi2_suspend(struct hl_device *hdev) 6426 { 6427 int rc; 6428 6429 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 6430 if (rc) 6431 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); 6432 6433 return rc; 6434 } 6435 6436 static int gaudi2_resume(struct hl_device *hdev) 6437 { 6438 return gaudi2_init_iatu(hdev); 6439 } 6440 6441 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 6442 void *cpu_addr, dma_addr_t dma_addr, size_t size) 6443 { 6444 int rc; 6445 6446 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 6447 VM_DONTCOPY | VM_NORESERVE); 6448 6449 #ifdef _HAS_DMA_MMAP_COHERENT 6450 6451 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); 6452 if (rc) 6453 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 6454 6455 #else 6456 6457 rc = remap_pfn_range(vma, vma->vm_start, 6458 virt_to_phys(cpu_addr) >> PAGE_SHIFT, 6459 size, vma->vm_page_prot); 6460 if (rc) 6461 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 6462 6463 #endif 6464 6465 return rc; 6466 } 6467 6468 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) 6469 { 6470 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6471 u64 hw_cap_mask = 0; 6472 u64 hw_tpc_cap_bit = 0; 6473 u64 hw_nic_cap_bit = 0; 6474 u64 hw_test_cap_bit = 0; 6475 6476 switch (hw_queue_id) { 6477 case GAUDI2_QUEUE_ID_PDMA_0_0: 6478 case GAUDI2_QUEUE_ID_PDMA_0_1: 6479 case GAUDI2_QUEUE_ID_PDMA_1_0: 6480 hw_cap_mask = HW_CAP_PDMA_MASK; 6481 break; 6482 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 6483 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 6484 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); 6485 break; 6486 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 6487 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE + 6488 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); 6489 break; 6490 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 6491 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE + 6492 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); 6493 break; 6494 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 6495 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE + 6496 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); 6497 break; 6498 6499 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 6500 hw_test_cap_bit = HW_CAP_MME_SHIFT; 6501 break; 6502 6503 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 6504 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1; 6505 break; 6506 6507 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 6508 hw_test_cap_bit = HW_CAP_MME_SHIFT + 2; 6509 break; 6510 6511 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 6512 hw_test_cap_bit = HW_CAP_MME_SHIFT + 3; 6513 break; 6514 6515 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3: 6516 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + 6517 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); 6518 6519 /* special case where cap bit refers to the first queue id */ 6520 if (!hw_tpc_cap_bit) 6521 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); 6522 break; 6523 6524 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 6525 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE + 6526 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); 6527 break; 6528 6529 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 6530 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) + 6531 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); 6532 break; 6533 6534 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 6535 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) + 6536 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); 6537 break; 6538 6539 case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 6540 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE); 6541 break; 6542 6543 case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3: 6544 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); 6545 break; 6546 6547 case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3: 6548 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); 6549 6550 /* special case where cap bit refers to the first queue id */ 6551 if (!hw_nic_cap_bit) 6552 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); 6553 break; 6554 6555 case GAUDI2_QUEUE_ID_CPU_PQ: 6556 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); 6557 6558 default: 6559 return false; 6560 } 6561 6562 if (hw_tpc_cap_bit) 6563 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); 6564 6565 if (hw_nic_cap_bit) 6566 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); 6567 6568 if (hw_test_cap_bit) 6569 hw_cap_mask = BIT_ULL(hw_test_cap_bit); 6570 6571 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); 6572 } 6573 6574 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) 6575 { 6576 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6577 6578 switch (arc_id) { 6579 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6580 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6581 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); 6582 6583 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6584 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6585 6586 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6587 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6588 6589 default: 6590 return false; 6591 } 6592 } 6593 6594 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6595 { 6596 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6597 6598 switch (arc_id) { 6599 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6600 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6601 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); 6602 break; 6603 6604 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6605 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6606 break; 6607 6608 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6609 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6610 break; 6611 6612 default: 6613 return; 6614 } 6615 } 6616 6617 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6618 { 6619 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6620 6621 switch (arc_id) { 6622 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6623 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6624 gaudi2->active_hw_arc |= BIT_ULL(arc_id); 6625 break; 6626 6627 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6628 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); 6629 break; 6630 6631 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6632 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); 6633 break; 6634 6635 default: 6636 return; 6637 } 6638 } 6639 6640 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 6641 { 6642 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6643 u32 pq_offset, reg_base, db_reg_offset, db_value; 6644 6645 if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) { 6646 /* 6647 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs. 6648 * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ 6649 * number. 6650 */ 6651 pq_offset = (hw_queue_id & 0x3) * 4; 6652 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6653 db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset; 6654 } else { 6655 db_reg_offset = mmCPU_IF_PF_PQ_PI; 6656 } 6657 6658 db_value = pi; 6659 6660 /* ring the doorbell */ 6661 WREG32(db_reg_offset, db_value); 6662 6663 if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) { 6664 /* make sure device CPU will read latest data from host */ 6665 mb(); 6666 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 6667 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 6668 } 6669 } 6670 6671 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) 6672 { 6673 __le64 *pbd = (__le64 *) bd; 6674 6675 /* The QMANs are on the host memory so a simple copy suffice */ 6676 pqe[0] = pbd[0]; 6677 pqe[1] = pbd[1]; 6678 } 6679 6680 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, 6681 dma_addr_t *dma_handle, gfp_t flags) 6682 { 6683 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); 6684 } 6685 6686 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, 6687 void *cpu_addr, dma_addr_t dma_handle) 6688 { 6689 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); 6690 } 6691 6692 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, 6693 u32 timeout, u64 *result) 6694 { 6695 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6696 6697 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { 6698 if (result) 6699 *result = 0; 6700 return 0; 6701 } 6702 6703 if (!timeout) 6704 timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC; 6705 6706 return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result); 6707 } 6708 6709 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, 6710 gfp_t mem_flags, dma_addr_t *dma_handle) 6711 { 6712 if (size > GAUDI2_DMA_POOL_BLK_SIZE) 6713 return NULL; 6714 6715 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 6716 } 6717 6718 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) 6719 { 6720 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); 6721 } 6722 6723 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 6724 dma_addr_t *dma_handle) 6725 { 6726 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 6727 } 6728 6729 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) 6730 { 6731 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 6732 } 6733 6734 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) 6735 { 6736 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 6737 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6738 6739 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { 6740 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); 6741 return -EINVAL; 6742 } 6743 6744 /* Just check if CB address is valid */ 6745 6746 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6747 parser->user_cb_size, 6748 asic_prop->sram_user_base_address, 6749 asic_prop->sram_end_address)) 6750 return 0; 6751 6752 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6753 parser->user_cb_size, 6754 asic_prop->dram_user_base_address, 6755 asic_prop->dram_end_address)) 6756 return 0; 6757 6758 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && 6759 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6760 parser->user_cb_size, 6761 asic_prop->dmmu.start_addr, 6762 asic_prop->dmmu.end_addr)) 6763 return 0; 6764 6765 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { 6766 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6767 parser->user_cb_size, 6768 asic_prop->pmmu.start_addr, 6769 asic_prop->pmmu.end_addr) || 6770 hl_mem_area_inside_range( 6771 (u64) (uintptr_t) parser->user_cb, 6772 parser->user_cb_size, 6773 asic_prop->pmmu_huge.start_addr, 6774 asic_prop->pmmu_huge.end_addr)) 6775 return 0; 6776 6777 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { 6778 if (!hdev->pdev) 6779 return 0; 6780 6781 if (!device_iommu_mapped(&hdev->pdev->dev)) 6782 return 0; 6783 } 6784 6785 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", 6786 parser->user_cb, parser->user_cb_size); 6787 6788 return -EFAULT; 6789 } 6790 6791 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 6792 { 6793 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6794 6795 if (!parser->is_kernel_allocated_cb) 6796 return gaudi2_validate_cb_address(hdev, parser); 6797 6798 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 6799 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); 6800 return -EINVAL; 6801 } 6802 6803 return 0; 6804 } 6805 6806 static int gaudi2_send_heartbeat(struct hl_device *hdev) 6807 { 6808 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6809 6810 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 6811 return 0; 6812 6813 return hl_fw_send_heartbeat(hdev); 6814 } 6815 6816 /* This is an internal helper function, used to update the KDMA mmu props. 6817 * Should be called with a proper kdma lock. 6818 */ 6819 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, 6820 bool mmu_bypass, u32 asid) 6821 { 6822 u32 rw_asid, rw_mmu_bp; 6823 6824 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 6825 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 6826 6827 rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) | 6828 (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT); 6829 6830 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid); 6831 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp); 6832 } 6833 6834 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, 6835 u32 mon_payload, u32 sync_value) 6836 { 6837 u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm; 6838 u8 mask; 6839 6840 sob_offset = sob_id * 4; 6841 mon_offset = mon_id * 4; 6842 6843 /* Reset the SOB value */ 6844 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 6845 6846 /* Configure this address with CQ_ID 0 because CQ_EN is set */ 6847 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id); 6848 6849 /* Configure this address with CS index because CQ_EN is set */ 6850 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload); 6851 6852 sync_group_id = sob_id / 8; 6853 mask = ~(1 << (sob_id & 0x7)); 6854 mode = 1; /* comparison mode is "equal to" */ 6855 6856 mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value); 6857 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode); 6858 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask); 6859 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id); 6860 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm); 6861 } 6862 6863 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */ 6864 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, 6865 u64 src_addr, u64 dst_addr, 6866 u32 size, bool is_memset) 6867 { 6868 u32 comp_val, commit_mask, *polling_addr, timeout, status = 0; 6869 struct hl_cq_entry *cq_base; 6870 struct hl_cq *cq; 6871 u64 comp_addr; 6872 int rc; 6873 6874 gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION, 6875 GAUDI2_RESERVED_MON_KDMA_COMPLETION, 6876 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1); 6877 6878 comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + 6879 (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32)); 6880 6881 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 6882 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 6883 6884 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr)); 6885 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr)); 6886 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr)); 6887 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr)); 6888 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr)); 6889 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr)); 6890 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val); 6891 WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size); 6892 6893 commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) | 6894 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1); 6895 6896 if (is_memset) 6897 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1); 6898 6899 WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask); 6900 6901 /* Wait for completion */ 6902 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; 6903 cq_base = cq->kernel_address; 6904 polling_addr = (u32 *)&cq_base[cq->ci]; 6905 6906 if (hdev->pldm) 6907 /* for each 1MB 20 second of timeout */ 6908 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20; 6909 else 6910 timeout = KDMA_TIMEOUT_USEC; 6911 6912 /* Polling */ 6913 rc = hl_poll_timeout_memory( 6914 hdev, 6915 polling_addr, 6916 status, 6917 (status == 1), 6918 1000, 6919 timeout, 6920 true); 6921 6922 *polling_addr = 0; 6923 6924 if (rc) { 6925 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); 6926 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT); 6927 return rc; 6928 } 6929 6930 cq->ci = hl_cq_inc_ptr(cq->ci); 6931 6932 return 0; 6933 } 6934 6935 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) 6936 { 6937 u32 i; 6938 6939 for (i = 0 ; i < size ; i += sizeof(u32)) 6940 WREG32(addr + i, val); 6941 } 6942 6943 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) 6944 { 6945 u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6946 6947 if (enable) { 6948 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE); 6949 WREG32(reg_base + QM_PQC_CFG_OFFSET, 0); 6950 } else { 6951 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED); 6952 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 6953 } 6954 } 6955 6956 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) 6957 { 6958 return hdev->asic_prop.first_available_user_sob[0] + 6959 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; 6960 } 6961 6962 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) 6963 { 6964 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6965 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 6966 6967 /* Reset the SOB value */ 6968 WREG32(sob_addr, 0); 6969 } 6970 6971 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, 6972 struct gaudi2_queues_test_info *msg_info) 6973 { 6974 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6975 u32 tmp, sob_base = 1; 6976 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; 6977 size_t pkt_size = sizeof(struct packet_msg_short); 6978 int rc; 6979 6980 tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) | 6981 (1 << GAUDI2_PKT_CTL_EB_SHIFT) | 6982 (1 << GAUDI2_PKT_CTL_MB_SHIFT) | 6983 (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) | 6984 (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT); 6985 6986 msg_short_pkt->value = cpu_to_le32(sob_val); 6987 msg_short_pkt->ctl = cpu_to_le32(tmp); 6988 6989 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); 6990 if (rc) 6991 dev_err(hdev->dev, 6992 "Failed to send msg_short packet to H/W queue %d\n", hw_queue_id); 6993 6994 return rc; 6995 } 6996 6997 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) 6998 { 6999 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7000 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 7001 u32 timeout_usec, tmp; 7002 int rc; 7003 7004 if (hdev->pldm) 7005 timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC; 7006 else 7007 timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC; 7008 7009 rc = hl_poll_timeout( 7010 hdev, 7011 sob_addr, 7012 tmp, 7013 (tmp == sob_val), 7014 1000, 7015 timeout_usec); 7016 7017 if (rc == -ETIMEDOUT) { 7018 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", 7019 hw_queue_id, tmp); 7020 rc = -EIO; 7021 } 7022 7023 return rc; 7024 } 7025 7026 static int gaudi2_test_cpu_queue(struct hl_device *hdev) 7027 { 7028 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7029 7030 /* 7031 * check capability here as send_cpu_message() won't update the result 7032 * value if no capability 7033 */ 7034 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7035 return 0; 7036 7037 return hl_fw_test_cpu_queue(hdev); 7038 } 7039 7040 static int gaudi2_test_queues(struct hl_device *hdev) 7041 { 7042 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7043 struct gaudi2_queues_test_info *msg_info; 7044 u32 sob_val = 0x5a5a; 7045 int i, rc; 7046 7047 /* send test message on all enabled Qs */ 7048 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7049 if (!gaudi2_is_queue_enabled(hdev, i)) 7050 continue; 7051 7052 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; 7053 gaudi2_qman_set_test_mode(hdev, i, true); 7054 gaudi2_test_queue_clear(hdev, i); 7055 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info); 7056 if (rc) 7057 goto done; 7058 } 7059 7060 rc = gaudi2_test_cpu_queue(hdev); 7061 if (rc) 7062 goto done; 7063 7064 /* verify that all messages were processed */ 7065 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7066 if (!gaudi2_is_queue_enabled(hdev, i)) 7067 continue; 7068 7069 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val); 7070 if (rc) 7071 /* chip is not usable, no need for cleanups, just bail-out with error */ 7072 goto done; 7073 7074 gaudi2_test_queue_clear(hdev, i); 7075 gaudi2_qman_set_test_mode(hdev, i, false); 7076 } 7077 7078 done: 7079 return rc; 7080 } 7081 7082 static int gaudi2_compute_reset_late_init(struct hl_device *hdev) 7083 { 7084 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7085 size_t irq_arr_size; 7086 int rc; 7087 7088 gaudi2_init_arcs(hdev); 7089 7090 rc = gaudi2_scrub_arcs_dccm(hdev); 7091 if (rc) { 7092 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 7093 return rc; 7094 } 7095 7096 gaudi2_init_security(hdev); 7097 7098 /* Unmask all IRQs since some could have been received during the soft reset */ 7099 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); 7100 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); 7101 } 7102 7103 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7104 struct engines_data *e) 7105 { 7106 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7107 struct asic_fixed_properties *prop = &hdev->asic_prop; 7108 unsigned long *mask = (unsigned long *) mask_arr; 7109 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; 7110 bool is_idle = true, is_eng_idle; 7111 int engine_idx, i, j; 7112 u64 offset; 7113 7114 if (e) 7115 hl_engine_data_sprintf(e, 7116 "\nCORE EDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7117 "---- ---- ------- ------------ ------------- -------------\n"); 7118 7119 for (i = 0; i < NUM_OF_DCORES; i++) { 7120 for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) { 7121 int seq = i * NUM_OF_EDMA_PER_DCORE + j; 7122 7123 if (!(prop->edma_enabled_mask & BIT(seq))) 7124 continue; 7125 7126 engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 + 7127 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7128 offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET; 7129 7130 dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset); 7131 dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset); 7132 7133 qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset); 7134 qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset); 7135 qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset); 7136 7137 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7138 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7139 is_idle &= is_eng_idle; 7140 7141 if (mask && !is_eng_idle) 7142 set_bit(engine_idx, mask); 7143 7144 if (e) 7145 hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N", 7146 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7147 } 7148 } 7149 7150 return is_idle; 7151 } 7152 7153 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7154 struct engines_data *e) 7155 { 7156 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7157 unsigned long *mask = (unsigned long *) mask_arr; 7158 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; 7159 bool is_idle = true, is_eng_idle; 7160 int engine_idx, i; 7161 u64 offset; 7162 7163 if (e) 7164 hl_engine_data_sprintf(e, 7165 "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7166 "---- ------- ------------ ------------- -------------\n"); 7167 7168 for (i = 0 ; i < NUM_OF_PDMA ; i++) { 7169 engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i; 7170 offset = i * PDMA_OFFSET; 7171 dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset); 7172 dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset); 7173 7174 qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset); 7175 qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset); 7176 qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset); 7177 7178 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7179 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7180 is_idle &= is_eng_idle; 7181 7182 if (mask && !is_eng_idle) 7183 set_bit(engine_idx, mask); 7184 7185 if (e) 7186 hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N", 7187 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7188 } 7189 7190 return is_idle; 7191 } 7192 7193 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7194 struct engines_data *e) 7195 { 7196 unsigned long *mask = (unsigned long *) mask_arr; 7197 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; 7198 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7199 bool is_idle = true, is_eng_idle; 7200 int engine_idx, i; 7201 u64 offset = 0; 7202 7203 /* NIC, twelve macros in Full chip */ 7204 if (e && hdev->nic_ports_mask) 7205 hl_engine_data_sprintf(e, 7206 "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" 7207 "--- ------- ------------ ----------\n"); 7208 7209 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 7210 if (!(i & 1)) 7211 offset = i / 2 * NIC_OFFSET; 7212 else 7213 offset += NIC_QM_OFFSET; 7214 7215 if (!(hdev->nic_ports_mask & BIT(i))) 7216 continue; 7217 7218 engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i; 7219 7220 7221 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); 7222 qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset); 7223 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); 7224 7225 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7226 is_idle &= is_eng_idle; 7227 7228 if (mask && !is_eng_idle) 7229 set_bit(engine_idx, mask); 7230 7231 if (e) 7232 hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N", 7233 qm_glbl_sts0, qm_cgm_sts); 7234 } 7235 7236 return is_idle; 7237 } 7238 7239 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7240 struct engines_data *e) 7241 { 7242 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts; 7243 unsigned long *mask = (unsigned long *) mask_arr; 7244 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; 7245 bool is_idle = true, is_eng_idle; 7246 int engine_idx, i; 7247 u64 offset; 7248 7249 if (e) 7250 hl_engine_data_sprintf(e, 7251 "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" 7252 "--- ---- ------- ------------ ---------------\n"); 7253 /* MME, one per Dcore */ 7254 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7255 engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET; 7256 offset = i * DCORE_OFFSET; 7257 7258 qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset); 7259 qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset); 7260 qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset); 7261 7262 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7263 is_idle &= is_eng_idle; 7264 7265 mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset); 7266 is_eng_idle &= IS_MME_IDLE(mme_arch_sts); 7267 is_idle &= is_eng_idle; 7268 7269 if (e) 7270 hl_engine_data_sprintf(e, mme_fmt, i, "N", 7271 is_eng_idle ? "Y" : "N", 7272 qm_glbl_sts0, 7273 mme_arch_sts); 7274 7275 if (mask && !is_eng_idle) 7276 set_bit(engine_idx, mask); 7277 } 7278 7279 return is_idle; 7280 } 7281 7282 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, 7283 struct iterate_module_ctx *ctx) 7284 { 7285 struct gaudi2_tpc_idle_data *idle_data = ctx->data; 7286 u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7287 bool is_eng_idle; 7288 int engine_idx; 7289 7290 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) 7291 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6; 7292 else 7293 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 + 7294 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst; 7295 7296 tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset); 7297 qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset); 7298 qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset); 7299 qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset); 7300 7301 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7302 IS_TPC_IDLE(tpc_cfg_sts); 7303 *(idle_data->is_idle) &= is_eng_idle; 7304 7305 if (idle_data->mask && !is_eng_idle) 7306 set_bit(engine_idx, idle_data->mask); 7307 7308 if (idle_data->e) 7309 hl_engine_data_sprintf(idle_data->e, 7310 idle_data->tpc_fmt, dcore, inst, 7311 is_eng_idle ? "Y" : "N", 7312 qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); 7313 } 7314 7315 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7316 struct engines_data *e) 7317 { 7318 struct asic_fixed_properties *prop = &hdev->asic_prop; 7319 unsigned long *mask = (unsigned long *) mask_arr; 7320 bool is_idle = true; 7321 7322 struct gaudi2_tpc_idle_data tpc_idle_data = { 7323 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", 7324 .e = e, 7325 .mask = mask, 7326 .is_idle = &is_idle, 7327 }; 7328 struct iterate_module_ctx tpc_iter = { 7329 .fn = &gaudi2_is_tpc_engine_idle, 7330 .data = &tpc_idle_data, 7331 }; 7332 7333 if (e && prop->tpc_enabled_mask) 7334 hl_engine_data_sprintf(e, 7335 "\nCORE TPC is_idle QM_GLBL_STS0 QM_CGM_STS STATUS\n" 7336 "---- --- ------- ------------ ---------- ------\n"); 7337 7338 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7339 7340 return *tpc_idle_data.is_idle; 7341 } 7342 7343 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7344 struct engines_data *e) 7345 { 7346 struct asic_fixed_properties *prop = &hdev->asic_prop; 7347 unsigned long *mask = (unsigned long *) mask_arr; 7348 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; 7349 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; 7350 bool is_idle = true, is_eng_idle; 7351 u32 dec_swreg15, dec_enabled_bit; 7352 int engine_idx, i, j; 7353 u64 offset; 7354 7355 /* Decoders, two each Dcore and two shared PCIe decoders */ 7356 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) 7357 hl_engine_data_sprintf(e, 7358 "\nCORE DEC is_idle VSI_CMD_SWREG15\n" 7359 "---- --- ------- ---------------\n"); 7360 7361 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7362 for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) { 7363 dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j); 7364 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) 7365 continue; 7366 7367 engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 + 7368 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7369 offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET; 7370 7371 dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset); 7372 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7373 is_idle &= is_eng_idle; 7374 7375 if (mask && !is_eng_idle) 7376 set_bit(engine_idx, mask); 7377 7378 if (e) 7379 hl_engine_data_sprintf(e, dec_fmt, i, j, 7380 is_eng_idle ? "Y" : "N", dec_swreg15); 7381 } 7382 } 7383 7384 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) 7385 hl_engine_data_sprintf(e, 7386 "\nPCIe DEC is_idle VSI_CMD_SWREG15\n" 7387 "-------- ------- ---------------\n"); 7388 7389 /* Check shared(PCIe) decoders */ 7390 for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) { 7391 dec_enabled_bit = PCIE_DEC_SHIFT + i; 7392 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) 7393 continue; 7394 7395 engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i; 7396 offset = i * DCORE_DEC_OFFSET; 7397 dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset); 7398 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7399 is_idle &= is_eng_idle; 7400 7401 if (mask && !is_eng_idle) 7402 set_bit(engine_idx, mask); 7403 7404 if (e) 7405 hl_engine_data_sprintf(e, pcie_dec_fmt, i, 7406 is_eng_idle ? "Y" : "N", dec_swreg15); 7407 } 7408 7409 return is_idle; 7410 } 7411 7412 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7413 struct engines_data *e) 7414 { 7415 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; 7416 unsigned long *mask = (unsigned long *) mask_arr; 7417 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7418 bool is_idle = true, is_eng_idle; 7419 int engine_idx, i; 7420 u64 offset; 7421 7422 if (e) 7423 hl_engine_data_sprintf(e, 7424 "\nCORE ROT is_idle QM_GLBL_STS0 QM_GLBL_STS1 QM_CGM_STS\n" 7425 "---- --- ------- ------------ ------------ ----------\n"); 7426 7427 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7428 engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i; 7429 7430 offset = i * ROT_OFFSET; 7431 7432 qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset); 7433 qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset); 7434 qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset); 7435 7436 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7437 is_idle &= is_eng_idle; 7438 7439 if (mask && !is_eng_idle) 7440 set_bit(engine_idx, mask); 7441 7442 if (e) 7443 hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", 7444 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7445 } 7446 7447 return is_idle; 7448 } 7449 7450 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7451 struct engines_data *e) 7452 { 7453 bool is_idle = true; 7454 7455 is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e); 7456 is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e); 7457 is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e); 7458 is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e); 7459 is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e); 7460 is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e); 7461 is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e); 7462 7463 return is_idle; 7464 } 7465 7466 static void gaudi2_hw_queues_lock(struct hl_device *hdev) 7467 __acquires(&gaudi2->hw_queues_lock) 7468 { 7469 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7470 7471 spin_lock(&gaudi2->hw_queues_lock); 7472 } 7473 7474 static void gaudi2_hw_queues_unlock(struct hl_device *hdev) 7475 __releases(&gaudi2->hw_queues_lock) 7476 { 7477 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7478 7479 spin_unlock(&gaudi2->hw_queues_lock); 7480 } 7481 7482 static u32 gaudi2_get_pci_id(struct hl_device *hdev) 7483 { 7484 return hdev->pdev->device; 7485 } 7486 7487 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) 7488 { 7489 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7490 7491 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7492 return 0; 7493 7494 return hl_fw_get_eeprom_data(hdev, data, max_size); 7495 } 7496 7497 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) 7498 { 7499 WREG32(mmCPU_IF_EQ_RD_OFFS, val); 7500 } 7501 7502 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 7503 { 7504 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7505 7506 if (aggregate) { 7507 *size = (u32) sizeof(gaudi2->events_stat_aggregate); 7508 return gaudi2->events_stat_aggregate; 7509 } 7510 7511 *size = (u32) sizeof(gaudi2->events_stat); 7512 return gaudi2->events_stat; 7513 } 7514 7515 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, 7516 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7517 { 7518 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * 7519 dcore_vdec_id + DCORE_OFFSET * dcore_id; 7520 7521 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7522 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7523 7524 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7525 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7526 7527 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7528 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7529 7530 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7531 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7532 7533 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7534 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7535 } 7536 7537 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) 7538 { 7539 u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7540 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7541 struct asic_fixed_properties *prop = &hdev->asic_prop; 7542 u32 dcore_offset = dcore_id * DCORE_OFFSET; 7543 u32 vdec_id, i, ports_offset, reg_val; 7544 u8 edma_seq_base; 7545 7546 /* EDMA */ 7547 edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE; 7548 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { 7549 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7550 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7551 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7552 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7553 } 7554 7555 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { 7556 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7557 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7558 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7559 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7560 } 7561 7562 /* Sync Mngr */ 7563 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid); 7564 /* 7565 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID 7566 * for any access type 7567 */ 7568 if (dcore_id > 0) { 7569 reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) | 7570 (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT); 7571 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val); 7572 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0); 7573 } 7574 7575 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0); 7576 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid); 7577 7578 for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) { 7579 ports_offset = i * DCORE_MME_SBTE_OFFSET; 7580 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP + 7581 dcore_offset + ports_offset, 0); 7582 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID + 7583 dcore_offset + ports_offset, rw_asid); 7584 } 7585 7586 for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) { 7587 ports_offset = i * DCORE_MME_WB_OFFSET; 7588 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP + 7589 dcore_offset + ports_offset, 0); 7590 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID + 7591 dcore_offset + ports_offset, rw_asid); 7592 } 7593 7594 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7595 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7596 7597 /* 7598 * Decoders 7599 */ 7600 for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) { 7601 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) 7602 gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0); 7603 } 7604 } 7605 7606 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, 7607 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7608 { 7609 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; 7610 7611 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7612 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7613 7614 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7615 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7616 7617 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7618 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7619 7620 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7621 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7622 7623 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7624 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7625 } 7626 7627 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, 7628 u32 rw_asid, u32 rw_mmu_bp) 7629 { 7630 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; 7631 7632 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp); 7633 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid); 7634 } 7635 7636 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) 7637 { 7638 u32 reg_base, reg_offset, reg_val = 0; 7639 7640 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 7641 7642 /* Enable MMU and configure asid for all relevant ARC regions */ 7643 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0); 7644 reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid); 7645 7646 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL); 7647 WREG32(reg_base + reg_offset, reg_val); 7648 7649 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW); 7650 WREG32(reg_base + reg_offset, reg_val); 7651 7652 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA); 7653 WREG32(reg_base + reg_offset, reg_val); 7654 7655 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA); 7656 WREG32(reg_base + reg_offset, reg_val); 7657 7658 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA); 7659 WREG32(reg_base + reg_offset, reg_val); 7660 7661 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE); 7662 WREG32(reg_base + reg_offset, reg_val); 7663 7664 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL); 7665 WREG32(reg_base + reg_offset, reg_val); 7666 7667 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL); 7668 WREG32(reg_base + reg_offset, reg_val); 7669 7670 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL); 7671 WREG32(reg_base + reg_offset, reg_val); 7672 7673 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL); 7674 WREG32(reg_base + reg_offset, reg_val); 7675 7676 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL); 7677 WREG32(reg_base + reg_offset, reg_val); 7678 } 7679 7680 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) 7681 { 7682 int i; 7683 7684 if (hdev->fw_components & FW_TYPE_BOOT_CPU) 7685 return hl_fw_cpucp_engine_core_asid_set(hdev, asid); 7686 7687 for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7688 gaudi2_arc_mmu_prepare(hdev, i, asid); 7689 7690 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 7691 if (!gaudi2_is_queue_enabled(hdev, i)) 7692 continue; 7693 7694 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid); 7695 } 7696 7697 return 0; 7698 } 7699 7700 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) 7701 { 7702 struct asic_fixed_properties *prop = &hdev->asic_prop; 7703 u32 rw_asid, offset; 7704 int rc, i; 7705 7706 rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) | 7707 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid); 7708 7709 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7710 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7711 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7712 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7713 7714 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7715 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7716 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7717 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7718 7719 /* ROT */ 7720 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7721 offset = i * ROT_OFFSET; 7722 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid); 7723 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7724 RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK); 7725 RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK); 7726 RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK); 7727 } 7728 7729 /* Shared Decoders are the last bits in the decoders mask */ 7730 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) 7731 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0); 7732 7733 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) 7734 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0); 7735 7736 /* arc farm arc dup eng */ 7737 for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7738 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0); 7739 7740 rc = gaudi2_arc_mmu_prepare_all(hdev, asid); 7741 if (rc) 7742 return rc; 7743 7744 return 0; 7745 } 7746 7747 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, 7748 struct iterate_module_ctx *ctx) 7749 { 7750 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; 7751 7752 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0); 7753 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); 7754 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7755 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); 7756 } 7757 7758 /* zero the MMUBP and set the ASID */ 7759 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) 7760 { 7761 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7762 struct gaudi2_tpc_mmu_data tpc_mmu_data; 7763 struct iterate_module_ctx tpc_iter = { 7764 .fn = &gaudi2_tpc_mmu_prepare, 7765 .data = &tpc_mmu_data, 7766 }; 7767 int rc, i; 7768 7769 if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) { 7770 dev_crit(hdev->dev, "asid %u is too big\n", asid); 7771 return -EINVAL; 7772 } 7773 7774 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 7775 return 0; 7776 7777 rc = gaudi2_mmu_shared_prepare(hdev, asid); 7778 if (rc) 7779 return rc; 7780 7781 /* configure DCORE MMUs */ 7782 tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7783 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7784 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7785 for (i = 0 ; i < NUM_OF_DCORES ; i++) 7786 gaudi2_mmu_dcore_prepare(hdev, i, asid); 7787 7788 return 0; 7789 } 7790 7791 static inline bool is_info_event(u32 event) 7792 { 7793 switch (event) { 7794 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 7795 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 7796 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY ... GAUDI2_EVENT_ARC_PWR_RD_MODE3: 7797 7798 /* return in case of NIC status event - these events are received periodically and not as 7799 * an indication to an error. 7800 */ 7801 case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1: 7802 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 7803 return true; 7804 default: 7805 return false; 7806 } 7807 } 7808 7809 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, 7810 bool ratelimited, const char *fmt, ...) 7811 { 7812 struct va_format vaf; 7813 va_list args; 7814 7815 va_start(args, fmt); 7816 vaf.fmt = fmt; 7817 vaf.va = &args; 7818 7819 if (ratelimited) 7820 dev_err_ratelimited(hdev->dev, "%s: %pV\n", 7821 gaudi2_irq_map_table[event_type].valid ? 7822 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7823 else 7824 dev_err(hdev->dev, "%s: %pV\n", 7825 gaudi2_irq_map_table[event_type].valid ? 7826 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7827 7828 va_end(args); 7829 } 7830 7831 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, 7832 struct hl_eq_ecc_data *ecc_data) 7833 { 7834 u64 ecc_address = 0, ecc_syndrome = 0; 7835 u8 memory_wrapper_idx = 0; 7836 bool has_block_id = false; 7837 u16 block_id; 7838 7839 if (!hl_is_fw_sw_ver_below(hdev, 1, 12)) 7840 has_block_id = true; 7841 7842 ecc_address = le64_to_cpu(ecc_data->ecc_address); 7843 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); 7844 memory_wrapper_idx = ecc_data->memory_wrapper_idx; 7845 7846 if (has_block_id) { 7847 block_id = le16_to_cpu(ecc_data->block_id); 7848 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7849 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. block id %#x. critical %u.", 7850 ecc_address, ecc_syndrome, memory_wrapper_idx, block_id, 7851 ecc_data->is_critical); 7852 } else { 7853 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7854 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. critical %u.", 7855 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); 7856 } 7857 7858 return !!ecc_data->is_critical; 7859 } 7860 7861 static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u64 event_mask) 7862 { 7863 u32 lo, hi, cq_ptr_size, arc_cq_ptr_size; 7864 u64 cq_ptr, arc_cq_ptr, cp_current_inst; 7865 7866 lo = RREG32(qman_base + QM_CQ_PTR_LO_4_OFFSET); 7867 hi = RREG32(qman_base + QM_CQ_PTR_HI_4_OFFSET); 7868 cq_ptr = ((u64) hi) << 32 | lo; 7869 cq_ptr_size = RREG32(qman_base + QM_CQ_TSIZE_4_OFFSET); 7870 7871 lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_OFFSET); 7872 hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_OFFSET); 7873 arc_cq_ptr = ((u64) hi) << 32 | lo; 7874 arc_cq_ptr_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_OFFSET); 7875 7876 lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET); 7877 hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET); 7878 cp_current_inst = ((u64) hi) << 32 | lo; 7879 7880 dev_info(hdev->dev, 7881 "LowerQM. CQ: {ptr %#llx, size %u}, ARC_CQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n", 7882 cq_ptr, cq_ptr_size, arc_cq_ptr, arc_cq_ptr_size, cp_current_inst); 7883 7884 if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) { 7885 if (arc_cq_ptr) { 7886 hdev->captured_err_info.undef_opcode.cq_addr = arc_cq_ptr; 7887 hdev->captured_err_info.undef_opcode.cq_size = arc_cq_ptr_size; 7888 } else { 7889 hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr; 7890 hdev->captured_err_info.undef_opcode.cq_size = cq_ptr_size; 7891 } 7892 7893 hdev->captured_err_info.undef_opcode.stream_id = QMAN_STREAMS; 7894 } 7895 } 7896 7897 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, 7898 u64 qman_base, u32 qid_base, u64 *event_mask) 7899 { 7900 u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0; 7901 u64 glbl_sts_addr, arb_err_addr; 7902 char reg_desc[32]; 7903 7904 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); 7905 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); 7906 7907 /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */ 7908 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { 7909 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i); 7910 7911 if (!glbl_sts_val) 7912 continue; 7913 7914 if (i == QMAN_STREAMS) { 7915 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM"); 7916 num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE; 7917 } else { 7918 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i); 7919 num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE; 7920 } 7921 7922 for (j = 0 ; j < num_error_causes ; j++) 7923 if (glbl_sts_val & BIT(j)) { 7924 gaudi2_print_event(hdev, event_type, true, 7925 "%s. err cause: %s", reg_desc, 7926 i == QMAN_STREAMS ? 7927 gaudi2_lower_qman_error_cause[j] : 7928 gaudi2_qman_error_cause[j]); 7929 error_count++; 7930 } 7931 7932 if (i == QMAN_STREAMS && error_count) { 7933 /* check for undefined opcode */ 7934 if (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK && 7935 hdev->captured_err_info.undef_opcode.write_enable) { 7936 memset(&hdev->captured_err_info.undef_opcode, 0, 7937 sizeof(hdev->captured_err_info.undef_opcode)); 7938 7939 hdev->captured_err_info.undef_opcode.write_enable = false; 7940 hdev->captured_err_info.undef_opcode.timestamp = ktime_get(); 7941 hdev->captured_err_info.undef_opcode.engine_id = 7942 gaudi2_queue_id_to_engine_id[qid_base]; 7943 *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE; 7944 } 7945 7946 handle_lower_qman_data_on_err(hdev, qman_base, *event_mask); 7947 } 7948 } 7949 7950 arb_err_val = RREG32(arb_err_addr); 7951 7952 if (!arb_err_val) 7953 goto out; 7954 7955 for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) { 7956 if (arb_err_val & BIT(j)) { 7957 gaudi2_print_event(hdev, event_type, true, 7958 "ARB_ERR. err cause: %s", 7959 gaudi2_qman_arb_error_cause[j]); 7960 error_count++; 7961 } 7962 } 7963 7964 out: 7965 return error_count; 7966 } 7967 7968 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, 7969 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 7970 enum gaudi2_engine_id id, u64 *event_mask) 7971 { 7972 u32 razwi_hi, razwi_lo, razwi_xy; 7973 u16 eng_id = id; 7974 u8 rd_wr_flag; 7975 7976 if (is_write) { 7977 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI); 7978 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO); 7979 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY); 7980 rd_wr_flag = HL_RAZWI_WRITE; 7981 } else { 7982 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI); 7983 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO); 7984 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY); 7985 rd_wr_flag = HL_RAZWI_READ; 7986 } 7987 7988 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1, 7989 rd_wr_flag | HL_RAZWI_HBW, event_mask); 7990 7991 dev_err_ratelimited(hdev->dev, 7992 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", 7993 name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy); 7994 } 7995 7996 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, 7997 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 7998 enum gaudi2_engine_id id, u64 *event_mask) 7999 { 8000 u64 razwi_addr = CFG_BASE; 8001 u32 razwi_xy; 8002 u16 eng_id = id; 8003 u8 rd_wr_flag; 8004 8005 if (is_write) { 8006 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI); 8007 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY); 8008 rd_wr_flag = HL_RAZWI_WRITE; 8009 } else { 8010 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI); 8011 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY); 8012 rd_wr_flag = HL_RAZWI_READ; 8013 } 8014 8015 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask); 8016 dev_err_ratelimited(hdev->dev, 8017 "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n", 8018 name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr, 8019 razwi_xy); 8020 } 8021 8022 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, 8023 enum razwi_event_sources module, u8 module_idx) 8024 { 8025 switch (module) { 8026 case RAZWI_TPC: 8027 if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES)) 8028 return GAUDI2_DCORE0_ENGINE_ID_TPC_6; 8029 return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8030 (module_idx % NUM_OF_TPC_PER_DCORE) + 8031 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8032 8033 case RAZWI_MME: 8034 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + 8035 (module_idx * ENGINE_ID_DCORE_OFFSET)); 8036 8037 case RAZWI_EDMA: 8038 return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8039 (module_idx % NUM_OF_EDMA_PER_DCORE)); 8040 8041 case RAZWI_PDMA: 8042 return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx); 8043 8044 case RAZWI_NIC: 8045 return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx)); 8046 8047 case RAZWI_DEC: 8048 if (module_idx == 8) 8049 return GAUDI2_PCIE_ENGINE_ID_DEC_0; 8050 8051 if (module_idx == 9) 8052 return GAUDI2_PCIE_ENGINE_ID_DEC_1; 8053 ; 8054 return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8055 (module_idx % NUM_OF_DEC_PER_DCORE) + 8056 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8057 8058 case RAZWI_ROT: 8059 return GAUDI2_ENGINE_ID_ROT_0 + module_idx; 8060 8061 case RAZWI_ARC_FARM: 8062 return GAUDI2_ENGINE_ID_ARC_FARM; 8063 8064 default: 8065 return GAUDI2_ENGINE_ID_SIZE; 8066 } 8067 } 8068 8069 /* 8070 * This function handles RR(Range register) hit events. 8071 * raised be initiators not PSOC RAZWI. 8072 */ 8073 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, 8074 enum razwi_event_sources module, u8 module_idx, 8075 u8 module_sub_idx, u64 *event_mask) 8076 { 8077 bool via_sft = false; 8078 u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx; 8079 u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr; 8080 u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0; 8081 u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0; 8082 char initiator_name[64]; 8083 8084 switch (module) { 8085 case RAZWI_TPC: 8086 sprintf(initiator_name, "TPC_%u", module_idx); 8087 if (hdev->tpc_binning) { 8088 binned_idx = __ffs(hdev->tpc_binning); 8089 if (binned_idx == module_idx) 8090 module_idx = TPC_ID_DCORE0_TPC6; 8091 } 8092 8093 hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx]; 8094 8095 if (hl_is_fw_sw_ver_below(hdev, 1, 9) && 8096 !hdev->asic_prop.fw_security_enabled && 8097 ((module_idx == 0) || (module_idx == 1))) 8098 lbw_rtr_id = DCORE0_RTR0; 8099 else 8100 lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx]; 8101 break; 8102 case RAZWI_MME: 8103 sprintf(initiator_name, "MME_%u", module_idx); 8104 switch (module_sub_idx) { 8105 case MME_WAP0: 8106 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0; 8107 break; 8108 case MME_WAP1: 8109 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1; 8110 break; 8111 case MME_WRITE: 8112 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write; 8113 break; 8114 case MME_READ: 8115 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read; 8116 break; 8117 case MME_SBTE0: 8118 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0; 8119 break; 8120 case MME_SBTE1: 8121 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1; 8122 break; 8123 case MME_SBTE2: 8124 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2; 8125 break; 8126 case MME_SBTE3: 8127 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3; 8128 break; 8129 case MME_SBTE4: 8130 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4; 8131 break; 8132 default: 8133 return; 8134 } 8135 lbw_rtr_id = hbw_rtr_id; 8136 break; 8137 case RAZWI_EDMA: 8138 hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx]; 8139 dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE; 8140 /* SFT has separate MSTR_IF for LBW, only there we can 8141 * read the LBW razwi related registers 8142 */ 8143 lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE + 8144 dcore_id * SFT_DCORE_OFFSET; 8145 via_sft = true; 8146 sprintf(initiator_name, "EDMA_%u", module_idx); 8147 break; 8148 case RAZWI_PDMA: 8149 hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx]; 8150 lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx]; 8151 sprintf(initiator_name, "PDMA_%u", module_idx); 8152 break; 8153 case RAZWI_NIC: 8154 hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx]; 8155 lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx]; 8156 sprintf(initiator_name, "NIC_%u", module_idx); 8157 break; 8158 case RAZWI_DEC: 8159 sprintf(initiator_name, "DEC_%u", module_idx); 8160 if (hdev->decoder_binning) { 8161 binned_idx = __ffs(hdev->decoder_binning); 8162 if (binned_idx == module_idx) 8163 module_idx = DEC_ID_PCIE_VDEC1; 8164 } 8165 hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx]; 8166 lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx]; 8167 break; 8168 case RAZWI_ROT: 8169 hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx]; 8170 lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx]; 8171 sprintf(initiator_name, "ROT_%u", module_idx); 8172 break; 8173 case RAZWI_ARC_FARM: 8174 lbw_rtr_id = DCORE1_RTR5; 8175 hbw_rtr_id = DCORE1_RTR7; 8176 sprintf(initiator_name, "ARC_FARM_%u", module_idx); 8177 break; 8178 default: 8179 return; 8180 } 8181 8182 /* Find router mstr_if register base */ 8183 if (!via_sft) { 8184 dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE; 8185 dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE; 8186 hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE + 8187 dcore_id * DCORE_OFFSET + 8188 dcore_rtr_id * DCORE_RTR_OFFSET + 8189 RTR_MSTR_IF_OFFSET; 8190 lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr + 8191 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); 8192 } 8193 8194 /* Find out event cause by reading "RAZWI_HAPPENED" registers */ 8195 hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED); 8196 hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED); 8197 lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED); 8198 lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED); 8199 8200 eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx); 8201 if (hbw_shrd_aw) { 8202 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true, 8203 initiator_name, eng_id, event_mask); 8204 8205 /* Clear event indication */ 8206 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw); 8207 } 8208 8209 if (hbw_shrd_ar) { 8210 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false, 8211 initiator_name, eng_id, event_mask); 8212 8213 /* Clear event indication */ 8214 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar); 8215 } 8216 8217 if (lbw_shrd_aw) { 8218 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true, 8219 initiator_name, eng_id, event_mask); 8220 8221 /* Clear event indication */ 8222 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw); 8223 } 8224 8225 if (lbw_shrd_ar) { 8226 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false, 8227 initiator_name, eng_id, event_mask); 8228 8229 /* Clear event indication */ 8230 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar); 8231 } 8232 } 8233 8234 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) 8235 { 8236 struct asic_fixed_properties *prop = &hdev->asic_prop; 8237 u8 mod_idx, sub_mod; 8238 8239 /* check all TPCs */ 8240 for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) { 8241 if (prop->tpc_enabled_mask & BIT(mod_idx)) 8242 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL); 8243 } 8244 8245 /* check all MMEs */ 8246 for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8247 for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++) 8248 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx, 8249 sub_mod, NULL); 8250 8251 /* check all EDMAs */ 8252 for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8253 if (prop->edma_enabled_mask & BIT(mod_idx)) 8254 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL); 8255 8256 /* check all PDMAs */ 8257 for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++) 8258 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL); 8259 8260 /* check all NICs */ 8261 for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++) 8262 if (hdev->nic_ports_mask & BIT(mod_idx)) 8263 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0, 8264 NULL); 8265 8266 /* check all DECs */ 8267 for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++) 8268 if (prop->decoder_enabled_mask & BIT(mod_idx)) 8269 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL); 8270 8271 /* check all ROTs */ 8272 for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++) 8273 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL); 8274 } 8275 8276 static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size, 8277 u32 axuser_xy, u32 *base, u16 *eng_id, 8278 char *eng_name) 8279 { 8280 8281 int i, num_of_eng = 0; 8282 u16 str_size = 0; 8283 8284 for (i = 0 ; i < array_size ; i++) { 8285 if (axuser_xy != razwi_info[i].axuser_xy) 8286 continue; 8287 8288 eng_id[num_of_eng] = razwi_info[i].eng_id; 8289 base[num_of_eng] = razwi_info[i].rtr_ctrl; 8290 if (!num_of_eng) 8291 str_size += scnprintf(eng_name + str_size, 8292 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", 8293 razwi_info[i].eng_name); 8294 else 8295 str_size += scnprintf(eng_name + str_size, 8296 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", 8297 razwi_info[i].eng_name); 8298 num_of_eng++; 8299 } 8300 8301 return num_of_eng; 8302 } 8303 8304 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, 8305 u64 *event_mask) 8306 { 8307 u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0; 8308 u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8309 u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8310 char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE]; 8311 bool razwi_happened = false; 8312 u64 addr; 8313 int i; 8314 8315 num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info), 8316 axuser_xy, base, eng_id, eng_name_str); 8317 8318 /* If no match for XY coordinates, try to find it in MME razwi table */ 8319 if (!num_of_eng) { 8320 axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg); 8321 num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info, 8322 ARRAY_SIZE(mme_razwi_info), 8323 axuser_xy, base, eng_id, 8324 eng_name_str); 8325 } 8326 8327 for (i = 0 ; i < num_of_eng ; i++) { 8328 if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) { 8329 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI); 8330 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO); 8331 addr = ((u64)addr_hi << 32) + addr_lo; 8332 if (addr) { 8333 dev_err(hdev->dev, 8334 "PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8335 eng_name_str, addr); 8336 hl_handle_razwi(hdev, addr, &eng_id[0], 8337 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask); 8338 razwi_happened = true; 8339 } 8340 } 8341 8342 if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) { 8343 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI); 8344 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO); 8345 addr = ((u64)addr_hi << 32) + addr_lo; 8346 if (addr) { 8347 dev_err(hdev->dev, 8348 "PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8349 eng_name_str, addr); 8350 hl_handle_razwi(hdev, addr, &eng_id[0], 8351 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask); 8352 razwi_happened = true; 8353 } 8354 } 8355 8356 if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) { 8357 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR); 8358 if (addr_lo) { 8359 dev_err(hdev->dev, 8360 "PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8361 eng_name_str, addr_lo); 8362 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8363 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask); 8364 razwi_happened = true; 8365 } 8366 } 8367 8368 if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) { 8369 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR); 8370 if (addr_lo) { 8371 dev_err(hdev->dev, 8372 "PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8373 eng_name_str, addr_lo); 8374 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8375 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask); 8376 razwi_happened = true; 8377 } 8378 } 8379 /* In common case the loop will break, when there is only one engine id, or 8380 * several engines with the same router. The exceptional case is with psoc razwi 8381 * from EDMA, where it's possible to get axuser id which fits 2 routers (2 8382 * interfaces of sft router). In this case, maybe the first router won't hold info 8383 * and we will need to iterate on the other router. 8384 */ 8385 if (razwi_happened) 8386 break; 8387 } 8388 8389 return razwi_happened; 8390 } 8391 8392 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */ 8393 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) 8394 { 8395 u32 razwi_mask_info, razwi_intr = 0, error_count = 0; 8396 8397 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { 8398 razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT); 8399 if (!razwi_intr) 8400 return 0; 8401 } 8402 8403 razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO); 8404 8405 dev_err_ratelimited(hdev->dev, 8406 "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", 8407 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info), 8408 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info), 8409 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info), 8410 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info), 8411 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info)); 8412 8413 if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask)) 8414 error_count++; 8415 else 8416 dev_err_ratelimited(hdev->dev, 8417 "PSOC RAZWI interrupt: invalid razwi info (0x%x)\n", 8418 razwi_mask_info); 8419 8420 /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */ 8421 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) 8422 WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr); 8423 8424 return error_count; 8425 } 8426 8427 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) 8428 { 8429 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8430 8431 sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET); 8432 8433 for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) { 8434 if (sts_val & BIT(i)) { 8435 gaudi2_print_event(hdev, event_type, true, 8436 "err cause: %s", gaudi2_qm_sei_error_cause[i]); 8437 sts_clr_val |= BIT(i); 8438 error_count++; 8439 } 8440 } 8441 8442 WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val); 8443 8444 return error_count; 8445 } 8446 8447 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, 8448 bool extended_err_check, u64 *event_mask) 8449 { 8450 enum razwi_event_sources module; 8451 u32 error_count = 0; 8452 u64 qman_base; 8453 u8 index; 8454 8455 switch (event_type) { 8456 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP: 8457 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 8458 qman_base = mmDCORE0_TPC0_QM_BASE + 8459 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET + 8460 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET; 8461 module = RAZWI_TPC; 8462 break; 8463 case GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 8464 qman_base = mmDCORE0_TPC6_QM_BASE; 8465 module = RAZWI_TPC; 8466 break; 8467 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 8468 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 8469 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 8470 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 8471 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 8472 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 8473 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 8474 qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET; 8475 module = RAZWI_MME; 8476 break; 8477 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 8478 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 8479 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; 8480 qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET; 8481 module = RAZWI_PDMA; 8482 break; 8483 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 8484 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 8485 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 8486 qman_base = mmROT0_QM_BASE + index * ROT_OFFSET; 8487 module = RAZWI_ROT; 8488 break; 8489 default: 8490 return 0; 8491 } 8492 8493 error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8494 8495 /* There is a single event per NIC macro, so should check its both QMAN blocks */ 8496 if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE && 8497 event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE) 8498 error_count += _gaudi2_handle_qm_sei_err(hdev, 8499 qman_base + NIC_QM_OFFSET, event_type); 8500 8501 if (extended_err_check) { 8502 /* check if RAZWI happened */ 8503 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask); 8504 hl_check_for_glbl_errors(hdev); 8505 } 8506 8507 return error_count; 8508 } 8509 8510 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8511 { 8512 u32 qid_base, error_count = 0; 8513 u64 qman_base; 8514 u8 index = 0; 8515 8516 switch (event_type) { 8517 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM: 8518 index = event_type - GAUDI2_EVENT_TPC0_QM; 8519 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS; 8520 qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8521 break; 8522 case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM: 8523 index = event_type - GAUDI2_EVENT_TPC6_QM; 8524 qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS; 8525 qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8526 break; 8527 case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM: 8528 index = event_type - GAUDI2_EVENT_TPC12_QM; 8529 qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS; 8530 qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8531 break; 8532 case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM: 8533 index = event_type - GAUDI2_EVENT_TPC18_QM; 8534 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS; 8535 qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8536 break; 8537 case GAUDI2_EVENT_TPC24_QM: 8538 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 8539 qman_base = mmDCORE0_TPC6_QM_BASE; 8540 break; 8541 case GAUDI2_EVENT_MME0_QM: 8542 qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 8543 qman_base = mmDCORE0_MME_QM_BASE; 8544 break; 8545 case GAUDI2_EVENT_MME1_QM: 8546 qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 8547 qman_base = mmDCORE1_MME_QM_BASE; 8548 break; 8549 case GAUDI2_EVENT_MME2_QM: 8550 qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 8551 qman_base = mmDCORE2_MME_QM_BASE; 8552 break; 8553 case GAUDI2_EVENT_MME3_QM: 8554 qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 8555 qman_base = mmDCORE3_MME_QM_BASE; 8556 break; 8557 case GAUDI2_EVENT_HDMA0_QM: 8558 index = 0; 8559 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0; 8560 qman_base = mmDCORE0_EDMA0_QM_BASE; 8561 break; 8562 case GAUDI2_EVENT_HDMA1_QM: 8563 index = 1; 8564 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0; 8565 qman_base = mmDCORE0_EDMA1_QM_BASE; 8566 break; 8567 case GAUDI2_EVENT_HDMA2_QM: 8568 index = 2; 8569 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0; 8570 qman_base = mmDCORE1_EDMA0_QM_BASE; 8571 break; 8572 case GAUDI2_EVENT_HDMA3_QM: 8573 index = 3; 8574 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0; 8575 qman_base = mmDCORE1_EDMA1_QM_BASE; 8576 break; 8577 case GAUDI2_EVENT_HDMA4_QM: 8578 index = 4; 8579 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0; 8580 qman_base = mmDCORE2_EDMA0_QM_BASE; 8581 break; 8582 case GAUDI2_EVENT_HDMA5_QM: 8583 index = 5; 8584 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0; 8585 qman_base = mmDCORE2_EDMA1_QM_BASE; 8586 break; 8587 case GAUDI2_EVENT_HDMA6_QM: 8588 index = 6; 8589 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0; 8590 qman_base = mmDCORE3_EDMA0_QM_BASE; 8591 break; 8592 case GAUDI2_EVENT_HDMA7_QM: 8593 index = 7; 8594 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0; 8595 qman_base = mmDCORE3_EDMA1_QM_BASE; 8596 break; 8597 case GAUDI2_EVENT_PDMA0_QM: 8598 qid_base = GAUDI2_QUEUE_ID_PDMA_0_0; 8599 qman_base = mmPDMA0_QM_BASE; 8600 break; 8601 case GAUDI2_EVENT_PDMA1_QM: 8602 qid_base = GAUDI2_QUEUE_ID_PDMA_1_0; 8603 qman_base = mmPDMA1_QM_BASE; 8604 break; 8605 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 8606 qid_base = GAUDI2_QUEUE_ID_ROT_0_0; 8607 qman_base = mmROT0_QM_BASE; 8608 break; 8609 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 8610 qid_base = GAUDI2_QUEUE_ID_ROT_1_0; 8611 qman_base = mmROT1_QM_BASE; 8612 break; 8613 default: 8614 return 0; 8615 } 8616 8617 error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, 8618 qid_base, event_mask); 8619 8620 /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */ 8621 if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) { 8622 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8623 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask); 8624 } 8625 8626 hl_check_for_glbl_errors(hdev); 8627 8628 return error_count; 8629 } 8630 8631 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8632 { 8633 u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm; 8634 8635 for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) { 8636 sts_clr_val = 0; 8637 sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS + 8638 (arc_farm * ARC_FARM_OFFSET)); 8639 8640 for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) { 8641 if (sts_val & BIT(i)) { 8642 gaudi2_print_event(hdev, event_type, true, 8643 "ARC FARM ARC %u err cause: %s", 8644 arc_farm, gaudi2_arc_sei_error_cause[i]); 8645 sts_clr_val |= BIT(i); 8646 error_count++; 8647 } 8648 } 8649 WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET), 8650 sts_clr_val); 8651 } 8652 8653 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ARC_FARM, 0, 0, event_mask); 8654 hl_check_for_glbl_errors(hdev); 8655 8656 return error_count; 8657 } 8658 8659 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) 8660 { 8661 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8662 8663 sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS); 8664 8665 for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) { 8666 if (sts_val & BIT(i)) { 8667 gaudi2_print_event(hdev, event_type, true, 8668 "err cause: %s", gaudi2_cpu_sei_error_cause[i]); 8669 sts_clr_val |= BIT(i); 8670 error_count++; 8671 } 8672 } 8673 8674 hl_check_for_glbl_errors(hdev); 8675 8676 WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val); 8677 8678 return error_count; 8679 } 8680 8681 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, 8682 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8683 u64 *event_mask) 8684 { 8685 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8686 u32 error_count = 0; 8687 int i; 8688 8689 for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++) 8690 if (intr_cause_data & BIT(i)) { 8691 gaudi2_print_event(hdev, event_type, true, 8692 "err cause: %s", guadi2_rot_error_cause[i]); 8693 error_count++; 8694 } 8695 8696 /* check if RAZWI happened */ 8697 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask); 8698 hl_check_for_glbl_errors(hdev); 8699 8700 return error_count; 8701 } 8702 8703 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, 8704 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8705 u64 *event_mask) 8706 { 8707 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8708 u32 error_count = 0; 8709 int i; 8710 8711 for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++) 8712 if (intr_cause_data & BIT(i)) { 8713 gaudi2_print_event(hdev, event_type, true, 8714 "interrupt cause: %s", gaudi2_tpc_interrupts_cause[i]); 8715 error_count++; 8716 } 8717 8718 /* check if RAZWI happened */ 8719 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask); 8720 hl_check_for_glbl_errors(hdev); 8721 8722 return error_count; 8723 } 8724 8725 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, 8726 u64 *event_mask) 8727 { 8728 u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0; 8729 int i; 8730 8731 if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES) 8732 /* DCORE DEC */ 8733 sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR + 8734 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) + 8735 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE); 8736 else 8737 /* PCIE DEC */ 8738 sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET * 8739 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); 8740 8741 sts_val = RREG32(sts_addr); 8742 8743 for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) { 8744 if (sts_val & BIT(i)) { 8745 gaudi2_print_event(hdev, event_type, true, 8746 "err cause: %s", gaudi2_dec_error_cause[i]); 8747 sts_clr_val |= BIT(i); 8748 error_count++; 8749 } 8750 } 8751 8752 /* check if RAZWI happened */ 8753 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask); 8754 hl_check_for_glbl_errors(hdev); 8755 8756 /* Write 1 clear errors */ 8757 WREG32(sts_addr, sts_clr_val); 8758 8759 return error_count; 8760 } 8761 8762 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8763 u64 *event_mask) 8764 { 8765 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8766 int i; 8767 8768 sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index; 8769 sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index; 8770 8771 sts_val = RREG32(sts_addr); 8772 8773 for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) { 8774 if (sts_val & BIT(i)) { 8775 gaudi2_print_event(hdev, event_type, true, 8776 "err cause: %s", guadi2_mme_error_cause[i]); 8777 sts_clr_val |= BIT(i); 8778 error_count++; 8779 } 8780 } 8781 8782 /* check if RAZWI happened */ 8783 for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++) 8784 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask); 8785 8786 hl_check_for_glbl_errors(hdev); 8787 8788 WREG32(sts_clr_addr, sts_clr_val); 8789 8790 return error_count; 8791 } 8792 8793 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type) 8794 { 8795 /* 8796 * We have a single error cause here but the report mechanism is 8797 * buggy. Hence there is no good reason to fetch the cause so we 8798 * just check for glbl_errors and exit. 8799 */ 8800 hl_check_for_glbl_errors(hdev); 8801 8802 return GAUDI2_NA_EVENT_CAUSE; 8803 } 8804 8805 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8806 u64 *event_mask) 8807 { 8808 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8809 int i; 8810 8811 sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index; 8812 sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index; 8813 8814 sts_val = RREG32(sts_addr); 8815 8816 for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) { 8817 if (sts_val & BIT(i)) { 8818 gaudi2_print_event(hdev, event_type, true, 8819 "err cause: %s", guadi2_mme_wap_error_cause[i]); 8820 sts_clr_val |= BIT(i); 8821 error_count++; 8822 } 8823 } 8824 8825 /* check if RAZWI happened on WAP0/1 */ 8826 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask); 8827 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask); 8828 hl_check_for_glbl_errors(hdev); 8829 8830 WREG32(sts_clr_addr, sts_clr_val); 8831 8832 return error_count; 8833 } 8834 8835 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, 8836 u64 intr_cause_data) 8837 { 8838 u32 error_count = 0; 8839 int i; 8840 8841 /* If an AXI read or write error is received, an error is reported and 8842 * interrupt message is sent. Due to an HW errata, when reading the cause 8843 * register of the KDMA engine, the reported error is always HBW even if 8844 * the actual error caused by a LBW KDMA transaction. 8845 */ 8846 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8847 if (intr_cause_data & BIT(i)) { 8848 gaudi2_print_event(hdev, event_type, true, 8849 "err cause: %s", gaudi2_kdma_core_interrupts_cause[i]); 8850 error_count++; 8851 } 8852 8853 hl_check_for_glbl_errors(hdev); 8854 8855 return error_count; 8856 } 8857 8858 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) 8859 { 8860 u32 error_count = 0; 8861 int i; 8862 8863 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8864 if (intr_cause & BIT(i)) { 8865 gaudi2_print_event(hdev, event_type, true, 8866 "err cause: %s", gaudi2_dma_core_interrupts_cause[i]); 8867 error_count++; 8868 } 8869 8870 hl_check_for_glbl_errors(hdev); 8871 8872 return error_count; 8873 } 8874 8875 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) 8876 { 8877 u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr; 8878 8879 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED; 8880 if (RREG32(razwi_happened_addr)) { 8881 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8882 GAUDI2_ENGINE_ID_PCIE, event_mask); 8883 WREG32(razwi_happened_addr, 0x1); 8884 } 8885 8886 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED; 8887 if (RREG32(razwi_happened_addr)) { 8888 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8889 GAUDI2_ENGINE_ID_PCIE, event_mask); 8890 WREG32(razwi_happened_addr, 0x1); 8891 } 8892 8893 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED; 8894 if (RREG32(razwi_happened_addr)) { 8895 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8896 GAUDI2_ENGINE_ID_PCIE, event_mask); 8897 WREG32(razwi_happened_addr, 0x1); 8898 } 8899 8900 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED; 8901 if (RREG32(razwi_happened_addr)) { 8902 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8903 GAUDI2_ENGINE_ID_PCIE, event_mask); 8904 WREG32(razwi_happened_addr, 0x1); 8905 } 8906 } 8907 8908 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, 8909 u64 intr_cause_data, u64 *event_mask) 8910 { 8911 u32 error_count = 0; 8912 int i; 8913 8914 gaudi2_print_event(hdev, event_type, true, 8915 "intr_cause_data: %#llx", intr_cause_data); 8916 8917 for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) { 8918 if (!(intr_cause_data & BIT_ULL(i))) 8919 continue; 8920 8921 gaudi2_print_event(hdev, event_type, true, 8922 "err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]); 8923 error_count++; 8924 8925 /* 8926 * Always check for LBW and HBW additional info as the indication itself is 8927 * sometimes missing 8928 */ 8929 } 8930 8931 hl_check_for_glbl_errors(hdev); 8932 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask); 8933 8934 return error_count; 8935 } 8936 8937 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, 8938 u64 intr_cause_data) 8939 8940 { 8941 u32 error_count = 0; 8942 int i; 8943 8944 for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) { 8945 if (intr_cause_data & BIT_ULL(i)) { 8946 gaudi2_print_event(hdev, event_type, true, 8947 "err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]); 8948 error_count++; 8949 } 8950 } 8951 8952 return error_count; 8953 } 8954 8955 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) 8956 { 8957 u32 error_count = 0; 8958 int i; 8959 8960 for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) { 8961 if (intr_cause_data & BIT_ULL(i)) { 8962 gaudi2_print_event(hdev, event_type, true, 8963 "err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]); 8964 error_count++; 8965 } 8966 } 8967 8968 return error_count; 8969 } 8970 8971 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, 8972 u64 *event_mask) 8973 { 8974 u32 valid, val; 8975 u64 addr; 8976 8977 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 8978 8979 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK)) 8980 return; 8981 8982 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE)); 8983 addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK; 8984 addr <<= 32; 8985 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA)); 8986 8987 if (is_pmmu) { 8988 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); 8989 } else { 8990 8991 addr = gaudi2_mmu_descramble_addr(hdev, addr); 8992 addr &= HW_UNSCRAMBLED_BITS_MASK; 8993 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", 8994 addr, addr + ~HW_UNSCRAMBLED_BITS_MASK); 8995 } 8996 8997 hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask); 8998 8999 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9000 } 9001 9002 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) 9003 { 9004 u32 valid, val; 9005 u64 addr; 9006 9007 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 9008 9009 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK)) 9010 return; 9011 9012 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE)); 9013 addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK; 9014 addr <<= 32; 9015 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA)); 9016 9017 if (!is_pmmu) 9018 addr = gaudi2_mmu_descramble_addr(hdev, addr); 9019 9020 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", 9021 is_pmmu ? "PMMU" : "HMMU", addr); 9022 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9023 } 9024 9025 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, 9026 u64 mmu_base, bool is_pmmu, u64 *event_mask) 9027 { 9028 u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0; 9029 int i; 9030 9031 spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET); 9032 9033 for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) { 9034 if (spi_sei_cause & BIT(i)) { 9035 gaudi2_print_event(hdev, event_type, true, 9036 "err cause: %s", gaudi2_mmu_spi_sei[i].cause); 9037 9038 if (i == 0) 9039 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask); 9040 else if (i == 1) 9041 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 9042 9043 if (gaudi2_mmu_spi_sei[i].clear_bit >= 0) 9044 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit); 9045 9046 error_count++; 9047 } 9048 } 9049 9050 /* Clear cause */ 9051 WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause); 9052 9053 /* Clear interrupt */ 9054 WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr); 9055 9056 return error_count; 9057 } 9058 9059 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) 9060 { 9061 u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log, 9062 cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0; 9063 int i; 9064 9065 sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index; 9066 cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index; 9067 9068 sei_cause_val = RREG32(sei_cause_addr); 9069 sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val); 9070 cq_intr_val = RREG32(cq_intr_addr); 9071 9072 /* SEI interrupt */ 9073 if (sei_cause_cause) { 9074 /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */ 9075 sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK, 9076 sei_cause_val); 9077 9078 for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) { 9079 if (!(sei_cause_cause & BIT(i))) 9080 continue; 9081 9082 gaudi2_print_event(hdev, event_type, true, 9083 "err cause: %s. %s: 0x%X", 9084 gaudi2_sm_sei_cause[i].cause_name, 9085 gaudi2_sm_sei_cause[i].log_name, 9086 sei_cause_log); 9087 error_count++; 9088 break; 9089 } 9090 9091 /* Clear SM_SEI_CAUSE */ 9092 WREG32(sei_cause_addr, 0); 9093 } 9094 9095 /* CQ interrupt */ 9096 if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) { 9097 cq_intr_queue_index = 9098 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK, 9099 cq_intr_val); 9100 9101 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", 9102 sm_index, cq_intr_queue_index); 9103 error_count++; 9104 9105 /* Clear CQ_INTR */ 9106 WREG32(cq_intr_addr, 0); 9107 } 9108 9109 hl_check_for_glbl_errors(hdev); 9110 9111 return error_count; 9112 } 9113 9114 static u64 get_hmmu_base(u16 event_type) 9115 { 9116 u8 dcore, index_in_dcore; 9117 9118 switch (event_type) { 9119 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP: 9120 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR: 9121 dcore = 0; 9122 index_in_dcore = 0; 9123 break; 9124 case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP: 9125 case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR: 9126 dcore = 1; 9127 index_in_dcore = 0; 9128 break; 9129 case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP: 9130 case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR: 9131 dcore = 0; 9132 index_in_dcore = 1; 9133 break; 9134 case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP: 9135 case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR: 9136 dcore = 1; 9137 index_in_dcore = 1; 9138 break; 9139 case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP: 9140 case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR: 9141 dcore = 3; 9142 index_in_dcore = 2; 9143 break; 9144 case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP: 9145 case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR: 9146 dcore = 2; 9147 index_in_dcore = 2; 9148 break; 9149 case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP: 9150 case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR: 9151 dcore = 3; 9152 index_in_dcore = 3; 9153 break; 9154 case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP: 9155 case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR: 9156 dcore = 2; 9157 index_in_dcore = 3; 9158 break; 9159 case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP: 9160 case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR: 9161 dcore = 0; 9162 index_in_dcore = 2; 9163 break; 9164 case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP: 9165 case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR: 9166 dcore = 1; 9167 index_in_dcore = 2; 9168 break; 9169 case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP: 9170 case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR: 9171 dcore = 0; 9172 index_in_dcore = 3; 9173 break; 9174 case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP: 9175 case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR: 9176 dcore = 1; 9177 index_in_dcore = 3; 9178 break; 9179 case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9180 case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9181 dcore = 3; 9182 index_in_dcore = 0; 9183 break; 9184 case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP: 9185 case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR: 9186 dcore = 2; 9187 index_in_dcore = 0; 9188 break; 9189 case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP: 9190 case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR: 9191 dcore = 3; 9192 index_in_dcore = 1; 9193 break; 9194 case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP: 9195 case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR: 9196 dcore = 2; 9197 index_in_dcore = 1; 9198 break; 9199 default: 9200 return ULONG_MAX; 9201 } 9202 9203 return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET; 9204 } 9205 9206 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9207 { 9208 bool is_pmmu = false; 9209 u32 error_count = 0; 9210 u64 mmu_base; 9211 9212 switch (event_type) { 9213 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9214 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9215 mmu_base = get_hmmu_base(event_type); 9216 break; 9217 9218 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9219 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9220 is_pmmu = true; 9221 mmu_base = mmPMMU_HBW_MMU_BASE; 9222 break; 9223 default: 9224 return 0; 9225 } 9226 9227 if (mmu_base == ULONG_MAX) 9228 return 0; 9229 9230 error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base, 9231 is_pmmu, event_mask); 9232 hl_check_for_glbl_errors(hdev); 9233 9234 return error_count; 9235 } 9236 9237 9238 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */ 9239 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, 9240 struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt) 9241 { 9242 u32 addr, beat, beat_shift; 9243 bool rc = false; 9244 9245 dev_err_ratelimited(hdev->dev, 9246 "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n", 9247 FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt), 9248 FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt), 9249 FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt)); 9250 9251 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); 9252 dev_err_ratelimited(hdev->dev, 9253 "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n", 9254 FIELD_GET(HBM_RD_ADDR_SID_MASK, addr), 9255 FIELD_GET(HBM_RD_ADDR_BG_MASK, addr), 9256 FIELD_GET(HBM_RD_ADDR_BA_MASK, addr), 9257 FIELD_GET(HBM_RD_ADDR_COL_MASK, addr), 9258 FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr)); 9259 9260 /* For each beat (RDQS edge), look for possible errors and print relevant info */ 9261 for (beat = 0 ; beat < 4 ; beat++) { 9262 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9263 (HBM_RD_ERR_SERR_BEAT0_MASK << beat)) 9264 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", 9265 beat, 9266 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9267 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9268 9269 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9270 (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) { 9271 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", 9272 beat, 9273 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9274 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9275 rc |= true; 9276 } 9277 9278 beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT; 9279 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9280 (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) { 9281 dev_err_ratelimited(hdev->dev, 9282 "Beat%d read PARITY: DM: %#x, PAR data: %#x\n", 9283 beat, 9284 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9285 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9286 (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >> 9287 (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift)); 9288 rc |= true; 9289 } 9290 9291 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); 9292 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9293 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); 9294 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9295 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); 9296 } 9297 9298 return rc; 9299 } 9300 9301 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, 9302 struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt) 9303 { 9304 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; 9305 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; 9306 9307 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); 9308 9309 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", 9310 derr & 0x3, derr & 0xc); 9311 9312 /* JIRA H6-3286 - the following prints may not be valid */ 9313 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); 9314 for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) { 9315 curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr); 9316 dev_err_ratelimited(hdev->dev, 9317 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n", 9318 i, 9319 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr), 9320 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr), 9321 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr), 9322 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr)); 9323 } 9324 } 9325 9326 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, 9327 struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt) 9328 { 9329 __le32 *col_cmd = ca_par_err_data->dbg_col; 9330 __le16 *row_cmd = ca_par_err_data->dbg_row; 9331 u32 i; 9332 9333 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); 9334 9335 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); 9336 for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++) 9337 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, 9338 le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0), 9339 le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0)); 9340 } 9341 9342 /* Returns true if hard reset is needed or false otherwise */ 9343 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, 9344 struct hl_eq_hbm_sei_data *sei_data) 9345 { 9346 bool require_hard_reset = false; 9347 u32 hbm_id, mc_id, cause_idx; 9348 9349 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; 9350 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; 9351 9352 cause_idx = sei_data->hdr.sei_cause; 9353 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { 9354 gaudi2_print_event(hdev, event_type, true, 9355 "err cause: %s", 9356 "Invalid HBM SEI event cause (%d) provided by FW", cause_idx); 9357 return true; 9358 } 9359 9360 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, 9361 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", 9362 sei_data->hdr.is_critical ? "Critical" : "Non-critical", 9363 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, 9364 hbm_mc_sei_cause[cause_idx]); 9365 9366 /* Print error-specific info */ 9367 switch (cause_idx) { 9368 case HBM_SEI_CATTRIP: 9369 require_hard_reset = true; 9370 break; 9371 9372 case HBM_SEI_CMD_PARITY_EVEN: 9373 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, 9374 le32_to_cpu(sei_data->hdr.cnt)); 9375 require_hard_reset = true; 9376 break; 9377 9378 case HBM_SEI_CMD_PARITY_ODD: 9379 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, 9380 le32_to_cpu(sei_data->hdr.cnt)); 9381 require_hard_reset = true; 9382 break; 9383 9384 case HBM_SEI_WRITE_DATA_PARITY_ERR: 9385 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, 9386 le32_to_cpu(sei_data->hdr.cnt)); 9387 require_hard_reset = true; 9388 break; 9389 9390 case HBM_SEI_READ_ERR: 9391 /* Unlike other SEI events, read error requires further processing of the 9392 * raw data in order to determine the root cause. 9393 */ 9394 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev, 9395 &sei_data->read_err_info, 9396 le32_to_cpu(sei_data->hdr.cnt)); 9397 break; 9398 9399 default: 9400 break; 9401 } 9402 9403 require_hard_reset |= !!sei_data->hdr.is_critical; 9404 9405 return require_hard_reset; 9406 } 9407 9408 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, 9409 u64 intr_cause_data) 9410 { 9411 if (intr_cause_data) { 9412 gaudi2_print_event(hdev, event_type, true, 9413 "temperature error cause: %#llx", intr_cause_data); 9414 return 1; 9415 } 9416 9417 return 0; 9418 } 9419 9420 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) 9421 { 9422 u32 i, error_count = 0; 9423 9424 for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++) 9425 if (intr_cause_data & hbm_mc_spi[i].mask) { 9426 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", 9427 hbm_mc_spi[i].cause); 9428 error_count++; 9429 } 9430 9431 return error_count; 9432 } 9433 9434 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9435 { 9436 ktime_t zero_time = ktime_set(0, 0); 9437 9438 mutex_lock(&hdev->clk_throttling.lock); 9439 9440 switch (event_type) { 9441 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 9442 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 9443 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 9444 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 9445 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 9446 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); 9447 break; 9448 9449 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 9450 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 9451 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 9452 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); 9453 break; 9454 9455 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 9456 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 9457 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 9458 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 9459 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 9460 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9461 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); 9462 break; 9463 9464 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 9465 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 9466 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 9467 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9468 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); 9469 break; 9470 9471 default: 9472 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); 9473 break; 9474 } 9475 9476 mutex_unlock(&hdev->clk_throttling.lock); 9477 } 9478 9479 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, 9480 struct cpucp_pkt_sync_err *sync_err) 9481 { 9482 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9483 9484 gaudi2_print_event(hdev, event_type, false, 9485 "FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9486 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), 9487 q->pi, atomic_read(&q->ci)); 9488 } 9489 9490 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) 9491 { 9492 u32 p2p_intr, msix_gw_intr, error_count = 0; 9493 9494 p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR); 9495 msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR); 9496 9497 if (p2p_intr) { 9498 gaudi2_print_event(hdev, event_type, true, 9499 "pcie p2p transaction terminated due to security, req_id(0x%x)", 9500 RREG32(mmPCIE_WRAP_P2P_REQ_ID)); 9501 9502 WREG32(mmPCIE_WRAP_P2P_INTR, 0x1); 9503 error_count++; 9504 } 9505 9506 if (msix_gw_intr) { 9507 gaudi2_print_event(hdev, event_type, true, 9508 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", 9509 RREG32(mmPCIE_WRAP_MSIX_GW_VEC)); 9510 9511 WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1); 9512 error_count++; 9513 } 9514 9515 return error_count; 9516 } 9517 9518 static int gaudi2_handle_pcie_drain(struct hl_device *hdev, 9519 struct hl_eq_pcie_drain_ind_data *drain_data) 9520 { 9521 u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause, error_count = 0; 9522 9523 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); 9524 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); 9525 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); 9526 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); 9527 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); 9528 9529 if (cause & BIT_ULL(0)) { 9530 dev_err_ratelimited(hdev->dev, 9531 "PCIE AXI drain LBW completed, read_err %u, write_err %u\n", 9532 !!lbw_rd, !!lbw_wr); 9533 error_count++; 9534 } 9535 9536 if (cause & BIT_ULL(1)) { 9537 dev_err_ratelimited(hdev->dev, 9538 "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n", 9539 hbw_rd, hbw_wr); 9540 error_count++; 9541 } 9542 9543 return error_count; 9544 } 9545 9546 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) 9547 { 9548 u32 error_count = 0; 9549 int i; 9550 9551 for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) { 9552 if (intr_cause_data & BIT_ULL(i)) { 9553 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", 9554 gaudi2_psoc_axi_drain_interrupts_cause[i]); 9555 error_count++; 9556 } 9557 } 9558 9559 hl_check_for_glbl_errors(hdev); 9560 9561 return error_count; 9562 } 9563 9564 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, 9565 struct cpucp_pkt_sync_err *sync_err) 9566 { 9567 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9568 9569 gaudi2_print_event(hdev, event_type, false, 9570 "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9571 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 9572 } 9573 9574 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, 9575 struct hl_eq_engine_arc_intr_data *data) 9576 { 9577 struct hl_engine_arc_dccm_queue_full_irq *q; 9578 u32 intr_type, engine_id; 9579 u64 payload; 9580 9581 intr_type = le32_to_cpu(data->intr_type); 9582 engine_id = le32_to_cpu(data->engine_id); 9583 payload = le64_to_cpu(data->payload); 9584 9585 switch (intr_type) { 9586 case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ: 9587 q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload; 9588 9589 gaudi2_print_event(hdev, event_type, true, 9590 "ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u", 9591 engine_id, intr_type, q->queue_index); 9592 return 1; 9593 default: 9594 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type"); 9595 return 0; 9596 } 9597 } 9598 9599 static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type) 9600 { 9601 enum gaudi2_block_types type = GAUDI2_BLOCK_TYPE_MAX; 9602 u16 index; 9603 9604 switch (event_type) { 9605 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9606 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9607 type = GAUDI2_BLOCK_TYPE_TPC; 9608 break; 9609 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC24_QM: 9610 index = event_type - GAUDI2_EVENT_TPC0_QM; 9611 type = GAUDI2_BLOCK_TYPE_TPC; 9612 break; 9613 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9614 case GAUDI2_EVENT_MME0_SPI_BASE ... GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9615 case GAUDI2_EVENT_MME0_QM: 9616 index = 0; 9617 type = GAUDI2_BLOCK_TYPE_MME; 9618 break; 9619 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9620 case GAUDI2_EVENT_MME1_SPI_BASE ... GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9621 case GAUDI2_EVENT_MME1_QM: 9622 index = 1; 9623 type = GAUDI2_BLOCK_TYPE_MME; 9624 break; 9625 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9626 case GAUDI2_EVENT_MME2_SPI_BASE ... GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9627 case GAUDI2_EVENT_MME2_QM: 9628 index = 2; 9629 type = GAUDI2_BLOCK_TYPE_MME; 9630 break; 9631 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9632 case GAUDI2_EVENT_MME3_SPI_BASE ... GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9633 case GAUDI2_EVENT_MME3_QM: 9634 index = 3; 9635 type = GAUDI2_BLOCK_TYPE_MME; 9636 break; 9637 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 9638 case GAUDI2_EVENT_KDMA_BM_SPMU: 9639 case GAUDI2_EVENT_KDMA0_CORE: 9640 return GAUDI2_ENGINE_ID_KDMA; 9641 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9642 case GAUDI2_EVENT_PDMA0_CORE: 9643 case GAUDI2_EVENT_PDMA0_BM_SPMU: 9644 case GAUDI2_EVENT_PDMA0_QM: 9645 return GAUDI2_ENGINE_ID_PDMA_0; 9646 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9647 case GAUDI2_EVENT_PDMA1_CORE: 9648 case GAUDI2_EVENT_PDMA1_BM_SPMU: 9649 case GAUDI2_EVENT_PDMA1_QM: 9650 return GAUDI2_ENGINE_ID_PDMA_1; 9651 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9652 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9653 type = GAUDI2_BLOCK_TYPE_DEC; 9654 break; 9655 case GAUDI2_EVENT_DEC0_SPI ... GAUDI2_EVENT_DEC9_BMON_SPMU: 9656 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; 9657 type = GAUDI2_BLOCK_TYPE_DEC; 9658 break; 9659 case GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE: 9660 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; 9661 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 9662 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9663 index = event_type - GAUDI2_EVENT_NIC0_QM0; 9664 return GAUDI2_ENGINE_ID_NIC0_0 + index; 9665 case GAUDI2_EVENT_NIC0_BMON_SPMU ... GAUDI2_EVENT_NIC11_SW_ERROR: 9666 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; 9667 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 9668 case GAUDI2_EVENT_TPC0_BMON_SPMU ... GAUDI2_EVENT_TPC24_KERNEL_ERR: 9669 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; 9670 type = GAUDI2_BLOCK_TYPE_TPC; 9671 break; 9672 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9673 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU: 9674 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 9675 return GAUDI2_ENGINE_ID_ROT_0; 9676 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9677 case GAUDI2_EVENT_ROTATOR1_BMON_SPMU: 9678 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9679 return GAUDI2_ENGINE_ID_ROT_1; 9680 case GAUDI2_EVENT_HDMA0_BM_SPMU: 9681 case GAUDI2_EVENT_HDMA0_QM: 9682 case GAUDI2_EVENT_HDMA0_CORE: 9683 return GAUDI2_DCORE0_ENGINE_ID_EDMA_0; 9684 case GAUDI2_EVENT_HDMA1_BM_SPMU: 9685 case GAUDI2_EVENT_HDMA1_QM: 9686 case GAUDI2_EVENT_HDMA1_CORE: 9687 return GAUDI2_DCORE0_ENGINE_ID_EDMA_1; 9688 case GAUDI2_EVENT_HDMA2_BM_SPMU: 9689 case GAUDI2_EVENT_HDMA2_QM: 9690 case GAUDI2_EVENT_HDMA2_CORE: 9691 return GAUDI2_DCORE1_ENGINE_ID_EDMA_0; 9692 case GAUDI2_EVENT_HDMA3_BM_SPMU: 9693 case GAUDI2_EVENT_HDMA3_QM: 9694 case GAUDI2_EVENT_HDMA3_CORE: 9695 return GAUDI2_DCORE1_ENGINE_ID_EDMA_1; 9696 case GAUDI2_EVENT_HDMA4_BM_SPMU: 9697 case GAUDI2_EVENT_HDMA4_QM: 9698 case GAUDI2_EVENT_HDMA4_CORE: 9699 return GAUDI2_DCORE2_ENGINE_ID_EDMA_0; 9700 case GAUDI2_EVENT_HDMA5_BM_SPMU: 9701 case GAUDI2_EVENT_HDMA5_QM: 9702 case GAUDI2_EVENT_HDMA5_CORE: 9703 return GAUDI2_DCORE2_ENGINE_ID_EDMA_1; 9704 case GAUDI2_EVENT_HDMA6_BM_SPMU: 9705 case GAUDI2_EVENT_HDMA6_QM: 9706 case GAUDI2_EVENT_HDMA6_CORE: 9707 return GAUDI2_DCORE3_ENGINE_ID_EDMA_0; 9708 case GAUDI2_EVENT_HDMA7_BM_SPMU: 9709 case GAUDI2_EVENT_HDMA7_QM: 9710 case GAUDI2_EVENT_HDMA7_CORE: 9711 return GAUDI2_DCORE3_ENGINE_ID_EDMA_1; 9712 default: 9713 break; 9714 } 9715 9716 switch (type) { 9717 case GAUDI2_BLOCK_TYPE_TPC: 9718 switch (index) { 9719 case TPC_ID_DCORE0_TPC0 ... TPC_ID_DCORE0_TPC5: 9720 return GAUDI2_DCORE0_ENGINE_ID_TPC_0 + index; 9721 case TPC_ID_DCORE1_TPC0 ... TPC_ID_DCORE1_TPC5: 9722 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; 9723 case TPC_ID_DCORE2_TPC0 ... TPC_ID_DCORE2_TPC5: 9724 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; 9725 case TPC_ID_DCORE3_TPC0 ... TPC_ID_DCORE3_TPC5: 9726 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; 9727 default: 9728 break; 9729 } 9730 break; 9731 case GAUDI2_BLOCK_TYPE_MME: 9732 switch (index) { 9733 case MME_ID_DCORE0: return GAUDI2_DCORE0_ENGINE_ID_MME; 9734 case MME_ID_DCORE1: return GAUDI2_DCORE1_ENGINE_ID_MME; 9735 case MME_ID_DCORE2: return GAUDI2_DCORE2_ENGINE_ID_MME; 9736 case MME_ID_DCORE3: return GAUDI2_DCORE3_ENGINE_ID_MME; 9737 default: 9738 break; 9739 } 9740 break; 9741 case GAUDI2_BLOCK_TYPE_DEC: 9742 switch (index) { 9743 case DEC_ID_DCORE0_DEC0: return GAUDI2_DCORE0_ENGINE_ID_DEC_0; 9744 case DEC_ID_DCORE0_DEC1: return GAUDI2_DCORE0_ENGINE_ID_DEC_1; 9745 case DEC_ID_DCORE1_DEC0: return GAUDI2_DCORE1_ENGINE_ID_DEC_0; 9746 case DEC_ID_DCORE1_DEC1: return GAUDI2_DCORE1_ENGINE_ID_DEC_1; 9747 case DEC_ID_DCORE2_DEC0: return GAUDI2_DCORE2_ENGINE_ID_DEC_0; 9748 case DEC_ID_DCORE2_DEC1: return GAUDI2_DCORE2_ENGINE_ID_DEC_1; 9749 case DEC_ID_DCORE3_DEC0: return GAUDI2_DCORE3_ENGINE_ID_DEC_0; 9750 case DEC_ID_DCORE3_DEC1: return GAUDI2_DCORE3_ENGINE_ID_DEC_1; 9751 case DEC_ID_PCIE_VDEC0: return GAUDI2_PCIE_ENGINE_ID_DEC_0; 9752 case DEC_ID_PCIE_VDEC1: return GAUDI2_PCIE_ENGINE_ID_DEC_1; 9753 default: 9754 break; 9755 } 9756 break; 9757 default: 9758 break; 9759 } 9760 9761 return U16_MAX; 9762 } 9763 9764 static void hl_eq_heartbeat_event_handle(struct hl_device *hdev) 9765 { 9766 hdev->eq_heartbeat_received = true; 9767 } 9768 9769 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 9770 { 9771 struct gaudi2_device *gaudi2 = hdev->asic_specific; 9772 bool reset_required = false, is_critical = false; 9773 u32 index, ctl, reset_flags = 0, error_count = 0; 9774 u64 event_mask = 0; 9775 u16 event_type; 9776 9777 ctl = le32_to_cpu(eq_entry->hdr.ctl); 9778 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT); 9779 9780 if (event_type >= GAUDI2_EVENT_SIZE) { 9781 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 9782 event_type, GAUDI2_EVENT_SIZE - 1); 9783 return; 9784 } 9785 9786 gaudi2->events_stat[event_type]++; 9787 gaudi2->events_stat_aggregate[event_type]++; 9788 9789 switch (event_type) { 9790 case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR: 9791 fallthrough; 9792 case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR: 9793 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9794 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9795 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 9796 is_critical = eq_entry->ecc_data.is_critical; 9797 error_count++; 9798 break; 9799 9800 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM: 9801 fallthrough; 9802 case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9803 fallthrough; 9804 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9805 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask); 9806 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9807 break; 9808 9809 case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0: 9810 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type, &event_mask); 9811 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9812 break; 9813 9814 case GAUDI2_EVENT_CPU_AXI_ERR_RSP: 9815 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type); 9816 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9817 event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR; 9818 break; 9819 9820 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9821 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9822 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask); 9823 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9824 break; 9825 9826 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9827 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9828 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 9829 error_count = gaudi2_handle_rot_err(hdev, index, event_type, 9830 &eq_entry->razwi_with_intr_cause, &event_mask); 9831 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9832 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9833 break; 9834 9835 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9836 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9837 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9838 &eq_entry->razwi_with_intr_cause, &event_mask); 9839 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9840 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9841 break; 9842 9843 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9844 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9845 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9846 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9847 break; 9848 9849 case GAUDI2_EVENT_TPC0_KERNEL_ERR: 9850 case GAUDI2_EVENT_TPC1_KERNEL_ERR: 9851 case GAUDI2_EVENT_TPC2_KERNEL_ERR: 9852 case GAUDI2_EVENT_TPC3_KERNEL_ERR: 9853 case GAUDI2_EVENT_TPC4_KERNEL_ERR: 9854 case GAUDI2_EVENT_TPC5_KERNEL_ERR: 9855 case GAUDI2_EVENT_TPC6_KERNEL_ERR: 9856 case GAUDI2_EVENT_TPC7_KERNEL_ERR: 9857 case GAUDI2_EVENT_TPC8_KERNEL_ERR: 9858 case GAUDI2_EVENT_TPC9_KERNEL_ERR: 9859 case GAUDI2_EVENT_TPC10_KERNEL_ERR: 9860 case GAUDI2_EVENT_TPC11_KERNEL_ERR: 9861 case GAUDI2_EVENT_TPC12_KERNEL_ERR: 9862 case GAUDI2_EVENT_TPC13_KERNEL_ERR: 9863 case GAUDI2_EVENT_TPC14_KERNEL_ERR: 9864 case GAUDI2_EVENT_TPC15_KERNEL_ERR: 9865 case GAUDI2_EVENT_TPC16_KERNEL_ERR: 9866 case GAUDI2_EVENT_TPC17_KERNEL_ERR: 9867 case GAUDI2_EVENT_TPC18_KERNEL_ERR: 9868 case GAUDI2_EVENT_TPC19_KERNEL_ERR: 9869 case GAUDI2_EVENT_TPC20_KERNEL_ERR: 9870 case GAUDI2_EVENT_TPC21_KERNEL_ERR: 9871 case GAUDI2_EVENT_TPC22_KERNEL_ERR: 9872 case GAUDI2_EVENT_TPC23_KERNEL_ERR: 9873 case GAUDI2_EVENT_TPC24_KERNEL_ERR: 9874 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / 9875 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); 9876 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9877 &eq_entry->razwi_with_intr_cause, &event_mask); 9878 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9879 break; 9880 9881 case GAUDI2_EVENT_DEC0_SPI: 9882 case GAUDI2_EVENT_DEC1_SPI: 9883 case GAUDI2_EVENT_DEC2_SPI: 9884 case GAUDI2_EVENT_DEC3_SPI: 9885 case GAUDI2_EVENT_DEC4_SPI: 9886 case GAUDI2_EVENT_DEC5_SPI: 9887 case GAUDI2_EVENT_DEC6_SPI: 9888 case GAUDI2_EVENT_DEC7_SPI: 9889 case GAUDI2_EVENT_DEC8_SPI: 9890 case GAUDI2_EVENT_DEC9_SPI: 9891 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / 9892 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); 9893 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9894 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9895 break; 9896 9897 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9898 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9899 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9900 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9901 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 9902 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 9903 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 9904 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9905 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9906 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9907 break; 9908 9909 case GAUDI2_EVENT_MME0_QMAN_SW_ERROR: 9910 case GAUDI2_EVENT_MME1_QMAN_SW_ERROR: 9911 case GAUDI2_EVENT_MME2_QMAN_SW_ERROR: 9912 case GAUDI2_EVENT_MME3_QMAN_SW_ERROR: 9913 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / 9914 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - 9915 GAUDI2_EVENT_MME0_QMAN_SW_ERROR); 9916 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9917 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9918 break; 9919 9920 case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9921 case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9922 case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9923 case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9924 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / 9925 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - 9926 GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID); 9927 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask); 9928 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9929 break; 9930 9931 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 9932 case GAUDI2_EVENT_KDMA0_CORE: 9933 error_count = gaudi2_handle_kdma_core_event(hdev, event_type, 9934 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9935 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9936 break; 9937 9938 case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE: 9939 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 9940 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9941 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9942 break; 9943 9944 case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE: 9945 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 9946 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9947 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9948 break; 9949 9950 case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR: 9951 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type, 9952 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); 9953 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9954 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9955 break; 9956 9957 case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9958 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9959 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9960 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9961 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask); 9962 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9963 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9964 break; 9965 9966 case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL: 9967 error_count = gaudi2_handle_hif_fatal(hdev, event_type, 9968 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9969 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9970 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9971 break; 9972 9973 case GAUDI2_EVENT_PMMU_FATAL_0: 9974 error_count = gaudi2_handle_pif_fatal(hdev, event_type, 9975 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9976 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9977 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9978 break; 9979 9980 case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT: 9981 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask); 9982 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9983 break; 9984 9985 case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE: 9986 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9987 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { 9988 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9989 reset_required = true; 9990 } 9991 error_count++; 9992 break; 9993 9994 case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5: 9995 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type, 9996 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9997 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9998 break; 9999 10000 case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI: 10001 error_count = gaudi2_handle_hbm_mc_spi(hdev, 10002 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10003 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10004 break; 10005 10006 case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE: 10007 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); 10008 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10009 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10010 break; 10011 10012 case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN: 10013 error_count = gaudi2_handle_psoc_drain(hdev, 10014 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10015 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10016 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10017 break; 10018 10019 case GAUDI2_EVENT_CPU_AXI_ECC: 10020 error_count = GAUDI2_NA_EVENT_CAUSE; 10021 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10022 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10023 break; 10024 case GAUDI2_EVENT_CPU_L2_RAM_ECC: 10025 error_count = GAUDI2_NA_EVENT_CAUSE; 10026 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10027 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10028 break; 10029 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP: 10030 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP: 10031 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP: 10032 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP: 10033 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type); 10034 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10035 break; 10036 case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B: 10037 error_count = GAUDI2_NA_EVENT_CAUSE; 10038 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10039 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10040 break; 10041 case GAUDI2_EVENT_PSOC_AXI_ERR_RSP: 10042 error_count = GAUDI2_NA_EVENT_CAUSE; 10043 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10044 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10045 break; 10046 case GAUDI2_EVENT_PSOC_PRSTN_FALL: 10047 error_count = GAUDI2_NA_EVENT_CAUSE; 10048 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10049 break; 10050 case GAUDI2_EVENT_PCIE_APB_TIMEOUT: 10051 error_count = GAUDI2_NA_EVENT_CAUSE; 10052 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10053 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10054 break; 10055 case GAUDI2_EVENT_PCIE_FATAL_ERR: 10056 error_count = GAUDI2_NA_EVENT_CAUSE; 10057 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10058 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10059 break; 10060 case GAUDI2_EVENT_TPC0_BMON_SPMU: 10061 case GAUDI2_EVENT_TPC1_BMON_SPMU: 10062 case GAUDI2_EVENT_TPC2_BMON_SPMU: 10063 case GAUDI2_EVENT_TPC3_BMON_SPMU: 10064 case GAUDI2_EVENT_TPC4_BMON_SPMU: 10065 case GAUDI2_EVENT_TPC5_BMON_SPMU: 10066 case GAUDI2_EVENT_TPC6_BMON_SPMU: 10067 case GAUDI2_EVENT_TPC7_BMON_SPMU: 10068 case GAUDI2_EVENT_TPC8_BMON_SPMU: 10069 case GAUDI2_EVENT_TPC9_BMON_SPMU: 10070 case GAUDI2_EVENT_TPC10_BMON_SPMU: 10071 case GAUDI2_EVENT_TPC11_BMON_SPMU: 10072 case GAUDI2_EVENT_TPC12_BMON_SPMU: 10073 case GAUDI2_EVENT_TPC13_BMON_SPMU: 10074 case GAUDI2_EVENT_TPC14_BMON_SPMU: 10075 case GAUDI2_EVENT_TPC15_BMON_SPMU: 10076 case GAUDI2_EVENT_TPC16_BMON_SPMU: 10077 case GAUDI2_EVENT_TPC17_BMON_SPMU: 10078 case GAUDI2_EVENT_TPC18_BMON_SPMU: 10079 case GAUDI2_EVENT_TPC19_BMON_SPMU: 10080 case GAUDI2_EVENT_TPC20_BMON_SPMU: 10081 case GAUDI2_EVENT_TPC21_BMON_SPMU: 10082 case GAUDI2_EVENT_TPC22_BMON_SPMU: 10083 case GAUDI2_EVENT_TPC23_BMON_SPMU: 10084 case GAUDI2_EVENT_TPC24_BMON_SPMU: 10085 case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU: 10086 case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU: 10087 case GAUDI2_EVENT_MME0_WAP_BMON_SPMU: 10088 case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU: 10089 case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU: 10090 case GAUDI2_EVENT_MME1_WAP_BMON_SPMU: 10091 case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU: 10092 case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU: 10093 case GAUDI2_EVENT_MME2_WAP_BMON_SPMU: 10094 case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU: 10095 case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU: 10096 case GAUDI2_EVENT_MME3_WAP_BMON_SPMU: 10097 case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU: 10098 fallthrough; 10099 case GAUDI2_EVENT_DEC0_BMON_SPMU: 10100 case GAUDI2_EVENT_DEC1_BMON_SPMU: 10101 case GAUDI2_EVENT_DEC2_BMON_SPMU: 10102 case GAUDI2_EVENT_DEC3_BMON_SPMU: 10103 case GAUDI2_EVENT_DEC4_BMON_SPMU: 10104 case GAUDI2_EVENT_DEC5_BMON_SPMU: 10105 case GAUDI2_EVENT_DEC6_BMON_SPMU: 10106 case GAUDI2_EVENT_DEC7_BMON_SPMU: 10107 case GAUDI2_EVENT_DEC8_BMON_SPMU: 10108 case GAUDI2_EVENT_DEC9_BMON_SPMU: 10109 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU: 10110 error_count = GAUDI2_NA_EVENT_CAUSE; 10111 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10112 break; 10113 10114 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 10115 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 10116 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 10117 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 10118 gaudi2_print_clk_change_info(hdev, event_type, &event_mask); 10119 error_count = GAUDI2_NA_EVENT_CAUSE; 10120 break; 10121 10122 case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC: 10123 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); 10124 error_count = GAUDI2_NA_EVENT_CAUSE; 10125 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10126 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10127 break; 10128 10129 case GAUDI2_EVENT_PCIE_FLR_REQUESTED: 10130 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10131 error_count = GAUDI2_NA_EVENT_CAUSE; 10132 /* Do nothing- FW will handle it */ 10133 break; 10134 10135 case GAUDI2_EVENT_PCIE_P2P_MSIX: 10136 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type); 10137 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10138 break; 10139 10140 case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE: 10141 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; 10142 error_count = gaudi2_handle_sm_err(hdev, event_type, index); 10143 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10144 break; 10145 10146 case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR: 10147 error_count = GAUDI2_NA_EVENT_CAUSE; 10148 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10149 break; 10150 10151 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 10152 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", 10153 le64_to_cpu(eq_entry->data[0])); 10154 error_count = GAUDI2_NA_EVENT_CAUSE; 10155 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10156 break; 10157 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT: 10158 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", 10159 le64_to_cpu(eq_entry->data[0])); 10160 error_count = GAUDI2_NA_EVENT_CAUSE; 10161 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10162 break; 10163 10164 case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED: 10165 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); 10166 error_count = GAUDI2_NA_EVENT_CAUSE; 10167 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10168 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10169 break; 10170 10171 case GAUDI2_EVENT_ARC_DCCM_FULL: 10172 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); 10173 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10174 break; 10175 10176 case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED: 10177 case GAUDI2_EVENT_CPU_DEV_RESET_REQ: 10178 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10179 error_count = GAUDI2_NA_EVENT_CAUSE; 10180 is_critical = true; 10181 break; 10182 10183 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY: 10184 case GAUDI2_EVENT_ARC_PWR_BRK_EXT: 10185 case GAUDI2_EVENT_ARC_PWR_RD_MODE0: 10186 case GAUDI2_EVENT_ARC_PWR_RD_MODE1: 10187 case GAUDI2_EVENT_ARC_PWR_RD_MODE2: 10188 case GAUDI2_EVENT_ARC_PWR_RD_MODE3: 10189 error_count = GAUDI2_NA_EVENT_CAUSE; 10190 dev_info_ratelimited(hdev->dev, "%s event received\n", 10191 gaudi2_irq_map_table[event_type].name); 10192 break; 10193 10194 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 10195 hl_eq_heartbeat_event_handle(hdev); 10196 error_count = GAUDI2_NA_EVENT_CAUSE; 10197 break; 10198 default: 10199 if (gaudi2_irq_map_table[event_type].valid) { 10200 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", 10201 event_type); 10202 error_count = GAUDI2_NA_EVENT_CAUSE; 10203 } 10204 } 10205 10206 if (event_mask & HL_NOTIFIER_EVENT_USER_ENGINE_ERR) 10207 hl_capture_engine_err(hdev, event_id_to_engine_id(hdev, event_type), error_count); 10208 10209 /* Make sure to dump an error in case no error cause was printed so far. 10210 * Note that although we have counted the errors, we use this number as 10211 * a boolean. 10212 */ 10213 if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type)) 10214 gaudi2_print_event(hdev, event_type, true, "%d", event_type); 10215 else if (error_count == 0) 10216 gaudi2_print_event(hdev, event_type, true, 10217 "No error cause for H/W event %u", event_type); 10218 10219 if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) || 10220 reset_required) { 10221 if (reset_required || 10222 (gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD)) 10223 reset_flags |= HL_DRV_RESET_HARD; 10224 10225 if (hdev->hard_reset_on_fw_events || 10226 (hdev->asic_prop.fw_security_enabled && is_critical)) 10227 goto reset_device; 10228 } 10229 10230 /* Send unmask irq only for interrupts not classified as MSG */ 10231 if (!gaudi2_irq_map_table[event_type].msg) 10232 hl_fw_unmask_irq(hdev, event_type); 10233 10234 if (event_mask) 10235 hl_notifier_event_send_all(hdev, event_mask); 10236 10237 return; 10238 10239 reset_device: 10240 if (hdev->asic_prop.fw_security_enabled && is_critical) { 10241 reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW; 10242 event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE; 10243 } else { 10244 reset_flags |= HL_DRV_RESET_DELAY; 10245 } 10246 /* escalate general hw errors to critical/fatal error */ 10247 if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR) 10248 hl_handle_critical_hw_err(hdev, event_type, &event_mask); 10249 10250 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 10251 hl_device_cond_reset(hdev, reset_flags, event_mask); 10252 } 10253 10254 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, 10255 struct packet_lin_dma *lin_dma_pkt, dma_addr_t pkt_dma_addr, 10256 u32 hw_queue_id, u32 size, u64 addr, u32 val) 10257 { 10258 u32 ctl, pkt_size; 10259 int rc = 0; 10260 10261 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 10262 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); 10263 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1); 10264 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1); 10265 10266 lin_dma_pkt->ctl = cpu_to_le32(ctl); 10267 lin_dma_pkt->src_addr = cpu_to_le64(val); 10268 lin_dma_pkt->dst_addr = cpu_to_le64(addr); 10269 lin_dma_pkt->tsize = cpu_to_le32(size); 10270 10271 pkt_size = sizeof(struct packet_lin_dma); 10272 10273 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr); 10274 if (rc) 10275 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", 10276 hw_queue_id); 10277 10278 return rc; 10279 } 10280 10281 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) 10282 { 10283 u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 10284 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 10285 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 10286 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0}; 10287 u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val, 10288 old_mmubp, mmubp, num_of_pkts, busy, pkt_size; 10289 u64 comp_addr, cur_addr = addr, end_addr = addr + size; 10290 struct asic_fixed_properties *prop = &hdev->asic_prop; 10291 void *lin_dma_pkts_arr; 10292 dma_addr_t pkt_dma_addr; 10293 int rc = 0, dma_num = 0; 10294 10295 if (prop->edma_enabled_mask == 0) { 10296 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); 10297 return -EIO; 10298 } 10299 10300 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10301 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 10302 comp_addr = CFG_BASE + sob_addr; 10303 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 10304 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 10305 mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) | 10306 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1); 10307 10308 /* Calculate how many lin dma pkts we'll need */ 10309 num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G); 10310 pkt_size = sizeof(struct packet_lin_dma); 10311 10312 lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts, 10313 &pkt_dma_addr, GFP_KERNEL); 10314 if (!lin_dma_pkts_arr) 10315 return -ENOMEM; 10316 10317 /* 10318 * set mmu bypass for the scrubbing - all ddmas are configured the same so save 10319 * only the first one to restore later 10320 * also set the sob addr for all edma cores for completion. 10321 * set QM as trusted to allow it to access physical address with MMU bp. 10322 */ 10323 old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP); 10324 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10325 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10326 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10327 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10328 10329 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10330 continue; 10331 10332 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + 10333 edma_offset, mmubp); 10334 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 10335 lower_32_bits(comp_addr)); 10336 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 10337 upper_32_bits(comp_addr)); 10338 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 10339 comp_val); 10340 gaudi2_qman_set_test_mode(hdev, 10341 edma_queues_id[dcore] + 4 * edma_idx, true); 10342 } 10343 } 10344 10345 WREG32(sob_addr, 0); 10346 10347 while (cur_addr < end_addr) { 10348 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10349 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10350 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10351 10352 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10353 continue; 10354 10355 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); 10356 10357 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev, 10358 (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num, 10359 pkt_dma_addr + dma_num * pkt_size, 10360 edma_queues_id[dcore] + edma_idx * 4, 10361 chunk_size, cur_addr, val); 10362 if (rc) 10363 goto end; 10364 10365 dma_num++; 10366 cur_addr += chunk_size; 10367 if (cur_addr == end_addr) 10368 break; 10369 } 10370 } 10371 } 10372 10373 rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000); 10374 if (rc) { 10375 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n"); 10376 goto end; 10377 } 10378 end: 10379 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10380 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10381 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10382 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10383 10384 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10385 continue; 10386 10387 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp); 10388 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0); 10389 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0); 10390 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0); 10391 gaudi2_qman_set_test_mode(hdev, 10392 edma_queues_id[dcore] + 4 * edma_idx, false); 10393 } 10394 } 10395 10396 WREG32(sob_addr, 0); 10397 hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr); 10398 10399 return rc; 10400 } 10401 10402 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) 10403 { 10404 int rc; 10405 struct asic_fixed_properties *prop = &hdev->asic_prop; 10406 u64 size = prop->dram_end_address - prop->dram_user_base_address; 10407 10408 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); 10409 10410 if (rc) 10411 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", 10412 prop->dram_user_base_address, size); 10413 return rc; 10414 } 10415 10416 static int gaudi2_scrub_device_mem(struct hl_device *hdev) 10417 { 10418 int rc; 10419 struct asic_fixed_properties *prop = &hdev->asic_prop; 10420 u64 val = hdev->memory_scrub_val; 10421 u64 addr, size; 10422 10423 if (!hdev->memory_scrub) 10424 return 0; 10425 10426 /* scrub SRAM */ 10427 addr = prop->sram_user_base_address; 10428 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); 10429 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", 10430 addr, addr + size, val); 10431 rc = gaudi2_memset_device_memory(hdev, addr, size, val); 10432 if (rc) { 10433 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); 10434 return rc; 10435 } 10436 10437 /* scrub DRAM */ 10438 rc = gaudi2_scrub_device_dram(hdev, val); 10439 if (rc) { 10440 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); 10441 return rc; 10442 } 10443 return 0; 10444 } 10445 10446 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) 10447 { 10448 u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr, 10449 cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr; 10450 u32 val, size, offset; 10451 int dcore_id; 10452 10453 offset = hdev->asic_prop.first_available_cq[0] * 4; 10454 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset; 10455 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset; 10456 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset; 10457 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset; 10458 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset; 10459 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset; 10460 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - 10461 (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset); 10462 10463 /* memset dcore0 CQ registers */ 10464 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10465 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10466 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10467 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10468 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10469 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10470 10471 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET; 10472 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET; 10473 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET; 10474 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET; 10475 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET; 10476 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET; 10477 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; 10478 10479 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10480 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10481 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10482 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10483 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10484 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10485 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10486 10487 cq_lbw_l_addr += DCORE_OFFSET; 10488 cq_lbw_h_addr += DCORE_OFFSET; 10489 cq_lbw_data_addr += DCORE_OFFSET; 10490 cq_base_l_addr += DCORE_OFFSET; 10491 cq_base_h_addr += DCORE_OFFSET; 10492 cq_size_addr += DCORE_OFFSET; 10493 } 10494 10495 offset = hdev->asic_prop.first_available_user_mon[0] * 4; 10496 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset; 10497 val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT; 10498 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); 10499 10500 /* memset dcore0 monitors */ 10501 gaudi2_memset_device_lbw(hdev, addr, size, val); 10502 10503 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset; 10504 gaudi2_memset_device_lbw(hdev, addr, size, 0); 10505 10506 mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET; 10507 mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET; 10508 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; 10509 10510 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10511 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val); 10512 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0); 10513 mon_sts_addr += DCORE_OFFSET; 10514 mon_cfg_addr += DCORE_OFFSET; 10515 } 10516 10517 offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10518 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset; 10519 val = 0; 10520 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - 10521 (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10522 10523 /* memset dcore0 sobs */ 10524 gaudi2_memset_device_lbw(hdev, addr, size, val); 10525 10526 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET; 10527 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; 10528 10529 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10530 gaudi2_memset_device_lbw(hdev, addr, size, val); 10531 addr += DCORE_OFFSET; 10532 } 10533 10534 /* Flush all WREG to prevent race */ 10535 val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10536 } 10537 10538 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) 10539 { 10540 u32 reg_base, hw_queue_id; 10541 10542 for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0; 10543 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10544 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10545 continue; 10546 10547 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10548 10549 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10550 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10551 } 10552 10553 /* Flush all WREG to prevent race */ 10554 RREG32(mmPDMA0_QM_ARB_CFG_0); 10555 } 10556 10557 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) 10558 { 10559 u32 reg_base, hw_queue_id; 10560 10561 for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3; 10562 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10563 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10564 continue; 10565 10566 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10567 10568 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10569 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10570 } 10571 10572 /* Flush all WREG to prevent race */ 10573 RREG32(mmPDMA0_QM_ARB_CFG_0); 10574 } 10575 10576 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) 10577 { 10578 return 0; 10579 } 10580 10581 static void gaudi2_restore_phase_topology(struct hl_device *hdev) 10582 { 10583 } 10584 10585 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, 10586 struct dup_block_ctx *cfg_ctx) 10587 { 10588 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; 10589 u8 seq; 10590 int i; 10591 10592 for (i = 0 ; i < cfg_ctx->instances ; i++) { 10593 seq = block_idx * cfg_ctx->instances + i; 10594 10595 /* skip disabled instance */ 10596 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) 10597 continue; 10598 10599 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, 10600 cfg_ctx->data); 10601 } 10602 } 10603 10604 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, 10605 u64 mask) 10606 { 10607 int i; 10608 10609 cfg_ctx->enabled_mask = mask; 10610 10611 for (i = 0 ; i < cfg_ctx->blocks ; i++) 10612 gaudi2_init_block_instances(hdev, i, cfg_ctx); 10613 } 10614 10615 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) 10616 { 10617 gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX); 10618 } 10619 10620 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) 10621 { 10622 void *host_mem_virtual_addr; 10623 dma_addr_t host_mem_dma_addr; 10624 u64 reserved_va_base; 10625 u32 pos, size_left, size_to_dma; 10626 struct hl_ctx *ctx; 10627 int rc = 0; 10628 10629 /* Fetch the ctx */ 10630 ctx = hl_get_compute_ctx(hdev); 10631 if (!ctx) { 10632 dev_err(hdev->dev, "No ctx available\n"); 10633 return -EINVAL; 10634 } 10635 10636 /* Allocate buffers for read and for poll */ 10637 host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr, 10638 GFP_KERNEL | __GFP_ZERO); 10639 if (host_mem_virtual_addr == NULL) { 10640 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); 10641 rc = -ENOMEM; 10642 goto put_ctx; 10643 } 10644 10645 /* Reserve VM region on asic side */ 10646 reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M, 10647 HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10648 if (!reserved_va_base) { 10649 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); 10650 rc = -ENOMEM; 10651 goto free_data_buffer; 10652 } 10653 10654 /* Create mapping on asic side */ 10655 mutex_lock(&hdev->mmu_lock); 10656 10657 rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M); 10658 if (rc) { 10659 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); 10660 goto unreserve_va; 10661 } 10662 10663 rc = hl_mmu_invalidate_cache_range(hdev, false, 10664 MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV, 10665 ctx->asid, reserved_va_base, SZ_2M); 10666 if (rc) { 10667 hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10668 goto unreserve_va; 10669 } 10670 10671 mutex_unlock(&hdev->mmu_lock); 10672 10673 /* Enable MMU on KDMA */ 10674 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); 10675 10676 pos = 0; 10677 size_left = size; 10678 size_to_dma = SZ_2M; 10679 10680 while (size_left > 0) { 10681 if (size_left < SZ_2M) 10682 size_to_dma = size_left; 10683 10684 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false); 10685 if (rc) 10686 break; 10687 10688 memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma); 10689 10690 if (size_left <= SZ_2M) 10691 break; 10692 10693 pos += SZ_2M; 10694 addr += SZ_2M; 10695 size_left -= SZ_2M; 10696 } 10697 10698 gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID); 10699 10700 mutex_lock(&hdev->mmu_lock); 10701 10702 rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10703 if (rc) 10704 goto unreserve_va; 10705 10706 rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR, 10707 ctx->asid, reserved_va_base, SZ_2M); 10708 10709 unreserve_va: 10710 mutex_unlock(&hdev->mmu_lock); 10711 hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M); 10712 free_data_buffer: 10713 hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr); 10714 put_ctx: 10715 hl_ctx_put(ctx); 10716 10717 return rc; 10718 } 10719 10720 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) 10721 { 10722 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10723 int min_alloc_order, rc; 10724 10725 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10726 return 0; 10727 10728 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, 10729 HOST_SPACE_INTERNAL_CB_SZ, 10730 &hdev->internal_cb_pool_dma_addr, 10731 GFP_KERNEL | __GFP_ZERO); 10732 10733 if (!hdev->internal_cb_pool_virt_addr) 10734 return -ENOMEM; 10735 10736 min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev), 10737 gaudi2_get_wait_cb_size(hdev))); 10738 10739 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); 10740 if (!hdev->internal_cb_pool) { 10741 dev_err(hdev->dev, "Failed to create internal CB pool\n"); 10742 rc = -ENOMEM; 10743 goto free_internal_cb_pool; 10744 } 10745 10746 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, 10747 HOST_SPACE_INTERNAL_CB_SZ, -1); 10748 if (rc) { 10749 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); 10750 rc = -EFAULT; 10751 goto destroy_internal_cb_pool; 10752 } 10753 10754 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 10755 HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10756 10757 if (!hdev->internal_cb_va_base) { 10758 rc = -ENOMEM; 10759 goto destroy_internal_cb_pool; 10760 } 10761 10762 mutex_lock(&hdev->mmu_lock); 10763 10764 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, 10765 HOST_SPACE_INTERNAL_CB_SZ); 10766 if (rc) 10767 goto unreserve_internal_cb_pool; 10768 10769 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); 10770 if (rc) 10771 goto unmap_internal_cb_pool; 10772 10773 mutex_unlock(&hdev->mmu_lock); 10774 10775 return 0; 10776 10777 unmap_internal_cb_pool: 10778 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10779 unreserve_internal_cb_pool: 10780 mutex_unlock(&hdev->mmu_lock); 10781 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10782 destroy_internal_cb_pool: 10783 gen_pool_destroy(hdev->internal_cb_pool); 10784 free_internal_cb_pool: 10785 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10786 hdev->internal_cb_pool_dma_addr); 10787 10788 return rc; 10789 } 10790 10791 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) 10792 { 10793 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10794 10795 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10796 return; 10797 10798 mutex_lock(&hdev->mmu_lock); 10799 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10800 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10801 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 10802 mutex_unlock(&hdev->mmu_lock); 10803 10804 gen_pool_destroy(hdev->internal_cb_pool); 10805 10806 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10807 hdev->internal_cb_pool_dma_addr); 10808 } 10809 10810 static void gaudi2_restore_user_registers(struct hl_device *hdev) 10811 { 10812 gaudi2_restore_user_sm_registers(hdev); 10813 gaudi2_restore_user_qm_registers(hdev); 10814 } 10815 10816 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10817 { 10818 struct hl_device *hdev = ctx->hdev; 10819 struct asic_fixed_properties *prop = &hdev->asic_prop; 10820 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10821 int rc; 10822 10823 rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10824 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); 10825 if (rc) 10826 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", 10827 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10828 10829 return rc; 10830 } 10831 10832 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10833 { 10834 struct hl_device *hdev = ctx->hdev; 10835 struct asic_fixed_properties *prop = &hdev->asic_prop; 10836 int rc; 10837 10838 rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10839 prop->pmmu.page_size, true); 10840 if (rc) 10841 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", 10842 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10843 } 10844 10845 static int gaudi2_ctx_init(struct hl_ctx *ctx) 10846 { 10847 int rc; 10848 10849 if (ctx->asid == HL_KERNEL_ASID_ID) 10850 return 0; 10851 10852 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); 10853 if (rc) 10854 return rc; 10855 10856 /* No need to clear user registers if the device has just 10857 * performed reset, we restore only nic qm registers 10858 */ 10859 if (ctx->hdev->reset_upon_device_release) 10860 gaudi2_restore_nic_qm_registers(ctx->hdev); 10861 else 10862 gaudi2_restore_user_registers(ctx->hdev); 10863 10864 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); 10865 if (rc) 10866 return rc; 10867 10868 rc = gaudi2_map_virtual_msix_doorbell_memory(ctx); 10869 if (rc) 10870 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10871 10872 return rc; 10873 } 10874 10875 static void gaudi2_ctx_fini(struct hl_ctx *ctx) 10876 { 10877 if (ctx->asid == HL_KERNEL_ASID_ID) 10878 return; 10879 10880 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10881 10882 gaudi2_unmap_virtual_msix_doorbell_memory(ctx); 10883 } 10884 10885 static int gaudi2_pre_schedule_cs(struct hl_cs *cs) 10886 { 10887 struct hl_device *hdev = cs->ctx->hdev; 10888 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); 10889 u32 mon_payload, sob_id, mon_id; 10890 10891 if (!cs_needs_completion(cs)) 10892 return 0; 10893 10894 /* 10895 * First 64 SOB/MON are reserved for driver for QMAN auto completion 10896 * mechanism. Each SOB/MON pair are used for a pending CS with the same 10897 * cyclic index. The SOB value is increased when each of the CS jobs is 10898 * completed. When the SOB reaches the number of CS jobs, the monitor 10899 * generates MSI-X interrupt. 10900 */ 10901 10902 sob_id = mon_id = index; 10903 mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) | 10904 (1 << CQ_ENTRY_READY_SHIFT) | index; 10905 10906 gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload, 10907 cs->jobs_cnt); 10908 10909 return 0; 10910 } 10911 10912 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 10913 { 10914 return HL_INVALID_QUEUE; 10915 } 10916 10917 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) 10918 { 10919 struct hl_cb *cb = data; 10920 struct packet_msg_short *pkt; 10921 u32 value, ctl, pkt_size = sizeof(*pkt); 10922 10923 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); 10924 memset(pkt, 0, pkt_size); 10925 10926 /* Inc by 1, Mode ADD */ 10927 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); 10928 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1); 10929 10930 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); 10931 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */ 10932 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10933 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb); 10934 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10935 10936 pkt->value = cpu_to_le32(value); 10937 pkt->ctl = cpu_to_le32(ctl); 10938 10939 return size + pkt_size; 10940 } 10941 10942 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) 10943 { 10944 u32 ctl, pkt_size = sizeof(*pkt); 10945 10946 memset(pkt, 0, pkt_size); 10947 10948 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 10949 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 10950 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10951 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10952 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0); 10953 10954 pkt->value = cpu_to_le32(value); 10955 pkt->ctl = cpu_to_le32(ctl); 10956 10957 return pkt_size; 10958 } 10959 10960 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, 10961 u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr) 10962 { 10963 u32 ctl, value, pkt_size = sizeof(*pkt); 10964 u8 mask; 10965 10966 if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) { 10967 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); 10968 return 0; 10969 } 10970 10971 memset(pkt, 0, pkt_size); 10972 10973 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8); 10974 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val); 10975 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/ 10976 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask); 10977 10978 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 10979 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 10980 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10981 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10982 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10983 10984 pkt->value = cpu_to_le32(value); 10985 pkt->ctl = cpu_to_le32(ctl); 10986 10987 return pkt_size; 10988 } 10989 10990 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) 10991 { 10992 u32 ctl, cfg, pkt_size = sizeof(*pkt); 10993 10994 memset(pkt, 0, pkt_size); 10995 10996 cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1); 10997 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); 10998 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2); 10999 11000 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE); 11001 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11002 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11003 11004 pkt->cfg = cpu_to_le32(cfg); 11005 pkt->ctl = cpu_to_le32(ctl); 11006 11007 return pkt_size; 11008 } 11009 11010 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) 11011 { 11012 struct hl_cb *cb = prop->data; 11013 void *buf = (void *) (uintptr_t) (cb->kernel_address); 11014 11015 u64 monitor_base, fence_addr = 0; 11016 u32 stream_index, size = prop->size; 11017 u16 msg_addr_offset; 11018 11019 stream_index = prop->q_idx % 4; 11020 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + 11021 QM_FENCE2_OFFSET + stream_index * 4; 11022 11023 /* 11024 * monitor_base should be the content of the base0 address registers, 11025 * so it will be added to the msg short offsets 11026 */ 11027 monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 11028 11029 /* First monitor config packet: low address of the sync */ 11030 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - 11031 monitor_base; 11032 11033 size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset); 11034 11035 /* Second monitor config packet: high address of the sync */ 11036 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - 11037 monitor_base; 11038 11039 size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset); 11040 11041 /* 11042 * Third monitor config packet: the payload, i.e. what to write when the 11043 * sync triggers 11044 */ 11045 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - 11046 monitor_base; 11047 11048 size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset); 11049 11050 /* Fourth monitor config packet: bind the monitor to a sync object */ 11051 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; 11052 11053 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, 11054 prop->sob_val, msg_addr_offset); 11055 11056 /* Fence packet */ 11057 size += gaudi2_add_fence_pkt(buf + size); 11058 11059 return size; 11060 } 11061 11062 static void gaudi2_reset_sob(struct hl_device *hdev, void *data) 11063 { 11064 struct hl_hw_sob *hw_sob = data; 11065 11066 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); 11067 11068 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); 11069 11070 kref_init(&hw_sob->kref); 11071 } 11072 11073 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) 11074 { 11075 } 11076 11077 static u64 gaudi2_get_device_time(struct hl_device *hdev) 11078 { 11079 u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 11080 11081 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 11082 } 11083 11084 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs) 11085 { 11086 return 0; 11087 } 11088 11089 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, 11090 struct hl_cs *cs, u32 wait_queue_id, 11091 u32 collective_engine_id, u32 encaps_signal_offset) 11092 { 11093 return -EINVAL; 11094 } 11095 11096 /* 11097 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address 11098 * to DMMU page-size address (64MB) before mapping it in 11099 * the MMU. 11100 * The operation is performed on both the virtual and physical addresses. 11101 * for device with 6 HBMs the scramble is: 11102 * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48] 11103 * 11104 * Example: 11105 * ============================================================================= 11106 * Allocated DRAM Reserved VA scrambled VA for MMU mapping Scrambled PA 11107 * Phys address in MMU last 11108 * HOP 11109 * ============================================================================= 11110 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1 11111 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3 11112 * ============================================================================= 11113 */ 11114 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) 11115 { 11116 struct asic_fixed_properties *prop = &hdev->asic_prop; 11117 u32 divisor, mod_va; 11118 u64 div_va; 11119 11120 /* accept any address in the DRAM address space */ 11121 if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE, 11122 VA_HBM_SPACE_END)) { 11123 11124 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11125 div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va); 11126 return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) | 11127 (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) | 11128 (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT); 11129 } 11130 11131 return raw_addr; 11132 } 11133 11134 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) 11135 { 11136 struct asic_fixed_properties *prop = &hdev->asic_prop; 11137 u32 divisor, mod_va; 11138 u64 div_va; 11139 11140 /* accept any address in the DRAM address space */ 11141 if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE, 11142 VA_HBM_SPACE_END)) { 11143 11144 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11145 div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, 11146 PAGE_SIZE_64MB, &mod_va); 11147 11148 return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) + 11149 (div_va * divisor + mod_va)); 11150 } 11151 11152 return scrambled_addr; 11153 } 11154 11155 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) 11156 { 11157 u32 base = 0, dcore_id, dec_id; 11158 11159 if (core_id >= NUMBER_OF_DEC) { 11160 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); 11161 goto out; 11162 } 11163 11164 if (core_id < 8) { 11165 dcore_id = core_id / NUM_OF_DEC_PER_DCORE; 11166 dec_id = core_id % NUM_OF_DEC_PER_DCORE; 11167 11168 base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET + 11169 dec_id * DCORE_VDEC_OFFSET; 11170 } else { 11171 /* PCIe Shared Decoder */ 11172 base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET); 11173 } 11174 out: 11175 return base; 11176 } 11177 11178 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 11179 u32 *block_size, u32 *block_id) 11180 { 11181 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11182 int i; 11183 11184 for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) { 11185 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { 11186 *block_id = i; 11187 if (block_size) 11188 *block_size = gaudi2->mapped_blocks[i].size; 11189 return 0; 11190 } 11191 } 11192 11193 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); 11194 11195 return -EINVAL; 11196 } 11197 11198 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 11199 u32 block_id, u32 block_size) 11200 { 11201 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11202 u64 offset_in_bar; 11203 u64 address; 11204 int rc; 11205 11206 if (block_id >= NUM_USER_MAPPED_BLOCKS) { 11207 dev_err(hdev->dev, "Invalid block id %u", block_id); 11208 return -EINVAL; 11209 } 11210 11211 /* we allow mapping only an entire block */ 11212 if (block_size != gaudi2->mapped_blocks[block_id].size) { 11213 dev_err(hdev->dev, "Invalid block size %u", block_size); 11214 return -EINVAL; 11215 } 11216 11217 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; 11218 11219 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; 11220 11221 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 11222 VM_DONTCOPY | VM_NORESERVE); 11223 11224 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, 11225 block_size, vma->vm_page_prot); 11226 if (rc) 11227 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 11228 11229 return rc; 11230 } 11231 11232 static void gaudi2_enable_events_from_fw(struct hl_device *hdev) 11233 { 11234 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11235 11236 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 11237 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); 11238 11239 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 11240 WREG32(irq_handler_offset, 11241 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id); 11242 } 11243 11244 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) 11245 { 11246 switch (mmu_id) { 11247 case HW_CAP_DCORE0_DMMU0: 11248 *mmu_base = mmDCORE0_HMMU0_MMU_BASE; 11249 break; 11250 case HW_CAP_DCORE0_DMMU1: 11251 *mmu_base = mmDCORE0_HMMU1_MMU_BASE; 11252 break; 11253 case HW_CAP_DCORE0_DMMU2: 11254 *mmu_base = mmDCORE0_HMMU2_MMU_BASE; 11255 break; 11256 case HW_CAP_DCORE0_DMMU3: 11257 *mmu_base = mmDCORE0_HMMU3_MMU_BASE; 11258 break; 11259 case HW_CAP_DCORE1_DMMU0: 11260 *mmu_base = mmDCORE1_HMMU0_MMU_BASE; 11261 break; 11262 case HW_CAP_DCORE1_DMMU1: 11263 *mmu_base = mmDCORE1_HMMU1_MMU_BASE; 11264 break; 11265 case HW_CAP_DCORE1_DMMU2: 11266 *mmu_base = mmDCORE1_HMMU2_MMU_BASE; 11267 break; 11268 case HW_CAP_DCORE1_DMMU3: 11269 *mmu_base = mmDCORE1_HMMU3_MMU_BASE; 11270 break; 11271 case HW_CAP_DCORE2_DMMU0: 11272 *mmu_base = mmDCORE2_HMMU0_MMU_BASE; 11273 break; 11274 case HW_CAP_DCORE2_DMMU1: 11275 *mmu_base = mmDCORE2_HMMU1_MMU_BASE; 11276 break; 11277 case HW_CAP_DCORE2_DMMU2: 11278 *mmu_base = mmDCORE2_HMMU2_MMU_BASE; 11279 break; 11280 case HW_CAP_DCORE2_DMMU3: 11281 *mmu_base = mmDCORE2_HMMU3_MMU_BASE; 11282 break; 11283 case HW_CAP_DCORE3_DMMU0: 11284 *mmu_base = mmDCORE3_HMMU0_MMU_BASE; 11285 break; 11286 case HW_CAP_DCORE3_DMMU1: 11287 *mmu_base = mmDCORE3_HMMU1_MMU_BASE; 11288 break; 11289 case HW_CAP_DCORE3_DMMU2: 11290 *mmu_base = mmDCORE3_HMMU2_MMU_BASE; 11291 break; 11292 case HW_CAP_DCORE3_DMMU3: 11293 *mmu_base = mmDCORE3_HMMU3_MMU_BASE; 11294 break; 11295 case HW_CAP_PMMU: 11296 *mmu_base = mmPMMU_HBW_MMU_BASE; 11297 break; 11298 default: 11299 return -EINVAL; 11300 } 11301 11302 return 0; 11303 } 11304 11305 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) 11306 { 11307 bool is_pmmu = (mmu_id == HW_CAP_PMMU); 11308 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11309 u32 mmu_base; 11310 11311 if (!(gaudi2->hw_cap_initialized & mmu_id)) 11312 return; 11313 11314 if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base)) 11315 return; 11316 11317 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL); 11318 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 11319 } 11320 11321 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 11322 { 11323 u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES; 11324 11325 /* check all HMMUs */ 11326 for (i = 0 ; i < num_of_hmmus ; i++) { 11327 mmu_id = HW_CAP_DCORE0_DMMU0 << i; 11328 11329 if (mmu_cap_mask & mmu_id) 11330 gaudi2_ack_mmu_error(hdev, mmu_id); 11331 } 11332 11333 /* check PMMU */ 11334 if (mmu_cap_mask & HW_CAP_PMMU) 11335 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU); 11336 11337 return 0; 11338 } 11339 11340 static void gaudi2_get_msi_info(__le32 *table) 11341 { 11342 table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX); 11343 table[CPUCP_EVENT_QUEUE_ERR_MSI_TYPE] = cpu_to_le32(GAUDI2_IRQ_NUM_EQ_ERROR); 11344 } 11345 11346 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx) 11347 { 11348 switch (pll_idx) { 11349 case HL_GAUDI2_CPU_PLL: return CPU_PLL; 11350 case HL_GAUDI2_PCI_PLL: return PCI_PLL; 11351 case HL_GAUDI2_NIC_PLL: return NIC_PLL; 11352 case HL_GAUDI2_DMA_PLL: return DMA_PLL; 11353 case HL_GAUDI2_MESH_PLL: return MESH_PLL; 11354 case HL_GAUDI2_MME_PLL: return MME_PLL; 11355 case HL_GAUDI2_TPC_PLL: return TPC_PLL; 11356 case HL_GAUDI2_IF_PLL: return IF_PLL; 11357 case HL_GAUDI2_SRAM_PLL: return SRAM_PLL; 11358 case HL_GAUDI2_HBM_PLL: return HBM_PLL; 11359 case HL_GAUDI2_VID_PLL: return VID_PLL; 11360 case HL_GAUDI2_MSS_PLL: return MSS_PLL; 11361 default: return -EINVAL; 11362 } 11363 } 11364 11365 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) 11366 { 11367 /* Not implemented */ 11368 return 0; 11369 } 11370 11371 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon) 11372 { 11373 /* Not implemented */ 11374 return 0; 11375 } 11376 11377 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset, 11378 struct hl_device *hdev, struct hl_mon_state_dump *mon) 11379 { 11380 /* Not implemented */ 11381 return 0; 11382 } 11383 11384 11385 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, 11386 u64 status_base_offset, enum hl_sync_engine_type engine_type, 11387 u32 engine_id, char **buf, size_t *size, size_t *offset) 11388 { 11389 /* Not implemented */ 11390 return 0; 11391 } 11392 11393 11394 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = { 11395 .monitor_valid = gaudi2_monitor_valid, 11396 .print_single_monitor = gaudi2_print_single_monitor, 11397 .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map, 11398 .print_fences_single_engine = gaudi2_print_fences_single_engine, 11399 }; 11400 11401 static void gaudi2_state_dump_init(struct hl_device *hdev) 11402 { 11403 /* Not implemented */ 11404 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; 11405 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; 11406 } 11407 11408 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) 11409 { 11410 return 0; 11411 } 11412 11413 static u32 *gaudi2_get_stream_master_qid_arr(void) 11414 { 11415 return NULL; 11416 } 11417 11418 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 11419 struct attribute_group *dev_vrm_attr_grp) 11420 { 11421 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp); 11422 hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp); 11423 } 11424 11425 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 11426 u32 page_size, u32 *real_page_size, bool is_dram_addr) 11427 { 11428 struct asic_fixed_properties *prop = &hdev->asic_prop; 11429 11430 /* for host pages the page size must be */ 11431 if (!is_dram_addr) { 11432 if (page_size % mmu_prop->page_size) 11433 goto page_size_err; 11434 11435 *real_page_size = mmu_prop->page_size; 11436 return 0; 11437 } 11438 11439 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) 11440 goto page_size_err; 11441 11442 /* 11443 * MMU page size is different from DRAM page size (more precisely, DMMU page is greater 11444 * than DRAM page size). 11445 * for this reason work with the DRAM page size and let the MMU scrambling routine handle 11446 * this mismatch when calculating the address to place in the MMU page table. 11447 * (in that case also make sure that the dram_page_size is not greater than the 11448 * mmu page size) 11449 */ 11450 *real_page_size = prop->dram_page_size; 11451 11452 return 0; 11453 11454 page_size_err: 11455 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", 11456 page_size, mmu_prop->page_size >> 10); 11457 return -EFAULT; 11458 } 11459 11460 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) 11461 { 11462 return -EOPNOTSUPP; 11463 } 11464 11465 int gaudi2_send_device_activity(struct hl_device *hdev, bool open) 11466 { 11467 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11468 11469 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 11470 return 0; 11471 11472 return hl_fw_send_device_activity(hdev, open); 11473 } 11474 11475 static const struct hl_asic_funcs gaudi2_funcs = { 11476 .early_init = gaudi2_early_init, 11477 .early_fini = gaudi2_early_fini, 11478 .late_init = gaudi2_late_init, 11479 .late_fini = gaudi2_late_fini, 11480 .sw_init = gaudi2_sw_init, 11481 .sw_fini = gaudi2_sw_fini, 11482 .hw_init = gaudi2_hw_init, 11483 .hw_fini = gaudi2_hw_fini, 11484 .halt_engines = gaudi2_halt_engines, 11485 .suspend = gaudi2_suspend, 11486 .resume = gaudi2_resume, 11487 .mmap = gaudi2_mmap, 11488 .ring_doorbell = gaudi2_ring_doorbell, 11489 .pqe_write = gaudi2_pqe_write, 11490 .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent, 11491 .asic_dma_free_coherent = gaudi2_dma_free_coherent, 11492 .scrub_device_mem = gaudi2_scrub_device_mem, 11493 .scrub_device_dram = gaudi2_scrub_device_dram, 11494 .get_int_queue_base = NULL, 11495 .test_queues = gaudi2_test_queues, 11496 .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc, 11497 .asic_dma_pool_free = gaudi2_dma_pool_free, 11498 .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc, 11499 .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free, 11500 .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable, 11501 .cs_parser = gaudi2_cs_parser, 11502 .dma_map_sgtable = hl_asic_dma_map_sgtable, 11503 .add_end_of_cb_packets = NULL, 11504 .update_eq_ci = gaudi2_update_eq_ci, 11505 .context_switch = gaudi2_context_switch, 11506 .restore_phase_topology = gaudi2_restore_phase_topology, 11507 .debugfs_read_dma = gaudi2_debugfs_read_dma, 11508 .add_device_attr = gaudi2_add_device_attr, 11509 .handle_eqe = gaudi2_handle_eqe, 11510 .get_events_stat = gaudi2_get_events_stat, 11511 .read_pte = NULL, 11512 .write_pte = NULL, 11513 .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache, 11514 .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range, 11515 .mmu_prefetch_cache_range = NULL, 11516 .send_heartbeat = gaudi2_send_heartbeat, 11517 .debug_coresight = gaudi2_debug_coresight, 11518 .is_device_idle = gaudi2_is_device_idle, 11519 .compute_reset_late_init = gaudi2_compute_reset_late_init, 11520 .hw_queues_lock = gaudi2_hw_queues_lock, 11521 .hw_queues_unlock = gaudi2_hw_queues_unlock, 11522 .get_pci_id = gaudi2_get_pci_id, 11523 .get_eeprom_data = gaudi2_get_eeprom_data, 11524 .get_monitor_dump = gaudi2_get_monitor_dump, 11525 .send_cpu_message = gaudi2_send_cpu_message, 11526 .pci_bars_map = gaudi2_pci_bars_map, 11527 .init_iatu = gaudi2_init_iatu, 11528 .rreg = hl_rreg, 11529 .wreg = hl_wreg, 11530 .halt_coresight = gaudi2_halt_coresight, 11531 .ctx_init = gaudi2_ctx_init, 11532 .ctx_fini = gaudi2_ctx_fini, 11533 .pre_schedule_cs = gaudi2_pre_schedule_cs, 11534 .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq, 11535 .load_firmware_to_device = NULL, 11536 .load_boot_fit_to_device = NULL, 11537 .get_signal_cb_size = gaudi2_get_signal_cb_size, 11538 .get_wait_cb_size = gaudi2_get_wait_cb_size, 11539 .gen_signal_cb = gaudi2_gen_signal_cb, 11540 .gen_wait_cb = gaudi2_gen_wait_cb, 11541 .reset_sob = gaudi2_reset_sob, 11542 .reset_sob_group = gaudi2_reset_sob_group, 11543 .get_device_time = gaudi2_get_device_time, 11544 .pb_print_security_errors = gaudi2_pb_print_security_errors, 11545 .collective_wait_init_cs = gaudi2_collective_wait_init_cs, 11546 .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs, 11547 .get_dec_base_addr = gaudi2_get_dec_base_addr, 11548 .scramble_addr = gaudi2_mmu_scramble_addr, 11549 .descramble_addr = gaudi2_mmu_descramble_addr, 11550 .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors, 11551 .get_hw_block_id = gaudi2_get_hw_block_id, 11552 .hw_block_mmap = gaudi2_block_mmap, 11553 .enable_events_from_fw = gaudi2_enable_events_from_fw, 11554 .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error, 11555 .get_msi_info = gaudi2_get_msi_info, 11556 .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx, 11557 .init_firmware_preload_params = gaudi2_init_firmware_preload_params, 11558 .init_firmware_loader = gaudi2_init_firmware_loader, 11559 .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm, 11560 .state_dump_init = gaudi2_state_dump_init, 11561 .get_sob_addr = &gaudi2_get_sob_addr, 11562 .set_pci_memory_regions = gaudi2_set_pci_memory_regions, 11563 .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr, 11564 .check_if_razwi_happened = gaudi2_check_if_razwi_happened, 11565 .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size, 11566 .access_dev_mem = hl_access_dev_mem, 11567 .set_dram_bar_base = gaudi2_set_hbm_bar_base, 11568 .set_engine_cores = gaudi2_set_engine_cores, 11569 .set_engines = gaudi2_set_engines, 11570 .send_device_activity = gaudi2_send_device_activity, 11571 .set_dram_properties = gaudi2_set_dram_properties, 11572 .set_binning_masks = gaudi2_set_binning_masks, 11573 }; 11574 11575 void gaudi2_set_asic_funcs(struct hl_device *hdev) 11576 { 11577 hdev->asic_funcs = &gaudi2_funcs; 11578 } 11579