1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2016-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include "gaudiP.h" 9 #include "../include/hw_ip/mmu/mmu_general.h" 10 #include "../include/hw_ip/mmu/mmu_v1_1.h" 11 #include "../include/gaudi/gaudi_masks.h" 12 #include "../include/gaudi/gaudi_fw_if.h" 13 #include "../include/gaudi/gaudi_reg_map.h" 14 #include "../include/gaudi/gaudi_async_ids_map_extended.h" 15 16 #include <linux/module.h> 17 #include <linux/pci.h> 18 #include <linux/firmware.h> 19 #include <linux/hwmon.h> 20 #include <linux/iommu.h> 21 #include <linux/seq_file.h> 22 23 /* 24 * Gaudi security scheme: 25 * 26 * 1. Host is protected by: 27 * - Range registers 28 * - MMU 29 * 30 * 2. DDR is protected by: 31 * - Range registers (protect the first 512MB) 32 * 33 * 3. Configuration is protected by: 34 * - Range registers 35 * - Protection bits 36 * 37 * MMU is always enabled. 38 * 39 * QMAN DMA channels 0,1 (PCI DMAN): 40 * - DMA is not secured. 41 * - PQ and CQ are secured. 42 * - CP is secured: The driver needs to parse CB but WREG should be allowed 43 * because of TDMA (tensor DMA). Hence, WREG is always not 44 * secured. 45 * 46 * When the driver needs to use DMA it will check that Gaudi is idle, set DMA 47 * channel 0 to be secured, execute the DMA and change it back to not secured. 48 * Currently, the driver doesn't use the DMA while there are compute jobs 49 * running. 50 * 51 * The current use cases for the driver to use the DMA are: 52 * - Clear SRAM on context switch (happens on context switch when device is 53 * idle) 54 * - MMU page tables area clear (happens on init) 55 * 56 * QMAN DMA 2-7, TPC, MME, NIC: 57 * PQ is secured and is located on the Host (HBM CON TPC3 bug) 58 * CQ, CP and the engine are not secured 59 * 60 */ 61 62 #define GAUDI_BOOT_FIT_FILE "habanalabs/gaudi/gaudi-boot-fit.itb" 63 #define GAUDI_LINUX_FW_FILE "habanalabs/gaudi/gaudi-fit.itb" 64 #define GAUDI_TPC_FW_FILE "habanalabs/gaudi/gaudi_tpc.bin" 65 66 #define GAUDI_DMA_POOL_BLK_SIZE 0x100 /* 256 bytes */ 67 68 #define GAUDI_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ 69 #define GAUDI_RESET_WAIT_MSEC 1 /* 1ms */ 70 #define GAUDI_CPU_RESET_WAIT_MSEC 200 /* 200ms */ 71 #define GAUDI_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 72 73 #define GAUDI_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 74 #define GAUDI_PLDM_HRESET_TIMEOUT_MSEC 20000 /* 20s */ 75 #define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */ 76 #define GAUDI_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100) 77 #define GAUDI_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) 78 #define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) 79 #define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 4000000 /* 4s */ 80 #define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 81 #define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */ 82 83 #define GAUDI_QMAN0_FENCE_VAL 0x72E91AB9 84 85 #define GAUDI_MAX_STRING_LEN 20 86 87 #define GAUDI_CB_POOL_CB_CNT 512 88 #define GAUDI_CB_POOL_CB_SIZE 0x20000 /* 128KB */ 89 90 #define GAUDI_ALLOC_CPU_MEM_RETRY_CNT 3 91 92 #define GAUDI_NUM_OF_TPC_INTR_CAUSE 20 93 94 #define GAUDI_NUM_OF_QM_ERR_CAUSE 16 95 96 #define GAUDI_NUM_OF_QM_ARB_ERR_CAUSE 3 97 98 #define GAUDI_ARB_WDT_TIMEOUT 0xEE6b27FF /* 8 seconds */ 99 100 #define HBM_SCRUBBING_TIMEOUT_US 1000000 /* 1s */ 101 102 #define BIN_REG_STRING_SIZE sizeof("0b10101010101010101010101010101010") 103 104 #define MONITOR_SOB_STRING_SIZE 256 105 106 static u32 gaudi_stream_master[GAUDI_STREAM_MASTER_ARR_SIZE] = { 107 GAUDI_QUEUE_ID_DMA_0_0, 108 GAUDI_QUEUE_ID_DMA_0_1, 109 GAUDI_QUEUE_ID_DMA_0_2, 110 GAUDI_QUEUE_ID_DMA_0_3, 111 GAUDI_QUEUE_ID_DMA_1_0, 112 GAUDI_QUEUE_ID_DMA_1_1, 113 GAUDI_QUEUE_ID_DMA_1_2, 114 GAUDI_QUEUE_ID_DMA_1_3 115 }; 116 117 static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = { 118 "gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3", 119 "gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3", 120 "gaudi cq 5_0", "gaudi cq 5_1", "gaudi cq 5_2", "gaudi cq 5_3", 121 "gaudi cpu eq" 122 }; 123 124 static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = { 125 [GAUDI_PCI_DMA_1] = GAUDI_ENGINE_ID_DMA_0, 126 [GAUDI_PCI_DMA_2] = GAUDI_ENGINE_ID_DMA_1, 127 [GAUDI_HBM_DMA_1] = GAUDI_ENGINE_ID_DMA_2, 128 [GAUDI_HBM_DMA_2] = GAUDI_ENGINE_ID_DMA_3, 129 [GAUDI_HBM_DMA_3] = GAUDI_ENGINE_ID_DMA_4, 130 [GAUDI_HBM_DMA_4] = GAUDI_ENGINE_ID_DMA_5, 131 [GAUDI_HBM_DMA_5] = GAUDI_ENGINE_ID_DMA_6, 132 [GAUDI_HBM_DMA_6] = GAUDI_ENGINE_ID_DMA_7 133 }; 134 135 static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = { 136 [0] = GAUDI_QUEUE_ID_DMA_0_0, 137 [1] = GAUDI_QUEUE_ID_DMA_0_1, 138 [2] = GAUDI_QUEUE_ID_DMA_0_2, 139 [3] = GAUDI_QUEUE_ID_DMA_0_3, 140 [4] = GAUDI_QUEUE_ID_DMA_1_0, 141 [5] = GAUDI_QUEUE_ID_DMA_1_1, 142 [6] = GAUDI_QUEUE_ID_DMA_1_2, 143 [7] = GAUDI_QUEUE_ID_DMA_1_3, 144 }; 145 146 static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = { 147 [PACKET_WREG_32] = sizeof(struct packet_wreg32), 148 [PACKET_WREG_BULK] = sizeof(struct packet_wreg_bulk), 149 [PACKET_MSG_LONG] = sizeof(struct packet_msg_long), 150 [PACKET_MSG_SHORT] = sizeof(struct packet_msg_short), 151 [PACKET_CP_DMA] = sizeof(struct packet_cp_dma), 152 [PACKET_REPEAT] = sizeof(struct packet_repeat), 153 [PACKET_MSG_PROT] = sizeof(struct packet_msg_prot), 154 [PACKET_FENCE] = sizeof(struct packet_fence), 155 [PACKET_LIN_DMA] = sizeof(struct packet_lin_dma), 156 [PACKET_NOP] = sizeof(struct packet_nop), 157 [PACKET_STOP] = sizeof(struct packet_stop), 158 [PACKET_ARB_POINT] = sizeof(struct packet_arb_point), 159 [PACKET_WAIT] = sizeof(struct packet_wait), 160 [PACKET_LOAD_AND_EXE] = sizeof(struct packet_load_and_exe) 161 }; 162 163 static inline bool validate_packet_id(enum packet_id id) 164 { 165 switch (id) { 166 case PACKET_WREG_32: 167 case PACKET_WREG_BULK: 168 case PACKET_MSG_LONG: 169 case PACKET_MSG_SHORT: 170 case PACKET_CP_DMA: 171 case PACKET_REPEAT: 172 case PACKET_MSG_PROT: 173 case PACKET_FENCE: 174 case PACKET_LIN_DMA: 175 case PACKET_NOP: 176 case PACKET_STOP: 177 case PACKET_ARB_POINT: 178 case PACKET_WAIT: 179 case PACKET_LOAD_AND_EXE: 180 return true; 181 default: 182 return false; 183 } 184 } 185 186 static const char * const 187 gaudi_tpc_interrupts_cause[GAUDI_NUM_OF_TPC_INTR_CAUSE] = { 188 "tpc_address_exceed_slm", 189 "tpc_div_by_0", 190 "tpc_spu_mac_overflow", 191 "tpc_spu_addsub_overflow", 192 "tpc_spu_abs_overflow", 193 "tpc_spu_fp_dst_nan_inf", 194 "tpc_spu_fp_dst_denorm", 195 "tpc_vpu_mac_overflow", 196 "tpc_vpu_addsub_overflow", 197 "tpc_vpu_abs_overflow", 198 "tpc_vpu_fp_dst_nan_inf", 199 "tpc_vpu_fp_dst_denorm", 200 "tpc_assertions", 201 "tpc_illegal_instruction", 202 "tpc_pc_wrap_around", 203 "tpc_qm_sw_err", 204 "tpc_hbw_rresp_err", 205 "tpc_hbw_bresp_err", 206 "tpc_lbw_rresp_err", 207 "tpc_lbw_bresp_err" 208 }; 209 210 static const char * const 211 gaudi_qman_error_cause[GAUDI_NUM_OF_QM_ERR_CAUSE] = { 212 "PQ AXI HBW error", 213 "CQ AXI HBW error", 214 "CP AXI HBW error", 215 "CP error due to undefined OPCODE", 216 "CP encountered STOP OPCODE", 217 "CP AXI LBW error", 218 "CP WRREG32 or WRBULK returned error", 219 "N/A", 220 "FENCE 0 inc over max value and clipped", 221 "FENCE 1 inc over max value and clipped", 222 "FENCE 2 inc over max value and clipped", 223 "FENCE 3 inc over max value and clipped", 224 "FENCE 0 dec under min value and clipped", 225 "FENCE 1 dec under min value and clipped", 226 "FENCE 2 dec under min value and clipped", 227 "FENCE 3 dec under min value and clipped" 228 }; 229 230 static const char * const 231 gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = { 232 "Choice push while full error", 233 "Choice Q watchdog error", 234 "MSG AXI LBW returned with error" 235 }; 236 237 static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = { 238 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */ 239 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */ 240 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_2 */ 241 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_3 */ 242 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_0 */ 243 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_1 */ 244 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_2 */ 245 QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_3 */ 246 QUEUE_TYPE_CPU, /* GAUDI_QUEUE_ID_CPU_PQ */ 247 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_0 */ 248 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_1 */ 249 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_2 */ 250 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_3 */ 251 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_0 */ 252 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_1 */ 253 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_2 */ 254 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_3 */ 255 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_0 */ 256 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_1 */ 257 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_2 */ 258 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_3 */ 259 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_0 */ 260 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_1 */ 261 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_2 */ 262 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_3 */ 263 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_0 */ 264 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_1 */ 265 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */ 266 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_3 */ 267 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_0 */ 268 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_1 */ 269 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_2 */ 270 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_3 */ 271 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_0 */ 272 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_1 */ 273 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_2 */ 274 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_3 */ 275 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_0 */ 276 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_1 */ 277 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_2 */ 278 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_3 */ 279 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_0 */ 280 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_1 */ 281 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_2 */ 282 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_3 */ 283 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_0 */ 284 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_1 */ 285 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_2 */ 286 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_3 */ 287 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_0 */ 288 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_1 */ 289 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_2 */ 290 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_3 */ 291 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_0 */ 292 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_1 */ 293 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_2 */ 294 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_3 */ 295 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_0 */ 296 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_1 */ 297 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_2 */ 298 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_3 */ 299 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_0 */ 300 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_1 */ 301 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_2 */ 302 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_3 */ 303 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_0 */ 304 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_1 */ 305 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_2 */ 306 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_3 */ 307 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_0 */ 308 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_1 */ 309 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_2 */ 310 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_3 */ 311 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_0 */ 312 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_1 */ 313 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_2 */ 314 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_3 */ 315 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_0 */ 316 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_1 */ 317 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_2 */ 318 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_3 */ 319 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_0 */ 320 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_1 */ 321 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_2 */ 322 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_3 */ 323 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_0 */ 324 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_1 */ 325 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_2 */ 326 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_3 */ 327 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_0 */ 328 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_1 */ 329 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_2 */ 330 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_3 */ 331 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_0 */ 332 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_1 */ 333 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_2 */ 334 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_3 */ 335 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_0 */ 336 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_1 */ 337 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_2 */ 338 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_3 */ 339 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_0 */ 340 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_1 */ 341 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_2 */ 342 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_3 */ 343 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_0 */ 344 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_1 */ 345 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_2 */ 346 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_3 */ 347 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_0 */ 348 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_1 */ 349 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_2 */ 350 QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_3 */ 351 }; 352 353 static struct hl_hw_obj_name_entry gaudi_so_id_to_str[] = { 354 { .id = 0, .name = "SYNC_OBJ_DMA_DOWN_FEEDBACK" }, 355 { .id = 1, .name = "SYNC_OBJ_DMA_UP_FEEDBACK" }, 356 { .id = 2, .name = "SYNC_OBJ_DMA_STATIC_DRAM_SRAM_FEEDBACK" }, 357 { .id = 3, .name = "SYNC_OBJ_DMA_SRAM_DRAM_FEEDBACK" }, 358 { .id = 4, .name = "SYNC_OBJ_FIRST_COMPUTE_FINISH" }, 359 { .id = 5, .name = "SYNC_OBJ_HOST_DRAM_DONE" }, 360 { .id = 6, .name = "SYNC_OBJ_DBG_CTR_DEPRECATED" }, 361 { .id = 7, .name = "SYNC_OBJ_DMA_ACTIVATIONS_DRAM_SRAM_FEEDBACK" }, 362 { .id = 8, .name = "SYNC_OBJ_ENGINE_SEM_MME_0" }, 363 { .id = 9, .name = "SYNC_OBJ_ENGINE_SEM_MME_1" }, 364 { .id = 10, .name = "SYNC_OBJ_ENGINE_SEM_TPC_0" }, 365 { .id = 11, .name = "SYNC_OBJ_ENGINE_SEM_TPC_1" }, 366 { .id = 12, .name = "SYNC_OBJ_ENGINE_SEM_TPC_2" }, 367 { .id = 13, .name = "SYNC_OBJ_ENGINE_SEM_TPC_3" }, 368 { .id = 14, .name = "SYNC_OBJ_ENGINE_SEM_TPC_4" }, 369 { .id = 15, .name = "SYNC_OBJ_ENGINE_SEM_TPC_5" }, 370 { .id = 16, .name = "SYNC_OBJ_ENGINE_SEM_TPC_6" }, 371 { .id = 17, .name = "SYNC_OBJ_ENGINE_SEM_TPC_7" }, 372 { .id = 18, .name = "SYNC_OBJ_ENGINE_SEM_DMA_1" }, 373 { .id = 19, .name = "SYNC_OBJ_ENGINE_SEM_DMA_2" }, 374 { .id = 20, .name = "SYNC_OBJ_ENGINE_SEM_DMA_3" }, 375 { .id = 21, .name = "SYNC_OBJ_ENGINE_SEM_DMA_4" }, 376 { .id = 22, .name = "SYNC_OBJ_ENGINE_SEM_DMA_5" }, 377 { .id = 23, .name = "SYNC_OBJ_ENGINE_SEM_DMA_6" }, 378 { .id = 24, .name = "SYNC_OBJ_ENGINE_SEM_DMA_7" }, 379 { .id = 25, .name = "SYNC_OBJ_DBG_CTR_0" }, 380 { .id = 26, .name = "SYNC_OBJ_DBG_CTR_1" }, 381 }; 382 383 static struct hl_hw_obj_name_entry gaudi_monitor_id_to_str[] = { 384 { .id = 200, .name = "MON_OBJ_DMA_DOWN_FEEDBACK_RESET" }, 385 { .id = 201, .name = "MON_OBJ_DMA_UP_FEEDBACK_RESET" }, 386 { .id = 203, .name = "MON_OBJ_DRAM_TO_SRAM_QUEUE_FENCE" }, 387 { .id = 204, .name = "MON_OBJ_TPC_0_CLK_GATE" }, 388 { .id = 205, .name = "MON_OBJ_TPC_1_CLK_GATE" }, 389 { .id = 206, .name = "MON_OBJ_TPC_2_CLK_GATE" }, 390 { .id = 207, .name = "MON_OBJ_TPC_3_CLK_GATE" }, 391 { .id = 208, .name = "MON_OBJ_TPC_4_CLK_GATE" }, 392 { .id = 209, .name = "MON_OBJ_TPC_5_CLK_GATE" }, 393 { .id = 210, .name = "MON_OBJ_TPC_6_CLK_GATE" }, 394 { .id = 211, .name = "MON_OBJ_TPC_7_CLK_GATE" }, 395 }; 396 397 static s64 gaudi_state_dump_specs_props[] = { 398 [SP_SYNC_OBJ_BASE_ADDR] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0, 399 [SP_NEXT_SYNC_OBJ_ADDR] = NEXT_SYNC_OBJ_ADDR_INTERVAL, 400 [SP_SYNC_OBJ_AMOUNT] = NUM_OF_SOB_IN_BLOCK, 401 [SP_MON_OBJ_WR_ADDR_LOW] = 402 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0, 403 [SP_MON_OBJ_WR_ADDR_HIGH] = 404 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0, 405 [SP_MON_OBJ_WR_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_DATA_0, 406 [SP_MON_OBJ_ARM_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_ARM_0, 407 [SP_MON_OBJ_STATUS] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0, 408 [SP_MONITORS_AMOUNT] = NUM_OF_MONITORS_IN_BLOCK, 409 [SP_TPC0_CMDQ] = mmTPC0_QM_GLBL_CFG0, 410 [SP_TPC0_CFG_SO] = mmTPC0_CFG_QM_SYNC_OBJECT_ADDR, 411 [SP_NEXT_TPC] = mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0, 412 [SP_MME_CMDQ] = mmMME0_QM_GLBL_CFG0, 413 [SP_MME_CFG_SO] = mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL, 414 [SP_NEXT_MME] = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0, 415 [SP_DMA_CMDQ] = mmDMA0_QM_GLBL_CFG0, 416 [SP_DMA_CFG_SO] = mmDMA0_CORE_WR_COMP_ADDR_LO, 417 [SP_DMA_QUEUES_OFFSET] = mmDMA1_QM_GLBL_CFG0 - mmDMA0_QM_GLBL_CFG0, 418 [SP_NUM_OF_MME_ENGINES] = NUM_OF_MME_ENGINES, 419 [SP_SUB_MME_ENG_NUM] = NUM_OF_MME_SUB_ENGINES, 420 [SP_NUM_OF_DMA_ENGINES] = NUM_OF_DMA_ENGINES, 421 [SP_NUM_OF_TPC_ENGINES] = NUM_OF_TPC_ENGINES, 422 [SP_ENGINE_NUM_OF_QUEUES] = NUM_OF_QUEUES, 423 [SP_ENGINE_NUM_OF_STREAMS] = NUM_OF_STREAMS, 424 [SP_ENGINE_NUM_OF_FENCES] = NUM_OF_FENCES, 425 [SP_FENCE0_CNT_OFFSET] = 426 mmDMA0_QM_CP_FENCE0_CNT_0 - mmDMA0_QM_GLBL_CFG0, 427 [SP_FENCE0_RDATA_OFFSET] = 428 mmDMA0_QM_CP_FENCE0_RDATA_0 - mmDMA0_QM_GLBL_CFG0, 429 [SP_CP_STS_OFFSET] = mmDMA0_QM_CP_STS_0 - mmDMA0_QM_GLBL_CFG0, 430 [SP_NUM_CORES] = 1, 431 }; 432 433 static const int gaudi_queue_id_to_engine_id[] = { 434 [GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3] = GAUDI_ENGINE_ID_DMA_0, 435 [GAUDI_QUEUE_ID_DMA_1_0...GAUDI_QUEUE_ID_DMA_1_3] = GAUDI_ENGINE_ID_DMA_1, 436 [GAUDI_QUEUE_ID_CPU_PQ] = GAUDI_ENGINE_ID_SIZE, 437 [GAUDI_QUEUE_ID_DMA_2_0...GAUDI_QUEUE_ID_DMA_2_3] = GAUDI_ENGINE_ID_DMA_2, 438 [GAUDI_QUEUE_ID_DMA_3_0...GAUDI_QUEUE_ID_DMA_3_3] = GAUDI_ENGINE_ID_DMA_3, 439 [GAUDI_QUEUE_ID_DMA_4_0...GAUDI_QUEUE_ID_DMA_4_3] = GAUDI_ENGINE_ID_DMA_4, 440 [GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3] = GAUDI_ENGINE_ID_DMA_5, 441 [GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3] = GAUDI_ENGINE_ID_DMA_6, 442 [GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3] = GAUDI_ENGINE_ID_DMA_7, 443 [GAUDI_QUEUE_ID_MME_0_0...GAUDI_QUEUE_ID_MME_0_3] = GAUDI_ENGINE_ID_MME_0, 444 [GAUDI_QUEUE_ID_MME_1_0...GAUDI_QUEUE_ID_MME_1_3] = GAUDI_ENGINE_ID_MME_2, 445 [GAUDI_QUEUE_ID_TPC_0_0...GAUDI_QUEUE_ID_TPC_0_3] = GAUDI_ENGINE_ID_TPC_0, 446 [GAUDI_QUEUE_ID_TPC_1_0...GAUDI_QUEUE_ID_TPC_1_3] = GAUDI_ENGINE_ID_TPC_1, 447 [GAUDI_QUEUE_ID_TPC_2_0...GAUDI_QUEUE_ID_TPC_2_3] = GAUDI_ENGINE_ID_TPC_2, 448 [GAUDI_QUEUE_ID_TPC_3_0...GAUDI_QUEUE_ID_TPC_3_3] = GAUDI_ENGINE_ID_TPC_3, 449 [GAUDI_QUEUE_ID_TPC_4_0...GAUDI_QUEUE_ID_TPC_4_3] = GAUDI_ENGINE_ID_TPC_4, 450 [GAUDI_QUEUE_ID_TPC_5_0...GAUDI_QUEUE_ID_TPC_5_3] = GAUDI_ENGINE_ID_TPC_5, 451 [GAUDI_QUEUE_ID_TPC_6_0...GAUDI_QUEUE_ID_TPC_6_3] = GAUDI_ENGINE_ID_TPC_6, 452 [GAUDI_QUEUE_ID_TPC_7_0...GAUDI_QUEUE_ID_TPC_7_3] = GAUDI_ENGINE_ID_TPC_7, 453 [GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3] = GAUDI_ENGINE_ID_NIC_0, 454 [GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3] = GAUDI_ENGINE_ID_NIC_1, 455 [GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3] = GAUDI_ENGINE_ID_NIC_2, 456 [GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3] = GAUDI_ENGINE_ID_NIC_3, 457 [GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3] = GAUDI_ENGINE_ID_NIC_4, 458 [GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3] = GAUDI_ENGINE_ID_NIC_5, 459 [GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3] = GAUDI_ENGINE_ID_NIC_6, 460 [GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3] = GAUDI_ENGINE_ID_NIC_7, 461 [GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3] = GAUDI_ENGINE_ID_NIC_8, 462 [GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3] = GAUDI_ENGINE_ID_NIC_9, 463 }; 464 465 /* The order here is opposite to the order of the indexing in the h/w. 466 * i.e. SYNC_MGR_W_S is actually 0, SYNC_MGR_E_S is 1, etc. 467 */ 468 static const char * const gaudi_sync_manager_names[] = { 469 "SYNC_MGR_E_N", 470 "SYNC_MGR_W_N", 471 "SYNC_MGR_E_S", 472 "SYNC_MGR_W_S", 473 NULL 474 }; 475 476 struct ecc_info_extract_params { 477 u64 block_address; 478 u32 num_memories; 479 bool derr; 480 }; 481 482 static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, 483 u64 phys_addr); 484 static int gaudi_send_job_on_qman0(struct hl_device *hdev, 485 struct hl_cs_job *job); 486 static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, 487 u32 size, u64 val); 488 static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, 489 u32 num_regs, u32 val); 490 static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, 491 u32 tpc_id); 492 static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev); 493 static int gaudi_cpucp_info_get(struct hl_device *hdev); 494 static void gaudi_disable_clock_gating(struct hl_device *hdev); 495 static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid); 496 static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, 497 u32 size, bool eb); 498 static u32 gaudi_gen_wait_cb(struct hl_device *hdev, 499 struct hl_gen_wait_properties *prop); 500 static inline enum hl_collective_mode 501 get_collective_mode(struct hl_device *hdev, u32 queue_id) 502 { 503 if (gaudi_queue_type[queue_id] == QUEUE_TYPE_EXT) 504 return HL_COLLECTIVE_MASTER; 505 506 if (queue_id >= GAUDI_QUEUE_ID_DMA_5_0 && 507 queue_id <= GAUDI_QUEUE_ID_DMA_5_3) 508 return HL_COLLECTIVE_SLAVE; 509 510 if (queue_id >= GAUDI_QUEUE_ID_TPC_7_0 && 511 queue_id <= GAUDI_QUEUE_ID_TPC_7_3) 512 return HL_COLLECTIVE_SLAVE; 513 514 if (queue_id >= GAUDI_QUEUE_ID_NIC_0_0 && 515 queue_id <= GAUDI_QUEUE_ID_NIC_9_3) 516 return HL_COLLECTIVE_SLAVE; 517 518 return HL_COLLECTIVE_NOT_SUPPORTED; 519 } 520 521 static inline void set_default_power_values(struct hl_device *hdev) 522 { 523 struct asic_fixed_properties *prop = &hdev->asic_prop; 524 525 if (hdev->card_type == cpucp_card_type_pmc) { 526 prop->max_power_default = MAX_POWER_DEFAULT_PMC; 527 528 if (prop->fw_security_enabled) 529 prop->dc_power_default = DC_POWER_DEFAULT_PMC_SEC; 530 else 531 prop->dc_power_default = DC_POWER_DEFAULT_PMC; 532 } else { 533 prop->max_power_default = MAX_POWER_DEFAULT_PCI; 534 prop->dc_power_default = DC_POWER_DEFAULT_PCI; 535 } 536 } 537 538 static int gaudi_set_fixed_properties(struct hl_device *hdev) 539 { 540 struct asic_fixed_properties *prop = &hdev->asic_prop; 541 u32 num_sync_stream_queues = 0; 542 int i; 543 544 prop->max_queues = GAUDI_QUEUE_ID_SIZE; 545 prop->hw_queues_props = kcalloc(prop->max_queues, 546 sizeof(struct hw_queue_properties), 547 GFP_KERNEL); 548 549 if (!prop->hw_queues_props) 550 return -ENOMEM; 551 552 for (i = 0 ; i < prop->max_queues ; i++) { 553 if (gaudi_queue_type[i] == QUEUE_TYPE_EXT) { 554 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; 555 prop->hw_queues_props[i].driver_only = 0; 556 prop->hw_queues_props[i].supports_sync_stream = 1; 557 prop->hw_queues_props[i].cb_alloc_flags = 558 CB_ALLOC_KERNEL; 559 num_sync_stream_queues++; 560 } else if (gaudi_queue_type[i] == QUEUE_TYPE_CPU) { 561 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; 562 prop->hw_queues_props[i].driver_only = 1; 563 prop->hw_queues_props[i].supports_sync_stream = 0; 564 prop->hw_queues_props[i].cb_alloc_flags = 565 CB_ALLOC_KERNEL; 566 } else if (gaudi_queue_type[i] == QUEUE_TYPE_INT) { 567 prop->hw_queues_props[i].type = QUEUE_TYPE_INT; 568 prop->hw_queues_props[i].driver_only = 0; 569 prop->hw_queues_props[i].supports_sync_stream = 0; 570 prop->hw_queues_props[i].cb_alloc_flags = 571 CB_ALLOC_USER; 572 573 } 574 prop->hw_queues_props[i].collective_mode = 575 get_collective_mode(hdev, i); 576 } 577 578 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; 579 prop->cfg_base_address = CFG_BASE; 580 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE; 581 prop->host_base_address = HOST_PHYS_BASE; 582 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE; 583 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; 584 prop->completion_mode = HL_COMPLETION_MODE_JOB; 585 prop->collective_first_sob = 0; 586 prop->collective_first_mon = 0; 587 588 /* 2 SOBs per internal queue stream are reserved for collective */ 589 prop->sync_stream_first_sob = 590 ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR) 591 * QMAN_STREAMS * HL_RSVD_SOBS; 592 593 /* 1 monitor per internal queue stream are reserved for collective 594 * 2 monitors per external queue stream are reserved for collective 595 */ 596 prop->sync_stream_first_mon = 597 (NUMBER_OF_COLLECTIVE_QUEUES * QMAN_STREAMS) + 598 (NUMBER_OF_EXT_HW_QUEUES * 2); 599 600 prop->dram_base_address = DRAM_PHYS_BASE; 601 prop->dram_size = GAUDI_HBM_SIZE_32GB; 602 prop->dram_end_address = prop->dram_base_address + prop->dram_size; 603 prop->dram_user_base_address = DRAM_BASE_ADDR_USER; 604 605 prop->sram_base_address = SRAM_BASE_ADDR; 606 prop->sram_size = SRAM_SIZE; 607 prop->sram_end_address = prop->sram_base_address + prop->sram_size; 608 prop->sram_user_base_address = 609 prop->sram_base_address + SRAM_USER_BASE_OFFSET; 610 611 prop->mmu_cache_mng_addr = MMU_CACHE_MNG_ADDR; 612 prop->mmu_cache_mng_size = MMU_CACHE_MNG_SIZE; 613 614 prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR; 615 if (hdev->pldm) 616 prop->mmu_pgt_size = 0x800000; /* 8MB */ 617 else 618 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE; 619 prop->mmu_pte_size = HL_PTE_SIZE; 620 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; 621 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; 622 prop->dram_page_size = PAGE_SIZE_2MB; 623 prop->device_mem_alloc_default_page_size = prop->dram_page_size; 624 prop->dram_supports_virtual_memory = false; 625 626 prop->pmmu.hop_shifts[MMU_HOP0] = MMU_V1_1_HOP0_SHIFT; 627 prop->pmmu.hop_shifts[MMU_HOP1] = MMU_V1_1_HOP1_SHIFT; 628 prop->pmmu.hop_shifts[MMU_HOP2] = MMU_V1_1_HOP2_SHIFT; 629 prop->pmmu.hop_shifts[MMU_HOP3] = MMU_V1_1_HOP3_SHIFT; 630 prop->pmmu.hop_shifts[MMU_HOP4] = MMU_V1_1_HOP4_SHIFT; 631 prop->pmmu.hop_masks[MMU_HOP0] = MMU_V1_1_HOP0_MASK; 632 prop->pmmu.hop_masks[MMU_HOP1] = MMU_V1_1_HOP1_MASK; 633 prop->pmmu.hop_masks[MMU_HOP2] = MMU_V1_1_HOP2_MASK; 634 prop->pmmu.hop_masks[MMU_HOP3] = MMU_V1_1_HOP3_MASK; 635 prop->pmmu.hop_masks[MMU_HOP4] = MMU_V1_1_HOP4_MASK; 636 prop->pmmu.start_addr = VA_HOST_SPACE_START; 637 prop->pmmu.end_addr = 638 (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1; 639 prop->pmmu.page_size = PAGE_SIZE_4KB; 640 prop->pmmu.num_hops = MMU_ARCH_5_HOPS; 641 prop->pmmu.last_mask = LAST_MASK; 642 /* TODO: will be duplicated until implementing per-MMU props */ 643 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; 644 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 645 646 /* PMMU and HPMMU are the same except of page size */ 647 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 648 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 649 650 /* shifts and masks are the same in PMMU and DMMU */ 651 memcpy(&prop->dmmu, &prop->pmmu, sizeof(prop->pmmu)); 652 prop->dmmu.start_addr = (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2); 653 prop->dmmu.end_addr = VA_HOST_SPACE_END; 654 prop->dmmu.page_size = PAGE_SIZE_2MB; 655 656 prop->cfg_size = CFG_SIZE; 657 prop->max_asid = MAX_ASID; 658 prop->num_of_events = GAUDI_EVENT_SIZE; 659 prop->tpc_enabled_mask = TPC_ENABLED_MASK; 660 661 set_default_power_values(hdev); 662 663 prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT; 664 prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE; 665 666 prop->pcie_dbi_base_address = mmPCIE_DBI_BASE; 667 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 668 669 strncpy(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME, 670 CARD_NAME_MAX_LEN); 671 672 prop->max_pending_cs = GAUDI_MAX_PENDING_CS; 673 674 prop->first_available_user_sob[HL_GAUDI_WS_DCORE] = 675 prop->sync_stream_first_sob + 676 (num_sync_stream_queues * HL_RSVD_SOBS); 677 prop->first_available_user_mon[HL_GAUDI_WS_DCORE] = 678 prop->sync_stream_first_mon + 679 (num_sync_stream_queues * HL_RSVD_MONS); 680 681 prop->first_available_user_interrupt = USHRT_MAX; 682 683 for (i = 0 ; i < HL_MAX_DCORES ; i++) 684 prop->first_available_cq[i] = USHRT_MAX; 685 686 prop->fw_cpu_boot_dev_sts0_valid = false; 687 prop->fw_cpu_boot_dev_sts1_valid = false; 688 prop->hard_reset_done_by_fw = false; 689 prop->gic_interrupts_enable = true; 690 691 prop->server_type = HL_SERVER_TYPE_UNKNOWN; 692 693 prop->clk_pll_index = HL_GAUDI_MME_PLL; 694 prop->max_freq_value = GAUDI_MAX_CLK_FREQ; 695 696 prop->use_get_power_for_reset_history = true; 697 698 prop->configurable_stop_on_err = true; 699 700 prop->set_max_power_on_device_init = true; 701 702 prop->dma_mask = 48; 703 704 prop->hbw_flush_reg = mmPCIE_WRAP_RR_ELBI_RD_SEC_REG_CTRL; 705 706 return 0; 707 } 708 709 static int gaudi_pci_bars_map(struct hl_device *hdev) 710 { 711 static const char * const name[] = {"SRAM", "CFG", "HBM"}; 712 bool is_wc[3] = {false, false, true}; 713 int rc; 714 715 rc = hl_pci_bars_map(hdev, name, is_wc); 716 if (rc) 717 return rc; 718 719 hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] + 720 (CFG_BASE - SPI_FLASH_BASE_ADDR); 721 722 return 0; 723 } 724 725 static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr) 726 { 727 struct gaudi_device *gaudi = hdev->asic_specific; 728 struct hl_inbound_pci_region pci_region; 729 u64 old_addr = addr; 730 int rc; 731 732 if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr)) 733 return old_addr; 734 735 if (hdev->asic_prop.iatu_done_by_fw) 736 return U64_MAX; 737 738 /* Inbound Region 2 - Bar 4 - Point to HBM */ 739 pci_region.mode = PCI_BAR_MATCH_MODE; 740 pci_region.bar = HBM_BAR_ID; 741 pci_region.addr = addr; 742 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region); 743 if (rc) 744 return U64_MAX; 745 746 if (gaudi) { 747 old_addr = gaudi->hbm_bar_cur_addr; 748 gaudi->hbm_bar_cur_addr = addr; 749 } 750 751 return old_addr; 752 } 753 754 static int gaudi_init_iatu(struct hl_device *hdev) 755 { 756 struct hl_inbound_pci_region inbound_region; 757 struct hl_outbound_pci_region outbound_region; 758 int rc; 759 760 if (hdev->asic_prop.iatu_done_by_fw) 761 return 0; 762 763 /* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */ 764 inbound_region.mode = PCI_BAR_MATCH_MODE; 765 inbound_region.bar = SRAM_BAR_ID; 766 inbound_region.addr = SRAM_BASE_ADDR; 767 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 768 if (rc) 769 goto done; 770 771 /* Inbound Region 1 - Bar 2 - Point to SPI FLASH */ 772 inbound_region.mode = PCI_BAR_MATCH_MODE; 773 inbound_region.bar = CFG_BAR_ID; 774 inbound_region.addr = SPI_FLASH_BASE_ADDR; 775 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 776 if (rc) 777 goto done; 778 779 /* Inbound Region 2 - Bar 4 - Point to HBM */ 780 inbound_region.mode = PCI_BAR_MATCH_MODE; 781 inbound_region.bar = HBM_BAR_ID; 782 inbound_region.addr = DRAM_PHYS_BASE; 783 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region); 784 if (rc) 785 goto done; 786 787 /* Outbound Region 0 - Point to Host */ 788 outbound_region.addr = HOST_PHYS_BASE; 789 outbound_region.size = HOST_PHYS_SIZE; 790 rc = hl_pci_set_outbound_region(hdev, &outbound_region); 791 792 done: 793 return rc; 794 } 795 796 static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev) 797 { 798 return RREG32(mmHW_STATE); 799 } 800 801 static int gaudi_early_init(struct hl_device *hdev) 802 { 803 struct asic_fixed_properties *prop = &hdev->asic_prop; 804 struct pci_dev *pdev = hdev->pdev; 805 resource_size_t pci_bar_size; 806 u32 fw_boot_status; 807 int rc; 808 809 rc = gaudi_set_fixed_properties(hdev); 810 if (rc) { 811 dev_err(hdev->dev, "Failed setting fixed properties\n"); 812 return rc; 813 } 814 815 /* Check BAR sizes */ 816 pci_bar_size = pci_resource_len(pdev, SRAM_BAR_ID); 817 818 if (pci_bar_size != SRAM_BAR_SIZE) { 819 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 820 SRAM_BAR_ID, &pci_bar_size, SRAM_BAR_SIZE); 821 rc = -ENODEV; 822 goto free_queue_props; 823 } 824 825 pci_bar_size = pci_resource_len(pdev, CFG_BAR_ID); 826 827 if (pci_bar_size != CFG_BAR_SIZE) { 828 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 829 CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 830 rc = -ENODEV; 831 goto free_queue_props; 832 } 833 834 prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID); 835 hdev->dram_pci_bar_start = pci_resource_start(pdev, HBM_BAR_ID); 836 837 /* If FW security is enabled at this point it means no access to ELBI */ 838 if (hdev->asic_prop.fw_security_enabled) { 839 hdev->asic_prop.iatu_done_by_fw = true; 840 841 /* 842 * GIC-security-bit can ONLY be set by CPUCP, so in this stage 843 * decision can only be taken based on PCI ID security. 844 */ 845 hdev->asic_prop.gic_interrupts_enable = false; 846 goto pci_init; 847 } 848 849 rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0, 850 &fw_boot_status); 851 if (rc) 852 goto free_queue_props; 853 854 /* Check whether FW is configuring iATU */ 855 if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) && 856 (fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN)) 857 hdev->asic_prop.iatu_done_by_fw = true; 858 859 pci_init: 860 rc = hl_pci_init(hdev); 861 if (rc) 862 goto free_queue_props; 863 864 /* Before continuing in the initialization, we need to read the preboot 865 * version to determine whether we run with a security-enabled firmware 866 */ 867 rc = hl_fw_read_preboot_status(hdev); 868 if (rc) { 869 if (hdev->reset_on_preboot_fail) 870 hdev->asic_funcs->hw_fini(hdev, true, false); 871 goto pci_fini; 872 } 873 874 if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 875 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 876 hdev->asic_funcs->hw_fini(hdev, true, false); 877 } 878 879 return 0; 880 881 pci_fini: 882 hl_pci_fini(hdev); 883 free_queue_props: 884 kfree(hdev->asic_prop.hw_queues_props); 885 return rc; 886 } 887 888 static int gaudi_early_fini(struct hl_device *hdev) 889 { 890 kfree(hdev->asic_prop.hw_queues_props); 891 hl_pci_fini(hdev); 892 893 return 0; 894 } 895 896 /** 897 * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values 898 * 899 * @hdev: pointer to hl_device structure 900 * 901 */ 902 static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) 903 { 904 u32 nr = 0, nf = 0, od = 0, div_fctr = 0, pll_clk, div_sel; 905 struct asic_fixed_properties *prop = &hdev->asic_prop; 906 u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; 907 int rc; 908 909 if ((hdev->fw_components & FW_TYPE_LINUX) && 910 (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PLL_INFO_EN)) { 911 struct gaudi_device *gaudi = hdev->asic_specific; 912 913 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) 914 return 0; 915 916 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr); 917 918 if (rc) 919 return rc; 920 921 freq = pll_freq_arr[2]; 922 } else { 923 /* Backward compatibility */ 924 div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2); 925 div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2); 926 nr = RREG32(mmPSOC_CPU_PLL_NR); 927 nf = RREG32(mmPSOC_CPU_PLL_NF); 928 od = RREG32(mmPSOC_CPU_PLL_OD); 929 930 if (div_sel == DIV_SEL_REF_CLK || 931 div_sel == DIV_SEL_DIVIDED_REF) { 932 if (div_sel == DIV_SEL_REF_CLK) 933 freq = PLL_REF_CLK; 934 else 935 freq = PLL_REF_CLK / (div_fctr + 1); 936 } else if (div_sel == DIV_SEL_PLL_CLK || 937 div_sel == DIV_SEL_DIVIDED_PLL) { 938 pll_clk = PLL_REF_CLK * (nf + 1) / 939 ((nr + 1) * (od + 1)); 940 if (div_sel == DIV_SEL_PLL_CLK) 941 freq = pll_clk; 942 else 943 freq = pll_clk / (div_fctr + 1); 944 } else { 945 dev_warn(hdev->dev, "Received invalid div select value: %#x", div_sel); 946 freq = 0; 947 } 948 } 949 950 prop->psoc_timestamp_frequency = freq; 951 prop->psoc_pci_pll_nr = nr; 952 prop->psoc_pci_pll_nf = nf; 953 prop->psoc_pci_pll_od = od; 954 prop->psoc_pci_pll_div_factor = div_fctr; 955 956 return 0; 957 } 958 959 static int _gaudi_init_tpc_mem(struct hl_device *hdev, 960 dma_addr_t tpc_kernel_src_addr, u32 tpc_kernel_size) 961 { 962 struct asic_fixed_properties *prop = &hdev->asic_prop; 963 struct packet_lin_dma *init_tpc_mem_pkt; 964 struct hl_cs_job *job; 965 struct hl_cb *cb; 966 u64 dst_addr; 967 u32 cb_size, ctl; 968 u8 tpc_id; 969 int rc; 970 971 cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false); 972 if (!cb) 973 return -EFAULT; 974 975 init_tpc_mem_pkt = cb->kernel_address; 976 cb_size = sizeof(*init_tpc_mem_pkt); 977 memset(init_tpc_mem_pkt, 0, cb_size); 978 979 init_tpc_mem_pkt->tsize = cpu_to_le32(tpc_kernel_size); 980 981 ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 982 ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1); 983 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 984 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 985 986 init_tpc_mem_pkt->ctl = cpu_to_le32(ctl); 987 988 init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr); 989 990 /* TPC_CMD is configured with I$ prefetch enabled, so address should be aligned to 8KB */ 991 dst_addr = FIELD_PREP(GAUDI_PKT_LIN_DMA_DST_ADDR_MASK, 992 round_up(prop->sram_user_base_address, SZ_8K)); 993 init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr); 994 995 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); 996 if (!job) { 997 dev_err(hdev->dev, "Failed to allocate a new job\n"); 998 rc = -ENOMEM; 999 goto release_cb; 1000 } 1001 1002 job->id = 0; 1003 job->user_cb = cb; 1004 atomic_inc(&job->user_cb->cs_cnt); 1005 job->user_cb_size = cb_size; 1006 job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; 1007 job->patched_cb = job->user_cb; 1008 job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot); 1009 1010 hl_debugfs_add_job(hdev, job); 1011 1012 rc = gaudi_send_job_on_qman0(hdev, job); 1013 1014 if (rc) 1015 goto free_job; 1016 1017 for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) { 1018 rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id); 1019 if (rc) 1020 break; 1021 } 1022 1023 free_job: 1024 hl_userptr_delete_list(hdev, &job->userptr_list); 1025 hl_debugfs_remove_job(hdev, job); 1026 kfree(job); 1027 atomic_dec(&cb->cs_cnt); 1028 1029 release_cb: 1030 hl_cb_put(cb); 1031 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); 1032 1033 return rc; 1034 } 1035 1036 /* 1037 * gaudi_init_tpc_mem() - Initialize TPC memories. 1038 * @hdev: Pointer to hl_device structure. 1039 * 1040 * Copy TPC kernel fw from firmware file and run it to initialize TPC memories. 1041 * 1042 * Return: 0 for success, negative value for error. 1043 */ 1044 static int gaudi_init_tpc_mem(struct hl_device *hdev) 1045 { 1046 const struct firmware *fw; 1047 size_t fw_size; 1048 void *cpu_addr; 1049 dma_addr_t dma_handle; 1050 int rc, count = 5; 1051 1052 again: 1053 rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev); 1054 if (rc == -EINTR && count-- > 0) { 1055 msleep(50); 1056 goto again; 1057 } 1058 1059 if (rc) { 1060 dev_err(hdev->dev, "Failed to load firmware file %s\n", 1061 GAUDI_TPC_FW_FILE); 1062 goto out; 1063 } 1064 1065 fw_size = fw->size; 1066 cpu_addr = hl_asic_dma_alloc_coherent(hdev, fw_size, &dma_handle, GFP_KERNEL | __GFP_ZERO); 1067 if (!cpu_addr) { 1068 dev_err(hdev->dev, 1069 "Failed to allocate %zu of dma memory for TPC kernel\n", 1070 fw_size); 1071 rc = -ENOMEM; 1072 goto out; 1073 } 1074 1075 memcpy(cpu_addr, fw->data, fw_size); 1076 1077 rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size); 1078 1079 hl_asic_dma_free_coherent(hdev, fw->size, cpu_addr, dma_handle); 1080 1081 out: 1082 release_firmware(fw); 1083 return rc; 1084 } 1085 1086 static void gaudi_collective_map_sobs(struct hl_device *hdev, u32 stream) 1087 { 1088 struct gaudi_device *gaudi = hdev->asic_specific; 1089 struct gaudi_collective_properties *prop = &gaudi->collective_props; 1090 struct hl_hw_queue *q; 1091 u32 i, sob_id, sob_group_id, queue_id; 1092 1093 /* Iterate through SOB groups and assign a SOB for each slave queue */ 1094 sob_group_id = 1095 stream * HL_RSVD_SOBS + prop->curr_sob_group_idx[stream]; 1096 sob_id = prop->hw_sob_group[sob_group_id].base_sob_id; 1097 1098 queue_id = GAUDI_QUEUE_ID_NIC_0_0 + stream; 1099 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 1100 q = &hdev->kernel_queues[queue_id + (4 * i)]; 1101 q->sync_stream_prop.collective_sob_id = sob_id + i; 1102 } 1103 1104 /* Both DMA5 and TPC7 use the same resources since only a single 1105 * engine need to participate in the reduction process 1106 */ 1107 queue_id = GAUDI_QUEUE_ID_DMA_5_0 + stream; 1108 q = &hdev->kernel_queues[queue_id]; 1109 q->sync_stream_prop.collective_sob_id = 1110 sob_id + NIC_NUMBER_OF_ENGINES; 1111 1112 queue_id = GAUDI_QUEUE_ID_TPC_7_0 + stream; 1113 q = &hdev->kernel_queues[queue_id]; 1114 q->sync_stream_prop.collective_sob_id = 1115 sob_id + NIC_NUMBER_OF_ENGINES; 1116 } 1117 1118 static void gaudi_sob_group_hw_reset(struct kref *ref) 1119 { 1120 struct gaudi_hw_sob_group *hw_sob_group = 1121 container_of(ref, struct gaudi_hw_sob_group, kref); 1122 struct hl_device *hdev = hw_sob_group->hdev; 1123 int i; 1124 1125 for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++) 1126 WREG32((mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + 1127 (hw_sob_group->base_sob_id * 4) + (i * 4)), 0); 1128 1129 kref_init(&hw_sob_group->kref); 1130 } 1131 1132 static void gaudi_sob_group_reset_error(struct kref *ref) 1133 { 1134 struct gaudi_hw_sob_group *hw_sob_group = 1135 container_of(ref, struct gaudi_hw_sob_group, kref); 1136 struct hl_device *hdev = hw_sob_group->hdev; 1137 1138 dev_crit(hdev->dev, 1139 "SOB release shouldn't be called here, base_sob_id: %d\n", 1140 hw_sob_group->base_sob_id); 1141 } 1142 1143 static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi) 1144 { 1145 struct gaudi_collective_properties *prop; 1146 int i; 1147 1148 prop = &gaudi->collective_props; 1149 1150 memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask)); 1151 1152 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) 1153 if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) 1154 prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= 1155 BIT(i % HL_MAX_SOBS_PER_MONITOR); 1156 /* Set collective engine bit */ 1157 prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= 1158 BIT(i % HL_MAX_SOBS_PER_MONITOR); 1159 } 1160 1161 static int gaudi_collective_init(struct hl_device *hdev) 1162 { 1163 u32 i, sob_id, reserved_sobs_per_group; 1164 struct gaudi_collective_properties *prop; 1165 struct gaudi_device *gaudi; 1166 1167 gaudi = hdev->asic_specific; 1168 prop = &gaudi->collective_props; 1169 sob_id = hdev->asic_prop.collective_first_sob; 1170 1171 /* First sob in group must be aligned to HL_MAX_SOBS_PER_MONITOR */ 1172 reserved_sobs_per_group = 1173 ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR); 1174 1175 /* Init SOB groups */ 1176 for (i = 0 ; i < NUM_SOB_GROUPS; i++) { 1177 prop->hw_sob_group[i].hdev = hdev; 1178 prop->hw_sob_group[i].base_sob_id = sob_id; 1179 sob_id += reserved_sobs_per_group; 1180 gaudi_sob_group_hw_reset(&prop->hw_sob_group[i].kref); 1181 } 1182 1183 for (i = 0 ; i < QMAN_STREAMS; i++) { 1184 prop->next_sob_group_val[i] = 1; 1185 prop->curr_sob_group_idx[i] = 0; 1186 gaudi_collective_map_sobs(hdev, i); 1187 } 1188 1189 gaudi_collective_mstr_sob_mask_set(gaudi); 1190 1191 return 0; 1192 } 1193 1194 static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_group) 1195 { 1196 struct gaudi_device *gaudi = hdev->asic_specific; 1197 struct gaudi_collective_properties *cprop = &gaudi->collective_props; 1198 1199 kref_put(&cprop->hw_sob_group[sob_group].kref, 1200 gaudi_sob_group_hw_reset); 1201 } 1202 1203 static void gaudi_collective_master_init_job(struct hl_device *hdev, 1204 struct hl_cs_job *job, u32 stream, u32 sob_group_offset) 1205 { 1206 u32 master_sob_base, master_monitor, queue_id, cb_size = 0; 1207 struct gaudi_collective_properties *cprop; 1208 struct hl_gen_wait_properties wait_prop; 1209 struct hl_sync_stream_properties *prop; 1210 struct gaudi_device *gaudi; 1211 1212 gaudi = hdev->asic_specific; 1213 cprop = &gaudi->collective_props; 1214 queue_id = job->hw_queue_id; 1215 prop = &hdev->kernel_queues[queue_id].sync_stream_prop; 1216 1217 master_sob_base = 1218 cprop->hw_sob_group[sob_group_offset].base_sob_id; 1219 master_monitor = prop->collective_mstr_mon_id[0]; 1220 1221 cprop->hw_sob_group[sob_group_offset].queue_id = queue_id; 1222 1223 dev_dbg(hdev->dev, 1224 "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n", 1225 master_sob_base, cprop->mstr_sob_mask[0], 1226 cprop->next_sob_group_val[stream], 1227 master_monitor, queue_id); 1228 1229 wait_prop.data = (void *) job->patched_cb; 1230 wait_prop.sob_base = master_sob_base; 1231 wait_prop.sob_mask = cprop->mstr_sob_mask[0]; 1232 wait_prop.sob_val = cprop->next_sob_group_val[stream]; 1233 wait_prop.mon_id = master_monitor; 1234 wait_prop.q_idx = queue_id; 1235 wait_prop.size = cb_size; 1236 cb_size += gaudi_gen_wait_cb(hdev, &wait_prop); 1237 1238 master_sob_base += HL_MAX_SOBS_PER_MONITOR; 1239 master_monitor = prop->collective_mstr_mon_id[1]; 1240 1241 dev_dbg(hdev->dev, 1242 "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n", 1243 master_sob_base, cprop->mstr_sob_mask[1], 1244 cprop->next_sob_group_val[stream], 1245 master_monitor, queue_id); 1246 1247 wait_prop.sob_base = master_sob_base; 1248 wait_prop.sob_mask = cprop->mstr_sob_mask[1]; 1249 wait_prop.mon_id = master_monitor; 1250 wait_prop.size = cb_size; 1251 cb_size += gaudi_gen_wait_cb(hdev, &wait_prop); 1252 } 1253 1254 static void gaudi_collective_slave_init_job(struct hl_device *hdev, 1255 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) 1256 { 1257 struct hl_gen_wait_properties wait_prop; 1258 struct hl_sync_stream_properties *prop; 1259 u32 queue_id, cb_size = 0; 1260 1261 queue_id = job->hw_queue_id; 1262 prop = &hdev->kernel_queues[queue_id].sync_stream_prop; 1263 1264 if (job->cs->encaps_signals) { 1265 /* use the encaps signal handle store earlier in the flow 1266 * and set the SOB information from the encaps 1267 * signals handle 1268 */ 1269 hl_hw_queue_encaps_sig_set_sob_info(hdev, job->cs, job, 1270 cs_cmpl); 1271 1272 dev_dbg(hdev->dev, "collective wait: Sequence %llu found, sob_id: %u, wait for sob_val: %u\n", 1273 job->cs->sequence, 1274 cs_cmpl->hw_sob->sob_id, 1275 cs_cmpl->sob_val); 1276 } 1277 1278 /* Add to wait CBs using slave monitor */ 1279 wait_prop.data = (void *) job->user_cb; 1280 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id; 1281 wait_prop.sob_mask = 0x1; 1282 wait_prop.sob_val = cs_cmpl->sob_val; 1283 wait_prop.mon_id = prop->collective_slave_mon_id; 1284 wait_prop.q_idx = queue_id; 1285 wait_prop.size = cb_size; 1286 1287 dev_dbg(hdev->dev, 1288 "Generate slave wait CB, sob %d, val:%x, mon %d, q %d\n", 1289 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, 1290 prop->collective_slave_mon_id, queue_id); 1291 1292 cb_size += gaudi_gen_wait_cb(hdev, &wait_prop); 1293 1294 dev_dbg(hdev->dev, 1295 "generate signal CB, sob_id: %d, sob val: 1, q_idx: %d\n", 1296 prop->collective_sob_id, queue_id); 1297 1298 cb_size += gaudi_gen_signal_cb(hdev, job->user_cb, 1299 prop->collective_sob_id, cb_size, false); 1300 } 1301 1302 static int gaudi_collective_wait_init_cs(struct hl_cs *cs) 1303 { 1304 struct hl_cs_compl *signal_cs_cmpl = 1305 container_of(cs->signal_fence, struct hl_cs_compl, base_fence); 1306 struct hl_cs_compl *cs_cmpl = 1307 container_of(cs->fence, struct hl_cs_compl, base_fence); 1308 struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl; 1309 struct gaudi_collective_properties *cprop; 1310 u32 stream, queue_id, sob_group_offset; 1311 struct gaudi_device *gaudi; 1312 struct hl_device *hdev; 1313 struct hl_cs_job *job; 1314 struct hl_ctx *ctx; 1315 1316 ctx = cs->ctx; 1317 hdev = ctx->hdev; 1318 gaudi = hdev->asic_specific; 1319 cprop = &gaudi->collective_props; 1320 1321 if (cs->encaps_signals) { 1322 cs_cmpl->hw_sob = handle->hw_sob; 1323 /* at this checkpoint we only need the hw_sob pointer 1324 * for the completion check before start going over the jobs 1325 * of the master/slaves, the sob_value will be taken later on 1326 * in gaudi_collective_slave_init_job depends on each 1327 * job wait offset value. 1328 */ 1329 cs_cmpl->sob_val = 0; 1330 } else { 1331 /* copy the SOB id and value of the signal CS */ 1332 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob; 1333 cs_cmpl->sob_val = signal_cs_cmpl->sob_val; 1334 } 1335 1336 /* check again if the signal cs already completed. 1337 * if yes then don't send any wait cs since the hw_sob 1338 * could be in reset already. if signal is not completed 1339 * then get refcount to hw_sob to prevent resetting the sob 1340 * while wait cs is not submitted. 1341 * note that this check is protected by two locks, 1342 * hw queue lock and completion object lock, 1343 * and the same completion object lock also protects 1344 * the hw_sob reset handler function. 1345 * The hw_queue lock prevent out of sync of hw_sob 1346 * refcount value, changed by signal/wait flows. 1347 */ 1348 spin_lock(&signal_cs_cmpl->lock); 1349 1350 if (completion_done(&cs->signal_fence->completion)) { 1351 spin_unlock(&signal_cs_cmpl->lock); 1352 return -EINVAL; 1353 } 1354 /* Increment kref since all slave queues are now waiting on it */ 1355 kref_get(&cs_cmpl->hw_sob->kref); 1356 1357 spin_unlock(&signal_cs_cmpl->lock); 1358 1359 /* Calculate the stream from collective master queue (1st job) */ 1360 job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node); 1361 stream = job->hw_queue_id % 4; 1362 sob_group_offset = 1363 stream * HL_RSVD_SOBS + cprop->curr_sob_group_idx[stream]; 1364 1365 list_for_each_entry(job, &cs->job_list, cs_node) { 1366 queue_id = job->hw_queue_id; 1367 1368 if (hdev->kernel_queues[queue_id].collective_mode == 1369 HL_COLLECTIVE_MASTER) 1370 gaudi_collective_master_init_job(hdev, job, stream, 1371 sob_group_offset); 1372 else 1373 gaudi_collective_slave_init_job(hdev, job, cs_cmpl); 1374 } 1375 1376 cs_cmpl->sob_group = sob_group_offset; 1377 1378 /* Handle sob group kref and wraparound */ 1379 kref_get(&cprop->hw_sob_group[sob_group_offset].kref); 1380 cprop->next_sob_group_val[stream]++; 1381 1382 if (cprop->next_sob_group_val[stream] == HL_MAX_SOB_VAL) { 1383 /* 1384 * Decrement as we reached the max value. 1385 * The release function won't be called here as we've 1386 * just incremented the refcount. 1387 */ 1388 kref_put(&cprop->hw_sob_group[sob_group_offset].kref, 1389 gaudi_sob_group_reset_error); 1390 cprop->next_sob_group_val[stream] = 1; 1391 /* only two SOBs are currently in use */ 1392 cprop->curr_sob_group_idx[stream] = 1393 (cprop->curr_sob_group_idx[stream] + 1) & 1394 (HL_RSVD_SOBS - 1); 1395 1396 gaudi_collective_map_sobs(hdev, stream); 1397 1398 dev_dbg(hdev->dev, "switched to SOB group %d, stream: %d\n", 1399 cprop->curr_sob_group_idx[stream], stream); 1400 } 1401 1402 mb(); 1403 hl_fence_put(cs->signal_fence); 1404 cs->signal_fence = NULL; 1405 1406 return 0; 1407 } 1408 1409 static u32 gaudi_get_patched_cb_extra_size(u32 user_cb_size) 1410 { 1411 u32 cacheline_end, additional_commands; 1412 1413 cacheline_end = round_up(user_cb_size, DEVICE_CACHE_LINE_SIZE); 1414 additional_commands = sizeof(struct packet_msg_prot) * 2; 1415 1416 if (user_cb_size + additional_commands > cacheline_end) 1417 return cacheline_end - user_cb_size + additional_commands; 1418 else 1419 return additional_commands; 1420 } 1421 1422 static int gaudi_collective_wait_create_job(struct hl_device *hdev, 1423 struct hl_ctx *ctx, struct hl_cs *cs, 1424 enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id, 1425 u32 encaps_signal_offset) 1426 { 1427 struct hw_queue_properties *hw_queue_prop; 1428 struct hl_cs_counters_atomic *cntr; 1429 struct hl_cs_job *job; 1430 struct hl_cb *cb; 1431 u32 cb_size; 1432 bool patched_cb; 1433 1434 cntr = &hdev->aggregated_cs_counters; 1435 1436 if (mode == HL_COLLECTIVE_MASTER) { 1437 /* CB size of collective master queue contains 1438 * 4 msg short packets for monitor 1 configuration 1439 * 1 fence packet 1440 * 4 msg short packets for monitor 2 configuration 1441 * 1 fence packet 1442 * 2 msg prot packets for completion and MSI 1443 */ 1444 cb_size = sizeof(struct packet_msg_short) * 8 + 1445 sizeof(struct packet_fence) * 2 + 1446 sizeof(struct packet_msg_prot) * 2; 1447 patched_cb = true; 1448 } else { 1449 /* CB size of collective slave queues contains 1450 * 4 msg short packets for monitor configuration 1451 * 1 fence packet 1452 * 1 additional msg short packet for sob signal 1453 */ 1454 cb_size = sizeof(struct packet_msg_short) * 5 + 1455 sizeof(struct packet_fence); 1456 patched_cb = false; 1457 } 1458 1459 hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id]; 1460 job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); 1461 if (!job) { 1462 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); 1463 atomic64_inc(&cntr->out_of_mem_drop_cnt); 1464 dev_err(hdev->dev, "Failed to allocate a new job\n"); 1465 return -ENOMEM; 1466 } 1467 1468 /* Allocate internal mapped CB for non patched CBs */ 1469 cb = hl_cb_kernel_create(hdev, cb_size, 1470 hdev->mmu_enable && !patched_cb); 1471 if (!cb) { 1472 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); 1473 atomic64_inc(&cntr->out_of_mem_drop_cnt); 1474 kfree(job); 1475 return -EFAULT; 1476 } 1477 1478 job->id = 0; 1479 job->cs = cs; 1480 job->user_cb = cb; 1481 atomic_inc(&job->user_cb->cs_cnt); 1482 job->user_cb_size = cb_size; 1483 job->hw_queue_id = queue_id; 1484 1485 /* since its guaranteed to have only one chunk in the collective wait 1486 * cs, we can use this chunk to set the encapsulated signal offset 1487 * in the jobs. 1488 */ 1489 if (cs->encaps_signals) 1490 job->encaps_sig_wait_offset = encaps_signal_offset; 1491 1492 /* 1493 * No need in parsing, user CB is the patched CB. 1494 * We call hl_cb_destroy() out of two reasons - we don't need 1495 * the CB in the CB idr anymore and to decrement its refcount as 1496 * it was incremented inside hl_cb_kernel_create(). 1497 */ 1498 if (patched_cb) 1499 job->patched_cb = job->user_cb; 1500 else 1501 job->patched_cb = NULL; 1502 1503 job->job_cb_size = job->user_cb_size; 1504 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); 1505 1506 /* increment refcount as for external queues we get completion */ 1507 if (hw_queue_prop->type == QUEUE_TYPE_EXT) 1508 cs_get(cs); 1509 1510 cs->jobs_in_queue_cnt[job->hw_queue_id]++; 1511 1512 list_add_tail(&job->cs_node, &cs->job_list); 1513 1514 hl_debugfs_add_job(hdev, job); 1515 1516 return 0; 1517 } 1518 1519 static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, 1520 struct hl_ctx *ctx, struct hl_cs *cs, 1521 u32 wait_queue_id, u32 collective_engine_id, 1522 u32 encaps_signal_offset) 1523 { 1524 struct gaudi_device *gaudi = hdev->asic_specific; 1525 struct hw_queue_properties *hw_queue_prop; 1526 u32 queue_id, collective_queue, num_jobs; 1527 u32 stream, nic_queue, nic_idx = 0; 1528 bool skip; 1529 int i, rc = 0; 1530 1531 /* Verify wait queue id is configured as master */ 1532 hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id]; 1533 if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) { 1534 dev_err(hdev->dev, 1535 "Queue %d is not configured as collective master\n", 1536 wait_queue_id); 1537 return -EINVAL; 1538 } 1539 1540 /* Verify engine id is supported */ 1541 if (collective_engine_id != GAUDI_ENGINE_ID_DMA_5 && 1542 collective_engine_id != GAUDI_ENGINE_ID_TPC_7) { 1543 dev_err(hdev->dev, 1544 "Collective wait does not support engine %u\n", 1545 collective_engine_id); 1546 return -EINVAL; 1547 } 1548 1549 stream = wait_queue_id % 4; 1550 1551 if (collective_engine_id == GAUDI_ENGINE_ID_DMA_5) 1552 collective_queue = GAUDI_QUEUE_ID_DMA_5_0 + stream; 1553 else 1554 collective_queue = GAUDI_QUEUE_ID_TPC_7_0 + stream; 1555 1556 num_jobs = NUMBER_OF_SOBS_IN_GRP + 1; 1557 nic_queue = GAUDI_QUEUE_ID_NIC_0_0 + stream; 1558 1559 /* First job goes to the collective master queue, it will wait for 1560 * the collective slave queues to finish execution. 1561 * The synchronization is done using two monitors: 1562 * First monitor for NICs 0-7, second monitor for NICs 8-9 and the 1563 * reduction engine (DMA5/TPC7). 1564 * 1565 * Rest of the jobs goes to the collective slave queues which will 1566 * all wait for the user to signal sob 'cs_cmpl->sob_val'. 1567 */ 1568 for (i = 0 ; i < num_jobs ; i++) { 1569 if (i == 0) { 1570 queue_id = wait_queue_id; 1571 rc = gaudi_collective_wait_create_job(hdev, ctx, cs, 1572 HL_COLLECTIVE_MASTER, queue_id, 1573 wait_queue_id, encaps_signal_offset); 1574 } else { 1575 if (nic_idx < NIC_NUMBER_OF_ENGINES) { 1576 if (gaudi->hw_cap_initialized & 1577 BIT(HW_CAP_NIC_SHIFT + nic_idx)) 1578 skip = false; 1579 else 1580 skip = true; 1581 1582 queue_id = nic_queue; 1583 nic_queue += 4; 1584 nic_idx++; 1585 1586 if (skip) 1587 continue; 1588 } else { 1589 queue_id = collective_queue; 1590 } 1591 1592 rc = gaudi_collective_wait_create_job(hdev, ctx, cs, 1593 HL_COLLECTIVE_SLAVE, queue_id, 1594 wait_queue_id, encaps_signal_offset); 1595 } 1596 1597 if (rc) 1598 return rc; 1599 } 1600 1601 return rc; 1602 } 1603 1604 static int gaudi_late_init(struct hl_device *hdev) 1605 { 1606 struct gaudi_device *gaudi = hdev->asic_specific; 1607 int rc; 1608 1609 rc = gaudi->cpucp_info_get(hdev); 1610 if (rc) { 1611 dev_err(hdev->dev, "Failed to get cpucp info\n"); 1612 return rc; 1613 } 1614 1615 if ((hdev->card_type == cpucp_card_type_pci) && 1616 (hdev->nic_ports_mask & 0x3)) { 1617 dev_info(hdev->dev, 1618 "PCI card detected, only 8 ports are enabled\n"); 1619 hdev->nic_ports_mask &= ~0x3; 1620 1621 /* Stop and disable unused NIC QMANs */ 1622 WREG32(mmNIC0_QM0_GLBL_CFG1, NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 1623 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 1624 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 1625 1626 WREG32(mmNIC0_QM1_GLBL_CFG1, NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 1627 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 1628 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 1629 1630 WREG32(mmNIC0_QM0_GLBL_CFG0, 0); 1631 WREG32(mmNIC0_QM1_GLBL_CFG0, 0); 1632 1633 gaudi->hw_cap_initialized &= ~(HW_CAP_NIC0 | HW_CAP_NIC1); 1634 } 1635 1636 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 0x0); 1637 if (rc) { 1638 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); 1639 return rc; 1640 } 1641 1642 /* Scrub both SRAM and DRAM */ 1643 rc = hdev->asic_funcs->scrub_device_mem(hdev); 1644 if (rc) 1645 goto disable_pci_access; 1646 1647 rc = gaudi_fetch_psoc_frequency(hdev); 1648 if (rc) { 1649 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); 1650 goto disable_pci_access; 1651 } 1652 1653 rc = gaudi_mmu_clear_pgt_range(hdev); 1654 if (rc) { 1655 dev_err(hdev->dev, "Failed to clear MMU page tables range\n"); 1656 goto disable_pci_access; 1657 } 1658 1659 rc = gaudi_init_tpc_mem(hdev); 1660 if (rc) { 1661 dev_err(hdev->dev, "Failed to initialize TPC memories\n"); 1662 goto disable_pci_access; 1663 } 1664 1665 rc = gaudi_collective_init(hdev); 1666 if (rc) { 1667 dev_err(hdev->dev, "Failed to init collective\n"); 1668 goto disable_pci_access; 1669 } 1670 1671 /* We only support a single ASID for the user, so for the sake of optimization, just 1672 * initialize the ASID one time during device initialization with the fixed value of 1 1673 */ 1674 gaudi_mmu_prepare(hdev, 1); 1675 1676 hl_fw_set_pll_profile(hdev); 1677 1678 return 0; 1679 1680 disable_pci_access: 1681 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 1682 1683 return rc; 1684 } 1685 1686 static void gaudi_late_fini(struct hl_device *hdev) 1687 { 1688 hl_hwmon_release_resources(hdev); 1689 } 1690 1691 static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) 1692 { 1693 dma_addr_t dma_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr; 1694 void *virt_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {}; 1695 int i, j, rc = 0; 1696 1697 /* 1698 * The device CPU works with 40-bits addresses, while bit 39 must be set 1699 * to '1' when accessing the host. 1700 * Bits 49:39 of the full host address are saved for a later 1701 * configuration of the HW to perform extension to 50 bits. 1702 * Because there is a single HW register that holds the extension bits, 1703 * these bits must be identical in all allocated range. 1704 */ 1705 1706 for (i = 0 ; i < GAUDI_ALLOC_CPU_MEM_RETRY_CNT ; i++) { 1707 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 1708 &dma_addr_arr[i], 1709 GFP_KERNEL | __GFP_ZERO); 1710 if (!virt_addr_arr[i]) { 1711 rc = -ENOMEM; 1712 goto free_dma_mem_arr; 1713 } 1714 1715 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; 1716 if (GAUDI_CPU_PCI_MSB_ADDR(dma_addr_arr[i]) == 1717 GAUDI_CPU_PCI_MSB_ADDR(end_addr)) 1718 break; 1719 } 1720 1721 if (i == GAUDI_ALLOC_CPU_MEM_RETRY_CNT) { 1722 dev_err(hdev->dev, 1723 "MSB of CPU accessible DMA memory are not identical in all range\n"); 1724 rc = -EFAULT; 1725 goto free_dma_mem_arr; 1726 } 1727 1728 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; 1729 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; 1730 hdev->cpu_pci_msb_addr = 1731 GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address); 1732 1733 if (!hdev->asic_prop.fw_security_enabled) 1734 GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address); 1735 1736 free_dma_mem_arr: 1737 for (j = 0 ; j < i ; j++) 1738 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j], 1739 dma_addr_arr[j]); 1740 1741 return rc; 1742 } 1743 1744 static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev) 1745 { 1746 struct gaudi_device *gaudi = hdev->asic_specific; 1747 struct gaudi_internal_qman_info *q; 1748 u32 i; 1749 1750 for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) { 1751 q = &gaudi->internal_qmans[i]; 1752 if (!q->pq_kernel_addr) 1753 continue; 1754 hl_asic_dma_free_coherent(hdev, q->pq_size, q->pq_kernel_addr, q->pq_dma_addr); 1755 } 1756 } 1757 1758 static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev) 1759 { 1760 struct gaudi_device *gaudi = hdev->asic_specific; 1761 struct gaudi_internal_qman_info *q; 1762 int rc, i; 1763 1764 for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) { 1765 if (gaudi_queue_type[i] != QUEUE_TYPE_INT) 1766 continue; 1767 1768 q = &gaudi->internal_qmans[i]; 1769 1770 switch (i) { 1771 case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_7_3: 1772 q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES; 1773 break; 1774 case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3: 1775 q->pq_size = MME_QMAN_SIZE_IN_BYTES; 1776 break; 1777 case GAUDI_QUEUE_ID_TPC_0_0 ... GAUDI_QUEUE_ID_TPC_7_3: 1778 q->pq_size = TPC_QMAN_SIZE_IN_BYTES; 1779 break; 1780 case GAUDI_QUEUE_ID_NIC_0_0 ... GAUDI_QUEUE_ID_NIC_9_3: 1781 q->pq_size = NIC_QMAN_SIZE_IN_BYTES; 1782 break; 1783 default: 1784 dev_err(hdev->dev, "Bad internal queue index %d", i); 1785 rc = -EINVAL; 1786 goto free_internal_qmans_pq_mem; 1787 } 1788 1789 q->pq_kernel_addr = hl_asic_dma_alloc_coherent(hdev, q->pq_size, &q->pq_dma_addr, 1790 GFP_KERNEL | __GFP_ZERO); 1791 if (!q->pq_kernel_addr) { 1792 rc = -ENOMEM; 1793 goto free_internal_qmans_pq_mem; 1794 } 1795 } 1796 1797 return 0; 1798 1799 free_internal_qmans_pq_mem: 1800 gaudi_free_internal_qmans_pq_mem(hdev); 1801 return rc; 1802 } 1803 1804 static void gaudi_set_pci_memory_regions(struct hl_device *hdev) 1805 { 1806 struct asic_fixed_properties *prop = &hdev->asic_prop; 1807 struct pci_mem_region *region; 1808 1809 /* CFG */ 1810 region = &hdev->pci_mem_region[PCI_REGION_CFG]; 1811 region->region_base = CFG_BASE; 1812 region->region_size = CFG_SIZE; 1813 region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR; 1814 region->bar_size = CFG_BAR_SIZE; 1815 region->bar_id = CFG_BAR_ID; 1816 region->used = 1; 1817 1818 /* SRAM */ 1819 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 1820 region->region_base = SRAM_BASE_ADDR; 1821 region->region_size = SRAM_SIZE; 1822 region->offset_in_bar = 0; 1823 region->bar_size = SRAM_BAR_SIZE; 1824 region->bar_id = SRAM_BAR_ID; 1825 region->used = 1; 1826 1827 /* DRAM */ 1828 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 1829 region->region_base = DRAM_PHYS_BASE; 1830 region->region_size = hdev->asic_prop.dram_size; 1831 region->offset_in_bar = 0; 1832 region->bar_size = prop->dram_pci_bar_size; 1833 region->bar_id = HBM_BAR_ID; 1834 region->used = 1; 1835 1836 /* SP SRAM */ 1837 region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM]; 1838 region->region_base = PSOC_SCRATCHPAD_ADDR; 1839 region->region_size = PSOC_SCRATCHPAD_SIZE; 1840 region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR; 1841 region->bar_size = CFG_BAR_SIZE; 1842 region->bar_id = CFG_BAR_ID; 1843 region->used = 1; 1844 } 1845 1846 static int gaudi_sw_init(struct hl_device *hdev) 1847 { 1848 struct gaudi_device *gaudi; 1849 u32 i, event_id = 0; 1850 int rc; 1851 1852 /* Allocate device structure */ 1853 gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL); 1854 if (!gaudi) 1855 return -ENOMEM; 1856 1857 for (i = 0 ; i < ARRAY_SIZE(gaudi_irq_map_table) ; i++) { 1858 if (gaudi_irq_map_table[i].valid) { 1859 if (event_id == GAUDI_EVENT_SIZE) { 1860 dev_err(hdev->dev, 1861 "Event array exceeds the limit of %u events\n", 1862 GAUDI_EVENT_SIZE); 1863 rc = -EINVAL; 1864 goto free_gaudi_device; 1865 } 1866 1867 gaudi->events[event_id++] = 1868 gaudi_irq_map_table[i].fc_id; 1869 } 1870 } 1871 1872 gaudi->cpucp_info_get = gaudi_cpucp_info_get; 1873 1874 hdev->asic_specific = gaudi; 1875 1876 /* Create DMA pool for small allocations */ 1877 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), 1878 &hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0); 1879 if (!hdev->dma_pool) { 1880 dev_err(hdev->dev, "failed to create DMA pool\n"); 1881 rc = -ENOMEM; 1882 goto free_gaudi_device; 1883 } 1884 1885 rc = gaudi_alloc_cpu_accessible_dma_mem(hdev); 1886 if (rc) 1887 goto free_dma_pool; 1888 1889 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 1890 if (!hdev->cpu_accessible_dma_pool) { 1891 dev_err(hdev->dev, 1892 "Failed to create CPU accessible DMA pool\n"); 1893 rc = -ENOMEM; 1894 goto free_cpu_dma_mem; 1895 } 1896 1897 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, 1898 (uintptr_t) hdev->cpu_accessible_dma_mem, 1899 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 1900 if (rc) { 1901 dev_err(hdev->dev, 1902 "Failed to add memory to CPU accessible DMA pool\n"); 1903 rc = -EFAULT; 1904 goto free_cpu_accessible_dma_pool; 1905 } 1906 1907 rc = gaudi_alloc_internal_qmans_pq_mem(hdev); 1908 if (rc) 1909 goto free_cpu_accessible_dma_pool; 1910 1911 spin_lock_init(&gaudi->hw_queues_lock); 1912 1913 hdev->supports_sync_stream = true; 1914 hdev->supports_coresight = true; 1915 hdev->supports_staged_submission = true; 1916 hdev->supports_wait_for_multi_cs = true; 1917 1918 hdev->asic_funcs->set_pci_memory_regions(hdev); 1919 hdev->stream_master_qid_arr = 1920 hdev->asic_funcs->get_stream_master_qid_arr(); 1921 hdev->stream_master_qid_arr_size = GAUDI_STREAM_MASTER_ARR_SIZE; 1922 1923 return 0; 1924 1925 free_cpu_accessible_dma_pool: 1926 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 1927 free_cpu_dma_mem: 1928 if (!hdev->asic_prop.fw_security_enabled) 1929 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, 1930 hdev->cpu_pci_msb_addr); 1931 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 1932 hdev->cpu_accessible_dma_address); 1933 free_dma_pool: 1934 dma_pool_destroy(hdev->dma_pool); 1935 free_gaudi_device: 1936 kfree(gaudi); 1937 return rc; 1938 } 1939 1940 static int gaudi_sw_fini(struct hl_device *hdev) 1941 { 1942 struct gaudi_device *gaudi = hdev->asic_specific; 1943 1944 gaudi_free_internal_qmans_pq_mem(hdev); 1945 1946 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 1947 1948 if (!hdev->asic_prop.fw_security_enabled) 1949 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, 1950 hdev->cpu_pci_msb_addr); 1951 1952 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 1953 hdev->cpu_accessible_dma_address); 1954 1955 dma_pool_destroy(hdev->dma_pool); 1956 1957 kfree(gaudi); 1958 1959 return 0; 1960 } 1961 1962 static irqreturn_t gaudi_irq_handler_single(int irq, void *arg) 1963 { 1964 struct hl_device *hdev = arg; 1965 int i; 1966 1967 if (hdev->disabled) 1968 return IRQ_HANDLED; 1969 1970 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) 1971 hl_irq_handler_cq(irq, &hdev->completion_queue[i]); 1972 1973 hl_irq_handler_eq(irq, &hdev->event_queue); 1974 1975 return IRQ_HANDLED; 1976 } 1977 1978 /* 1979 * For backward compatibility, new MSI interrupts should be set after the 1980 * existing CPU and NIC interrupts. 1981 */ 1982 static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr, 1983 bool cpu_eq) 1984 { 1985 int msi_vec; 1986 1987 if ((nr != GAUDI_EVENT_QUEUE_MSI_IDX) && (cpu_eq)) 1988 dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n", 1989 GAUDI_EVENT_QUEUE_MSI_IDX); 1990 1991 msi_vec = ((nr < GAUDI_EVENT_QUEUE_MSI_IDX) || (cpu_eq)) ? nr : 1992 (nr + NIC_NUMBER_OF_ENGINES + 1); 1993 1994 return pci_irq_vector(hdev->pdev, msi_vec); 1995 } 1996 1997 static int gaudi_enable_msi_single(struct hl_device *hdev) 1998 { 1999 int rc, irq; 2000 2001 dev_dbg(hdev->dev, "Working in single MSI IRQ mode\n"); 2002 2003 irq = gaudi_pci_irq_vector(hdev, 0, false); 2004 rc = request_irq(irq, gaudi_irq_handler_single, 0, 2005 "gaudi single msi", hdev); 2006 if (rc) 2007 dev_err(hdev->dev, 2008 "Failed to request single MSI IRQ\n"); 2009 2010 return rc; 2011 } 2012 2013 static int gaudi_enable_msi_multi(struct hl_device *hdev) 2014 { 2015 int cq_cnt = hdev->asic_prop.completion_queues_count; 2016 int rc, i, irq_cnt_init, irq; 2017 2018 for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) { 2019 irq = gaudi_pci_irq_vector(hdev, i, false); 2020 rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi_irq_name[i], 2021 &hdev->completion_queue[i]); 2022 if (rc) { 2023 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 2024 goto free_irqs; 2025 } 2026 } 2027 2028 irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, true); 2029 rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi_irq_name[cq_cnt], 2030 &hdev->event_queue); 2031 if (rc) { 2032 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 2033 goto free_irqs; 2034 } 2035 2036 return 0; 2037 2038 free_irqs: 2039 for (i = 0 ; i < irq_cnt_init ; i++) 2040 free_irq(gaudi_pci_irq_vector(hdev, i, false), 2041 &hdev->completion_queue[i]); 2042 return rc; 2043 } 2044 2045 static int gaudi_enable_msi(struct hl_device *hdev) 2046 { 2047 struct gaudi_device *gaudi = hdev->asic_specific; 2048 int rc; 2049 2050 if (gaudi->hw_cap_initialized & HW_CAP_MSI) 2051 return 0; 2052 2053 rc = pci_alloc_irq_vectors(hdev->pdev, 1, 1, PCI_IRQ_MSI); 2054 if (rc < 0) { 2055 dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc); 2056 return rc; 2057 } 2058 2059 if (rc < NUMBER_OF_INTERRUPTS) { 2060 gaudi->multi_msi_mode = false; 2061 rc = gaudi_enable_msi_single(hdev); 2062 } else { 2063 gaudi->multi_msi_mode = true; 2064 rc = gaudi_enable_msi_multi(hdev); 2065 } 2066 2067 if (rc) 2068 goto free_pci_irq_vectors; 2069 2070 gaudi->hw_cap_initialized |= HW_CAP_MSI; 2071 2072 return 0; 2073 2074 free_pci_irq_vectors: 2075 pci_free_irq_vectors(hdev->pdev); 2076 return rc; 2077 } 2078 2079 static void gaudi_sync_irqs(struct hl_device *hdev) 2080 { 2081 struct gaudi_device *gaudi = hdev->asic_specific; 2082 int i, cq_cnt = hdev->asic_prop.completion_queues_count; 2083 2084 if (!(gaudi->hw_cap_initialized & HW_CAP_MSI)) 2085 return; 2086 2087 /* Wait for all pending IRQs to be finished */ 2088 if (gaudi->multi_msi_mode) { 2089 for (i = 0 ; i < cq_cnt ; i++) 2090 synchronize_irq(gaudi_pci_irq_vector(hdev, i, false)); 2091 2092 synchronize_irq(gaudi_pci_irq_vector(hdev, 2093 GAUDI_EVENT_QUEUE_MSI_IDX, 2094 true)); 2095 } else { 2096 synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false)); 2097 } 2098 } 2099 2100 static void gaudi_disable_msi(struct hl_device *hdev) 2101 { 2102 struct gaudi_device *gaudi = hdev->asic_specific; 2103 int i, irq, cq_cnt = hdev->asic_prop.completion_queues_count; 2104 2105 if (!(gaudi->hw_cap_initialized & HW_CAP_MSI)) 2106 return; 2107 2108 gaudi_sync_irqs(hdev); 2109 2110 if (gaudi->multi_msi_mode) { 2111 irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, 2112 true); 2113 free_irq(irq, &hdev->event_queue); 2114 2115 for (i = 0 ; i < cq_cnt ; i++) { 2116 irq = gaudi_pci_irq_vector(hdev, i, false); 2117 free_irq(irq, &hdev->completion_queue[i]); 2118 } 2119 } else { 2120 free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev); 2121 } 2122 2123 pci_free_irq_vectors(hdev->pdev); 2124 2125 gaudi->hw_cap_initialized &= ~HW_CAP_MSI; 2126 } 2127 2128 static void gaudi_init_scrambler_sram(struct hl_device *hdev) 2129 { 2130 struct gaudi_device *gaudi = hdev->asic_specific; 2131 2132 if (hdev->asic_prop.fw_security_enabled) 2133 return; 2134 2135 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & 2136 CPU_BOOT_DEV_STS0_SRAM_SCR_EN) 2137 return; 2138 2139 if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER) 2140 return; 2141 2142 WREG32(mmNIF_RTR_CTRL_0_SCRAM_SRAM_EN, 2143 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2144 WREG32(mmNIF_RTR_CTRL_1_SCRAM_SRAM_EN, 2145 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2146 WREG32(mmNIF_RTR_CTRL_2_SCRAM_SRAM_EN, 2147 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2148 WREG32(mmNIF_RTR_CTRL_3_SCRAM_SRAM_EN, 2149 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2150 WREG32(mmNIF_RTR_CTRL_4_SCRAM_SRAM_EN, 2151 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2152 WREG32(mmNIF_RTR_CTRL_5_SCRAM_SRAM_EN, 2153 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2154 WREG32(mmNIF_RTR_CTRL_6_SCRAM_SRAM_EN, 2155 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2156 WREG32(mmNIF_RTR_CTRL_7_SCRAM_SRAM_EN, 2157 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2158 2159 WREG32(mmSIF_RTR_CTRL_0_SCRAM_SRAM_EN, 2160 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2161 WREG32(mmSIF_RTR_CTRL_1_SCRAM_SRAM_EN, 2162 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2163 WREG32(mmSIF_RTR_CTRL_2_SCRAM_SRAM_EN, 2164 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2165 WREG32(mmSIF_RTR_CTRL_3_SCRAM_SRAM_EN, 2166 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2167 WREG32(mmSIF_RTR_CTRL_4_SCRAM_SRAM_EN, 2168 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2169 WREG32(mmSIF_RTR_CTRL_5_SCRAM_SRAM_EN, 2170 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2171 WREG32(mmSIF_RTR_CTRL_6_SCRAM_SRAM_EN, 2172 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2173 WREG32(mmSIF_RTR_CTRL_7_SCRAM_SRAM_EN, 2174 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); 2175 2176 WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_SRAM_EN, 2177 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2178 WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_SRAM_EN, 2179 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2180 WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_SRAM_EN, 2181 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2182 WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_SRAM_EN, 2183 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2184 WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_SRAM_EN, 2185 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2186 WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_SRAM_EN, 2187 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2188 WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_SRAM_EN, 2189 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2190 WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_SRAM_EN, 2191 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); 2192 2193 gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER; 2194 } 2195 2196 static void gaudi_init_scrambler_hbm(struct hl_device *hdev) 2197 { 2198 struct gaudi_device *gaudi = hdev->asic_specific; 2199 2200 if (hdev->asic_prop.fw_security_enabled) 2201 return; 2202 2203 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & 2204 CPU_BOOT_DEV_STS0_DRAM_SCR_EN) 2205 return; 2206 2207 if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER) 2208 return; 2209 2210 WREG32(mmNIF_RTR_CTRL_0_SCRAM_HBM_EN, 2211 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2212 WREG32(mmNIF_RTR_CTRL_1_SCRAM_HBM_EN, 2213 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2214 WREG32(mmNIF_RTR_CTRL_2_SCRAM_HBM_EN, 2215 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2216 WREG32(mmNIF_RTR_CTRL_3_SCRAM_HBM_EN, 2217 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2218 WREG32(mmNIF_RTR_CTRL_4_SCRAM_HBM_EN, 2219 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2220 WREG32(mmNIF_RTR_CTRL_5_SCRAM_HBM_EN, 2221 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2222 WREG32(mmNIF_RTR_CTRL_6_SCRAM_HBM_EN, 2223 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2224 WREG32(mmNIF_RTR_CTRL_7_SCRAM_HBM_EN, 2225 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2226 2227 WREG32(mmSIF_RTR_CTRL_0_SCRAM_HBM_EN, 2228 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2229 WREG32(mmSIF_RTR_CTRL_1_SCRAM_HBM_EN, 2230 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2231 WREG32(mmSIF_RTR_CTRL_2_SCRAM_HBM_EN, 2232 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2233 WREG32(mmSIF_RTR_CTRL_3_SCRAM_HBM_EN, 2234 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2235 WREG32(mmSIF_RTR_CTRL_4_SCRAM_HBM_EN, 2236 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2237 WREG32(mmSIF_RTR_CTRL_5_SCRAM_HBM_EN, 2238 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2239 WREG32(mmSIF_RTR_CTRL_6_SCRAM_HBM_EN, 2240 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2241 WREG32(mmSIF_RTR_CTRL_7_SCRAM_HBM_EN, 2242 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); 2243 2244 WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_HBM_EN, 2245 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2246 WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_HBM_EN, 2247 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2248 WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_HBM_EN, 2249 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2250 WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_HBM_EN, 2251 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2252 WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_HBM_EN, 2253 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2254 WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_HBM_EN, 2255 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2256 WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_HBM_EN, 2257 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2258 WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_HBM_EN, 2259 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); 2260 2261 gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER; 2262 } 2263 2264 static void gaudi_init_e2e(struct hl_device *hdev) 2265 { 2266 if (hdev->asic_prop.fw_security_enabled) 2267 return; 2268 2269 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & 2270 CPU_BOOT_DEV_STS0_E2E_CRED_EN) 2271 return; 2272 2273 WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3); 2274 WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 785 >> 3); 2275 WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 49); 2276 WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 101); 2277 2278 WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3); 2279 WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3); 2280 WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1); 2281 WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39); 2282 2283 WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1); 2284 WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1); 2285 WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1); 2286 WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32); 2287 2288 WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3); 2289 WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3); 2290 WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19); 2291 WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32); 2292 2293 WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3); 2294 WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3); 2295 WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19); 2296 WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32); 2297 2298 WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1); 2299 WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1); 2300 WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1); 2301 WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32); 2302 2303 WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3); 2304 WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3); 2305 WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1); 2306 WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39); 2307 2308 WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 297 >> 3); 2309 WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 908 >> 3); 2310 WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 19); 2311 WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 19); 2312 2313 WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 318 >> 3); 2314 WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 956 >> 3); 2315 WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 79); 2316 WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 163); 2317 2318 WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3); 2319 WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3); 2320 WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1); 2321 WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39); 2322 2323 WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1); 2324 WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1); 2325 WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1); 2326 WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32); 2327 2328 WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3); 2329 WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3); 2330 WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19); 2331 WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32); 2332 2333 WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3); 2334 WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3); 2335 WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19); 2336 WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32); 2337 2338 WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1); 2339 WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1); 2340 WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1); 2341 WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32); 2342 2343 WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3); 2344 WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3); 2345 WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1); 2346 WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39); 2347 2348 WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 318 >> 3); 2349 WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 956 >> 3); 2350 WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 79); 2351 WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 79); 2352 2353 WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3); 2354 WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3); 2355 WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162); 2356 WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338); 2357 2358 WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3); 2359 WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3); 2360 WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162); 2361 WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338); 2362 2363 WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3); 2364 WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3); 2365 WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162); 2366 WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338); 2367 2368 WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3); 2369 WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3); 2370 WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162); 2371 WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338); 2372 2373 WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3); 2374 WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3); 2375 WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162); 2376 WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338); 2377 2378 WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3); 2379 WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3); 2380 WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162); 2381 WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338); 2382 2383 WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3); 2384 WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3); 2385 WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162); 2386 WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338); 2387 2388 WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3); 2389 WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3); 2390 WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162); 2391 WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338); 2392 2393 WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_EN, 2394 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2395 WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_EN, 2396 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2397 2398 WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_EN, 2399 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2400 WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_EN, 2401 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2402 2403 WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_EN, 2404 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2405 WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_EN, 2406 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2407 2408 WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_EN, 2409 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2410 WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_EN, 2411 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2412 2413 WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_EN, 2414 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2415 WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_EN, 2416 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2417 2418 WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_EN, 2419 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2420 WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_EN, 2421 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2422 2423 WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_EN, 2424 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2425 WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_EN, 2426 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2427 2428 WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_EN, 2429 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2430 WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_EN, 2431 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2432 2433 WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_EN, 2434 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2435 WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_EN, 2436 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2437 2438 WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_EN, 2439 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2440 WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_EN, 2441 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2442 2443 WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_EN, 2444 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2445 WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_EN, 2446 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2447 2448 WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_EN, 2449 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2450 WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_EN, 2451 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2452 2453 WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_EN, 2454 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2455 WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_EN, 2456 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2457 2458 WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_EN, 2459 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2460 WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_EN, 2461 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2462 2463 WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_EN, 2464 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2465 WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_EN, 2466 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2467 2468 WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_EN, 2469 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); 2470 WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_EN, 2471 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); 2472 2473 WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_EN, 2474 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2475 WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_EN, 2476 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2477 2478 WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_EN, 2479 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2480 WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_EN, 2481 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2482 2483 WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_EN, 2484 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2485 WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_EN, 2486 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2487 2488 WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_EN, 2489 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2490 WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_EN, 2491 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2492 2493 WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_EN, 2494 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2495 WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_EN, 2496 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2497 2498 WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_EN, 2499 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2500 WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_EN, 2501 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2502 2503 WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_EN, 2504 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2505 WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_EN, 2506 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2507 2508 WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_EN, 2509 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); 2510 WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_EN, 2511 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); 2512 } 2513 2514 static void gaudi_init_hbm_cred(struct hl_device *hdev) 2515 { 2516 u32 hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd; 2517 2518 if (hdev->asic_prop.fw_security_enabled) 2519 return; 2520 2521 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & 2522 CPU_BOOT_DEV_STS0_HBM_CRED_EN) 2523 return; 2524 2525 hbm0_wr = 0x33333333; 2526 hbm0_rd = 0x77777777; 2527 hbm1_wr = 0x55555555; 2528 hbm1_rd = 0xDDDDDDDD; 2529 2530 WREG32(mmDMA_IF_E_N_HBM0_WR_CRED_CNT, hbm0_wr); 2531 WREG32(mmDMA_IF_E_N_HBM1_WR_CRED_CNT, hbm1_wr); 2532 WREG32(mmDMA_IF_E_N_HBM0_RD_CRED_CNT, hbm0_rd); 2533 WREG32(mmDMA_IF_E_N_HBM1_RD_CRED_CNT, hbm1_rd); 2534 2535 WREG32(mmDMA_IF_E_S_HBM0_WR_CRED_CNT, hbm0_wr); 2536 WREG32(mmDMA_IF_E_S_HBM1_WR_CRED_CNT, hbm1_wr); 2537 WREG32(mmDMA_IF_E_S_HBM0_RD_CRED_CNT, hbm0_rd); 2538 WREG32(mmDMA_IF_E_S_HBM1_RD_CRED_CNT, hbm1_rd); 2539 2540 WREG32(mmDMA_IF_W_N_HBM0_WR_CRED_CNT, hbm0_wr); 2541 WREG32(mmDMA_IF_W_N_HBM1_WR_CRED_CNT, hbm1_wr); 2542 WREG32(mmDMA_IF_W_N_HBM0_RD_CRED_CNT, hbm0_rd); 2543 WREG32(mmDMA_IF_W_N_HBM1_RD_CRED_CNT, hbm1_rd); 2544 2545 WREG32(mmDMA_IF_W_S_HBM0_WR_CRED_CNT, hbm0_wr); 2546 WREG32(mmDMA_IF_W_S_HBM1_WR_CRED_CNT, hbm1_wr); 2547 WREG32(mmDMA_IF_W_S_HBM0_RD_CRED_CNT, hbm0_rd); 2548 WREG32(mmDMA_IF_W_S_HBM1_RD_CRED_CNT, hbm1_rd); 2549 2550 WREG32(mmDMA_IF_E_N_HBM_CRED_EN_0, 2551 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2552 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2553 WREG32(mmDMA_IF_E_S_HBM_CRED_EN_0, 2554 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2555 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2556 WREG32(mmDMA_IF_W_N_HBM_CRED_EN_0, 2557 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2558 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2559 WREG32(mmDMA_IF_W_S_HBM_CRED_EN_0, 2560 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2561 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2562 2563 WREG32(mmDMA_IF_E_N_HBM_CRED_EN_1, 2564 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2565 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2566 WREG32(mmDMA_IF_E_S_HBM_CRED_EN_1, 2567 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2568 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2569 WREG32(mmDMA_IF_W_N_HBM_CRED_EN_1, 2570 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2571 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2572 WREG32(mmDMA_IF_W_S_HBM_CRED_EN_1, 2573 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | 2574 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); 2575 } 2576 2577 static void gaudi_init_golden_registers(struct hl_device *hdev) 2578 { 2579 u32 tpc_offset; 2580 int tpc_id, i; 2581 2582 gaudi_init_e2e(hdev); 2583 gaudi_init_hbm_cred(hdev); 2584 2585 for (tpc_id = 0, tpc_offset = 0; 2586 tpc_id < TPC_NUMBER_OF_ENGINES; 2587 tpc_id++, tpc_offset += TPC_CFG_OFFSET) { 2588 /* Mask all arithmetic interrupts from TPC */ 2589 WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFE); 2590 /* Set 16 cache lines */ 2591 WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset, 2592 ICACHE_FETCH_LINE_NUM, 2); 2593 } 2594 2595 /* Make sure 1st 128 bytes in SRAM are 0 for Tensor DMA */ 2596 for (i = 0 ; i < 128 ; i += 8) 2597 writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i); 2598 2599 WREG32(mmMME0_CTRL_EUS_ROLLUP_CNT_ADD, 3); 2600 WREG32(mmMME1_CTRL_EUS_ROLLUP_CNT_ADD, 3); 2601 WREG32(mmMME2_CTRL_EUS_ROLLUP_CNT_ADD, 3); 2602 WREG32(mmMME3_CTRL_EUS_ROLLUP_CNT_ADD, 3); 2603 } 2604 2605 static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, 2606 int qman_id, dma_addr_t qman_pq_addr) 2607 { 2608 struct cpu_dyn_regs *dyn_regs = 2609 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 2610 u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; 2611 u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; 2612 u32 q_off, dma_qm_offset; 2613 u32 dma_qm_err_cfg, irq_handler_offset; 2614 2615 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 2616 2617 mtr_base_en_lo = lower_32_bits(CFG_BASE + 2618 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2619 mtr_base_en_hi = upper_32_bits(CFG_BASE + 2620 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2621 so_base_en_lo = lower_32_bits(CFG_BASE + 2622 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 2623 so_base_en_hi = upper_32_bits(CFG_BASE + 2624 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 2625 mtr_base_ws_lo = lower_32_bits(CFG_BASE + 2626 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2627 mtr_base_ws_hi = upper_32_bits(CFG_BASE + 2628 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2629 so_base_ws_lo = lower_32_bits(CFG_BASE + 2630 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 2631 so_base_ws_hi = upper_32_bits(CFG_BASE + 2632 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 2633 2634 q_off = dma_qm_offset + qman_id * 4; 2635 2636 WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_pq_addr)); 2637 WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_pq_addr)); 2638 2639 WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HL_QUEUE_LENGTH)); 2640 WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0); 2641 WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0); 2642 2643 WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); 2644 WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 2645 QMAN_LDMA_SRC_OFFSET); 2646 WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 2647 QMAN_LDMA_DST_OFFSET); 2648 2649 WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); 2650 WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); 2651 WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); 2652 WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); 2653 WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo); 2654 WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi); 2655 WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo); 2656 WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi); 2657 2658 WREG32(mmDMA0_QM_CP_BARRIER_CFG_0 + q_off, 0x100); 2659 2660 /* The following configuration is needed only once per QMAN */ 2661 if (qman_id == 0) { 2662 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 2663 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 2664 le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 2665 2666 /* Configure RAZWI IRQ */ 2667 dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; 2668 if (hdev->stop_on_err) 2669 dma_qm_err_cfg |= 2670 PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; 2671 2672 WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg); 2673 2674 WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset, 2675 lower_32_bits(CFG_BASE + irq_handler_offset)); 2676 WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset, 2677 upper_32_bits(CFG_BASE + irq_handler_offset)); 2678 2679 WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset, 2680 gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id + 2681 dma_id); 2682 2683 WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset, 2684 QM_ARB_ERR_MSG_EN_MASK); 2685 2686 /* Set timeout to maximum */ 2687 WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset, GAUDI_ARB_WDT_TIMEOUT); 2688 2689 WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset, 2690 QMAN_EXTERNAL_MAKE_TRUSTED); 2691 2692 WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0); 2693 } 2694 } 2695 2696 static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id) 2697 { 2698 struct cpu_dyn_regs *dyn_regs = 2699 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 2700 u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT; 2701 u32 dma_offset = dma_id * DMA_CORE_OFFSET; 2702 u32 irq_handler_offset; 2703 2704 /* Set to maximum possible according to physical size */ 2705 WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0); 2706 WREG32(mmDMA0_CORE_RD_MAX_SIZE + dma_offset, 0); 2707 2708 /* WA for H/W bug H3-2116 */ 2709 WREG32(mmDMA0_CORE_LBW_MAX_OUTSTAND + dma_offset, 15); 2710 2711 /* STOP_ON bit implies no completion to operation in case of RAZWI */ 2712 if (hdev->stop_on_err) 2713 dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT; 2714 2715 WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg); 2716 2717 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 2718 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 2719 le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); 2720 2721 WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset, 2722 lower_32_bits(CFG_BASE + irq_handler_offset)); 2723 WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset, 2724 upper_32_bits(CFG_BASE + irq_handler_offset)); 2725 2726 WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset, 2727 gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id); 2728 WREG32(mmDMA0_CORE_PROT + dma_offset, 2729 1 << DMA0_CORE_PROT_ERR_VAL_SHIFT); 2730 /* If the channel is secured, it should be in MMU bypass mode */ 2731 WREG32(mmDMA0_CORE_SECURE_PROPS + dma_offset, 2732 1 << DMA0_CORE_SECURE_PROPS_MMBP_SHIFT); 2733 WREG32(mmDMA0_CORE_CFG_0 + dma_offset, 1 << DMA0_CORE_CFG_0_EN_SHIFT); 2734 } 2735 2736 static void gaudi_enable_qman(struct hl_device *hdev, int dma_id, 2737 u32 enable_mask) 2738 { 2739 u32 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 2740 2741 WREG32(mmDMA0_QM_GLBL_CFG0 + dma_qm_offset, enable_mask); 2742 } 2743 2744 static void gaudi_init_pci_dma_qmans(struct hl_device *hdev) 2745 { 2746 struct gaudi_device *gaudi = hdev->asic_specific; 2747 struct hl_hw_queue *q; 2748 int i, j, dma_id, cpu_skip, nic_skip, cq_id = 0, q_idx, msi_vec = 0; 2749 2750 if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA) 2751 return; 2752 2753 for (i = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) { 2754 dma_id = gaudi_dma_assignment[i]; 2755 /* 2756 * For queues after the CPU Q need to add 1 to get the correct 2757 * queue. In addition, need to add the CPU EQ and NIC IRQs in 2758 * order to get the correct MSI register. 2759 */ 2760 if (dma_id > 1) { 2761 cpu_skip = 1; 2762 nic_skip = NIC_NUMBER_OF_ENGINES; 2763 } else { 2764 cpu_skip = 0; 2765 nic_skip = 0; 2766 } 2767 2768 for (j = 0 ; j < QMAN_STREAMS ; j++) { 2769 q_idx = 4 * dma_id + j + cpu_skip; 2770 q = &hdev->kernel_queues[q_idx]; 2771 q->cq_id = cq_id++; 2772 q->msi_vec = nic_skip + cpu_skip + msi_vec++; 2773 gaudi_init_pci_dma_qman(hdev, dma_id, j, 2774 q->bus_address); 2775 } 2776 2777 gaudi_init_dma_core(hdev, dma_id); 2778 2779 gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE); 2780 } 2781 2782 gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA; 2783 } 2784 2785 static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, 2786 int qman_id, u64 qman_base_addr) 2787 { 2788 struct cpu_dyn_regs *dyn_regs = 2789 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 2790 u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; 2791 u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; 2792 u32 dma_qm_err_cfg, irq_handler_offset; 2793 u32 q_off, dma_qm_offset; 2794 2795 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 2796 2797 mtr_base_en_lo = lower_32_bits(CFG_BASE + 2798 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2799 mtr_base_en_hi = upper_32_bits(CFG_BASE + 2800 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2801 so_base_en_lo = lower_32_bits(CFG_BASE + 2802 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 2803 so_base_en_hi = upper_32_bits(CFG_BASE + 2804 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 2805 mtr_base_ws_lo = lower_32_bits(CFG_BASE + 2806 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2807 mtr_base_ws_hi = upper_32_bits(CFG_BASE + 2808 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2809 so_base_ws_lo = lower_32_bits(CFG_BASE + 2810 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 2811 so_base_ws_hi = upper_32_bits(CFG_BASE + 2812 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 2813 2814 q_off = dma_qm_offset + qman_id * 4; 2815 2816 if (qman_id < 4) { 2817 WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off, 2818 lower_32_bits(qman_base_addr)); 2819 WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off, 2820 upper_32_bits(qman_base_addr)); 2821 2822 WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HBM_DMA_QMAN_LENGTH)); 2823 WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0); 2824 WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0); 2825 2826 WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 2827 QMAN_CPDMA_SIZE_OFFSET); 2828 WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 2829 QMAN_CPDMA_SRC_OFFSET); 2830 WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 2831 QMAN_CPDMA_DST_OFFSET); 2832 } else { 2833 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 2834 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 2835 le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 2836 2837 WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 2838 QMAN_LDMA_SIZE_OFFSET); 2839 WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 2840 QMAN_LDMA_SRC_OFFSET); 2841 WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 2842 QMAN_LDMA_DST_OFFSET); 2843 2844 /* Configure RAZWI IRQ */ 2845 dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; 2846 if (hdev->stop_on_err) 2847 dma_qm_err_cfg |= 2848 HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; 2849 2850 WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg); 2851 2852 WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset, 2853 lower_32_bits(CFG_BASE + irq_handler_offset)); 2854 WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset, 2855 upper_32_bits(CFG_BASE + irq_handler_offset)); 2856 2857 WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset, 2858 gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id + 2859 dma_id); 2860 2861 WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset, 2862 QM_ARB_ERR_MSG_EN_MASK); 2863 2864 /* Set timeout to maximum */ 2865 WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset, GAUDI_ARB_WDT_TIMEOUT); 2866 2867 WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0); 2868 WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset, 2869 QMAN_INTERNAL_MAKE_TRUSTED); 2870 } 2871 2872 WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); 2873 WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); 2874 WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); 2875 WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); 2876 2877 /* Configure DMA5 CP_MSG_BASE 2/3 for sync stream collective */ 2878 if (gaudi_dma_assignment[dma_id] == GAUDI_ENGINE_ID_DMA_5) { 2879 WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, 2880 mtr_base_ws_lo); 2881 WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, 2882 mtr_base_ws_hi); 2883 WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, 2884 so_base_ws_lo); 2885 WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, 2886 so_base_ws_hi); 2887 } 2888 } 2889 2890 static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev) 2891 { 2892 struct gaudi_device *gaudi = hdev->asic_specific; 2893 struct gaudi_internal_qman_info *q; 2894 u64 qman_base_addr; 2895 int i, j, dma_id, internal_q_index; 2896 2897 if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA) 2898 return; 2899 2900 for (i = 0 ; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) { 2901 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1 + i]; 2902 2903 for (j = 0 ; j < QMAN_STREAMS ; j++) { 2904 /* 2905 * Add the CPU queue in order to get the correct queue 2906 * number as all internal queue are placed after it 2907 */ 2908 internal_q_index = dma_id * QMAN_STREAMS + j + 1; 2909 2910 q = &gaudi->internal_qmans[internal_q_index]; 2911 qman_base_addr = (u64) q->pq_dma_addr; 2912 gaudi_init_hbm_dma_qman(hdev, dma_id, j, 2913 qman_base_addr); 2914 } 2915 2916 /* Initializing lower CP for HBM DMA QMAN */ 2917 gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0); 2918 2919 gaudi_init_dma_core(hdev, dma_id); 2920 2921 gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE); 2922 } 2923 2924 gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA; 2925 } 2926 2927 static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, 2928 int qman_id, u64 qman_base_addr) 2929 { 2930 struct cpu_dyn_regs *dyn_regs = 2931 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 2932 u32 mtr_base_lo, mtr_base_hi; 2933 u32 so_base_lo, so_base_hi; 2934 u32 irq_handler_offset; 2935 u32 q_off, mme_id; 2936 u32 mme_qm_err_cfg; 2937 2938 mtr_base_lo = lower_32_bits(CFG_BASE + 2939 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2940 mtr_base_hi = upper_32_bits(CFG_BASE + 2941 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 2942 so_base_lo = lower_32_bits(CFG_BASE + 2943 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 2944 so_base_hi = upper_32_bits(CFG_BASE + 2945 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 2946 2947 q_off = mme_offset + qman_id * 4; 2948 2949 if (qman_id < 4) { 2950 WREG32(mmMME0_QM_PQ_BASE_LO_0 + q_off, 2951 lower_32_bits(qman_base_addr)); 2952 WREG32(mmMME0_QM_PQ_BASE_HI_0 + q_off, 2953 upper_32_bits(qman_base_addr)); 2954 2955 WREG32(mmMME0_QM_PQ_SIZE_0 + q_off, ilog2(MME_QMAN_LENGTH)); 2956 WREG32(mmMME0_QM_PQ_PI_0 + q_off, 0); 2957 WREG32(mmMME0_QM_PQ_CI_0 + q_off, 0); 2958 2959 WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 2960 QMAN_CPDMA_SIZE_OFFSET); 2961 WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 2962 QMAN_CPDMA_SRC_OFFSET); 2963 WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 2964 QMAN_CPDMA_DST_OFFSET); 2965 } else { 2966 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 2967 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 2968 le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); 2969 2970 WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 2971 QMAN_LDMA_SIZE_OFFSET); 2972 WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 2973 QMAN_LDMA_SRC_OFFSET); 2974 WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 2975 QMAN_LDMA_DST_OFFSET); 2976 2977 /* Configure RAZWI IRQ */ 2978 mme_id = mme_offset / 2979 (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2; 2980 2981 mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; 2982 if (hdev->stop_on_err) 2983 mme_qm_err_cfg |= 2984 MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; 2985 2986 WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg); 2987 2988 WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset, 2989 lower_32_bits(CFG_BASE + irq_handler_offset)); 2990 WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset, 2991 upper_32_bits(CFG_BASE + irq_handler_offset)); 2992 2993 WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset, 2994 gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id + 2995 mme_id); 2996 2997 WREG32(mmMME0_QM_ARB_ERR_MSG_EN + mme_offset, 2998 QM_ARB_ERR_MSG_EN_MASK); 2999 3000 /* Set timeout to maximum */ 3001 WREG32(mmMME0_QM_ARB_SLV_CHOISE_WDT + mme_offset, GAUDI_ARB_WDT_TIMEOUT); 3002 3003 WREG32(mmMME0_QM_GLBL_CFG1 + mme_offset, 0); 3004 WREG32(mmMME0_QM_GLBL_PROT + mme_offset, 3005 QMAN_INTERNAL_MAKE_TRUSTED); 3006 } 3007 3008 WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo); 3009 WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi); 3010 WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo); 3011 WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi); 3012 } 3013 3014 static void gaudi_init_mme_qmans(struct hl_device *hdev) 3015 { 3016 struct gaudi_device *gaudi = hdev->asic_specific; 3017 struct gaudi_internal_qman_info *q; 3018 u64 qman_base_addr; 3019 u32 mme_offset; 3020 int i, internal_q_index; 3021 3022 if (gaudi->hw_cap_initialized & HW_CAP_MME) 3023 return; 3024 3025 /* 3026 * map GAUDI_QUEUE_ID_MME_0_X to the N_W_MME (mmMME2_QM_BASE) 3027 * and GAUDI_QUEUE_ID_MME_1_X to the S_W_MME (mmMME0_QM_BASE) 3028 */ 3029 3030 mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0; 3031 3032 for (i = 0 ; i < MME_NUMBER_OF_QMANS ; i++) { 3033 internal_q_index = GAUDI_QUEUE_ID_MME_0_0 + i; 3034 q = &gaudi->internal_qmans[internal_q_index]; 3035 qman_base_addr = (u64) q->pq_dma_addr; 3036 gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3), 3037 qman_base_addr); 3038 if (i == 3) 3039 mme_offset = 0; 3040 } 3041 3042 /* Initializing lower CP for MME QMANs */ 3043 mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0; 3044 gaudi_init_mme_qman(hdev, mme_offset, 4, 0); 3045 gaudi_init_mme_qman(hdev, 0, 4, 0); 3046 3047 WREG32(mmMME2_QM_GLBL_CFG0, QMAN_MME_ENABLE); 3048 WREG32(mmMME0_QM_GLBL_CFG0, QMAN_MME_ENABLE); 3049 3050 gaudi->hw_cap_initialized |= HW_CAP_MME; 3051 } 3052 3053 static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, 3054 int qman_id, u64 qman_base_addr) 3055 { 3056 struct cpu_dyn_regs *dyn_regs = 3057 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3058 u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; 3059 u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; 3060 u32 tpc_qm_err_cfg, irq_handler_offset; 3061 u32 q_off, tpc_id; 3062 3063 mtr_base_en_lo = lower_32_bits(CFG_BASE + 3064 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3065 mtr_base_en_hi = upper_32_bits(CFG_BASE + 3066 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3067 so_base_en_lo = lower_32_bits(CFG_BASE + 3068 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 3069 so_base_en_hi = upper_32_bits(CFG_BASE + 3070 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 3071 mtr_base_ws_lo = lower_32_bits(CFG_BASE + 3072 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3073 mtr_base_ws_hi = upper_32_bits(CFG_BASE + 3074 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3075 so_base_ws_lo = lower_32_bits(CFG_BASE + 3076 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 3077 so_base_ws_hi = upper_32_bits(CFG_BASE + 3078 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 3079 3080 q_off = tpc_offset + qman_id * 4; 3081 3082 tpc_id = tpc_offset / 3083 (mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0); 3084 3085 if (qman_id < 4) { 3086 WREG32(mmTPC0_QM_PQ_BASE_LO_0 + q_off, 3087 lower_32_bits(qman_base_addr)); 3088 WREG32(mmTPC0_QM_PQ_BASE_HI_0 + q_off, 3089 upper_32_bits(qman_base_addr)); 3090 3091 WREG32(mmTPC0_QM_PQ_SIZE_0 + q_off, ilog2(TPC_QMAN_LENGTH)); 3092 WREG32(mmTPC0_QM_PQ_PI_0 + q_off, 0); 3093 WREG32(mmTPC0_QM_PQ_CI_0 + q_off, 0); 3094 3095 WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 3096 QMAN_CPDMA_SIZE_OFFSET); 3097 WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 3098 QMAN_CPDMA_SRC_OFFSET); 3099 WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 3100 QMAN_CPDMA_DST_OFFSET); 3101 } else { 3102 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 3103 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 3104 le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); 3105 3106 WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 3107 QMAN_LDMA_SIZE_OFFSET); 3108 WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 3109 QMAN_LDMA_SRC_OFFSET); 3110 WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 3111 QMAN_LDMA_DST_OFFSET); 3112 3113 /* Configure RAZWI IRQ */ 3114 tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; 3115 if (hdev->stop_on_err) 3116 tpc_qm_err_cfg |= 3117 TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; 3118 3119 WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg); 3120 3121 WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset, 3122 lower_32_bits(CFG_BASE + irq_handler_offset)); 3123 WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset, 3124 upper_32_bits(CFG_BASE + irq_handler_offset)); 3125 3126 WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset, 3127 gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id + 3128 tpc_id); 3129 3130 WREG32(mmTPC0_QM_ARB_ERR_MSG_EN + tpc_offset, 3131 QM_ARB_ERR_MSG_EN_MASK); 3132 3133 /* Set timeout to maximum */ 3134 WREG32(mmTPC0_QM_ARB_SLV_CHOISE_WDT + tpc_offset, GAUDI_ARB_WDT_TIMEOUT); 3135 3136 WREG32(mmTPC0_QM_GLBL_CFG1 + tpc_offset, 0); 3137 WREG32(mmTPC0_QM_GLBL_PROT + tpc_offset, 3138 QMAN_INTERNAL_MAKE_TRUSTED); 3139 } 3140 3141 WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); 3142 WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); 3143 WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); 3144 WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); 3145 3146 /* Configure TPC7 CP_MSG_BASE 2/3 for sync stream collective */ 3147 if (tpc_id == 6) { 3148 WREG32(mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, 3149 mtr_base_ws_lo); 3150 WREG32(mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, 3151 mtr_base_ws_hi); 3152 WREG32(mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, 3153 so_base_ws_lo); 3154 WREG32(mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, 3155 so_base_ws_hi); 3156 } 3157 } 3158 3159 static void gaudi_init_tpc_qmans(struct hl_device *hdev) 3160 { 3161 struct gaudi_device *gaudi = hdev->asic_specific; 3162 struct gaudi_internal_qman_info *q; 3163 u64 qman_base_addr; 3164 u32 so_base_hi, tpc_offset = 0; 3165 u32 tpc_delta = mmTPC1_CFG_SM_BASE_ADDRESS_HIGH - 3166 mmTPC0_CFG_SM_BASE_ADDRESS_HIGH; 3167 int i, tpc_id, internal_q_index; 3168 3169 if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK) 3170 return; 3171 3172 so_base_hi = upper_32_bits(CFG_BASE + 3173 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 3174 3175 for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) { 3176 for (i = 0 ; i < QMAN_STREAMS ; i++) { 3177 internal_q_index = GAUDI_QUEUE_ID_TPC_0_0 + 3178 tpc_id * QMAN_STREAMS + i; 3179 q = &gaudi->internal_qmans[internal_q_index]; 3180 qman_base_addr = (u64) q->pq_dma_addr; 3181 gaudi_init_tpc_qman(hdev, tpc_offset, i, 3182 qman_base_addr); 3183 3184 if (i == 3) { 3185 /* Initializing lower CP for TPC QMAN */ 3186 gaudi_init_tpc_qman(hdev, tpc_offset, 4, 0); 3187 3188 /* Enable the QMAN and TPC channel */ 3189 WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset, 3190 QMAN_TPC_ENABLE); 3191 } 3192 } 3193 3194 WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + tpc_id * tpc_delta, 3195 so_base_hi); 3196 3197 tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0; 3198 3199 gaudi->hw_cap_initialized |= 3200 FIELD_PREP(HW_CAP_TPC_MASK, 1 << tpc_id); 3201 } 3202 } 3203 3204 static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, 3205 int qman_id, u64 qman_base_addr, int nic_id) 3206 { 3207 struct cpu_dyn_regs *dyn_regs = 3208 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3209 u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; 3210 u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; 3211 u32 nic_qm_err_cfg, irq_handler_offset; 3212 u32 q_off; 3213 3214 mtr_base_en_lo = lower_32_bits((CFG_BASE & U32_MAX) + 3215 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3216 mtr_base_en_hi = upper_32_bits(CFG_BASE + 3217 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3218 so_base_en_lo = lower_32_bits((CFG_BASE & U32_MAX) + 3219 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 3220 so_base_en_hi = upper_32_bits(CFG_BASE + 3221 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); 3222 mtr_base_ws_lo = lower_32_bits((CFG_BASE & U32_MAX) + 3223 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3224 mtr_base_ws_hi = upper_32_bits(CFG_BASE + 3225 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 3226 so_base_ws_lo = lower_32_bits((CFG_BASE & U32_MAX) + 3227 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 3228 so_base_ws_hi = upper_32_bits(CFG_BASE + 3229 mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); 3230 3231 q_off = nic_offset + qman_id * 4; 3232 3233 WREG32(mmNIC0_QM0_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_base_addr)); 3234 WREG32(mmNIC0_QM0_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_base_addr)); 3235 3236 WREG32(mmNIC0_QM0_PQ_SIZE_0 + q_off, ilog2(NIC_QMAN_LENGTH)); 3237 WREG32(mmNIC0_QM0_PQ_PI_0 + q_off, 0); 3238 WREG32(mmNIC0_QM0_PQ_CI_0 + q_off, 0); 3239 3240 WREG32(mmNIC0_QM0_CP_LDMA_TSIZE_OFFSET_0 + q_off, 3241 QMAN_LDMA_SIZE_OFFSET); 3242 WREG32(mmNIC0_QM0_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 3243 QMAN_LDMA_SRC_OFFSET); 3244 WREG32(mmNIC0_QM0_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 3245 QMAN_LDMA_DST_OFFSET); 3246 3247 WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); 3248 WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); 3249 WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); 3250 WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); 3251 3252 /* Configure NIC CP_MSG_BASE 2/3 for sync stream collective */ 3253 WREG32(mmNIC0_QM0_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo); 3254 WREG32(mmNIC0_QM0_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi); 3255 WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo); 3256 WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi); 3257 3258 if (qman_id == 0) { 3259 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 3260 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 3261 le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); 3262 3263 /* Configure RAZWI IRQ */ 3264 nic_qm_err_cfg = NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; 3265 if (hdev->stop_on_err) 3266 nic_qm_err_cfg |= 3267 NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; 3268 3269 WREG32(mmNIC0_QM0_GLBL_ERR_CFG + nic_offset, nic_qm_err_cfg); 3270 3271 WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_LO + nic_offset, 3272 lower_32_bits(CFG_BASE + irq_handler_offset)); 3273 WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_HI + nic_offset, 3274 upper_32_bits(CFG_BASE + irq_handler_offset)); 3275 3276 WREG32(mmNIC0_QM0_GLBL_ERR_WDATA + nic_offset, 3277 gaudi_irq_map_table[GAUDI_EVENT_NIC0_QM0].cpu_id + 3278 nic_id); 3279 3280 WREG32(mmNIC0_QM0_ARB_ERR_MSG_EN + nic_offset, 3281 QM_ARB_ERR_MSG_EN_MASK); 3282 3283 /* Set timeout to maximum */ 3284 WREG32(mmNIC0_QM0_ARB_SLV_CHOISE_WDT + nic_offset, GAUDI_ARB_WDT_TIMEOUT); 3285 3286 WREG32(mmNIC0_QM0_GLBL_CFG1 + nic_offset, 0); 3287 WREG32(mmNIC0_QM0_GLBL_PROT + nic_offset, 3288 QMAN_INTERNAL_MAKE_TRUSTED); 3289 } 3290 } 3291 3292 static void gaudi_init_nic_qmans(struct hl_device *hdev) 3293 { 3294 struct gaudi_device *gaudi = hdev->asic_specific; 3295 struct gaudi_internal_qman_info *q; 3296 u64 qman_base_addr; 3297 u32 nic_offset = 0; 3298 u32 nic_delta_between_qmans = 3299 mmNIC0_QM1_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; 3300 u32 nic_delta_between_nics = 3301 mmNIC1_QM0_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; 3302 int i, nic_id, internal_q_index; 3303 3304 if (!hdev->nic_ports_mask) 3305 return; 3306 3307 if (gaudi->hw_cap_initialized & HW_CAP_NIC_MASK) 3308 return; 3309 3310 dev_dbg(hdev->dev, "Initializing NIC QMANs\n"); 3311 3312 for (nic_id = 0 ; nic_id < NIC_NUMBER_OF_ENGINES ; nic_id++) { 3313 if (!(hdev->nic_ports_mask & (1 << nic_id))) { 3314 nic_offset += nic_delta_between_qmans; 3315 if (nic_id & 1) { 3316 nic_offset -= (nic_delta_between_qmans * 2); 3317 nic_offset += nic_delta_between_nics; 3318 } 3319 continue; 3320 } 3321 3322 for (i = 0 ; i < QMAN_STREAMS ; i++) { 3323 internal_q_index = GAUDI_QUEUE_ID_NIC_0_0 + 3324 nic_id * QMAN_STREAMS + i; 3325 q = &gaudi->internal_qmans[internal_q_index]; 3326 qman_base_addr = (u64) q->pq_dma_addr; 3327 gaudi_init_nic_qman(hdev, nic_offset, (i & 0x3), 3328 qman_base_addr, nic_id); 3329 } 3330 3331 /* Enable the QMAN */ 3332 WREG32(mmNIC0_QM0_GLBL_CFG0 + nic_offset, NIC_QMAN_ENABLE); 3333 3334 nic_offset += nic_delta_between_qmans; 3335 if (nic_id & 1) { 3336 nic_offset -= (nic_delta_between_qmans * 2); 3337 nic_offset += nic_delta_between_nics; 3338 } 3339 3340 gaudi->hw_cap_initialized |= 1 << (HW_CAP_NIC_SHIFT + nic_id); 3341 } 3342 } 3343 3344 static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev) 3345 { 3346 struct gaudi_device *gaudi = hdev->asic_specific; 3347 3348 if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)) 3349 return; 3350 3351 WREG32(mmDMA0_QM_GLBL_CFG0, 0); 3352 WREG32(mmDMA1_QM_GLBL_CFG0, 0); 3353 WREG32(mmDMA5_QM_GLBL_CFG0, 0); 3354 } 3355 3356 static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev) 3357 { 3358 struct gaudi_device *gaudi = hdev->asic_specific; 3359 3360 if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)) 3361 return; 3362 3363 WREG32(mmDMA2_QM_GLBL_CFG0, 0); 3364 WREG32(mmDMA3_QM_GLBL_CFG0, 0); 3365 WREG32(mmDMA4_QM_GLBL_CFG0, 0); 3366 WREG32(mmDMA6_QM_GLBL_CFG0, 0); 3367 WREG32(mmDMA7_QM_GLBL_CFG0, 0); 3368 } 3369 3370 static void gaudi_disable_mme_qmans(struct hl_device *hdev) 3371 { 3372 struct gaudi_device *gaudi = hdev->asic_specific; 3373 3374 if (!(gaudi->hw_cap_initialized & HW_CAP_MME)) 3375 return; 3376 3377 WREG32(mmMME2_QM_GLBL_CFG0, 0); 3378 WREG32(mmMME0_QM_GLBL_CFG0, 0); 3379 } 3380 3381 static void gaudi_disable_tpc_qmans(struct hl_device *hdev) 3382 { 3383 struct gaudi_device *gaudi = hdev->asic_specific; 3384 u32 tpc_offset = 0; 3385 int tpc_id; 3386 3387 if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)) 3388 return; 3389 3390 for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) { 3391 WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset, 0); 3392 tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0; 3393 } 3394 } 3395 3396 static void gaudi_disable_nic_qmans(struct hl_device *hdev) 3397 { 3398 struct gaudi_device *gaudi = hdev->asic_specific; 3399 u32 nic_mask, nic_offset = 0; 3400 u32 nic_delta_between_qmans = 3401 mmNIC0_QM1_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; 3402 u32 nic_delta_between_nics = 3403 mmNIC1_QM0_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; 3404 int nic_id; 3405 3406 for (nic_id = 0 ; nic_id < NIC_NUMBER_OF_ENGINES ; nic_id++) { 3407 nic_mask = 1 << (HW_CAP_NIC_SHIFT + nic_id); 3408 3409 if (gaudi->hw_cap_initialized & nic_mask) 3410 WREG32(mmNIC0_QM0_GLBL_CFG0 + nic_offset, 0); 3411 3412 nic_offset += nic_delta_between_qmans; 3413 if (nic_id & 1) { 3414 nic_offset -= (nic_delta_between_qmans * 2); 3415 nic_offset += nic_delta_between_nics; 3416 } 3417 } 3418 } 3419 3420 static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev) 3421 { 3422 struct gaudi_device *gaudi = hdev->asic_specific; 3423 3424 if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)) 3425 return; 3426 3427 /* Stop upper CPs of QMANs 0.0 to 1.3 and 5.0 to 5.3 */ 3428 WREG32(mmDMA0_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3429 WREG32(mmDMA1_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3430 WREG32(mmDMA5_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3431 } 3432 3433 static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev) 3434 { 3435 struct gaudi_device *gaudi = hdev->asic_specific; 3436 3437 if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)) 3438 return; 3439 3440 /* Stop CPs of HBM DMA QMANs */ 3441 3442 WREG32(mmDMA2_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3443 WREG32(mmDMA3_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3444 WREG32(mmDMA4_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3445 WREG32(mmDMA6_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3446 WREG32(mmDMA7_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3447 } 3448 3449 static void gaudi_stop_mme_qmans(struct hl_device *hdev) 3450 { 3451 struct gaudi_device *gaudi = hdev->asic_specific; 3452 3453 if (!(gaudi->hw_cap_initialized & HW_CAP_MME)) 3454 return; 3455 3456 /* Stop CPs of MME QMANs */ 3457 WREG32(mmMME2_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3458 WREG32(mmMME0_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3459 } 3460 3461 static void gaudi_stop_tpc_qmans(struct hl_device *hdev) 3462 { 3463 struct gaudi_device *gaudi = hdev->asic_specific; 3464 3465 if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)) 3466 return; 3467 3468 WREG32(mmTPC0_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3469 WREG32(mmTPC1_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3470 WREG32(mmTPC2_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3471 WREG32(mmTPC3_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3472 WREG32(mmTPC4_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3473 WREG32(mmTPC5_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3474 WREG32(mmTPC6_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3475 WREG32(mmTPC7_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 3476 } 3477 3478 static void gaudi_stop_nic_qmans(struct hl_device *hdev) 3479 { 3480 struct gaudi_device *gaudi = hdev->asic_specific; 3481 3482 /* Stop upper CPs of QMANs */ 3483 3484 if (gaudi->hw_cap_initialized & HW_CAP_NIC0) 3485 WREG32(mmNIC0_QM0_GLBL_CFG1, 3486 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3487 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3488 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3489 3490 if (gaudi->hw_cap_initialized & HW_CAP_NIC1) 3491 WREG32(mmNIC0_QM1_GLBL_CFG1, 3492 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3493 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3494 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3495 3496 if (gaudi->hw_cap_initialized & HW_CAP_NIC2) 3497 WREG32(mmNIC1_QM0_GLBL_CFG1, 3498 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3499 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3500 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3501 3502 if (gaudi->hw_cap_initialized & HW_CAP_NIC3) 3503 WREG32(mmNIC1_QM1_GLBL_CFG1, 3504 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3505 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3506 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3507 3508 if (gaudi->hw_cap_initialized & HW_CAP_NIC4) 3509 WREG32(mmNIC2_QM0_GLBL_CFG1, 3510 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3511 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3512 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3513 3514 if (gaudi->hw_cap_initialized & HW_CAP_NIC5) 3515 WREG32(mmNIC2_QM1_GLBL_CFG1, 3516 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3517 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3518 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3519 3520 if (gaudi->hw_cap_initialized & HW_CAP_NIC6) 3521 WREG32(mmNIC3_QM0_GLBL_CFG1, 3522 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3523 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3524 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3525 3526 if (gaudi->hw_cap_initialized & HW_CAP_NIC7) 3527 WREG32(mmNIC3_QM1_GLBL_CFG1, 3528 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3529 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3530 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3531 3532 if (gaudi->hw_cap_initialized & HW_CAP_NIC8) 3533 WREG32(mmNIC4_QM0_GLBL_CFG1, 3534 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3535 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3536 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3537 3538 if (gaudi->hw_cap_initialized & HW_CAP_NIC9) 3539 WREG32(mmNIC4_QM1_GLBL_CFG1, 3540 NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK | 3541 NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK | 3542 NIC0_QM0_GLBL_CFG1_CP_STOP_MASK); 3543 } 3544 3545 static void gaudi_pci_dma_stall(struct hl_device *hdev) 3546 { 3547 struct gaudi_device *gaudi = hdev->asic_specific; 3548 3549 if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)) 3550 return; 3551 3552 WREG32(mmDMA0_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3553 WREG32(mmDMA1_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3554 WREG32(mmDMA5_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3555 } 3556 3557 static void gaudi_hbm_dma_stall(struct hl_device *hdev) 3558 { 3559 struct gaudi_device *gaudi = hdev->asic_specific; 3560 3561 if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)) 3562 return; 3563 3564 WREG32(mmDMA2_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3565 WREG32(mmDMA3_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3566 WREG32(mmDMA4_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3567 WREG32(mmDMA6_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3568 WREG32(mmDMA7_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); 3569 } 3570 3571 static void gaudi_mme_stall(struct hl_device *hdev) 3572 { 3573 struct gaudi_device *gaudi = hdev->asic_specific; 3574 3575 if (!(gaudi->hw_cap_initialized & HW_CAP_MME)) 3576 return; 3577 3578 /* WA for H3-1800 bug: do ACC and SBAB writes twice */ 3579 WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3580 WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3581 WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3582 WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3583 WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3584 WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3585 WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3586 WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3587 WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3588 WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3589 WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3590 WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3591 WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3592 WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); 3593 WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3594 WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); 3595 } 3596 3597 static void gaudi_tpc_stall(struct hl_device *hdev) 3598 { 3599 struct gaudi_device *gaudi = hdev->asic_specific; 3600 3601 if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)) 3602 return; 3603 3604 WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3605 WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3606 WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3607 WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3608 WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3609 WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3610 WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3611 WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 3612 } 3613 3614 static void gaudi_disable_clock_gating(struct hl_device *hdev) 3615 { 3616 u32 qman_offset; 3617 int i; 3618 3619 if (hdev->asic_prop.fw_security_enabled) 3620 return; 3621 3622 for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) { 3623 WREG32(mmDMA0_QM_CGM_CFG + qman_offset, 0); 3624 WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, 0); 3625 3626 qman_offset += (mmDMA1_QM_CGM_CFG - mmDMA0_QM_CGM_CFG); 3627 } 3628 3629 WREG32(mmMME0_QM_CGM_CFG, 0); 3630 WREG32(mmMME0_QM_CGM_CFG1, 0); 3631 WREG32(mmMME2_QM_CGM_CFG, 0); 3632 WREG32(mmMME2_QM_CGM_CFG1, 0); 3633 3634 for (i = 0, qman_offset = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) { 3635 WREG32(mmTPC0_QM_CGM_CFG + qman_offset, 0); 3636 WREG32(mmTPC0_QM_CGM_CFG1 + qman_offset, 0); 3637 3638 qman_offset += (mmTPC1_QM_CGM_CFG - mmTPC0_QM_CGM_CFG); 3639 } 3640 } 3641 3642 static void gaudi_enable_timestamp(struct hl_device *hdev) 3643 { 3644 /* Disable the timestamp counter */ 3645 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); 3646 3647 /* Zero the lower/upper parts of the 64-bit counter */ 3648 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0); 3649 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0); 3650 3651 /* Enable the counter */ 3652 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1); 3653 } 3654 3655 static void gaudi_disable_timestamp(struct hl_device *hdev) 3656 { 3657 /* Disable the timestamp counter */ 3658 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); 3659 } 3660 3661 static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 3662 { 3663 u32 wait_timeout_ms; 3664 3665 if (hdev->pldm) 3666 wait_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC; 3667 else 3668 wait_timeout_ms = GAUDI_RESET_WAIT_MSEC; 3669 3670 if (fw_reset) 3671 goto skip_engines; 3672 3673 gaudi_stop_nic_qmans(hdev); 3674 gaudi_stop_mme_qmans(hdev); 3675 gaudi_stop_tpc_qmans(hdev); 3676 gaudi_stop_hbm_dma_qmans(hdev); 3677 gaudi_stop_pci_dma_qmans(hdev); 3678 3679 msleep(wait_timeout_ms); 3680 3681 gaudi_pci_dma_stall(hdev); 3682 gaudi_hbm_dma_stall(hdev); 3683 gaudi_tpc_stall(hdev); 3684 gaudi_mme_stall(hdev); 3685 3686 msleep(wait_timeout_ms); 3687 3688 gaudi_disable_nic_qmans(hdev); 3689 gaudi_disable_mme_qmans(hdev); 3690 gaudi_disable_tpc_qmans(hdev); 3691 gaudi_disable_hbm_dma_qmans(hdev); 3692 gaudi_disable_pci_dma_qmans(hdev); 3693 3694 gaudi_disable_timestamp(hdev); 3695 3696 skip_engines: 3697 gaudi_disable_msi(hdev); 3698 } 3699 3700 static int gaudi_mmu_init(struct hl_device *hdev) 3701 { 3702 struct asic_fixed_properties *prop = &hdev->asic_prop; 3703 struct gaudi_device *gaudi = hdev->asic_specific; 3704 u64 hop0_addr; 3705 int rc, i; 3706 3707 if (!hdev->mmu_enable) 3708 return 0; 3709 3710 if (gaudi->hw_cap_initialized & HW_CAP_MMU) 3711 return 0; 3712 3713 for (i = 0 ; i < prop->max_asid ; i++) { 3714 hop0_addr = prop->mmu_pgt_addr + 3715 (i * prop->mmu_hop_table_size); 3716 3717 rc = gaudi_mmu_update_asid_hop0_addr(hdev, i, hop0_addr); 3718 if (rc) { 3719 dev_err(hdev->dev, 3720 "failed to set hop0 addr for asid %d\n", i); 3721 goto err; 3722 } 3723 } 3724 3725 /* init MMU cache manage page */ 3726 WREG32(mmSTLB_CACHE_INV_BASE_39_8, prop->mmu_cache_mng_addr >> 8); 3727 WREG32(mmSTLB_CACHE_INV_BASE_49_40, prop->mmu_cache_mng_addr >> 40); 3728 3729 /* mem cache invalidation */ 3730 WREG32(mmSTLB_MEM_CACHE_INVALIDATION, 1); 3731 3732 hl_mmu_invalidate_cache(hdev, true, 0); 3733 3734 WREG32(mmMMU_UP_MMU_ENABLE, 1); 3735 WREG32(mmMMU_UP_SPI_MASK, 0xF); 3736 3737 WREG32(mmSTLB_HOP_CONFIGURATION, 0x30440); 3738 3739 /* 3740 * The H/W expects the first PI after init to be 1. After wraparound 3741 * we'll write 0. 3742 */ 3743 gaudi->mmu_cache_inv_pi = 1; 3744 3745 gaudi->hw_cap_initialized |= HW_CAP_MMU; 3746 3747 return 0; 3748 3749 err: 3750 return rc; 3751 } 3752 3753 static int gaudi_load_firmware_to_device(struct hl_device *hdev) 3754 { 3755 void __iomem *dst; 3756 3757 dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET; 3758 3759 return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst, 0, 0); 3760 } 3761 3762 static int gaudi_load_boot_fit_to_device(struct hl_device *hdev) 3763 { 3764 void __iomem *dst; 3765 3766 dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET; 3767 3768 return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst, 0, 0); 3769 } 3770 3771 static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev) 3772 { 3773 struct dynamic_fw_load_mgr *dynamic_loader; 3774 struct cpu_dyn_regs *dyn_regs; 3775 3776 dynamic_loader = &hdev->fw_loader.dynamic_loader; 3777 3778 /* 3779 * here we update initial values for few specific dynamic regs (as 3780 * before reading the first descriptor from FW those value has to be 3781 * hard-coded) in later stages of the protocol those values will be 3782 * updated automatically by reading the FW descriptor so data there 3783 * will always be up-to-date 3784 */ 3785 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 3786 dyn_regs->kmd_msg_to_cpu = 3787 cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 3788 dyn_regs->cpu_cmd_status_to_host = 3789 cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 3790 3791 dynamic_loader->wait_for_bl_timeout = GAUDI_WAIT_FOR_BL_TIMEOUT_USEC; 3792 } 3793 3794 static void gaudi_init_static_firmware_loader(struct hl_device *hdev) 3795 { 3796 struct static_fw_load_mgr *static_loader; 3797 3798 static_loader = &hdev->fw_loader.static_loader; 3799 3800 static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; 3801 static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; 3802 static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; 3803 static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; 3804 static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 3805 static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; 3806 static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; 3807 static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; 3808 static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; 3809 static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; 3810 static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; 3811 static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); 3812 static_loader->cpu_reset_wait_msec = hdev->pldm ? 3813 GAUDI_PLDM_RESET_WAIT_MSEC : 3814 GAUDI_CPU_RESET_WAIT_MSEC; 3815 } 3816 3817 static void gaudi_init_firmware_preload_params(struct hl_device *hdev) 3818 { 3819 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 3820 3821 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 3822 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 3823 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 3824 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 3825 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 3826 pre_fw_load->wait_for_preboot_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC; 3827 } 3828 3829 static void gaudi_init_firmware_loader(struct hl_device *hdev) 3830 { 3831 struct asic_fixed_properties *prop = &hdev->asic_prop; 3832 struct fw_load_mgr *fw_loader = &hdev->fw_loader; 3833 3834 /* fill common fields */ 3835 fw_loader->fw_comp_loaded = FW_TYPE_NONE; 3836 fw_loader->boot_fit_img.image_name = GAUDI_BOOT_FIT_FILE; 3837 fw_loader->linux_img.image_name = GAUDI_LINUX_FW_FILE; 3838 fw_loader->cpu_timeout = GAUDI_CPU_TIMEOUT_USEC; 3839 fw_loader->boot_fit_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC; 3840 fw_loader->skip_bmc = !hdev->bmc_enable; 3841 fw_loader->sram_bar_id = SRAM_BAR_ID; 3842 fw_loader->dram_bar_id = HBM_BAR_ID; 3843 3844 if (prop->dynamic_fw_load) 3845 gaudi_init_dynamic_firmware_loader(hdev); 3846 else 3847 gaudi_init_static_firmware_loader(hdev); 3848 } 3849 3850 static int gaudi_init_cpu(struct hl_device *hdev) 3851 { 3852 struct gaudi_device *gaudi = hdev->asic_specific; 3853 int rc; 3854 3855 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 3856 return 0; 3857 3858 if (gaudi->hw_cap_initialized & HW_CAP_CPU) 3859 return 0; 3860 3861 /* 3862 * The device CPU works with 40 bits addresses. 3863 * This register sets the extension to 50 bits. 3864 */ 3865 if (!hdev->asic_prop.fw_security_enabled) 3866 WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr); 3867 3868 rc = hl_fw_init_cpu(hdev); 3869 3870 if (rc) 3871 return rc; 3872 3873 gaudi->hw_cap_initialized |= HW_CAP_CPU; 3874 3875 return 0; 3876 } 3877 3878 static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) 3879 { 3880 struct cpu_dyn_regs *dyn_regs = 3881 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3882 struct asic_fixed_properties *prop = &hdev->asic_prop; 3883 struct gaudi_device *gaudi = hdev->asic_specific; 3884 u32 status, irq_handler_offset; 3885 struct hl_eq *eq; 3886 struct hl_hw_queue *cpu_pq = 3887 &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ]; 3888 int err; 3889 3890 if (!hdev->cpu_queues_enable) 3891 return 0; 3892 3893 if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q) 3894 return 0; 3895 3896 eq = &hdev->event_queue; 3897 3898 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 3899 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 3900 3901 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 3902 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 3903 3904 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, 3905 lower_32_bits(hdev->cpu_accessible_dma_address)); 3906 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, 3907 upper_32_bits(hdev->cpu_accessible_dma_address)); 3908 3909 WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 3910 WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 3911 WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 3912 3913 /* Used for EQ CI */ 3914 WREG32(mmCPU_IF_EQ_RD_OFFS, 0); 3915 3916 WREG32(mmCPU_IF_PF_PQ_PI, 0); 3917 3918 if (gaudi->multi_msi_mode) 3919 WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP); 3920 else 3921 WREG32(mmCPU_IF_QUEUE_INIT, 3922 PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI); 3923 3924 irq_handler_offset = prop->gic_interrupts_enable ? 3925 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 3926 le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); 3927 3928 WREG32(irq_handler_offset, 3929 gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id); 3930 3931 err = hl_poll_timeout( 3932 hdev, 3933 mmCPU_IF_QUEUE_INIT, 3934 status, 3935 (status == PQ_INIT_STATUS_READY_FOR_HOST), 3936 1000, 3937 cpu_timeout); 3938 3939 if (err) { 3940 dev_err(hdev->dev, 3941 "Failed to communicate with Device CPU (CPU-CP timeout)\n"); 3942 return -EIO; 3943 } 3944 3945 /* update FW application security bits */ 3946 if (prop->fw_cpu_boot_dev_sts0_valid) 3947 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 3948 if (prop->fw_cpu_boot_dev_sts1_valid) 3949 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 3950 3951 gaudi->hw_cap_initialized |= HW_CAP_CPU_Q; 3952 return 0; 3953 } 3954 3955 static void gaudi_pre_hw_init(struct hl_device *hdev) 3956 { 3957 /* Perform read from the device to make sure device is up */ 3958 RREG32(mmHW_STATE); 3959 3960 if (!hdev->asic_prop.fw_security_enabled) { 3961 /* Set the access through PCI bars (Linux driver only) as 3962 * secured 3963 */ 3964 WREG32(mmPCIE_WRAP_LBW_PROT_OVR, 3965 (PCIE_WRAP_LBW_PROT_OVR_RD_EN_MASK | 3966 PCIE_WRAP_LBW_PROT_OVR_WR_EN_MASK)); 3967 3968 /* Perform read to flush the waiting writes to ensure 3969 * configuration was set in the device 3970 */ 3971 RREG32(mmPCIE_WRAP_LBW_PROT_OVR); 3972 } 3973 3974 /* 3975 * Let's mark in the H/W that we have reached this point. We check 3976 * this value in the reset_before_init function to understand whether 3977 * we need to reset the chip before doing H/W init. This register is 3978 * cleared by the H/W upon H/W reset 3979 */ 3980 WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 3981 } 3982 3983 static int gaudi_hw_init(struct hl_device *hdev) 3984 { 3985 struct gaudi_device *gaudi = hdev->asic_specific; 3986 int rc; 3987 3988 gaudi_pre_hw_init(hdev); 3989 3990 /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. 3991 * So we set it here and if anyone tries to move it later to 3992 * a different address, there will be an error 3993 */ 3994 if (hdev->asic_prop.iatu_done_by_fw) 3995 gaudi->hbm_bar_cur_addr = DRAM_PHYS_BASE; 3996 3997 /* 3998 * Before pushing u-boot/linux to device, need to set the hbm bar to 3999 * base address of dram 4000 */ 4001 if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 4002 dev_err(hdev->dev, 4003 "failed to map HBM bar to DRAM base address\n"); 4004 return -EIO; 4005 } 4006 4007 rc = gaudi_init_cpu(hdev); 4008 if (rc) { 4009 dev_err(hdev->dev, "failed to initialize CPU\n"); 4010 return rc; 4011 } 4012 4013 /* In case the clock gating was enabled in preboot we need to disable 4014 * it here before touching the MME/TPC registers. 4015 */ 4016 gaudi_disable_clock_gating(hdev); 4017 4018 /* SRAM scrambler must be initialized after CPU is running from HBM */ 4019 gaudi_init_scrambler_sram(hdev); 4020 4021 /* This is here just in case we are working without CPU */ 4022 gaudi_init_scrambler_hbm(hdev); 4023 4024 gaudi_init_golden_registers(hdev); 4025 4026 rc = gaudi_mmu_init(hdev); 4027 if (rc) 4028 return rc; 4029 4030 gaudi_init_security(hdev); 4031 4032 gaudi_init_pci_dma_qmans(hdev); 4033 4034 gaudi_init_hbm_dma_qmans(hdev); 4035 4036 gaudi_init_mme_qmans(hdev); 4037 4038 gaudi_init_tpc_qmans(hdev); 4039 4040 gaudi_init_nic_qmans(hdev); 4041 4042 gaudi_enable_timestamp(hdev); 4043 4044 /* MSI must be enabled before CPU queues and NIC are initialized */ 4045 rc = gaudi_enable_msi(hdev); 4046 if (rc) 4047 goto disable_queues; 4048 4049 /* must be called after MSI was enabled */ 4050 rc = gaudi_init_cpu_queues(hdev, GAUDI_CPU_TIMEOUT_USEC); 4051 if (rc) { 4052 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", 4053 rc); 4054 goto disable_msi; 4055 } 4056 4057 /* Perform read from the device to flush all configuration */ 4058 RREG32(mmHW_STATE); 4059 4060 return 0; 4061 4062 disable_msi: 4063 gaudi_disable_msi(hdev); 4064 disable_queues: 4065 gaudi_disable_mme_qmans(hdev); 4066 gaudi_disable_pci_dma_qmans(hdev); 4067 4068 return rc; 4069 } 4070 4071 static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 4072 { 4073 struct cpu_dyn_regs *dyn_regs = 4074 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4075 u32 status, reset_timeout_ms, cpu_timeout_ms, irq_handler_offset; 4076 struct gaudi_device *gaudi = hdev->asic_specific; 4077 bool driver_performs_reset; 4078 4079 if (!hard_reset) { 4080 dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n"); 4081 return; 4082 } 4083 4084 if (hdev->pldm) { 4085 reset_timeout_ms = GAUDI_PLDM_HRESET_TIMEOUT_MSEC; 4086 cpu_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC; 4087 } else { 4088 reset_timeout_ms = GAUDI_RESET_TIMEOUT_MSEC; 4089 cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC; 4090 } 4091 4092 if (fw_reset) { 4093 dev_dbg(hdev->dev, 4094 "Firmware performs HARD reset, going to wait %dms\n", 4095 reset_timeout_ms); 4096 4097 goto skip_reset; 4098 } 4099 4100 driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled && 4101 !hdev->asic_prop.hard_reset_done_by_fw); 4102 4103 /* Set device to handle FLR by H/W as we will put the device CPU to 4104 * halt mode 4105 */ 4106 if (driver_performs_reset) 4107 WREG32(mmPCIE_AUX_FLR_CTRL, (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | 4108 PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); 4109 4110 /* If linux is loaded in the device CPU we need to communicate with it 4111 * via the GIC. Otherwise, we need to use COMMS or the MSG_TO_CPU 4112 * registers in case of old F/Ws 4113 */ 4114 if (hdev->fw_loader.fw_comp_loaded & FW_TYPE_LINUX) { 4115 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 4116 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 4117 le32_to_cpu(dyn_regs->gic_host_halt_irq); 4118 4119 WREG32(irq_handler_offset, 4120 gaudi_irq_map_table[GAUDI_EVENT_HALT_MACHINE].cpu_id); 4121 4122 /* This is a hail-mary attempt to revive the card in the small chance that the 4123 * f/w has experienced a watchdog event, which caused it to return back to preboot. 4124 * In that case, triggering reset through GIC won't help. We need to trigger the 4125 * reset as if Linux wasn't loaded. 4126 * 4127 * We do it only if the reset cause was HB, because that would be the indication 4128 * of such an event. 4129 * 4130 * In case watchdog hasn't expired but we still got HB, then this won't do any 4131 * damage. 4132 */ 4133 if (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT) { 4134 if (hdev->asic_prop.hard_reset_done_by_fw) 4135 hl_fw_ask_hard_reset_without_linux(hdev); 4136 else 4137 hl_fw_ask_halt_machine_without_linux(hdev); 4138 } 4139 } else { 4140 if (hdev->asic_prop.hard_reset_done_by_fw) 4141 hl_fw_ask_hard_reset_without_linux(hdev); 4142 else 4143 hl_fw_ask_halt_machine_without_linux(hdev); 4144 } 4145 4146 if (driver_performs_reset) { 4147 4148 /* Configure the reset registers. Must be done as early as 4149 * possible in case we fail during H/W initialization 4150 */ 4151 WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_H, 4152 (CFG_RST_H_DMA_MASK | 4153 CFG_RST_H_MME_MASK | 4154 CFG_RST_H_SM_MASK | 4155 CFG_RST_H_TPC_7_MASK)); 4156 4157 WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_L, CFG_RST_L_TPC_MASK); 4158 4159 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H, 4160 (CFG_RST_H_HBM_MASK | 4161 CFG_RST_H_TPC_7_MASK | 4162 CFG_RST_H_NIC_MASK | 4163 CFG_RST_H_SM_MASK | 4164 CFG_RST_H_DMA_MASK | 4165 CFG_RST_H_MME_MASK | 4166 CFG_RST_H_CPU_MASK | 4167 CFG_RST_H_MMU_MASK)); 4168 4169 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L, 4170 (CFG_RST_L_IF_MASK | 4171 CFG_RST_L_PSOC_MASK | 4172 CFG_RST_L_TPC_MASK)); 4173 4174 msleep(cpu_timeout_ms); 4175 4176 /* Tell ASIC not to re-initialize PCIe */ 4177 WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC); 4178 4179 /* Restart BTL/BLR upon hard-reset */ 4180 WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1); 4181 4182 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST, 4183 1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT); 4184 4185 dev_dbg(hdev->dev, 4186 "Issued HARD reset command, going to wait %dms\n", 4187 reset_timeout_ms); 4188 } else { 4189 dev_dbg(hdev->dev, 4190 "Firmware performs HARD reset, going to wait %dms\n", 4191 reset_timeout_ms); 4192 } 4193 4194 skip_reset: 4195 /* 4196 * After hard reset, we can't poll the BTM_FSM register because the PSOC 4197 * itself is in reset. Need to wait until the reset is deasserted 4198 */ 4199 msleep(reset_timeout_ms); 4200 4201 status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM); 4202 if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK) 4203 dev_err(hdev->dev, 4204 "Timeout while waiting for device to reset 0x%x\n", 4205 status); 4206 4207 if (gaudi) { 4208 gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | HW_CAP_HBM | 4209 HW_CAP_PCI_DMA | HW_CAP_MME | HW_CAP_TPC_MASK | 4210 HW_CAP_HBM_DMA | HW_CAP_PLL | HW_CAP_NIC_MASK | 4211 HW_CAP_MMU | HW_CAP_SRAM_SCRAMBLER | 4212 HW_CAP_HBM_SCRAMBLER); 4213 4214 memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat)); 4215 4216 hdev->device_cpu_is_halted = false; 4217 } 4218 } 4219 4220 static int gaudi_suspend(struct hl_device *hdev) 4221 { 4222 int rc; 4223 4224 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 4225 if (rc) 4226 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); 4227 4228 return rc; 4229 } 4230 4231 static int gaudi_resume(struct hl_device *hdev) 4232 { 4233 return gaudi_init_iatu(hdev); 4234 } 4235 4236 static int gaudi_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 4237 void *cpu_addr, dma_addr_t dma_addr, size_t size) 4238 { 4239 int rc; 4240 4241 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 4242 VM_DONTCOPY | VM_NORESERVE); 4243 4244 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, 4245 (dma_addr - HOST_PHYS_BASE), size); 4246 if (rc) 4247 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 4248 4249 return rc; 4250 } 4251 4252 static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 4253 { 4254 struct cpu_dyn_regs *dyn_regs = 4255 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4256 u32 db_reg_offset, db_value, dma_qm_offset, q_off, irq_handler_offset; 4257 struct gaudi_device *gaudi = hdev->asic_specific; 4258 bool invalid_queue = false; 4259 int dma_id; 4260 4261 switch (hw_queue_id) { 4262 case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3: 4263 dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1]; 4264 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4265 q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4; 4266 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4267 break; 4268 4269 case GAUDI_QUEUE_ID_DMA_1_0...GAUDI_QUEUE_ID_DMA_1_3: 4270 dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2]; 4271 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4272 q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4; 4273 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4274 break; 4275 4276 case GAUDI_QUEUE_ID_DMA_2_0...GAUDI_QUEUE_ID_DMA_2_3: 4277 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1]; 4278 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4279 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; 4280 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4281 break; 4282 4283 case GAUDI_QUEUE_ID_DMA_3_0...GAUDI_QUEUE_ID_DMA_3_3: 4284 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_2]; 4285 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4286 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; 4287 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4288 break; 4289 4290 case GAUDI_QUEUE_ID_DMA_4_0...GAUDI_QUEUE_ID_DMA_4_3: 4291 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_3]; 4292 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4293 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; 4294 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4295 break; 4296 4297 case GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3: 4298 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4]; 4299 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4300 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; 4301 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4302 break; 4303 4304 case GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3: 4305 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5]; 4306 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4307 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; 4308 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4309 break; 4310 4311 case GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3: 4312 dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_6]; 4313 dma_qm_offset = dma_id * DMA_QMAN_OFFSET; 4314 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; 4315 db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; 4316 break; 4317 4318 case GAUDI_QUEUE_ID_CPU_PQ: 4319 if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q) 4320 db_reg_offset = mmCPU_IF_PF_PQ_PI; 4321 else 4322 invalid_queue = true; 4323 break; 4324 4325 case GAUDI_QUEUE_ID_MME_0_0: 4326 db_reg_offset = mmMME2_QM_PQ_PI_0; 4327 break; 4328 4329 case GAUDI_QUEUE_ID_MME_0_1: 4330 db_reg_offset = mmMME2_QM_PQ_PI_1; 4331 break; 4332 4333 case GAUDI_QUEUE_ID_MME_0_2: 4334 db_reg_offset = mmMME2_QM_PQ_PI_2; 4335 break; 4336 4337 case GAUDI_QUEUE_ID_MME_0_3: 4338 db_reg_offset = mmMME2_QM_PQ_PI_3; 4339 break; 4340 4341 case GAUDI_QUEUE_ID_MME_1_0: 4342 db_reg_offset = mmMME0_QM_PQ_PI_0; 4343 break; 4344 4345 case GAUDI_QUEUE_ID_MME_1_1: 4346 db_reg_offset = mmMME0_QM_PQ_PI_1; 4347 break; 4348 4349 case GAUDI_QUEUE_ID_MME_1_2: 4350 db_reg_offset = mmMME0_QM_PQ_PI_2; 4351 break; 4352 4353 case GAUDI_QUEUE_ID_MME_1_3: 4354 db_reg_offset = mmMME0_QM_PQ_PI_3; 4355 break; 4356 4357 case GAUDI_QUEUE_ID_TPC_0_0: 4358 db_reg_offset = mmTPC0_QM_PQ_PI_0; 4359 break; 4360 4361 case GAUDI_QUEUE_ID_TPC_0_1: 4362 db_reg_offset = mmTPC0_QM_PQ_PI_1; 4363 break; 4364 4365 case GAUDI_QUEUE_ID_TPC_0_2: 4366 db_reg_offset = mmTPC0_QM_PQ_PI_2; 4367 break; 4368 4369 case GAUDI_QUEUE_ID_TPC_0_3: 4370 db_reg_offset = mmTPC0_QM_PQ_PI_3; 4371 break; 4372 4373 case GAUDI_QUEUE_ID_TPC_1_0: 4374 db_reg_offset = mmTPC1_QM_PQ_PI_0; 4375 break; 4376 4377 case GAUDI_QUEUE_ID_TPC_1_1: 4378 db_reg_offset = mmTPC1_QM_PQ_PI_1; 4379 break; 4380 4381 case GAUDI_QUEUE_ID_TPC_1_2: 4382 db_reg_offset = mmTPC1_QM_PQ_PI_2; 4383 break; 4384 4385 case GAUDI_QUEUE_ID_TPC_1_3: 4386 db_reg_offset = mmTPC1_QM_PQ_PI_3; 4387 break; 4388 4389 case GAUDI_QUEUE_ID_TPC_2_0: 4390 db_reg_offset = mmTPC2_QM_PQ_PI_0; 4391 break; 4392 4393 case GAUDI_QUEUE_ID_TPC_2_1: 4394 db_reg_offset = mmTPC2_QM_PQ_PI_1; 4395 break; 4396 4397 case GAUDI_QUEUE_ID_TPC_2_2: 4398 db_reg_offset = mmTPC2_QM_PQ_PI_2; 4399 break; 4400 4401 case GAUDI_QUEUE_ID_TPC_2_3: 4402 db_reg_offset = mmTPC2_QM_PQ_PI_3; 4403 break; 4404 4405 case GAUDI_QUEUE_ID_TPC_3_0: 4406 db_reg_offset = mmTPC3_QM_PQ_PI_0; 4407 break; 4408 4409 case GAUDI_QUEUE_ID_TPC_3_1: 4410 db_reg_offset = mmTPC3_QM_PQ_PI_1; 4411 break; 4412 4413 case GAUDI_QUEUE_ID_TPC_3_2: 4414 db_reg_offset = mmTPC3_QM_PQ_PI_2; 4415 break; 4416 4417 case GAUDI_QUEUE_ID_TPC_3_3: 4418 db_reg_offset = mmTPC3_QM_PQ_PI_3; 4419 break; 4420 4421 case GAUDI_QUEUE_ID_TPC_4_0: 4422 db_reg_offset = mmTPC4_QM_PQ_PI_0; 4423 break; 4424 4425 case GAUDI_QUEUE_ID_TPC_4_1: 4426 db_reg_offset = mmTPC4_QM_PQ_PI_1; 4427 break; 4428 4429 case GAUDI_QUEUE_ID_TPC_4_2: 4430 db_reg_offset = mmTPC4_QM_PQ_PI_2; 4431 break; 4432 4433 case GAUDI_QUEUE_ID_TPC_4_3: 4434 db_reg_offset = mmTPC4_QM_PQ_PI_3; 4435 break; 4436 4437 case GAUDI_QUEUE_ID_TPC_5_0: 4438 db_reg_offset = mmTPC5_QM_PQ_PI_0; 4439 break; 4440 4441 case GAUDI_QUEUE_ID_TPC_5_1: 4442 db_reg_offset = mmTPC5_QM_PQ_PI_1; 4443 break; 4444 4445 case GAUDI_QUEUE_ID_TPC_5_2: 4446 db_reg_offset = mmTPC5_QM_PQ_PI_2; 4447 break; 4448 4449 case GAUDI_QUEUE_ID_TPC_5_3: 4450 db_reg_offset = mmTPC5_QM_PQ_PI_3; 4451 break; 4452 4453 case GAUDI_QUEUE_ID_TPC_6_0: 4454 db_reg_offset = mmTPC6_QM_PQ_PI_0; 4455 break; 4456 4457 case GAUDI_QUEUE_ID_TPC_6_1: 4458 db_reg_offset = mmTPC6_QM_PQ_PI_1; 4459 break; 4460 4461 case GAUDI_QUEUE_ID_TPC_6_2: 4462 db_reg_offset = mmTPC6_QM_PQ_PI_2; 4463 break; 4464 4465 case GAUDI_QUEUE_ID_TPC_6_3: 4466 db_reg_offset = mmTPC6_QM_PQ_PI_3; 4467 break; 4468 4469 case GAUDI_QUEUE_ID_TPC_7_0: 4470 db_reg_offset = mmTPC7_QM_PQ_PI_0; 4471 break; 4472 4473 case GAUDI_QUEUE_ID_TPC_7_1: 4474 db_reg_offset = mmTPC7_QM_PQ_PI_1; 4475 break; 4476 4477 case GAUDI_QUEUE_ID_TPC_7_2: 4478 db_reg_offset = mmTPC7_QM_PQ_PI_2; 4479 break; 4480 4481 case GAUDI_QUEUE_ID_TPC_7_3: 4482 db_reg_offset = mmTPC7_QM_PQ_PI_3; 4483 break; 4484 4485 case GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3: 4486 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC0)) 4487 invalid_queue = true; 4488 4489 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4490 db_reg_offset = mmNIC0_QM0_PQ_PI_0 + q_off; 4491 break; 4492 4493 case GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3: 4494 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC1)) 4495 invalid_queue = true; 4496 4497 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4498 db_reg_offset = mmNIC0_QM1_PQ_PI_0 + q_off; 4499 break; 4500 4501 case GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3: 4502 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC2)) 4503 invalid_queue = true; 4504 4505 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4506 db_reg_offset = mmNIC1_QM0_PQ_PI_0 + q_off; 4507 break; 4508 4509 case GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3: 4510 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC3)) 4511 invalid_queue = true; 4512 4513 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4514 db_reg_offset = mmNIC1_QM1_PQ_PI_0 + q_off; 4515 break; 4516 4517 case GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3: 4518 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC4)) 4519 invalid_queue = true; 4520 4521 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4522 db_reg_offset = mmNIC2_QM0_PQ_PI_0 + q_off; 4523 break; 4524 4525 case GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3: 4526 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC5)) 4527 invalid_queue = true; 4528 4529 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4530 db_reg_offset = mmNIC2_QM1_PQ_PI_0 + q_off; 4531 break; 4532 4533 case GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3: 4534 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC6)) 4535 invalid_queue = true; 4536 4537 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4538 db_reg_offset = mmNIC3_QM0_PQ_PI_0 + q_off; 4539 break; 4540 4541 case GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3: 4542 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC7)) 4543 invalid_queue = true; 4544 4545 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4546 db_reg_offset = mmNIC3_QM1_PQ_PI_0 + q_off; 4547 break; 4548 4549 case GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3: 4550 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC8)) 4551 invalid_queue = true; 4552 4553 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4554 db_reg_offset = mmNIC4_QM0_PQ_PI_0 + q_off; 4555 break; 4556 4557 case GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3: 4558 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC9)) 4559 invalid_queue = true; 4560 4561 q_off = ((hw_queue_id - 1) & 0x3) * 4; 4562 db_reg_offset = mmNIC4_QM1_PQ_PI_0 + q_off; 4563 break; 4564 4565 default: 4566 invalid_queue = true; 4567 } 4568 4569 if (invalid_queue) { 4570 /* Should never get here */ 4571 dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n", 4572 hw_queue_id); 4573 return; 4574 } 4575 4576 db_value = pi; 4577 4578 /* ring the doorbell */ 4579 WREG32(db_reg_offset, db_value); 4580 4581 if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) { 4582 /* make sure device CPU will read latest data from host */ 4583 mb(); 4584 4585 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 4586 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 4587 le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); 4588 4589 WREG32(irq_handler_offset, 4590 gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id); 4591 } 4592 } 4593 4594 static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe, 4595 struct hl_bd *bd) 4596 { 4597 __le64 *pbd = (__le64 *) bd; 4598 4599 /* The QMANs are on the host memory so a simple copy suffice */ 4600 pqe[0] = pbd[0]; 4601 pqe[1] = pbd[1]; 4602 } 4603 4604 static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size, 4605 dma_addr_t *dma_handle, gfp_t flags) 4606 { 4607 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, 4608 dma_handle, flags); 4609 4610 /* Shift to the device's base physical address of host memory */ 4611 if (kernel_addr) 4612 *dma_handle += HOST_PHYS_BASE; 4613 4614 return kernel_addr; 4615 } 4616 4617 static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size, 4618 void *cpu_addr, dma_addr_t dma_handle) 4619 { 4620 /* Cancel the device's base physical address of host memory */ 4621 dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE; 4622 4623 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); 4624 } 4625 4626 static int gaudi_scrub_device_dram(struct hl_device *hdev, u64 val) 4627 { 4628 struct asic_fixed_properties *prop = &hdev->asic_prop; 4629 u64 cur_addr = prop->dram_user_base_address; 4630 u32 chunk_size, busy; 4631 int rc, dma_id; 4632 4633 while (cur_addr < prop->dram_end_address) { 4634 for (dma_id = 0 ; dma_id < DMA_NUMBER_OF_CHANNELS ; dma_id++) { 4635 u32 dma_offset = dma_id * DMA_CORE_OFFSET; 4636 4637 chunk_size = 4638 min((u64)SZ_2G, prop->dram_end_address - cur_addr); 4639 4640 dev_dbg(hdev->dev, 4641 "Doing HBM scrubbing for 0x%09llx - 0x%09llx\n", 4642 cur_addr, cur_addr + chunk_size); 4643 4644 WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, 4645 lower_32_bits(val)); 4646 WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, 4647 upper_32_bits(val)); 4648 WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset, 4649 lower_32_bits(cur_addr)); 4650 WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset, 4651 upper_32_bits(cur_addr)); 4652 WREG32(mmDMA0_CORE_DST_TSIZE_0 + dma_offset, 4653 chunk_size); 4654 WREG32(mmDMA0_CORE_COMMIT + dma_offset, 4655 ((1 << DMA0_CORE_COMMIT_LIN_SHIFT) | 4656 (1 << DMA0_CORE_COMMIT_MEM_SET_SHIFT))); 4657 4658 cur_addr += chunk_size; 4659 4660 if (cur_addr == prop->dram_end_address) 4661 break; 4662 } 4663 4664 for (dma_id = 0 ; dma_id < DMA_NUMBER_OF_CHANNELS ; dma_id++) { 4665 u32 dma_offset = dma_id * DMA_CORE_OFFSET; 4666 4667 rc = hl_poll_timeout( 4668 hdev, 4669 mmDMA0_CORE_STS0 + dma_offset, 4670 busy, 4671 ((busy & DMA0_CORE_STS0_BUSY_MASK) == 0), 4672 1000, 4673 HBM_SCRUBBING_TIMEOUT_US); 4674 4675 if (rc) { 4676 dev_err(hdev->dev, 4677 "DMA Timeout during HBM scrubbing of DMA #%d\n", 4678 dma_id); 4679 return -EIO; 4680 } 4681 } 4682 } 4683 4684 return 0; 4685 } 4686 4687 static int gaudi_scrub_device_mem(struct hl_device *hdev) 4688 { 4689 struct asic_fixed_properties *prop = &hdev->asic_prop; 4690 u64 wait_to_idle_time = hdev->pdev ? HBM_SCRUBBING_TIMEOUT_US : 4691 min_t(u64, HBM_SCRUBBING_TIMEOUT_US * 10, HL_SIM_MAX_TIMEOUT_US); 4692 u64 addr, size, val = hdev->memory_scrub_val; 4693 ktime_t timeout; 4694 int rc = 0; 4695 4696 if (!hdev->memory_scrub) 4697 return 0; 4698 4699 timeout = ktime_add_us(ktime_get(), wait_to_idle_time); 4700 while (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { 4701 if (ktime_compare(ktime_get(), timeout) > 0) { 4702 dev_err(hdev->dev, "waiting for idle timeout\n"); 4703 return -ETIMEDOUT; 4704 } 4705 usleep_range((1000 >> 2) + 1, 1000); 4706 } 4707 4708 /* Scrub SRAM */ 4709 addr = prop->sram_user_base_address; 4710 size = hdev->pldm ? 0x10000 : prop->sram_size - SRAM_USER_BASE_OFFSET; 4711 4712 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx val: 0x%llx\n", 4713 addr, addr + size, val); 4714 rc = gaudi_memset_device_memory(hdev, addr, size, val); 4715 if (rc) { 4716 dev_err(hdev->dev, "Failed to clear SRAM (%d)\n", rc); 4717 return rc; 4718 } 4719 4720 /* Scrub HBM using all DMA channels in parallel */ 4721 rc = gaudi_scrub_device_dram(hdev, val); 4722 if (rc) { 4723 dev_err(hdev->dev, "Failed to clear HBM (%d)\n", rc); 4724 return rc; 4725 } 4726 4727 return 0; 4728 } 4729 4730 static void *gaudi_get_int_queue_base(struct hl_device *hdev, 4731 u32 queue_id, dma_addr_t *dma_handle, 4732 u16 *queue_len) 4733 { 4734 struct gaudi_device *gaudi = hdev->asic_specific; 4735 struct gaudi_internal_qman_info *q; 4736 4737 if (queue_id >= GAUDI_QUEUE_ID_SIZE || 4738 gaudi_queue_type[queue_id] != QUEUE_TYPE_INT) { 4739 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); 4740 return NULL; 4741 } 4742 4743 q = &gaudi->internal_qmans[queue_id]; 4744 *dma_handle = q->pq_dma_addr; 4745 *queue_len = q->pq_size / QMAN_PQ_ENTRY_SIZE; 4746 4747 return q->pq_kernel_addr; 4748 } 4749 4750 static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg, 4751 u16 len, u32 timeout, u64 *result) 4752 { 4753 struct gaudi_device *gaudi = hdev->asic_specific; 4754 4755 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) { 4756 if (result) 4757 *result = 0; 4758 return 0; 4759 } 4760 4761 if (!timeout) 4762 timeout = GAUDI_MSG_TO_CPU_TIMEOUT_USEC; 4763 4764 return hl_fw_send_cpu_message(hdev, GAUDI_QUEUE_ID_CPU_PQ, msg, len, 4765 timeout, result); 4766 } 4767 4768 static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id) 4769 { 4770 struct packet_msg_prot *fence_pkt; 4771 dma_addr_t pkt_dma_addr; 4772 u32 fence_val, tmp, timeout_usec; 4773 dma_addr_t fence_dma_addr; 4774 u32 *fence_ptr; 4775 int rc; 4776 4777 if (hdev->pldm) 4778 timeout_usec = GAUDI_PLDM_TEST_QUEUE_WAIT_USEC; 4779 else 4780 timeout_usec = GAUDI_TEST_QUEUE_WAIT_USEC; 4781 4782 fence_val = GAUDI_QMAN0_FENCE_VAL; 4783 4784 fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr); 4785 if (!fence_ptr) { 4786 dev_err(hdev->dev, 4787 "Failed to allocate memory for H/W queue %d testing\n", 4788 hw_queue_id); 4789 return -ENOMEM; 4790 } 4791 4792 *fence_ptr = 0; 4793 4794 fence_pkt = hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_prot), GFP_KERNEL, 4795 &pkt_dma_addr); 4796 if (!fence_pkt) { 4797 dev_err(hdev->dev, 4798 "Failed to allocate packet for H/W queue %d testing\n", 4799 hw_queue_id); 4800 rc = -ENOMEM; 4801 goto free_fence_ptr; 4802 } 4803 4804 tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT); 4805 tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); 4806 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 4807 4808 fence_pkt->ctl = cpu_to_le32(tmp); 4809 fence_pkt->value = cpu_to_le32(fence_val); 4810 fence_pkt->addr = cpu_to_le64(fence_dma_addr); 4811 4812 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, 4813 sizeof(struct packet_msg_prot), 4814 pkt_dma_addr); 4815 if (rc) { 4816 dev_err(hdev->dev, 4817 "Failed to send fence packet to H/W queue %d\n", 4818 hw_queue_id); 4819 goto free_pkt; 4820 } 4821 4822 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val), 4823 1000, timeout_usec, true); 4824 4825 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); 4826 4827 if (rc == -ETIMEDOUT) { 4828 dev_err(hdev->dev, 4829 "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n", 4830 hw_queue_id, (unsigned long long) fence_dma_addr, tmp); 4831 rc = -EIO; 4832 } 4833 4834 free_pkt: 4835 hl_asic_dma_pool_free(hdev, (void *) fence_pkt, pkt_dma_addr); 4836 free_fence_ptr: 4837 hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr); 4838 return rc; 4839 } 4840 4841 static int gaudi_test_cpu_queue(struct hl_device *hdev) 4842 { 4843 struct gaudi_device *gaudi = hdev->asic_specific; 4844 4845 /* 4846 * check capability here as send_cpu_message() won't update the result 4847 * value if no capability 4848 */ 4849 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) 4850 return 0; 4851 4852 return hl_fw_test_cpu_queue(hdev); 4853 } 4854 4855 static int gaudi_test_queues(struct hl_device *hdev) 4856 { 4857 int i, rc, ret_val = 0; 4858 4859 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++) { 4860 if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) { 4861 rc = gaudi_test_queue(hdev, i); 4862 if (rc) 4863 ret_val = -EINVAL; 4864 } 4865 } 4866 4867 rc = gaudi_test_cpu_queue(hdev); 4868 if (rc) 4869 ret_val = -EINVAL; 4870 4871 return ret_val; 4872 } 4873 4874 static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size, 4875 gfp_t mem_flags, dma_addr_t *dma_handle) 4876 { 4877 void *kernel_addr; 4878 4879 if (size > GAUDI_DMA_POOL_BLK_SIZE) 4880 return NULL; 4881 4882 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 4883 4884 /* Shift to the device's base physical address of host memory */ 4885 if (kernel_addr) 4886 *dma_handle += HOST_PHYS_BASE; 4887 4888 return kernel_addr; 4889 } 4890 4891 static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr, 4892 dma_addr_t dma_addr) 4893 { 4894 /* Cancel the device's base physical address of host memory */ 4895 dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE; 4896 4897 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); 4898 } 4899 4900 static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, 4901 size_t size, dma_addr_t *dma_handle) 4902 { 4903 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 4904 } 4905 4906 static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev, 4907 size_t size, void *vaddr) 4908 { 4909 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 4910 } 4911 4912 static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) 4913 { 4914 struct scatterlist *sg, *sg_next_iter; 4915 u32 count, dma_desc_cnt; 4916 u64 len, len_next; 4917 dma_addr_t addr, addr_next; 4918 4919 dma_desc_cnt = 0; 4920 4921 for_each_sgtable_dma_sg(sgt, sg, count) { 4922 len = sg_dma_len(sg); 4923 addr = sg_dma_address(sg); 4924 4925 if (len == 0) 4926 break; 4927 4928 while ((count + 1) < sgt->nents) { 4929 sg_next_iter = sg_next(sg); 4930 len_next = sg_dma_len(sg_next_iter); 4931 addr_next = sg_dma_address(sg_next_iter); 4932 4933 if (len_next == 0) 4934 break; 4935 4936 if ((addr + len == addr_next) && 4937 (len + len_next <= DMA_MAX_TRANSFER_SIZE)) { 4938 len += len_next; 4939 count++; 4940 sg = sg_next_iter; 4941 } else { 4942 break; 4943 } 4944 } 4945 4946 dma_desc_cnt++; 4947 } 4948 4949 return dma_desc_cnt * sizeof(struct packet_lin_dma); 4950 } 4951 4952 static int gaudi_pin_memory_before_cs(struct hl_device *hdev, 4953 struct hl_cs_parser *parser, 4954 struct packet_lin_dma *user_dma_pkt, 4955 u64 addr, enum dma_data_direction dir) 4956 { 4957 struct hl_userptr *userptr; 4958 int rc; 4959 4960 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), 4961 parser->job_userptr_list, &userptr)) 4962 goto already_pinned; 4963 4964 userptr = kzalloc(sizeof(*userptr), GFP_KERNEL); 4965 if (!userptr) 4966 return -ENOMEM; 4967 4968 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), 4969 userptr); 4970 if (rc) 4971 goto free_userptr; 4972 4973 list_add_tail(&userptr->job_node, parser->job_userptr_list); 4974 4975 rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, dir); 4976 if (rc) { 4977 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); 4978 goto unpin_memory; 4979 } 4980 4981 userptr->dma_mapped = true; 4982 userptr->dir = dir; 4983 4984 already_pinned: 4985 parser->patched_cb_size += 4986 gaudi_get_dma_desc_list_size(hdev, userptr->sgt); 4987 4988 return 0; 4989 4990 unpin_memory: 4991 list_del(&userptr->job_node); 4992 hl_unpin_host_memory(hdev, userptr); 4993 free_userptr: 4994 kfree(userptr); 4995 return rc; 4996 } 4997 4998 static int gaudi_validate_dma_pkt_host(struct hl_device *hdev, 4999 struct hl_cs_parser *parser, 5000 struct packet_lin_dma *user_dma_pkt, 5001 bool src_in_host) 5002 { 5003 enum dma_data_direction dir; 5004 bool skip_host_mem_pin = false, user_memset; 5005 u64 addr; 5006 int rc = 0; 5007 5008 user_memset = (le32_to_cpu(user_dma_pkt->ctl) & 5009 GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >> 5010 GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT; 5011 5012 if (src_in_host) { 5013 if (user_memset) 5014 skip_host_mem_pin = true; 5015 5016 dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n"); 5017 dir = DMA_TO_DEVICE; 5018 addr = le64_to_cpu(user_dma_pkt->src_addr); 5019 } else { 5020 dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n"); 5021 dir = DMA_FROM_DEVICE; 5022 addr = (le64_to_cpu(user_dma_pkt->dst_addr) & 5023 GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >> 5024 GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT; 5025 } 5026 5027 if (skip_host_mem_pin) 5028 parser->patched_cb_size += sizeof(*user_dma_pkt); 5029 else 5030 rc = gaudi_pin_memory_before_cs(hdev, parser, user_dma_pkt, 5031 addr, dir); 5032 5033 return rc; 5034 } 5035 5036 static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev, 5037 struct hl_cs_parser *parser, 5038 struct packet_lin_dma *user_dma_pkt) 5039 { 5040 bool src_in_host = false; 5041 u64 dst_addr = (le64_to_cpu(user_dma_pkt->dst_addr) & 5042 GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >> 5043 GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT; 5044 5045 dev_dbg(hdev->dev, "DMA packet details:\n"); 5046 dev_dbg(hdev->dev, "source == 0x%llx\n", 5047 le64_to_cpu(user_dma_pkt->src_addr)); 5048 dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr); 5049 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); 5050 5051 /* 5052 * Special handling for DMA with size 0. Bypass all validations 5053 * because no transactions will be done except for WR_COMP, which 5054 * is not a security issue 5055 */ 5056 if (!le32_to_cpu(user_dma_pkt->tsize)) { 5057 parser->patched_cb_size += sizeof(*user_dma_pkt); 5058 return 0; 5059 } 5060 5061 if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3) 5062 src_in_host = true; 5063 5064 return gaudi_validate_dma_pkt_host(hdev, parser, user_dma_pkt, 5065 src_in_host); 5066 } 5067 5068 static int gaudi_validate_load_and_exe_pkt(struct hl_device *hdev, 5069 struct hl_cs_parser *parser, 5070 struct packet_load_and_exe *user_pkt) 5071 { 5072 u32 cfg; 5073 5074 cfg = le32_to_cpu(user_pkt->cfg); 5075 5076 if (cfg & GAUDI_PKT_LOAD_AND_EXE_CFG_DST_MASK) { 5077 dev_err(hdev->dev, 5078 "User not allowed to use Load and Execute\n"); 5079 return -EPERM; 5080 } 5081 5082 parser->patched_cb_size += sizeof(struct packet_load_and_exe); 5083 5084 return 0; 5085 } 5086 5087 static int gaudi_validate_cb(struct hl_device *hdev, 5088 struct hl_cs_parser *parser, bool is_mmu) 5089 { 5090 u32 cb_parsed_length = 0; 5091 int rc = 0; 5092 5093 parser->patched_cb_size = 0; 5094 5095 /* cb_user_size is more than 0 so loop will always be executed */ 5096 while (cb_parsed_length < parser->user_cb_size) { 5097 enum packet_id pkt_id; 5098 u16 pkt_size; 5099 struct gaudi_packet *user_pkt; 5100 5101 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; 5102 5103 pkt_id = (enum packet_id) ( 5104 (le64_to_cpu(user_pkt->header) & 5105 PACKET_HEADER_PACKET_ID_MASK) >> 5106 PACKET_HEADER_PACKET_ID_SHIFT); 5107 5108 if (!validate_packet_id(pkt_id)) { 5109 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); 5110 rc = -EINVAL; 5111 break; 5112 } 5113 5114 pkt_size = gaudi_packet_sizes[pkt_id]; 5115 cb_parsed_length += pkt_size; 5116 if (cb_parsed_length > parser->user_cb_size) { 5117 dev_err(hdev->dev, 5118 "packet 0x%x is out of CB boundary\n", pkt_id); 5119 rc = -EINVAL; 5120 break; 5121 } 5122 5123 switch (pkt_id) { 5124 case PACKET_MSG_PROT: 5125 dev_err(hdev->dev, 5126 "User not allowed to use MSG_PROT\n"); 5127 rc = -EPERM; 5128 break; 5129 5130 case PACKET_CP_DMA: 5131 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); 5132 rc = -EPERM; 5133 break; 5134 5135 case PACKET_STOP: 5136 dev_err(hdev->dev, "User not allowed to use STOP\n"); 5137 rc = -EPERM; 5138 break; 5139 5140 case PACKET_WREG_BULK: 5141 dev_err(hdev->dev, 5142 "User not allowed to use WREG_BULK\n"); 5143 rc = -EPERM; 5144 break; 5145 5146 case PACKET_LOAD_AND_EXE: 5147 rc = gaudi_validate_load_and_exe_pkt(hdev, parser, 5148 (struct packet_load_and_exe *) user_pkt); 5149 break; 5150 5151 case PACKET_LIN_DMA: 5152 parser->contains_dma_pkt = true; 5153 if (is_mmu) 5154 parser->patched_cb_size += pkt_size; 5155 else 5156 rc = gaudi_validate_dma_pkt_no_mmu(hdev, parser, 5157 (struct packet_lin_dma *) user_pkt); 5158 break; 5159 5160 case PACKET_WREG_32: 5161 case PACKET_MSG_LONG: 5162 case PACKET_MSG_SHORT: 5163 case PACKET_REPEAT: 5164 case PACKET_FENCE: 5165 case PACKET_NOP: 5166 case PACKET_ARB_POINT: 5167 parser->patched_cb_size += pkt_size; 5168 break; 5169 5170 default: 5171 dev_err(hdev->dev, "Invalid packet header 0x%x\n", 5172 pkt_id); 5173 rc = -EINVAL; 5174 break; 5175 } 5176 5177 if (rc) 5178 break; 5179 } 5180 5181 /* 5182 * The new CB should have space at the end for two MSG_PROT packets: 5183 * 1. Optional NOP padding for cacheline alignment 5184 * 2. A packet that will act as a completion packet 5185 * 3. A packet that will generate MSI interrupt 5186 */ 5187 if (parser->completion) 5188 parser->patched_cb_size += gaudi_get_patched_cb_extra_size( 5189 parser->patched_cb_size); 5190 5191 return rc; 5192 } 5193 5194 static int gaudi_patch_dma_packet(struct hl_device *hdev, 5195 struct hl_cs_parser *parser, 5196 struct packet_lin_dma *user_dma_pkt, 5197 struct packet_lin_dma *new_dma_pkt, 5198 u32 *new_dma_pkt_size) 5199 { 5200 struct hl_userptr *userptr; 5201 struct scatterlist *sg, *sg_next_iter; 5202 u32 count, dma_desc_cnt, user_wrcomp_en_mask, ctl; 5203 u64 len, len_next; 5204 dma_addr_t dma_addr, dma_addr_next; 5205 u64 device_memory_addr, addr; 5206 enum dma_data_direction dir; 5207 struct sg_table *sgt; 5208 bool src_in_host = false; 5209 bool skip_host_mem_pin = false; 5210 bool user_memset; 5211 5212 ctl = le32_to_cpu(user_dma_pkt->ctl); 5213 5214 if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3) 5215 src_in_host = true; 5216 5217 user_memset = (ctl & GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >> 5218 GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT; 5219 5220 if (src_in_host) { 5221 addr = le64_to_cpu(user_dma_pkt->src_addr); 5222 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); 5223 dir = DMA_TO_DEVICE; 5224 if (user_memset) 5225 skip_host_mem_pin = true; 5226 } else { 5227 addr = le64_to_cpu(user_dma_pkt->dst_addr); 5228 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); 5229 dir = DMA_FROM_DEVICE; 5230 } 5231 5232 if ((!skip_host_mem_pin) && 5233 (!hl_userptr_is_pinned(hdev, addr, 5234 le32_to_cpu(user_dma_pkt->tsize), 5235 parser->job_userptr_list, &userptr))) { 5236 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", 5237 addr, user_dma_pkt->tsize); 5238 return -EFAULT; 5239 } 5240 5241 if ((user_memset) && (dir == DMA_TO_DEVICE)) { 5242 memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt)); 5243 *new_dma_pkt_size = sizeof(*user_dma_pkt); 5244 return 0; 5245 } 5246 5247 user_wrcomp_en_mask = ctl & GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK; 5248 5249 sgt = userptr->sgt; 5250 dma_desc_cnt = 0; 5251 5252 for_each_sgtable_dma_sg(sgt, sg, count) { 5253 len = sg_dma_len(sg); 5254 dma_addr = sg_dma_address(sg); 5255 5256 if (len == 0) 5257 break; 5258 5259 while ((count + 1) < sgt->nents) { 5260 sg_next_iter = sg_next(sg); 5261 len_next = sg_dma_len(sg_next_iter); 5262 dma_addr_next = sg_dma_address(sg_next_iter); 5263 5264 if (len_next == 0) 5265 break; 5266 5267 if ((dma_addr + len == dma_addr_next) && 5268 (len + len_next <= DMA_MAX_TRANSFER_SIZE)) { 5269 len += len_next; 5270 count++; 5271 sg = sg_next_iter; 5272 } else { 5273 break; 5274 } 5275 } 5276 5277 ctl = le32_to_cpu(user_dma_pkt->ctl); 5278 if (likely(dma_desc_cnt)) 5279 ctl &= ~GAUDI_PKT_CTL_EB_MASK; 5280 ctl &= ~GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK; 5281 new_dma_pkt->ctl = cpu_to_le32(ctl); 5282 new_dma_pkt->tsize = cpu_to_le32(len); 5283 5284 if (dir == DMA_TO_DEVICE) { 5285 new_dma_pkt->src_addr = cpu_to_le64(dma_addr); 5286 new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr); 5287 } else { 5288 new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr); 5289 new_dma_pkt->dst_addr = cpu_to_le64(dma_addr); 5290 } 5291 5292 if (!user_memset) 5293 device_memory_addr += len; 5294 dma_desc_cnt++; 5295 new_dma_pkt++; 5296 } 5297 5298 if (!dma_desc_cnt) { 5299 dev_err(hdev->dev, 5300 "Error of 0 SG entries when patching DMA packet\n"); 5301 return -EFAULT; 5302 } 5303 5304 /* Fix the last dma packet - wrcomp must be as user set it */ 5305 new_dma_pkt--; 5306 new_dma_pkt->ctl |= cpu_to_le32(user_wrcomp_en_mask); 5307 5308 *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma); 5309 5310 return 0; 5311 } 5312 5313 static int gaudi_patch_cb(struct hl_device *hdev, 5314 struct hl_cs_parser *parser) 5315 { 5316 u32 cb_parsed_length = 0; 5317 u32 cb_patched_cur_length = 0; 5318 int rc = 0; 5319 5320 /* cb_user_size is more than 0 so loop will always be executed */ 5321 while (cb_parsed_length < parser->user_cb_size) { 5322 enum packet_id pkt_id; 5323 u16 pkt_size; 5324 u32 new_pkt_size = 0; 5325 struct gaudi_packet *user_pkt, *kernel_pkt; 5326 5327 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; 5328 kernel_pkt = parser->patched_cb->kernel_address + 5329 cb_patched_cur_length; 5330 5331 pkt_id = (enum packet_id) ( 5332 (le64_to_cpu(user_pkt->header) & 5333 PACKET_HEADER_PACKET_ID_MASK) >> 5334 PACKET_HEADER_PACKET_ID_SHIFT); 5335 5336 if (!validate_packet_id(pkt_id)) { 5337 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); 5338 rc = -EINVAL; 5339 break; 5340 } 5341 5342 pkt_size = gaudi_packet_sizes[pkt_id]; 5343 cb_parsed_length += pkt_size; 5344 if (cb_parsed_length > parser->user_cb_size) { 5345 dev_err(hdev->dev, 5346 "packet 0x%x is out of CB boundary\n", pkt_id); 5347 rc = -EINVAL; 5348 break; 5349 } 5350 5351 switch (pkt_id) { 5352 case PACKET_LIN_DMA: 5353 rc = gaudi_patch_dma_packet(hdev, parser, 5354 (struct packet_lin_dma *) user_pkt, 5355 (struct packet_lin_dma *) kernel_pkt, 5356 &new_pkt_size); 5357 cb_patched_cur_length += new_pkt_size; 5358 break; 5359 5360 case PACKET_MSG_PROT: 5361 dev_err(hdev->dev, 5362 "User not allowed to use MSG_PROT\n"); 5363 rc = -EPERM; 5364 break; 5365 5366 case PACKET_CP_DMA: 5367 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); 5368 rc = -EPERM; 5369 break; 5370 5371 case PACKET_STOP: 5372 dev_err(hdev->dev, "User not allowed to use STOP\n"); 5373 rc = -EPERM; 5374 break; 5375 5376 case PACKET_WREG_32: 5377 case PACKET_WREG_BULK: 5378 case PACKET_MSG_LONG: 5379 case PACKET_MSG_SHORT: 5380 case PACKET_REPEAT: 5381 case PACKET_FENCE: 5382 case PACKET_NOP: 5383 case PACKET_ARB_POINT: 5384 case PACKET_LOAD_AND_EXE: 5385 memcpy(kernel_pkt, user_pkt, pkt_size); 5386 cb_patched_cur_length += pkt_size; 5387 break; 5388 5389 default: 5390 dev_err(hdev->dev, "Invalid packet header 0x%x\n", 5391 pkt_id); 5392 rc = -EINVAL; 5393 break; 5394 } 5395 5396 if (rc) 5397 break; 5398 } 5399 5400 return rc; 5401 } 5402 5403 static int gaudi_parse_cb_mmu(struct hl_device *hdev, 5404 struct hl_cs_parser *parser) 5405 { 5406 u64 handle; 5407 u32 patched_cb_size; 5408 struct hl_cb *user_cb; 5409 int rc; 5410 5411 /* 5412 * The new CB should have space at the end for two MSG_PROT packets: 5413 * 1. Optional NOP padding for cacheline alignment 5414 * 2. A packet that will act as a completion packet 5415 * 3. A packet that will generate MSI interrupt 5416 */ 5417 if (parser->completion) 5418 parser->patched_cb_size = parser->user_cb_size + 5419 gaudi_get_patched_cb_extra_size(parser->user_cb_size); 5420 else 5421 parser->patched_cb_size = parser->user_cb_size; 5422 5423 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, 5424 parser->patched_cb_size, false, false, 5425 &handle); 5426 5427 if (rc) { 5428 dev_err(hdev->dev, 5429 "Failed to allocate patched CB for DMA CS %d\n", 5430 rc); 5431 return rc; 5432 } 5433 5434 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); 5435 /* hl_cb_get should never fail */ 5436 if (!parser->patched_cb) { 5437 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); 5438 rc = -EFAULT; 5439 goto out; 5440 } 5441 5442 /* 5443 * We are protected from overflow because the check 5444 * "parser->user_cb_size <= parser->user_cb->size" was done in get_cb_from_cs_chunk() 5445 * in the common code. That check is done only if is_kernel_allocated_cb is true. 5446 * 5447 * There is no option to reach here without going through that check because: 5448 * 1. validate_queue_index() assigns true to is_kernel_allocated_cb for any submission to 5449 * an external queue. 5450 * 2. For Gaudi, we only parse CBs that were submitted to the external queues. 5451 */ 5452 memcpy(parser->patched_cb->kernel_address, 5453 parser->user_cb->kernel_address, 5454 parser->user_cb_size); 5455 5456 patched_cb_size = parser->patched_cb_size; 5457 5458 /* Validate patched CB instead of user CB */ 5459 user_cb = parser->user_cb; 5460 parser->user_cb = parser->patched_cb; 5461 rc = gaudi_validate_cb(hdev, parser, true); 5462 parser->user_cb = user_cb; 5463 5464 if (rc) { 5465 hl_cb_put(parser->patched_cb); 5466 goto out; 5467 } 5468 5469 if (patched_cb_size != parser->patched_cb_size) { 5470 dev_err(hdev->dev, "user CB size mismatch\n"); 5471 hl_cb_put(parser->patched_cb); 5472 rc = -EINVAL; 5473 goto out; 5474 } 5475 5476 out: 5477 /* 5478 * Always call cb destroy here because we still have 1 reference 5479 * to it by calling cb_get earlier. After the job will be completed, 5480 * cb_put will release it, but here we want to remove it from the 5481 * idr 5482 */ 5483 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); 5484 5485 return rc; 5486 } 5487 5488 static int gaudi_parse_cb_no_mmu(struct hl_device *hdev, 5489 struct hl_cs_parser *parser) 5490 { 5491 u64 handle; 5492 int rc; 5493 5494 rc = gaudi_validate_cb(hdev, parser, false); 5495 5496 if (rc) 5497 goto free_userptr; 5498 5499 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, 5500 parser->patched_cb_size, false, false, 5501 &handle); 5502 if (rc) { 5503 dev_err(hdev->dev, 5504 "Failed to allocate patched CB for DMA CS %d\n", rc); 5505 goto free_userptr; 5506 } 5507 5508 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); 5509 /* hl_cb_get should never fail here */ 5510 if (!parser->patched_cb) { 5511 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); 5512 rc = -EFAULT; 5513 goto out; 5514 } 5515 5516 rc = gaudi_patch_cb(hdev, parser); 5517 5518 if (rc) 5519 hl_cb_put(parser->patched_cb); 5520 5521 out: 5522 /* 5523 * Always call cb destroy here because we still have 1 reference 5524 * to it by calling cb_get earlier. After the job will be completed, 5525 * cb_put will release it, but here we want to remove it from the 5526 * idr 5527 */ 5528 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); 5529 5530 free_userptr: 5531 if (rc) 5532 hl_userptr_delete_list(hdev, parser->job_userptr_list); 5533 return rc; 5534 } 5535 5536 static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev, 5537 struct hl_cs_parser *parser) 5538 { 5539 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 5540 struct gaudi_device *gaudi = hdev->asic_specific; 5541 u32 nic_queue_offset, nic_mask_q_id; 5542 5543 if ((parser->hw_queue_id >= GAUDI_QUEUE_ID_NIC_0_0) && 5544 (parser->hw_queue_id <= GAUDI_QUEUE_ID_NIC_9_3)) { 5545 nic_queue_offset = parser->hw_queue_id - GAUDI_QUEUE_ID_NIC_0_0; 5546 nic_mask_q_id = 1 << (HW_CAP_NIC_SHIFT + (nic_queue_offset >> 2)); 5547 5548 if (!(gaudi->hw_cap_initialized & nic_mask_q_id)) { 5549 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); 5550 return -EINVAL; 5551 } 5552 } 5553 5554 /* For internal queue jobs just check if CB address is valid */ 5555 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 5556 parser->user_cb_size, 5557 asic_prop->sram_user_base_address, 5558 asic_prop->sram_end_address)) 5559 return 0; 5560 5561 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 5562 parser->user_cb_size, 5563 asic_prop->dram_user_base_address, 5564 asic_prop->dram_end_address)) 5565 return 0; 5566 5567 /* PMMU and HPMMU addresses are equal, check only one of them */ 5568 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 5569 parser->user_cb_size, 5570 asic_prop->pmmu.start_addr, 5571 asic_prop->pmmu.end_addr)) 5572 return 0; 5573 5574 dev_err(hdev->dev, 5575 "CB address 0x%px + 0x%x for internal QMAN is not valid\n", 5576 parser->user_cb, parser->user_cb_size); 5577 5578 return -EFAULT; 5579 } 5580 5581 static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 5582 { 5583 struct gaudi_device *gaudi = hdev->asic_specific; 5584 5585 if (parser->queue_type == QUEUE_TYPE_INT) 5586 return gaudi_parse_cb_no_ext_queue(hdev, parser); 5587 5588 if (gaudi->hw_cap_initialized & HW_CAP_MMU) 5589 return gaudi_parse_cb_mmu(hdev, parser); 5590 else 5591 return gaudi_parse_cb_no_mmu(hdev, parser); 5592 } 5593 5594 static void gaudi_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address, 5595 u32 len, u32 original_len, u64 cq_addr, u32 cq_val, 5596 u32 msi_vec, bool eb) 5597 { 5598 struct gaudi_device *gaudi = hdev->asic_specific; 5599 struct packet_msg_prot *cq_pkt; 5600 struct packet_nop *cq_padding; 5601 u64 msi_addr; 5602 u32 tmp; 5603 5604 cq_padding = kernel_address + original_len; 5605 cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2); 5606 5607 while ((void *)cq_padding < (void *)cq_pkt) { 5608 cq_padding->ctl = cpu_to_le32(FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_NOP)); 5609 cq_padding++; 5610 } 5611 5612 tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT); 5613 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 5614 5615 if (eb) 5616 tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); 5617 5618 cq_pkt->ctl = cpu_to_le32(tmp); 5619 cq_pkt->value = cpu_to_le32(cq_val); 5620 cq_pkt->addr = cpu_to_le64(cq_addr); 5621 5622 cq_pkt++; 5623 5624 tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT); 5625 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 5626 cq_pkt->ctl = cpu_to_le32(tmp); 5627 cq_pkt->value = cpu_to_le32(1); 5628 5629 if (gaudi->multi_msi_mode) 5630 msi_addr = mmPCIE_MSI_INTR_0 + msi_vec * 4; 5631 else 5632 msi_addr = mmPCIE_CORE_MSI_REQ; 5633 5634 cq_pkt->addr = cpu_to_le64(CFG_BASE + msi_addr); 5635 } 5636 5637 static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val) 5638 { 5639 WREG32(mmCPU_IF_EQ_RD_OFFS, val); 5640 } 5641 5642 static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, 5643 u32 size, u64 val) 5644 { 5645 struct packet_lin_dma *lin_dma_pkt; 5646 struct hl_cs_job *job; 5647 u32 cb_size, ctl, err_cause; 5648 struct hl_cb *cb; 5649 int rc; 5650 5651 cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false); 5652 if (!cb) 5653 return -EFAULT; 5654 5655 lin_dma_pkt = cb->kernel_address; 5656 memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt)); 5657 cb_size = sizeof(*lin_dma_pkt); 5658 5659 ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 5660 ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); 5661 ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1); 5662 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 5663 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 5664 5665 lin_dma_pkt->ctl = cpu_to_le32(ctl); 5666 lin_dma_pkt->src_addr = cpu_to_le64(val); 5667 lin_dma_pkt->dst_addr |= cpu_to_le64(addr); 5668 lin_dma_pkt->tsize = cpu_to_le32(size); 5669 5670 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); 5671 if (!job) { 5672 dev_err(hdev->dev, "Failed to allocate a new job\n"); 5673 rc = -ENOMEM; 5674 goto release_cb; 5675 } 5676 5677 /* Verify DMA is OK */ 5678 err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE); 5679 if (err_cause && !hdev->init_done) { 5680 dev_dbg(hdev->dev, 5681 "Clearing DMA0 engine from errors (cause 0x%x)\n", 5682 err_cause); 5683 WREG32(mmDMA0_CORE_ERR_CAUSE, err_cause); 5684 } 5685 5686 job->id = 0; 5687 job->user_cb = cb; 5688 atomic_inc(&job->user_cb->cs_cnt); 5689 job->user_cb_size = cb_size; 5690 job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; 5691 job->patched_cb = job->user_cb; 5692 job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot); 5693 5694 hl_debugfs_add_job(hdev, job); 5695 5696 rc = gaudi_send_job_on_qman0(hdev, job); 5697 hl_debugfs_remove_job(hdev, job); 5698 kfree(job); 5699 atomic_dec(&cb->cs_cnt); 5700 5701 /* Verify DMA is OK */ 5702 err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE); 5703 if (err_cause) { 5704 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause); 5705 rc = -EIO; 5706 if (!hdev->init_done) { 5707 dev_dbg(hdev->dev, 5708 "Clearing DMA0 engine from errors (cause 0x%x)\n", 5709 err_cause); 5710 WREG32(mmDMA0_CORE_ERR_CAUSE, err_cause); 5711 } 5712 } 5713 5714 release_cb: 5715 hl_cb_put(cb); 5716 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); 5717 5718 return rc; 5719 } 5720 5721 static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, 5722 u32 num_regs, u32 val) 5723 { 5724 struct packet_msg_long *pkt; 5725 struct hl_cs_job *job; 5726 u32 cb_size, ctl; 5727 struct hl_cb *cb; 5728 int i, rc; 5729 5730 cb_size = (sizeof(*pkt) * num_regs) + sizeof(struct packet_msg_prot); 5731 5732 if (cb_size > SZ_2M) { 5733 dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M); 5734 return -ENOMEM; 5735 } 5736 5737 cb = hl_cb_kernel_create(hdev, cb_size, false); 5738 if (!cb) 5739 return -EFAULT; 5740 5741 pkt = cb->kernel_address; 5742 5743 ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */ 5744 ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG); 5745 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); 5746 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 5747 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 5748 5749 for (i = 0; i < num_regs ; i++, pkt++) { 5750 pkt->ctl = cpu_to_le32(ctl); 5751 pkt->value = cpu_to_le32(val); 5752 pkt->addr = cpu_to_le64(reg_base + (i * 4)); 5753 } 5754 5755 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); 5756 if (!job) { 5757 dev_err(hdev->dev, "Failed to allocate a new job\n"); 5758 rc = -ENOMEM; 5759 goto release_cb; 5760 } 5761 5762 job->id = 0; 5763 job->user_cb = cb; 5764 atomic_inc(&job->user_cb->cs_cnt); 5765 job->user_cb_size = cb_size; 5766 job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; 5767 job->patched_cb = job->user_cb; 5768 job->job_cb_size = cb_size; 5769 5770 hl_debugfs_add_job(hdev, job); 5771 5772 rc = gaudi_send_job_on_qman0(hdev, job); 5773 hl_debugfs_remove_job(hdev, job); 5774 kfree(job); 5775 atomic_dec(&cb->cs_cnt); 5776 5777 release_cb: 5778 hl_cb_put(cb); 5779 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); 5780 5781 return rc; 5782 } 5783 5784 static int gaudi_restore_sm_registers(struct hl_device *hdev) 5785 { 5786 u64 base_addr; 5787 u32 num_regs; 5788 int rc; 5789 5790 base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0; 5791 num_regs = NUM_OF_SOB_IN_BLOCK; 5792 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5793 if (rc) { 5794 dev_err(hdev->dev, "failed resetting SM registers"); 5795 return -ENOMEM; 5796 } 5797 5798 base_addr = CFG_BASE + mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0; 5799 num_regs = NUM_OF_SOB_IN_BLOCK; 5800 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5801 if (rc) { 5802 dev_err(hdev->dev, "failed resetting SM registers"); 5803 return -ENOMEM; 5804 } 5805 5806 base_addr = CFG_BASE + mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0; 5807 num_regs = NUM_OF_SOB_IN_BLOCK; 5808 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5809 if (rc) { 5810 dev_err(hdev->dev, "failed resetting SM registers"); 5811 return -ENOMEM; 5812 } 5813 5814 base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0; 5815 num_regs = NUM_OF_MONITORS_IN_BLOCK; 5816 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5817 if (rc) { 5818 dev_err(hdev->dev, "failed resetting SM registers"); 5819 return -ENOMEM; 5820 } 5821 5822 base_addr = CFG_BASE + mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0; 5823 num_regs = NUM_OF_MONITORS_IN_BLOCK; 5824 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5825 if (rc) { 5826 dev_err(hdev->dev, "failed resetting SM registers"); 5827 return -ENOMEM; 5828 } 5829 5830 base_addr = CFG_BASE + mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0; 5831 num_regs = NUM_OF_MONITORS_IN_BLOCK; 5832 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5833 if (rc) { 5834 dev_err(hdev->dev, "failed resetting SM registers"); 5835 return -ENOMEM; 5836 } 5837 5838 base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + 5839 (GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4); 5840 num_regs = NUM_OF_SOB_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT; 5841 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5842 if (rc) { 5843 dev_err(hdev->dev, "failed resetting SM registers"); 5844 return -ENOMEM; 5845 } 5846 5847 base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + 5848 (GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4); 5849 num_regs = NUM_OF_MONITORS_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_MONITOR; 5850 rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); 5851 if (rc) { 5852 dev_err(hdev->dev, "failed resetting SM registers"); 5853 return -ENOMEM; 5854 } 5855 5856 return 0; 5857 } 5858 5859 static void gaudi_restore_dma_registers(struct hl_device *hdev) 5860 { 5861 u32 sob_delta = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 - 5862 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0; 5863 int i; 5864 5865 for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) { 5866 u64 sob_addr = CFG_BASE + 5867 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + 5868 (i * sob_delta); 5869 u32 dma_offset = i * DMA_CORE_OFFSET; 5870 5871 WREG32(mmDMA0_CORE_WR_COMP_ADDR_LO + dma_offset, 5872 lower_32_bits(sob_addr)); 5873 WREG32(mmDMA0_CORE_WR_COMP_ADDR_HI + dma_offset, 5874 upper_32_bits(sob_addr)); 5875 WREG32(mmDMA0_CORE_WR_COMP_WDATA + dma_offset, 0x80000001); 5876 5877 /* For DMAs 2-7, need to restore WR_AWUSER_31_11 as it can be 5878 * modified by the user for SRAM reduction 5879 */ 5880 if (i > 1) 5881 WREG32(mmDMA0_CORE_WR_AWUSER_31_11 + dma_offset, 5882 0x00000001); 5883 } 5884 } 5885 5886 static void gaudi_restore_qm_registers(struct hl_device *hdev) 5887 { 5888 u32 qman_offset; 5889 int i; 5890 5891 for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) { 5892 qman_offset = i * DMA_QMAN_OFFSET; 5893 WREG32(mmDMA0_QM_ARB_CFG_0 + qman_offset, 0); 5894 } 5895 5896 for (i = 0 ; i < MME_NUMBER_OF_MASTER_ENGINES ; i++) { 5897 qman_offset = i * (mmMME2_QM_BASE - mmMME0_QM_BASE); 5898 WREG32(mmMME0_QM_ARB_CFG_0 + qman_offset, 0); 5899 } 5900 5901 for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) { 5902 qman_offset = i * TPC_QMAN_OFFSET; 5903 WREG32(mmTPC0_QM_ARB_CFG_0 + qman_offset, 0); 5904 } 5905 5906 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 5907 qman_offset = (i >> 1) * NIC_MACRO_QMAN_OFFSET + 5908 (i & 0x1) * NIC_ENGINE_QMAN_OFFSET; 5909 WREG32(mmNIC0_QM0_ARB_CFG_0 + qman_offset, 0); 5910 } 5911 } 5912 5913 static int gaudi_restore_user_registers(struct hl_device *hdev) 5914 { 5915 int rc; 5916 5917 rc = gaudi_restore_sm_registers(hdev); 5918 if (rc) 5919 return rc; 5920 5921 gaudi_restore_dma_registers(hdev); 5922 gaudi_restore_qm_registers(hdev); 5923 5924 return 0; 5925 } 5926 5927 static int gaudi_context_switch(struct hl_device *hdev, u32 asid) 5928 { 5929 return 0; 5930 } 5931 5932 static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev) 5933 { 5934 u32 size = hdev->asic_prop.mmu_pgt_size + 5935 hdev->asic_prop.mmu_cache_mng_size; 5936 struct gaudi_device *gaudi = hdev->asic_specific; 5937 u64 addr = hdev->asic_prop.mmu_pgt_addr; 5938 5939 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) 5940 return 0; 5941 5942 return gaudi_memset_device_memory(hdev, addr, size, 0); 5943 } 5944 5945 static void gaudi_restore_phase_topology(struct hl_device *hdev) 5946 { 5947 5948 } 5949 5950 static int gaudi_dma_core_transfer(struct hl_device *hdev, int dma_id, u64 addr, 5951 u32 size_to_dma, dma_addr_t dma_addr) 5952 { 5953 u32 err_cause, val; 5954 u64 dma_offset; 5955 int rc; 5956 5957 dma_offset = dma_id * DMA_CORE_OFFSET; 5958 5959 WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, lower_32_bits(addr)); 5960 WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, upper_32_bits(addr)); 5961 WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset, lower_32_bits(dma_addr)); 5962 WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset, upper_32_bits(dma_addr)); 5963 WREG32(mmDMA0_CORE_DST_TSIZE_0 + dma_offset, size_to_dma); 5964 WREG32(mmDMA0_CORE_COMMIT + dma_offset, 5965 (1 << DMA0_CORE_COMMIT_LIN_SHIFT)); 5966 5967 rc = hl_poll_timeout( 5968 hdev, 5969 mmDMA0_CORE_STS0 + dma_offset, 5970 val, 5971 ((val & DMA0_CORE_STS0_BUSY_MASK) == 0), 5972 0, 5973 1000000); 5974 5975 if (rc) { 5976 dev_err(hdev->dev, 5977 "DMA %d timed-out during reading of 0x%llx\n", 5978 dma_id, addr); 5979 return -EIO; 5980 } 5981 5982 /* Verify DMA is OK */ 5983 err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset); 5984 if (err_cause) { 5985 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause); 5986 dev_dbg(hdev->dev, 5987 "Clearing DMA0 engine from errors (cause 0x%x)\n", 5988 err_cause); 5989 WREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset, err_cause); 5990 5991 return -EIO; 5992 } 5993 5994 return 0; 5995 } 5996 5997 static int gaudi_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, 5998 void *blob_addr) 5999 { 6000 u32 dma_core_sts0, err_cause, cfg1, size_left, pos, size_to_dma; 6001 u32 qm_glbl_sts0, qm_cgm_sts; 6002 u64 dma_offset, qm_offset; 6003 dma_addr_t dma_addr; 6004 void *kernel_addr; 6005 bool is_eng_idle; 6006 int rc = 0, dma_id; 6007 6008 kernel_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &dma_addr, GFP_KERNEL | __GFP_ZERO); 6009 6010 if (!kernel_addr) 6011 return -ENOMEM; 6012 6013 hdev->asic_funcs->hw_queues_lock(hdev); 6014 6015 dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1]; 6016 dma_offset = dma_id * DMA_CORE_OFFSET; 6017 qm_offset = dma_id * DMA_QMAN_OFFSET; 6018 dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + dma_offset); 6019 qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + qm_offset); 6020 qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + qm_offset); 6021 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) && 6022 IS_DMA_IDLE(dma_core_sts0); 6023 6024 if (!is_eng_idle) { 6025 dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2]; 6026 dma_offset = dma_id * DMA_CORE_OFFSET; 6027 qm_offset = dma_id * DMA_QMAN_OFFSET; 6028 dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + dma_offset); 6029 qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + qm_offset); 6030 qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + qm_offset); 6031 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) && 6032 IS_DMA_IDLE(dma_core_sts0); 6033 6034 if (!is_eng_idle) { 6035 dev_err_ratelimited(hdev->dev, 6036 "Can't read via DMA because it is BUSY\n"); 6037 rc = -EAGAIN; 6038 goto out; 6039 } 6040 } 6041 6042 cfg1 = RREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset); 6043 WREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset, 6044 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT); 6045 6046 /* TODO: remove this by mapping the DMA temporary buffer to the MMU 6047 * using the compute ctx ASID, if exists. If not, use the kernel ctx 6048 * ASID 6049 */ 6050 WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT)); 6051 6052 /* Verify DMA is OK */ 6053 err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset); 6054 if (err_cause) { 6055 dev_dbg(hdev->dev, 6056 "Clearing DMA0 engine from errors (cause 0x%x)\n", 6057 err_cause); 6058 WREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset, err_cause); 6059 } 6060 6061 pos = 0; 6062 size_left = size; 6063 size_to_dma = SZ_2M; 6064 6065 while (size_left > 0) { 6066 6067 if (size_left < SZ_2M) 6068 size_to_dma = size_left; 6069 6070 rc = gaudi_dma_core_transfer(hdev, dma_id, addr, size_to_dma, 6071 dma_addr); 6072 if (rc) 6073 break; 6074 6075 memcpy(blob_addr + pos, kernel_addr, size_to_dma); 6076 6077 if (size_left <= SZ_2M) 6078 break; 6079 6080 pos += SZ_2M; 6081 addr += SZ_2M; 6082 size_left -= SZ_2M; 6083 } 6084 6085 /* TODO: remove this by mapping the DMA temporary buffer to the MMU 6086 * using the compute ctx ASID, if exists. If not, use the kernel ctx 6087 * ASID 6088 */ 6089 WREG32_AND(mmDMA0_CORE_PROT + dma_offset, 6090 ~BIT(DMA0_CORE_PROT_VAL_SHIFT)); 6091 6092 WREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset, cfg1); 6093 6094 out: 6095 hdev->asic_funcs->hw_queues_unlock(hdev); 6096 6097 hl_asic_dma_free_coherent(hdev, SZ_2M, kernel_addr, dma_addr); 6098 6099 return rc; 6100 } 6101 6102 static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr) 6103 { 6104 struct gaudi_device *gaudi = hdev->asic_specific; 6105 6106 if (hdev->reset_info.hard_reset_pending) 6107 return U64_MAX; 6108 6109 return readq(hdev->pcie_bar[HBM_BAR_ID] + 6110 (addr - gaudi->hbm_bar_cur_addr)); 6111 } 6112 6113 static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val) 6114 { 6115 struct gaudi_device *gaudi = hdev->asic_specific; 6116 6117 if (hdev->reset_info.hard_reset_pending) 6118 return; 6119 6120 writeq(val, hdev->pcie_bar[HBM_BAR_ID] + 6121 (addr - gaudi->hbm_bar_cur_addr)); 6122 } 6123 6124 void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) 6125 { 6126 /* mask to zero the MMBP and ASID bits */ 6127 WREG32_AND(reg, ~0x7FF); 6128 WREG32_OR(reg, asid); 6129 } 6130 6131 static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) 6132 { 6133 struct gaudi_device *gaudi = hdev->asic_specific; 6134 6135 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) 6136 return; 6137 6138 if (asid & ~DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK) { 6139 dev_crit(hdev->dev, "asid %u is too big\n", asid); 6140 return; 6141 } 6142 6143 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_0, asid); 6144 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_1, asid); 6145 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_2, asid); 6146 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_3, asid); 6147 gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_4, asid); 6148 6149 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_0, asid); 6150 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_1, asid); 6151 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_2, asid); 6152 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_3, asid); 6153 gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_4, asid); 6154 6155 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_0, asid); 6156 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_1, asid); 6157 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_2, asid); 6158 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_3, asid); 6159 gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_4, asid); 6160 6161 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_0, asid); 6162 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_1, asid); 6163 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_2, asid); 6164 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_3, asid); 6165 gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_4, asid); 6166 6167 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_0, asid); 6168 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_1, asid); 6169 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_2, asid); 6170 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_3, asid); 6171 gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_4, asid); 6172 6173 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_0, asid); 6174 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_1, asid); 6175 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_2, asid); 6176 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_3, asid); 6177 gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_4, asid); 6178 6179 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_0, asid); 6180 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_1, asid); 6181 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_2, asid); 6182 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_3, asid); 6183 gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_4, asid); 6184 6185 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_0, asid); 6186 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_1, asid); 6187 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_2, asid); 6188 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_3, asid); 6189 gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_4, asid); 6190 6191 gaudi_mmu_prepare_reg(hdev, mmDMA0_CORE_NON_SECURE_PROPS, asid); 6192 gaudi_mmu_prepare_reg(hdev, mmDMA1_CORE_NON_SECURE_PROPS, asid); 6193 gaudi_mmu_prepare_reg(hdev, mmDMA2_CORE_NON_SECURE_PROPS, asid); 6194 gaudi_mmu_prepare_reg(hdev, mmDMA3_CORE_NON_SECURE_PROPS, asid); 6195 gaudi_mmu_prepare_reg(hdev, mmDMA4_CORE_NON_SECURE_PROPS, asid); 6196 gaudi_mmu_prepare_reg(hdev, mmDMA5_CORE_NON_SECURE_PROPS, asid); 6197 gaudi_mmu_prepare_reg(hdev, mmDMA6_CORE_NON_SECURE_PROPS, asid); 6198 gaudi_mmu_prepare_reg(hdev, mmDMA7_CORE_NON_SECURE_PROPS, asid); 6199 6200 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_0, asid); 6201 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_1, asid); 6202 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_2, asid); 6203 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_3, asid); 6204 gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_4, asid); 6205 gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_ARUSER_LO, asid); 6206 gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_AWUSER_LO, asid); 6207 6208 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_0, asid); 6209 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_1, asid); 6210 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_2, asid); 6211 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_3, asid); 6212 gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_4, asid); 6213 gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_ARUSER_LO, asid); 6214 gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_AWUSER_LO, asid); 6215 6216 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_0, asid); 6217 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_1, asid); 6218 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_2, asid); 6219 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_3, asid); 6220 gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_4, asid); 6221 gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_ARUSER_LO, asid); 6222 gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_AWUSER_LO, asid); 6223 6224 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_0, asid); 6225 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_1, asid); 6226 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_2, asid); 6227 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_3, asid); 6228 gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_4, asid); 6229 gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_ARUSER_LO, asid); 6230 gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_AWUSER_LO, asid); 6231 6232 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_0, asid); 6233 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_1, asid); 6234 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_2, asid); 6235 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_3, asid); 6236 gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_4, asid); 6237 gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_ARUSER_LO, asid); 6238 gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_AWUSER_LO, asid); 6239 6240 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_0, asid); 6241 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_1, asid); 6242 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_2, asid); 6243 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_3, asid); 6244 gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_4, asid); 6245 gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_ARUSER_LO, asid); 6246 gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_AWUSER_LO, asid); 6247 6248 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_0, asid); 6249 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_1, asid); 6250 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_2, asid); 6251 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_3, asid); 6252 gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_4, asid); 6253 gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_ARUSER_LO, asid); 6254 gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_AWUSER_LO, asid); 6255 6256 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_0, asid); 6257 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_1, asid); 6258 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_2, asid); 6259 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_3, asid); 6260 gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_4, asid); 6261 gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_ARUSER_LO, asid); 6262 gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_AWUSER_LO, asid); 6263 6264 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_0, asid); 6265 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_1, asid); 6266 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_2, asid); 6267 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_3, asid); 6268 gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_4, asid); 6269 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_0, asid); 6270 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_1, asid); 6271 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_2, asid); 6272 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_3, asid); 6273 gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_4, asid); 6274 6275 gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER0, asid); 6276 gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER1, asid); 6277 gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER0, asid); 6278 gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER1, asid); 6279 gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER0, asid); 6280 gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER1, asid); 6281 gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER0, asid); 6282 gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER1, asid); 6283 gaudi_mmu_prepare_reg(hdev, mmMME0_ACC_WBC, asid); 6284 gaudi_mmu_prepare_reg(hdev, mmMME1_ACC_WBC, asid); 6285 gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid); 6286 gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid); 6287 6288 if (gaudi->hw_cap_initialized & HW_CAP_NIC0) { 6289 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_0, 6290 asid); 6291 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_1, 6292 asid); 6293 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_2, 6294 asid); 6295 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_3, 6296 asid); 6297 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_4, 6298 asid); 6299 } 6300 6301 if (gaudi->hw_cap_initialized & HW_CAP_NIC1) { 6302 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_0, 6303 asid); 6304 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_1, 6305 asid); 6306 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_2, 6307 asid); 6308 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_3, 6309 asid); 6310 gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_4, 6311 asid); 6312 } 6313 6314 if (gaudi->hw_cap_initialized & HW_CAP_NIC2) { 6315 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_0, 6316 asid); 6317 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_1, 6318 asid); 6319 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_2, 6320 asid); 6321 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_3, 6322 asid); 6323 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_4, 6324 asid); 6325 } 6326 6327 if (gaudi->hw_cap_initialized & HW_CAP_NIC3) { 6328 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_0, 6329 asid); 6330 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_1, 6331 asid); 6332 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_2, 6333 asid); 6334 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_3, 6335 asid); 6336 gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_4, 6337 asid); 6338 } 6339 6340 if (gaudi->hw_cap_initialized & HW_CAP_NIC4) { 6341 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_0, 6342 asid); 6343 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_1, 6344 asid); 6345 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_2, 6346 asid); 6347 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_3, 6348 asid); 6349 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_4, 6350 asid); 6351 } 6352 6353 if (gaudi->hw_cap_initialized & HW_CAP_NIC5) { 6354 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_0, 6355 asid); 6356 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_1, 6357 asid); 6358 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_2, 6359 asid); 6360 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_3, 6361 asid); 6362 gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_4, 6363 asid); 6364 } 6365 6366 if (gaudi->hw_cap_initialized & HW_CAP_NIC6) { 6367 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_0, 6368 asid); 6369 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_1, 6370 asid); 6371 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_2, 6372 asid); 6373 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_3, 6374 asid); 6375 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_4, 6376 asid); 6377 } 6378 6379 if (gaudi->hw_cap_initialized & HW_CAP_NIC7) { 6380 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_0, 6381 asid); 6382 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_1, 6383 asid); 6384 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_2, 6385 asid); 6386 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_3, 6387 asid); 6388 gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_4, 6389 asid); 6390 } 6391 6392 if (gaudi->hw_cap_initialized & HW_CAP_NIC8) { 6393 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_0, 6394 asid); 6395 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_1, 6396 asid); 6397 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_2, 6398 asid); 6399 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_3, 6400 asid); 6401 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_4, 6402 asid); 6403 } 6404 6405 if (gaudi->hw_cap_initialized & HW_CAP_NIC9) { 6406 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_0, 6407 asid); 6408 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_1, 6409 asid); 6410 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_2, 6411 asid); 6412 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_3, 6413 asid); 6414 gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_4, 6415 asid); 6416 } 6417 6418 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid); 6419 gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid); 6420 } 6421 6422 static int gaudi_send_job_on_qman0(struct hl_device *hdev, 6423 struct hl_cs_job *job) 6424 { 6425 struct packet_msg_prot *fence_pkt; 6426 u32 *fence_ptr; 6427 dma_addr_t fence_dma_addr; 6428 struct hl_cb *cb; 6429 u32 tmp, timeout, dma_offset; 6430 int rc; 6431 6432 if (hdev->pldm) 6433 timeout = GAUDI_PLDM_QMAN0_TIMEOUT_USEC; 6434 else 6435 timeout = HL_DEVICE_TIMEOUT_USEC; 6436 6437 fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr); 6438 if (!fence_ptr) { 6439 dev_err(hdev->dev, 6440 "Failed to allocate fence memory for QMAN0\n"); 6441 return -ENOMEM; 6442 } 6443 6444 cb = job->patched_cb; 6445 6446 fence_pkt = cb->kernel_address + 6447 job->job_cb_size - sizeof(struct packet_msg_prot); 6448 6449 tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT); 6450 tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); 6451 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 6452 6453 fence_pkt->ctl = cpu_to_le32(tmp); 6454 fence_pkt->value = cpu_to_le32(GAUDI_QMAN0_FENCE_VAL); 6455 fence_pkt->addr = cpu_to_le64(fence_dma_addr); 6456 6457 dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET; 6458 6459 WREG32(mmDMA0_CORE_PROT + dma_offset, 6460 BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT) | BIT(DMA0_CORE_PROT_VAL_SHIFT)); 6461 6462 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0, 6463 job->job_cb_size, cb->bus_address); 6464 if (rc) { 6465 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); 6466 goto free_fence_ptr; 6467 } 6468 6469 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, 6470 (tmp == GAUDI_QMAN0_FENCE_VAL), 1000, 6471 timeout, true); 6472 6473 hl_hw_queue_inc_ci_kernel(hdev, GAUDI_QUEUE_ID_DMA_0_0); 6474 6475 if (rc == -ETIMEDOUT) { 6476 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); 6477 goto free_fence_ptr; 6478 } 6479 6480 free_fence_ptr: 6481 WREG32(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT)); 6482 6483 hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr); 6484 return rc; 6485 } 6486 6487 static void gaudi_get_event_desc(u16 event_type, char *desc, size_t size) 6488 { 6489 if (event_type >= GAUDI_EVENT_SIZE) 6490 goto event_not_supported; 6491 6492 if (!gaudi_irq_map_table[event_type].valid) 6493 goto event_not_supported; 6494 6495 snprintf(desc, size, gaudi_irq_map_table[event_type].name); 6496 6497 return; 6498 6499 event_not_supported: 6500 snprintf(desc, size, "N/A"); 6501 } 6502 6503 static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev, u32 x_y, 6504 bool is_write, u16 *engine_id_1, 6505 u16 *engine_id_2) 6506 { 6507 u32 dma_id[2], dma_offset, err_cause[2], mask, i; 6508 6509 mask = is_write ? DMA0_CORE_ERR_CAUSE_HBW_WR_ERR_MASK : 6510 DMA0_CORE_ERR_CAUSE_HBW_RD_ERR_MASK; 6511 6512 switch (x_y) { 6513 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0: 6514 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1: 6515 dma_id[0] = 0; 6516 dma_id[1] = 2; 6517 break; 6518 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0: 6519 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1: 6520 dma_id[0] = 1; 6521 dma_id[1] = 3; 6522 break; 6523 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0: 6524 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1: 6525 dma_id[0] = 4; 6526 dma_id[1] = 6; 6527 break; 6528 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0: 6529 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1: 6530 dma_id[0] = 5; 6531 dma_id[1] = 7; 6532 break; 6533 default: 6534 goto unknown_initiator; 6535 } 6536 6537 for (i = 0 ; i < 2 ; i++) { 6538 dma_offset = dma_id[i] * DMA_CORE_OFFSET; 6539 err_cause[i] = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset); 6540 } 6541 6542 switch (x_y) { 6543 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0: 6544 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1: 6545 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { 6546 *engine_id_1 = GAUDI_ENGINE_ID_DMA_0; 6547 return "DMA0"; 6548 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { 6549 *engine_id_1 = GAUDI_ENGINE_ID_DMA_2; 6550 return "DMA2"; 6551 } else { 6552 *engine_id_1 = GAUDI_ENGINE_ID_DMA_0; 6553 *engine_id_2 = GAUDI_ENGINE_ID_DMA_2; 6554 return "DMA0 or DMA2"; 6555 } 6556 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0: 6557 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1: 6558 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { 6559 *engine_id_1 = GAUDI_ENGINE_ID_DMA_1; 6560 return "DMA1"; 6561 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { 6562 *engine_id_1 = GAUDI_ENGINE_ID_DMA_3; 6563 return "DMA3"; 6564 } else { 6565 *engine_id_1 = GAUDI_ENGINE_ID_DMA_1; 6566 *engine_id_2 = GAUDI_ENGINE_ID_DMA_3; 6567 return "DMA1 or DMA3"; 6568 } 6569 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0: 6570 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1: 6571 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { 6572 *engine_id_1 = GAUDI_ENGINE_ID_DMA_4; 6573 return "DMA4"; 6574 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { 6575 *engine_id_1 = GAUDI_ENGINE_ID_DMA_6; 6576 return "DMA6"; 6577 } else { 6578 *engine_id_1 = GAUDI_ENGINE_ID_DMA_4; 6579 *engine_id_2 = GAUDI_ENGINE_ID_DMA_6; 6580 return "DMA4 or DMA6"; 6581 } 6582 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0: 6583 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1: 6584 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { 6585 *engine_id_1 = GAUDI_ENGINE_ID_DMA_5; 6586 return "DMA5"; 6587 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { 6588 *engine_id_1 = GAUDI_ENGINE_ID_DMA_7; 6589 return "DMA7"; 6590 } else { 6591 *engine_id_1 = GAUDI_ENGINE_ID_DMA_5; 6592 *engine_id_2 = GAUDI_ENGINE_ID_DMA_7; 6593 return "DMA5 or DMA7"; 6594 } 6595 } 6596 6597 unknown_initiator: 6598 return "unknown initiator"; 6599 } 6600 6601 static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev, bool is_write, 6602 u16 *engine_id_1, u16 *engine_id_2) 6603 { 6604 u32 val, x_y, axi_id; 6605 6606 val = is_write ? RREG32(mmMMU_UP_RAZWI_WRITE_ID) : 6607 RREG32(mmMMU_UP_RAZWI_READ_ID); 6608 x_y = val & ((RAZWI_INITIATOR_Y_MASK << RAZWI_INITIATOR_Y_SHIFT) | 6609 (RAZWI_INITIATOR_X_MASK << RAZWI_INITIATOR_X_SHIFT)); 6610 axi_id = val & (RAZWI_INITIATOR_AXI_ID_MASK << 6611 RAZWI_INITIATOR_AXI_ID_SHIFT); 6612 6613 switch (x_y) { 6614 case RAZWI_INITIATOR_ID_X_Y_TPC0_NIC0: 6615 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) { 6616 *engine_id_1 = GAUDI_ENGINE_ID_TPC_0; 6617 return "TPC0"; 6618 } 6619 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC)) { 6620 *engine_id_1 = GAUDI_ENGINE_ID_NIC_0; 6621 return "NIC0"; 6622 } 6623 break; 6624 case RAZWI_INITIATOR_ID_X_Y_TPC1: 6625 *engine_id_1 = GAUDI_ENGINE_ID_TPC_1; 6626 return "TPC1"; 6627 case RAZWI_INITIATOR_ID_X_Y_MME0_0: 6628 case RAZWI_INITIATOR_ID_X_Y_MME0_1: 6629 *engine_id_1 = GAUDI_ENGINE_ID_MME_0; 6630 return "MME0"; 6631 case RAZWI_INITIATOR_ID_X_Y_MME1_0: 6632 case RAZWI_INITIATOR_ID_X_Y_MME1_1: 6633 *engine_id_1 = GAUDI_ENGINE_ID_MME_1; 6634 return "MME1"; 6635 case RAZWI_INITIATOR_ID_X_Y_TPC2: 6636 *engine_id_1 = GAUDI_ENGINE_ID_TPC_2; 6637 return "TPC2"; 6638 case RAZWI_INITIATOR_ID_X_Y_TPC3_PCI_CPU_PSOC: 6639 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) { 6640 *engine_id_1 = GAUDI_ENGINE_ID_TPC_3; 6641 return "TPC3"; 6642 } 6643 /* PCI, CPU or PSOC does not have engine id*/ 6644 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PCI)) 6645 return "PCI"; 6646 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_CPU)) 6647 return "CPU"; 6648 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PSOC)) 6649 return "PSOC"; 6650 break; 6651 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0: 6652 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1: 6653 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0: 6654 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1: 6655 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0: 6656 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1: 6657 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0: 6658 case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1: 6659 return gaudi_get_razwi_initiator_dma_name(hdev, x_y, is_write, 6660 engine_id_1, engine_id_2); 6661 case RAZWI_INITIATOR_ID_X_Y_TPC4_NIC1_NIC2: 6662 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) { 6663 *engine_id_1 = GAUDI_ENGINE_ID_TPC_4; 6664 return "TPC4"; 6665 } 6666 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC)) { 6667 *engine_id_1 = GAUDI_ENGINE_ID_NIC_1; 6668 return "NIC1"; 6669 } 6670 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT)) { 6671 *engine_id_1 = GAUDI_ENGINE_ID_NIC_2; 6672 return "NIC2"; 6673 } 6674 break; 6675 case RAZWI_INITIATOR_ID_X_Y_TPC5: 6676 *engine_id_1 = GAUDI_ENGINE_ID_TPC_5; 6677 return "TPC5"; 6678 case RAZWI_INITIATOR_ID_X_Y_MME2_0: 6679 case RAZWI_INITIATOR_ID_X_Y_MME2_1: 6680 *engine_id_1 = GAUDI_ENGINE_ID_MME_2; 6681 return "MME2"; 6682 case RAZWI_INITIATOR_ID_X_Y_MME3_0: 6683 case RAZWI_INITIATOR_ID_X_Y_MME3_1: 6684 *engine_id_1 = GAUDI_ENGINE_ID_MME_3; 6685 return "MME3"; 6686 case RAZWI_INITIATOR_ID_X_Y_TPC6: 6687 *engine_id_1 = GAUDI_ENGINE_ID_TPC_6; 6688 return "TPC6"; 6689 case RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5: 6690 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) { 6691 *engine_id_1 = GAUDI_ENGINE_ID_TPC_7; 6692 return "TPC7"; 6693 } 6694 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC)) { 6695 *engine_id_1 = GAUDI_ENGINE_ID_NIC_4; 6696 return "NIC4"; 6697 } 6698 if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT)) { 6699 *engine_id_1 = GAUDI_ENGINE_ID_NIC_5; 6700 return "NIC5"; 6701 } 6702 break; 6703 default: 6704 break; 6705 } 6706 6707 dev_err(hdev->dev, 6708 "Unknown RAZWI initiator ID 0x%x [Y=%d, X=%d, AXI_ID=%d]\n", 6709 val, 6710 (val >> RAZWI_INITIATOR_Y_SHIFT) & RAZWI_INITIATOR_Y_MASK, 6711 (val >> RAZWI_INITIATOR_X_SHIFT) & RAZWI_INITIATOR_X_MASK, 6712 (val >> RAZWI_INITIATOR_AXI_ID_SHIFT) & 6713 RAZWI_INITIATOR_AXI_ID_MASK); 6714 6715 return "unknown initiator"; 6716 } 6717 6718 static void gaudi_print_and_get_razwi_info(struct hl_device *hdev, u16 *engine_id_1, 6719 u16 *engine_id_2, bool *is_read, bool *is_write) 6720 { 6721 6722 if (RREG32(mmMMU_UP_RAZWI_WRITE_VLD)) { 6723 dev_err_ratelimited(hdev->dev, 6724 "RAZWI event caused by illegal write of %s\n", 6725 gaudi_get_razwi_initiator_name(hdev, true, engine_id_1, engine_id_2)); 6726 WREG32(mmMMU_UP_RAZWI_WRITE_VLD, 0); 6727 *is_write = true; 6728 } 6729 6730 if (RREG32(mmMMU_UP_RAZWI_READ_VLD)) { 6731 dev_err_ratelimited(hdev->dev, 6732 "RAZWI event caused by illegal read of %s\n", 6733 gaudi_get_razwi_initiator_name(hdev, false, engine_id_1, engine_id_2)); 6734 WREG32(mmMMU_UP_RAZWI_READ_VLD, 0); 6735 *is_read = true; 6736 } 6737 } 6738 6739 static void gaudi_print_and_get_mmu_error_info(struct hl_device *hdev, u64 *addr, u64 *event_mask) 6740 { 6741 struct gaudi_device *gaudi = hdev->asic_specific; 6742 u32 val; 6743 6744 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) 6745 return; 6746 6747 val = RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE); 6748 if (val & MMU_UP_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) { 6749 *addr = val & MMU_UP_PAGE_ERROR_CAPTURE_VA_49_32_MASK; 6750 *addr <<= 32; 6751 *addr |= RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE_VA); 6752 6753 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", *addr); 6754 hl_handle_page_fault(hdev, *addr, 0, true, event_mask); 6755 6756 WREG32(mmMMU_UP_PAGE_ERROR_CAPTURE, 0); 6757 } 6758 6759 val = RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE); 6760 if (val & MMU_UP_ACCESS_ERROR_CAPTURE_ENTRY_VALID_MASK) { 6761 *addr = val & MMU_UP_ACCESS_ERROR_CAPTURE_VA_49_32_MASK; 6762 *addr <<= 32; 6763 *addr |= RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE_VA); 6764 6765 dev_err_ratelimited(hdev->dev, "MMU access error on va 0x%llx\n", *addr); 6766 6767 WREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE, 0); 6768 } 6769 } 6770 6771 /* 6772 * +-------------------+------------------------------------------------------+ 6773 * | Configuration Reg | Description | 6774 * | Address | | 6775 * +-------------------+------------------------------------------------------+ 6776 * | 0xF30 - 0xF3F |ECC single error indication (1 bit per memory wrapper)| 6777 * | |0xF30 memory wrappers 31:0 (MSB to LSB) | 6778 * | |0xF34 memory wrappers 63:32 | 6779 * | |0xF38 memory wrappers 95:64 | 6780 * | |0xF3C memory wrappers 127:96 | 6781 * +-------------------+------------------------------------------------------+ 6782 * | 0xF40 - 0xF4F |ECC double error indication (1 bit per memory wrapper)| 6783 * | |0xF40 memory wrappers 31:0 (MSB to LSB) | 6784 * | |0xF44 memory wrappers 63:32 | 6785 * | |0xF48 memory wrappers 95:64 | 6786 * | |0xF4C memory wrappers 127:96 | 6787 * +-------------------+------------------------------------------------------+ 6788 */ 6789 static int gaudi_extract_ecc_info(struct hl_device *hdev, 6790 struct ecc_info_extract_params *params, u64 *ecc_address, 6791 u64 *ecc_syndrom, u8 *memory_wrapper_idx) 6792 { 6793 u32 i, num_mem_regs, reg, err_bit; 6794 u64 err_addr, err_word = 0; 6795 6796 num_mem_regs = params->num_memories / 32 + 6797 ((params->num_memories % 32) ? 1 : 0); 6798 6799 if (params->block_address >= CFG_BASE) 6800 params->block_address -= CFG_BASE; 6801 6802 if (params->derr) 6803 err_addr = params->block_address + GAUDI_ECC_DERR0_OFFSET; 6804 else 6805 err_addr = params->block_address + GAUDI_ECC_SERR0_OFFSET; 6806 6807 /* Set invalid wrapper index */ 6808 *memory_wrapper_idx = 0xFF; 6809 6810 /* Iterate through memory wrappers, a single bit must be set */ 6811 for (i = 0 ; i < num_mem_regs ; i++) { 6812 err_addr += i * 4; 6813 err_word = RREG32(err_addr); 6814 if (err_word) { 6815 err_bit = __ffs(err_word); 6816 *memory_wrapper_idx = err_bit + (32 * i); 6817 break; 6818 } 6819 } 6820 6821 if (*memory_wrapper_idx == 0xFF) { 6822 dev_err(hdev->dev, "ECC error information cannot be found\n"); 6823 return -EINVAL; 6824 } 6825 6826 WREG32(params->block_address + GAUDI_ECC_MEM_SEL_OFFSET, 6827 *memory_wrapper_idx); 6828 6829 *ecc_address = 6830 RREG32(params->block_address + GAUDI_ECC_ADDRESS_OFFSET); 6831 *ecc_syndrom = 6832 RREG32(params->block_address + GAUDI_ECC_SYNDROME_OFFSET); 6833 6834 /* Clear error indication */ 6835 reg = RREG32(params->block_address + GAUDI_ECC_MEM_INFO_CLR_OFFSET); 6836 if (params->derr) 6837 reg |= FIELD_PREP(GAUDI_ECC_MEM_INFO_CLR_DERR_MASK, 1); 6838 else 6839 reg |= FIELD_PREP(GAUDI_ECC_MEM_INFO_CLR_SERR_MASK, 1); 6840 6841 WREG32(params->block_address + GAUDI_ECC_MEM_INFO_CLR_OFFSET, reg); 6842 6843 return 0; 6844 } 6845 6846 /* 6847 * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap 6848 * 6849 * @idx: the current pi/ci value 6850 * @q_len: the queue length (power of 2) 6851 * 6852 * @return the cyclically decremented index 6853 */ 6854 static inline u32 gaudi_queue_idx_dec(u32 idx, u32 q_len) 6855 { 6856 u32 mask = q_len - 1; 6857 6858 /* 6859 * modular decrement is equivalent to adding (queue_size -1) 6860 * later we take LSBs to make sure the value is in the 6861 * range [0, queue_len - 1] 6862 */ 6863 return (idx + q_len - 1) & mask; 6864 } 6865 6866 /** 6867 * gaudi_handle_sw_config_stream_data - print SW config stream data 6868 * 6869 * @hdev: pointer to the habanalabs device structure 6870 * @stream: the QMAN's stream 6871 * @qman_base: base address of QMAN registers block 6872 * @event_mask: mask of the last events occurred 6873 */ 6874 static void gaudi_handle_sw_config_stream_data(struct hl_device *hdev, u32 stream, 6875 u64 qman_base, u64 event_mask) 6876 { 6877 u64 cq_ptr_lo, cq_ptr_hi, cq_tsize, cq_ptr; 6878 u32 cq_ptr_lo_off, size; 6879 6880 cq_ptr_lo_off = mmTPC0_QM_CQ_PTR_LO_1 - mmTPC0_QM_CQ_PTR_LO_0; 6881 6882 cq_ptr_lo = qman_base + (mmTPC0_QM_CQ_PTR_LO_0 - mmTPC0_QM_BASE) + 6883 stream * cq_ptr_lo_off; 6884 cq_ptr_hi = cq_ptr_lo + 6885 (mmTPC0_QM_CQ_PTR_HI_0 - mmTPC0_QM_CQ_PTR_LO_0); 6886 cq_tsize = cq_ptr_lo + 6887 (mmTPC0_QM_CQ_TSIZE_0 - mmTPC0_QM_CQ_PTR_LO_0); 6888 6889 cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo); 6890 size = RREG32(cq_tsize); 6891 dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %u\n", 6892 stream, cq_ptr, size); 6893 6894 if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) { 6895 hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr; 6896 hdev->captured_err_info.undef_opcode.cq_size = size; 6897 hdev->captured_err_info.undef_opcode.stream_id = stream; 6898 } 6899 } 6900 6901 /** 6902 * gaudi_handle_last_pqes_on_err - print last PQEs on error 6903 * 6904 * @hdev: pointer to the habanalabs device structure 6905 * @qid_base: first QID of the QMAN (out of 4 streams) 6906 * @stream: the QMAN's stream 6907 * @qman_base: base address of QMAN registers block 6908 * @event_mask: mask of the last events occurred 6909 * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE) 6910 */ 6911 static void gaudi_handle_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, 6912 u32 stream, u64 qman_base, 6913 u64 event_mask, 6914 bool pr_sw_conf) 6915 { 6916 u32 ci, qm_ci_stream_off, queue_len; 6917 struct hl_hw_queue *q; 6918 u64 pq_ci, addr[PQ_FETCHER_CACHE_SIZE]; 6919 int i; 6920 6921 q = &hdev->kernel_queues[qid_base + stream]; 6922 6923 qm_ci_stream_off = mmTPC0_QM_PQ_CI_1 - mmTPC0_QM_PQ_CI_0; 6924 pq_ci = qman_base + (mmTPC0_QM_PQ_CI_0 - mmTPC0_QM_BASE) + 6925 stream * qm_ci_stream_off; 6926 6927 queue_len = (q->queue_type == QUEUE_TYPE_INT) ? 6928 q->int_queue_len : HL_QUEUE_LENGTH; 6929 6930 hdev->asic_funcs->hw_queues_lock(hdev); 6931 6932 if (pr_sw_conf) 6933 gaudi_handle_sw_config_stream_data(hdev, stream, qman_base, event_mask); 6934 6935 ci = RREG32(pq_ci); 6936 6937 /* we should start printing form ci -1 */ 6938 ci = gaudi_queue_idx_dec(ci, queue_len); 6939 memset(addr, 0, sizeof(addr)); 6940 6941 for (i = 0; i < PQ_FETCHER_CACHE_SIZE; i++) { 6942 struct hl_bd *bd; 6943 u32 len; 6944 6945 bd = q->kernel_address; 6946 bd += ci; 6947 6948 len = le32_to_cpu(bd->len); 6949 /* len 0 means uninitialized entry- break */ 6950 if (!len) 6951 break; 6952 6953 addr[i] = le64_to_cpu(bd->ptr); 6954 6955 dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %u\n", 6956 stream, ci, addr[i], len); 6957 6958 /* get previous ci, wrap if needed */ 6959 ci = gaudi_queue_idx_dec(ci, queue_len); 6960 } 6961 6962 if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) { 6963 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; 6964 u32 arr_idx = undef_opcode->cb_addr_streams_len; 6965 6966 if (arr_idx == 0) { 6967 undef_opcode->timestamp = ktime_get(); 6968 undef_opcode->engine_id = gaudi_queue_id_to_engine_id[qid_base]; 6969 } 6970 6971 memcpy(undef_opcode->cb_addr_streams[arr_idx], addr, sizeof(addr)); 6972 undef_opcode->cb_addr_streams_len++; 6973 } 6974 6975 hdev->asic_funcs->hw_queues_unlock(hdev); 6976 } 6977 6978 /** 6979 * handle_qman_data_on_err - extract QMAN data on error 6980 * 6981 * @hdev: pointer to the habanalabs device structure 6982 * @qid_base: first QID of the QMAN (out of 4 streams) 6983 * @stream: the QMAN's stream 6984 * @qman_base: base address of QMAN registers block 6985 * @event_mask: mask of the last events occurred 6986 * 6987 * This function attempt to exatract as much data as possible on QMAN error. 6988 * On upper CP print the SW config stream data and last 8 PQEs. 6989 * On lower CP print SW config data and last PQEs of ALL 4 upper CPs 6990 */ 6991 static void handle_qman_data_on_err(struct hl_device *hdev, u32 qid_base, 6992 u32 stream, u64 qman_base, u64 event_mask) 6993 { 6994 u32 i; 6995 6996 if (stream != QMAN_STREAMS) { 6997 gaudi_handle_last_pqes_on_err(hdev, qid_base, stream, 6998 qman_base, event_mask, true); 6999 return; 7000 } 7001 7002 /* handle Lower-CP */ 7003 gaudi_handle_sw_config_stream_data(hdev, stream, qman_base, event_mask); 7004 7005 for (i = 0; i < QMAN_STREAMS; i++) 7006 gaudi_handle_last_pqes_on_err(hdev, qid_base, i, 7007 qman_base, event_mask, false); 7008 } 7009 7010 static void gaudi_handle_qman_err_generic(struct hl_device *hdev, 7011 const char *qm_name, 7012 u64 qman_base, 7013 u32 qid_base, 7014 u64 *event_mask) 7015 { 7016 u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val; 7017 u64 glbl_sts_addr, arb_err_addr; 7018 char reg_desc[32]; 7019 7020 glbl_sts_addr = qman_base + (mmTPC0_QM_GLBL_STS1_0 - mmTPC0_QM_BASE); 7021 arb_err_addr = qman_base + (mmTPC0_QM_ARB_ERR_CAUSE - mmTPC0_QM_BASE); 7022 7023 /* Iterate through all stream GLBL_STS1 registers + Lower CP */ 7024 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { 7025 glbl_sts_clr_val = 0; 7026 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i); 7027 7028 if (!glbl_sts_val) 7029 continue; 7030 7031 if (i == QMAN_STREAMS) 7032 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerCP"); 7033 else 7034 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i); 7035 7036 for (j = 0 ; j < GAUDI_NUM_OF_QM_ERR_CAUSE ; j++) { 7037 if (glbl_sts_val & BIT(j)) { 7038 dev_err_ratelimited(hdev->dev, 7039 "%s %s. err cause: %s\n", 7040 qm_name, reg_desc, 7041 gaudi_qman_error_cause[j]); 7042 glbl_sts_clr_val |= BIT(j); 7043 } 7044 } 7045 /* check for undefined opcode */ 7046 if (glbl_sts_val & TPC0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_MASK && 7047 hdev->captured_err_info.undef_opcode.write_enable) { 7048 memset(&hdev->captured_err_info.undef_opcode, 0, 7049 sizeof(hdev->captured_err_info.undef_opcode)); 7050 7051 hdev->captured_err_info.undef_opcode.write_enable = false; 7052 *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE; 7053 } 7054 7055 /* Write 1 clear errors */ 7056 if (!hdev->stop_on_err) 7057 WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val); 7058 else 7059 handle_qman_data_on_err(hdev, qid_base, i, qman_base, *event_mask); 7060 } 7061 7062 arb_err_val = RREG32(arb_err_addr); 7063 7064 if (!arb_err_val) 7065 return; 7066 7067 for (j = 0 ; j < GAUDI_NUM_OF_QM_ARB_ERR_CAUSE ; j++) { 7068 if (arb_err_val & BIT(j)) { 7069 dev_err_ratelimited(hdev->dev, 7070 "%s ARB_ERR. err cause: %s\n", 7071 qm_name, 7072 gaudi_qman_arb_error_cause[j]); 7073 } 7074 } 7075 } 7076 7077 static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, 7078 struct hl_eq_sm_sei_data *sei_data) 7079 { 7080 u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0; 7081 7082 /* Flip the bits as the enum is ordered in the opposite way */ 7083 index = (index ^ 0x3) & 0x3; 7084 7085 switch (sei_data->sei_cause) { 7086 case SM_SEI_SO_OVERFLOW: 7087 dev_err_ratelimited(hdev->dev, 7088 "%s SEI Error: SOB Group %u overflow/underflow", 7089 gaudi_sync_manager_names[index], 7090 le32_to_cpu(sei_data->sei_log)); 7091 break; 7092 case SM_SEI_LBW_4B_UNALIGNED: 7093 dev_err_ratelimited(hdev->dev, 7094 "%s SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x", 7095 gaudi_sync_manager_names[index], 7096 le32_to_cpu(sei_data->sei_log)); 7097 break; 7098 case SM_SEI_AXI_RESPONSE_ERR: 7099 dev_err_ratelimited(hdev->dev, 7100 "%s SEI Error: AXI ID %u response error", 7101 gaudi_sync_manager_names[index], 7102 le32_to_cpu(sei_data->sei_log)); 7103 break; 7104 default: 7105 dev_err_ratelimited(hdev->dev, "Unknown SM SEI cause %u", 7106 le32_to_cpu(sei_data->sei_log)); 7107 break; 7108 } 7109 } 7110 7111 static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, 7112 struct hl_eq_ecc_data *ecc_data) 7113 { 7114 struct ecc_info_extract_params params; 7115 u64 ecc_address = 0, ecc_syndrom = 0; 7116 u8 index, memory_wrapper_idx = 0; 7117 bool extract_info_from_fw; 7118 int rc; 7119 7120 if (hdev->asic_prop.fw_security_enabled) { 7121 extract_info_from_fw = true; 7122 goto extract_ecc_info; 7123 } 7124 7125 switch (event_type) { 7126 case GAUDI_EVENT_PCIE_CORE_SERR ... GAUDI_EVENT_PCIE_PHY_DERR: 7127 case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_MMU_DERR: 7128 extract_info_from_fw = true; 7129 break; 7130 case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR: 7131 index = event_type - GAUDI_EVENT_TPC0_SERR; 7132 params.block_address = mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET; 7133 params.num_memories = 90; 7134 params.derr = false; 7135 extract_info_from_fw = false; 7136 break; 7137 case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR: 7138 index = event_type - GAUDI_EVENT_TPC0_DERR; 7139 params.block_address = 7140 mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET; 7141 params.num_memories = 90; 7142 params.derr = true; 7143 extract_info_from_fw = false; 7144 break; 7145 case GAUDI_EVENT_MME0_ACC_SERR: 7146 case GAUDI_EVENT_MME1_ACC_SERR: 7147 case GAUDI_EVENT_MME2_ACC_SERR: 7148 case GAUDI_EVENT_MME3_ACC_SERR: 7149 index = (event_type - GAUDI_EVENT_MME0_ACC_SERR) / 4; 7150 params.block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET; 7151 params.num_memories = 128; 7152 params.derr = false; 7153 extract_info_from_fw = false; 7154 break; 7155 case GAUDI_EVENT_MME0_ACC_DERR: 7156 case GAUDI_EVENT_MME1_ACC_DERR: 7157 case GAUDI_EVENT_MME2_ACC_DERR: 7158 case GAUDI_EVENT_MME3_ACC_DERR: 7159 index = (event_type - GAUDI_EVENT_MME0_ACC_DERR) / 4; 7160 params.block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET; 7161 params.num_memories = 128; 7162 params.derr = true; 7163 extract_info_from_fw = false; 7164 break; 7165 case GAUDI_EVENT_MME0_SBAB_SERR: 7166 case GAUDI_EVENT_MME1_SBAB_SERR: 7167 case GAUDI_EVENT_MME2_SBAB_SERR: 7168 case GAUDI_EVENT_MME3_SBAB_SERR: 7169 index = (event_type - GAUDI_EVENT_MME0_SBAB_SERR) / 4; 7170 params.block_address = 7171 mmMME0_SBAB_BASE + index * MME_ACC_OFFSET; 7172 params.num_memories = 33; 7173 params.derr = false; 7174 extract_info_from_fw = false; 7175 break; 7176 case GAUDI_EVENT_MME0_SBAB_DERR: 7177 case GAUDI_EVENT_MME1_SBAB_DERR: 7178 case GAUDI_EVENT_MME2_SBAB_DERR: 7179 case GAUDI_EVENT_MME3_SBAB_DERR: 7180 index = (event_type - GAUDI_EVENT_MME0_SBAB_DERR) / 4; 7181 params.block_address = 7182 mmMME0_SBAB_BASE + index * MME_ACC_OFFSET; 7183 params.num_memories = 33; 7184 params.derr = true; 7185 extract_info_from_fw = false; 7186 break; 7187 default: 7188 return; 7189 } 7190 7191 extract_ecc_info: 7192 if (extract_info_from_fw) { 7193 ecc_address = le64_to_cpu(ecc_data->ecc_address); 7194 ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom); 7195 memory_wrapper_idx = ecc_data->memory_wrapper_idx; 7196 } else { 7197 rc = gaudi_extract_ecc_info(hdev, ¶ms, &ecc_address, 7198 &ecc_syndrom, &memory_wrapper_idx); 7199 if (rc) 7200 return; 7201 } 7202 7203 dev_err(hdev->dev, 7204 "ECC error detected. address: %#llx. Syndrom: %#llx. block id %u\n", 7205 ecc_address, ecc_syndrom, memory_wrapper_idx); 7206 } 7207 7208 static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 7209 { 7210 u64 qman_base; 7211 char desc[32]; 7212 u32 qid_base; 7213 u8 index; 7214 7215 switch (event_type) { 7216 case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: 7217 index = event_type - GAUDI_EVENT_TPC0_QM; 7218 qid_base = GAUDI_QUEUE_ID_TPC_0_0 + index * QMAN_STREAMS; 7219 qman_base = mmTPC0_QM_BASE + index * TPC_QMAN_OFFSET; 7220 snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index); 7221 break; 7222 case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM: 7223 if (event_type == GAUDI_EVENT_MME0_QM) { 7224 index = 0; 7225 qid_base = GAUDI_QUEUE_ID_MME_0_0; 7226 } else { /* event_type == GAUDI_EVENT_MME2_QM */ 7227 index = 2; 7228 qid_base = GAUDI_QUEUE_ID_MME_1_0; 7229 } 7230 qman_base = mmMME0_QM_BASE + index * MME_QMAN_OFFSET; 7231 snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index); 7232 break; 7233 case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM: 7234 index = event_type - GAUDI_EVENT_DMA0_QM; 7235 qid_base = GAUDI_QUEUE_ID_DMA_0_0 + index * QMAN_STREAMS; 7236 /* skip GAUDI_QUEUE_ID_CPU_PQ if necessary */ 7237 if (index > 1) 7238 qid_base++; 7239 qman_base = mmDMA0_QM_BASE + index * DMA_QMAN_OFFSET; 7240 snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index); 7241 break; 7242 case GAUDI_EVENT_NIC0_QM0: 7243 qid_base = GAUDI_QUEUE_ID_NIC_0_0; 7244 qman_base = mmNIC0_QM0_BASE; 7245 snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM0"); 7246 break; 7247 case GAUDI_EVENT_NIC0_QM1: 7248 qid_base = GAUDI_QUEUE_ID_NIC_1_0; 7249 qman_base = mmNIC0_QM1_BASE; 7250 snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM1"); 7251 break; 7252 case GAUDI_EVENT_NIC1_QM0: 7253 qid_base = GAUDI_QUEUE_ID_NIC_2_0; 7254 qman_base = mmNIC1_QM0_BASE; 7255 snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM0"); 7256 break; 7257 case GAUDI_EVENT_NIC1_QM1: 7258 qid_base = GAUDI_QUEUE_ID_NIC_3_0; 7259 qman_base = mmNIC1_QM1_BASE; 7260 snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM1"); 7261 break; 7262 case GAUDI_EVENT_NIC2_QM0: 7263 qid_base = GAUDI_QUEUE_ID_NIC_4_0; 7264 qman_base = mmNIC2_QM0_BASE; 7265 snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM0"); 7266 break; 7267 case GAUDI_EVENT_NIC2_QM1: 7268 qid_base = GAUDI_QUEUE_ID_NIC_5_0; 7269 qman_base = mmNIC2_QM1_BASE; 7270 snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM1"); 7271 break; 7272 case GAUDI_EVENT_NIC3_QM0: 7273 qid_base = GAUDI_QUEUE_ID_NIC_6_0; 7274 qman_base = mmNIC3_QM0_BASE; 7275 snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM0"); 7276 break; 7277 case GAUDI_EVENT_NIC3_QM1: 7278 qid_base = GAUDI_QUEUE_ID_NIC_7_0; 7279 qman_base = mmNIC3_QM1_BASE; 7280 snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM1"); 7281 break; 7282 case GAUDI_EVENT_NIC4_QM0: 7283 qid_base = GAUDI_QUEUE_ID_NIC_8_0; 7284 qman_base = mmNIC4_QM0_BASE; 7285 snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM0"); 7286 break; 7287 case GAUDI_EVENT_NIC4_QM1: 7288 qid_base = GAUDI_QUEUE_ID_NIC_9_0; 7289 qman_base = mmNIC4_QM1_BASE; 7290 snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM1"); 7291 break; 7292 default: 7293 return; 7294 } 7295 7296 gaudi_handle_qman_err_generic(hdev, desc, qman_base, qid_base, event_mask); 7297 } 7298 7299 static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type, 7300 bool razwi, u64 *event_mask) 7301 { 7302 bool is_read = false, is_write = false; 7303 u16 engine_id[2], num_of_razwi_eng = 0; 7304 char desc[64] = ""; 7305 u64 razwi_addr = 0; 7306 u8 razwi_flags = 0; 7307 7308 /* 7309 * Init engine id by default as not valid and only if razwi initiated from engine with 7310 * engine id it will get valid value. 7311 */ 7312 engine_id[0] = HL_RAZWI_NA_ENG_ID; 7313 engine_id[1] = HL_RAZWI_NA_ENG_ID; 7314 7315 gaudi_get_event_desc(event_type, desc, sizeof(desc)); 7316 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", 7317 event_type, desc); 7318 7319 if (razwi) { 7320 gaudi_print_and_get_razwi_info(hdev, &engine_id[0], &engine_id[1], &is_read, 7321 &is_write); 7322 gaudi_print_and_get_mmu_error_info(hdev, &razwi_addr, event_mask); 7323 7324 if (is_read) 7325 razwi_flags |= HL_RAZWI_READ; 7326 if (is_write) 7327 razwi_flags |= HL_RAZWI_WRITE; 7328 7329 if (engine_id[0] != HL_RAZWI_NA_ENG_ID) { 7330 if (engine_id[1] != HL_RAZWI_NA_ENG_ID) 7331 num_of_razwi_eng = 2; 7332 else 7333 num_of_razwi_eng = 1; 7334 } 7335 7336 hl_handle_razwi(hdev, razwi_addr, engine_id, num_of_razwi_eng, razwi_flags, 7337 event_mask); 7338 } 7339 } 7340 7341 static void gaudi_print_out_of_sync_info(struct hl_device *hdev, 7342 struct cpucp_pkt_sync_err *sync_err) 7343 { 7344 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ]; 7345 7346 dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n", 7347 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 7348 } 7349 7350 static void gaudi_print_fw_alive_info(struct hl_device *hdev, 7351 struct hl_eq_fw_alive *fw_alive) 7352 { 7353 dev_err(hdev->dev, 7354 "FW alive report: severity=%s, process_id=%u, thread_id=%u, uptime=%llu seconds\n", 7355 (fw_alive->severity == FW_ALIVE_SEVERITY_MINOR) ? "Minor" : "Critical", 7356 le32_to_cpu(fw_alive->process_id), 7357 le32_to_cpu(fw_alive->thread_id), 7358 le64_to_cpu(fw_alive->uptime_seconds)); 7359 } 7360 7361 static void gaudi_print_nic_axi_irq_info(struct hl_device *hdev, u16 event_type, 7362 void *data) 7363 { 7364 char desc[64] = "", *type; 7365 struct eq_nic_sei_event *eq_nic_sei = data; 7366 u16 nic_id = event_type - GAUDI_EVENT_NIC_SEI_0; 7367 7368 switch (eq_nic_sei->axi_error_cause) { 7369 case RXB: 7370 type = "RXB"; 7371 break; 7372 case RXE: 7373 type = "RXE"; 7374 break; 7375 case TXS: 7376 type = "TXS"; 7377 break; 7378 case TXE: 7379 type = "TXE"; 7380 break; 7381 case QPC_RESP: 7382 type = "QPC_RESP"; 7383 break; 7384 case NON_AXI_ERR: 7385 type = "NON_AXI_ERR"; 7386 break; 7387 case TMR: 7388 type = "TMR"; 7389 break; 7390 default: 7391 dev_err(hdev->dev, "unknown NIC AXI cause %d\n", 7392 eq_nic_sei->axi_error_cause); 7393 type = "N/A"; 7394 break; 7395 } 7396 7397 snprintf(desc, sizeof(desc), "NIC%d_%s%d", nic_id, type, 7398 eq_nic_sei->id); 7399 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", 7400 event_type, desc); 7401 } 7402 7403 static int gaudi_compute_reset_late_init(struct hl_device *hdev) 7404 { 7405 /* GAUDI doesn't support any reset except hard-reset */ 7406 return -EPERM; 7407 } 7408 7409 static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, 7410 struct hl_eq_hbm_ecc_data *hbm_ecc_data) 7411 { 7412 u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch; 7413 int rc = 0; 7414 7415 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & 7416 CPU_BOOT_DEV_STS0_HBM_ECC_EN) { 7417 if (!hbm_ecc_data) { 7418 dev_err(hdev->dev, "No FW ECC data"); 7419 return 0; 7420 } 7421 7422 wr_par = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_WR_PAR_MASK, 7423 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7424 rd_par = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_RD_PAR_MASK, 7425 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7426 ca_par = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_CA_PAR_MASK, 7427 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7428 derr = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_DERR_MASK, 7429 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7430 serr = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_SERR_MASK, 7431 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7432 type = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_TYPE_MASK, 7433 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7434 ch = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_HBM_CH_MASK, 7435 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); 7436 7437 dev_err(hdev->dev, 7438 "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", 7439 device, ch, wr_par, rd_par, ca_par, serr, derr); 7440 dev_err(hdev->dev, 7441 "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%u, SEC_CNT=%d, DEC_CNT=%d\n", 7442 device, ch, hbm_ecc_data->first_addr, type, 7443 hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt, 7444 hbm_ecc_data->dec_cnt); 7445 return 0; 7446 } 7447 7448 if (hdev->asic_prop.fw_security_enabled) { 7449 dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n"); 7450 return 0; 7451 } 7452 7453 base = GAUDI_HBM_CFG_BASE + device * GAUDI_HBM_CFG_OFFSET; 7454 for (ch = 0 ; ch < GAUDI_HBM_CHANNELS ; ch++) { 7455 val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF); 7456 val = (val & 0xFF) | ((val >> 8) & 0xFF); 7457 if (val) { 7458 rc = -EIO; 7459 dev_err(hdev->dev, 7460 "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", 7461 device, ch * 2, val & 0x1, (val >> 1) & 0x1, 7462 (val >> 2) & 0x1, (val >> 3) & 0x1, 7463 (val >> 4) & 0x1); 7464 7465 val2 = RREG32(base + ch * 0x1000 + 0x060); 7466 dev_err(hdev->dev, 7467 "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DEC_CNT=%d\n", 7468 device, ch * 2, 7469 RREG32(base + ch * 0x1000 + 0x064), 7470 (val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10, 7471 (val2 & 0xFF0000) >> 16, 7472 (val2 & 0xFF000000) >> 24); 7473 } 7474 7475 val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF); 7476 val = (val & 0xFF) | ((val >> 8) & 0xFF); 7477 if (val) { 7478 rc = -EIO; 7479 dev_err(hdev->dev, 7480 "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", 7481 device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1, 7482 (val >> 2) & 0x1, (val >> 3) & 0x1, 7483 (val >> 4) & 0x1); 7484 7485 val2 = RREG32(base + ch * 0x1000 + 0x070); 7486 dev_err(hdev->dev, 7487 "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DEC_CNT=%d\n", 7488 device, ch * 2 + 1, 7489 RREG32(base + ch * 0x1000 + 0x074), 7490 (val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10, 7491 (val2 & 0xFF0000) >> 16, 7492 (val2 & 0xFF000000) >> 24); 7493 } 7494 7495 /* Clear interrupts */ 7496 RMWREG32(base + (ch * 0x1000) + 0x060, 0x1C8, 0x1FF); 7497 RMWREG32(base + (ch * 0x1000) + 0x070, 0x1C8, 0x1FF); 7498 WREG32(base + (ch * 0x1000) + 0x06C, 0x1F1F); 7499 WREG32(base + (ch * 0x1000) + 0x07C, 0x1F1F); 7500 RMWREG32(base + (ch * 0x1000) + 0x060, 0x0, 0xF); 7501 RMWREG32(base + (ch * 0x1000) + 0x070, 0x0, 0xF); 7502 } 7503 7504 val = RREG32(base + 0x8F30); 7505 val2 = RREG32(base + 0x8F34); 7506 if (val | val2) { 7507 rc = -EIO; 7508 dev_err(hdev->dev, 7509 "HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n", 7510 device, val, val2); 7511 } 7512 val = RREG32(base + 0x8F40); 7513 val2 = RREG32(base + 0x8F44); 7514 if (val | val2) { 7515 rc = -EIO; 7516 dev_err(hdev->dev, 7517 "HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n", 7518 device, val, val2); 7519 } 7520 7521 return rc; 7522 } 7523 7524 static int gaudi_hbm_event_to_dev(u16 hbm_event_type) 7525 { 7526 switch (hbm_event_type) { 7527 case GAUDI_EVENT_HBM0_SPI_0: 7528 case GAUDI_EVENT_HBM0_SPI_1: 7529 return 0; 7530 case GAUDI_EVENT_HBM1_SPI_0: 7531 case GAUDI_EVENT_HBM1_SPI_1: 7532 return 1; 7533 case GAUDI_EVENT_HBM2_SPI_0: 7534 case GAUDI_EVENT_HBM2_SPI_1: 7535 return 2; 7536 case GAUDI_EVENT_HBM3_SPI_0: 7537 case GAUDI_EVENT_HBM3_SPI_1: 7538 return 3; 7539 default: 7540 break; 7541 } 7542 7543 /* Should never happen */ 7544 return 0; 7545 } 7546 7547 static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id, 7548 char *interrupt_name) 7549 { 7550 u32 tpc_offset = tpc_id * TPC_CFG_OFFSET, tpc_interrupts_cause, i; 7551 bool soft_reset_required = false; 7552 7553 tpc_interrupts_cause = RREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset) & 7554 TPC0_CFG_TPC_INTR_CAUSE_CAUSE_MASK; 7555 7556 for (i = 0 ; i < GAUDI_NUM_OF_TPC_INTR_CAUSE ; i++) 7557 if (tpc_interrupts_cause & BIT(i)) { 7558 dev_err_ratelimited(hdev->dev, 7559 "TPC%d_%s interrupt cause: %s\n", 7560 tpc_id, interrupt_name, 7561 gaudi_tpc_interrupts_cause[i]); 7562 /* If this is QM error, we need to soft-reset */ 7563 if (i == 15) 7564 soft_reset_required = true; 7565 } 7566 7567 /* Clear interrupts */ 7568 WREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset, 0); 7569 7570 return soft_reset_required; 7571 } 7572 7573 static int tpc_dec_event_to_tpc_id(u16 tpc_dec_event_type) 7574 { 7575 return (tpc_dec_event_type - GAUDI_EVENT_TPC0_DEC) >> 1; 7576 } 7577 7578 static int tpc_krn_event_to_tpc_id(u16 tpc_dec_event_type) 7579 { 7580 return (tpc_dec_event_type - GAUDI_EVENT_TPC0_KRN_ERR) / 6; 7581 } 7582 7583 static void gaudi_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) 7584 { 7585 ktime_t zero_time = ktime_set(0, 0); 7586 7587 mutex_lock(&hdev->clk_throttling.lock); 7588 7589 switch (event_type) { 7590 case GAUDI_EVENT_FIX_POWER_ENV_S: 7591 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 7592 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 7593 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 7594 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 7595 dev_info_ratelimited(hdev->dev, 7596 "Clock throttling due to power consumption\n"); 7597 break; 7598 7599 case GAUDI_EVENT_FIX_POWER_ENV_E: 7600 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 7601 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 7602 dev_info_ratelimited(hdev->dev, 7603 "Power envelop is safe, back to optimal clock\n"); 7604 break; 7605 7606 case GAUDI_EVENT_FIX_THERMAL_ENV_S: 7607 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 7608 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 7609 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 7610 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 7611 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7612 dev_info_ratelimited(hdev->dev, 7613 "Clock throttling due to overheating\n"); 7614 break; 7615 7616 case GAUDI_EVENT_FIX_THERMAL_ENV_E: 7617 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 7618 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 7619 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7620 dev_info_ratelimited(hdev->dev, 7621 "Thermal envelop is safe, back to optimal clock\n"); 7622 break; 7623 7624 default: 7625 dev_err(hdev->dev, "Received invalid clock change event %d\n", 7626 event_type); 7627 break; 7628 } 7629 7630 mutex_unlock(&hdev->clk_throttling.lock); 7631 } 7632 7633 static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 7634 { 7635 struct gaudi_device *gaudi = hdev->asic_specific; 7636 u64 data = le64_to_cpu(eq_entry->data[0]), event_mask = 0; 7637 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl); 7638 u32 fw_fatal_err_flag = 0, flags = 0; 7639 u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) 7640 >> EQ_CTL_EVENT_TYPE_SHIFT); 7641 bool reset_required, reset_direct = false; 7642 u8 cause; 7643 int rc; 7644 7645 if (event_type >= GAUDI_EVENT_SIZE) { 7646 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 7647 event_type, GAUDI_EVENT_SIZE - 1); 7648 return; 7649 } 7650 7651 gaudi->events_stat[event_type]++; 7652 gaudi->events_stat_aggregate[event_type]++; 7653 7654 switch (event_type) { 7655 case GAUDI_EVENT_PCIE_CORE_DERR: 7656 case GAUDI_EVENT_PCIE_IF_DERR: 7657 case GAUDI_EVENT_PCIE_PHY_DERR: 7658 case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR: 7659 case GAUDI_EVENT_MME0_ACC_DERR: 7660 case GAUDI_EVENT_MME0_SBAB_DERR: 7661 case GAUDI_EVENT_MME1_ACC_DERR: 7662 case GAUDI_EVENT_MME1_SBAB_DERR: 7663 case GAUDI_EVENT_MME2_ACC_DERR: 7664 case GAUDI_EVENT_MME2_SBAB_DERR: 7665 case GAUDI_EVENT_MME3_ACC_DERR: 7666 case GAUDI_EVENT_MME3_SBAB_DERR: 7667 case GAUDI_EVENT_DMA0_DERR_ECC ... GAUDI_EVENT_DMA7_DERR_ECC: 7668 fallthrough; 7669 case GAUDI_EVENT_CPU_IF_ECC_DERR: 7670 case GAUDI_EVENT_PSOC_MEM_DERR: 7671 case GAUDI_EVENT_PSOC_CORESIGHT_DERR: 7672 case GAUDI_EVENT_SRAM0_DERR ... GAUDI_EVENT_SRAM28_DERR: 7673 case GAUDI_EVENT_NIC0_DERR ... GAUDI_EVENT_NIC4_DERR: 7674 case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR: 7675 case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR: 7676 case GAUDI_EVENT_MMU_DERR: 7677 case GAUDI_EVENT_NIC0_CS_DBG_DERR ... GAUDI_EVENT_NIC4_CS_DBG_DERR: 7678 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7679 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 7680 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7681 fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR; 7682 goto reset_device; 7683 7684 case GAUDI_EVENT_GIC500: 7685 case GAUDI_EVENT_AXI_ECC: 7686 case GAUDI_EVENT_L2_RAM_ECC: 7687 case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17: 7688 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7689 fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR; 7690 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7691 goto reset_device; 7692 7693 case GAUDI_EVENT_HBM0_SPI_0: 7694 case GAUDI_EVENT_HBM1_SPI_0: 7695 case GAUDI_EVENT_HBM2_SPI_0: 7696 case GAUDI_EVENT_HBM3_SPI_0: 7697 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7698 gaudi_hbm_read_interrupts(hdev, 7699 gaudi_hbm_event_to_dev(event_type), 7700 &eq_entry->hbm_ecc_data); 7701 fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR; 7702 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7703 goto reset_device; 7704 7705 case GAUDI_EVENT_HBM0_SPI_1: 7706 case GAUDI_EVENT_HBM1_SPI_1: 7707 case GAUDI_EVENT_HBM2_SPI_1: 7708 case GAUDI_EVENT_HBM3_SPI_1: 7709 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7710 gaudi_hbm_read_interrupts(hdev, 7711 gaudi_hbm_event_to_dev(event_type), 7712 &eq_entry->hbm_ecc_data); 7713 hl_fw_unmask_irq(hdev, event_type); 7714 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7715 break; 7716 7717 case GAUDI_EVENT_TPC0_DEC: 7718 case GAUDI_EVENT_TPC1_DEC: 7719 case GAUDI_EVENT_TPC2_DEC: 7720 case GAUDI_EVENT_TPC3_DEC: 7721 case GAUDI_EVENT_TPC4_DEC: 7722 case GAUDI_EVENT_TPC5_DEC: 7723 case GAUDI_EVENT_TPC6_DEC: 7724 case GAUDI_EVENT_TPC7_DEC: 7725 /* In TPC DEC event, notify on TPC assertion. While there isn't 7726 * a specific event for assertion yet, the FW generates TPC DEC event. 7727 * The SW upper layer will inspect an internal mapped area to indicate 7728 * if the event is a TPC Assertion or a "real" TPC DEC. 7729 */ 7730 event_mask |= HL_NOTIFIER_EVENT_TPC_ASSERT; 7731 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7732 reset_required = gaudi_tpc_read_interrupts(hdev, 7733 tpc_dec_event_to_tpc_id(event_type), 7734 "AXI_SLV_DEC_Error"); 7735 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7736 if (reset_required) { 7737 dev_err(hdev->dev, "reset required due to %s\n", 7738 gaudi_irq_map_table[event_type].name); 7739 7740 reset_direct = true; 7741 goto reset_device; 7742 } else { 7743 hl_fw_unmask_irq(hdev, event_type); 7744 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 7745 } 7746 break; 7747 7748 case GAUDI_EVENT_TPC0_KRN_ERR: 7749 case GAUDI_EVENT_TPC1_KRN_ERR: 7750 case GAUDI_EVENT_TPC2_KRN_ERR: 7751 case GAUDI_EVENT_TPC3_KRN_ERR: 7752 case GAUDI_EVENT_TPC4_KRN_ERR: 7753 case GAUDI_EVENT_TPC5_KRN_ERR: 7754 case GAUDI_EVENT_TPC6_KRN_ERR: 7755 case GAUDI_EVENT_TPC7_KRN_ERR: 7756 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7757 reset_required = gaudi_tpc_read_interrupts(hdev, 7758 tpc_krn_event_to_tpc_id(event_type), 7759 "KRN_ERR"); 7760 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7761 if (reset_required) { 7762 dev_err(hdev->dev, "reset required due to %s\n", 7763 gaudi_irq_map_table[event_type].name); 7764 7765 reset_direct = true; 7766 goto reset_device; 7767 } else { 7768 hl_fw_unmask_irq(hdev, event_type); 7769 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 7770 } 7771 break; 7772 7773 case GAUDI_EVENT_PCIE_CORE_SERR: 7774 case GAUDI_EVENT_PCIE_IF_SERR: 7775 case GAUDI_EVENT_PCIE_PHY_SERR: 7776 case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR: 7777 case GAUDI_EVENT_MME0_ACC_SERR: 7778 case GAUDI_EVENT_MME0_SBAB_SERR: 7779 case GAUDI_EVENT_MME1_ACC_SERR: 7780 case GAUDI_EVENT_MME1_SBAB_SERR: 7781 case GAUDI_EVENT_MME2_ACC_SERR: 7782 case GAUDI_EVENT_MME2_SBAB_SERR: 7783 case GAUDI_EVENT_MME3_ACC_SERR: 7784 case GAUDI_EVENT_MME3_SBAB_SERR: 7785 case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_DMA7_SERR_ECC: 7786 case GAUDI_EVENT_CPU_IF_ECC_SERR: 7787 case GAUDI_EVENT_PSOC_MEM_SERR: 7788 case GAUDI_EVENT_PSOC_CORESIGHT_SERR: 7789 case GAUDI_EVENT_SRAM0_SERR ... GAUDI_EVENT_SRAM28_SERR: 7790 case GAUDI_EVENT_NIC0_SERR ... GAUDI_EVENT_NIC4_SERR: 7791 case GAUDI_EVENT_DMA_IF0_SERR ... GAUDI_EVENT_DMA_IF3_SERR: 7792 case GAUDI_EVENT_HBM_0_SERR ... GAUDI_EVENT_HBM_3_SERR: 7793 fallthrough; 7794 case GAUDI_EVENT_MMU_SERR: 7795 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7796 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 7797 hl_fw_unmask_irq(hdev, event_type); 7798 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7799 break; 7800 7801 case GAUDI_EVENT_PCIE_DEC: 7802 case GAUDI_EVENT_CPU_AXI_SPLITTER: 7803 case GAUDI_EVENT_PSOC_AXI_DEC: 7804 case GAUDI_EVENT_PSOC_PRSTN_FALL: 7805 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7806 hl_fw_unmask_irq(hdev, event_type); 7807 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7808 break; 7809 7810 case GAUDI_EVENT_MMU_PAGE_FAULT: 7811 case GAUDI_EVENT_MMU_WR_PERM: 7812 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7813 hl_fw_unmask_irq(hdev, event_type); 7814 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7815 break; 7816 7817 case GAUDI_EVENT_MME0_WBC_RSP: 7818 case GAUDI_EVENT_MME0_SBAB0_RSP: 7819 case GAUDI_EVENT_MME1_WBC_RSP: 7820 case GAUDI_EVENT_MME1_SBAB0_RSP: 7821 case GAUDI_EVENT_MME2_WBC_RSP: 7822 case GAUDI_EVENT_MME2_SBAB0_RSP: 7823 case GAUDI_EVENT_MME3_WBC_RSP: 7824 case GAUDI_EVENT_MME3_SBAB0_RSP: 7825 case GAUDI_EVENT_RAZWI_OR_ADC: 7826 case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM: 7827 case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM: 7828 fallthrough; 7829 case GAUDI_EVENT_NIC0_QM0: 7830 case GAUDI_EVENT_NIC0_QM1: 7831 case GAUDI_EVENT_NIC1_QM0: 7832 case GAUDI_EVENT_NIC1_QM1: 7833 case GAUDI_EVENT_NIC2_QM0: 7834 case GAUDI_EVENT_NIC2_QM1: 7835 case GAUDI_EVENT_NIC3_QM0: 7836 case GAUDI_EVENT_NIC3_QM1: 7837 case GAUDI_EVENT_NIC4_QM0: 7838 case GAUDI_EVENT_NIC4_QM1: 7839 case GAUDI_EVENT_DMA0_CORE ... GAUDI_EVENT_DMA7_CORE: 7840 case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: 7841 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7842 gaudi_handle_qman_err(hdev, event_type, &event_mask); 7843 hl_fw_unmask_irq(hdev, event_type); 7844 event_mask |= (HL_NOTIFIER_EVENT_USER_ENGINE_ERR | HL_NOTIFIER_EVENT_DEVICE_RESET); 7845 break; 7846 7847 case GAUDI_EVENT_RAZWI_OR_ADC_SW: 7848 gaudi_print_irq_info(hdev, event_type, true, &event_mask); 7849 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7850 goto reset_device; 7851 7852 case GAUDI_EVENT_TPC0_BMON_SPMU: 7853 case GAUDI_EVENT_TPC1_BMON_SPMU: 7854 case GAUDI_EVENT_TPC2_BMON_SPMU: 7855 case GAUDI_EVENT_TPC3_BMON_SPMU: 7856 case GAUDI_EVENT_TPC4_BMON_SPMU: 7857 case GAUDI_EVENT_TPC5_BMON_SPMU: 7858 case GAUDI_EVENT_TPC6_BMON_SPMU: 7859 case GAUDI_EVENT_TPC7_BMON_SPMU: 7860 case GAUDI_EVENT_DMA_BM_CH0 ... GAUDI_EVENT_DMA_BM_CH7: 7861 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7862 hl_fw_unmask_irq(hdev, event_type); 7863 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7864 break; 7865 7866 case GAUDI_EVENT_NIC_SEI_0 ... GAUDI_EVENT_NIC_SEI_4: 7867 gaudi_print_nic_axi_irq_info(hdev, event_type, &data); 7868 hl_fw_unmask_irq(hdev, event_type); 7869 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7870 break; 7871 7872 case GAUDI_EVENT_DMA_IF_SEI_0 ... GAUDI_EVENT_DMA_IF_SEI_3: 7873 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7874 gaudi_print_sm_sei_info(hdev, event_type, 7875 &eq_entry->sm_sei_data); 7876 rc = hl_state_dump(hdev); 7877 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7878 if (rc) 7879 dev_err(hdev->dev, 7880 "Error during system state dump %d\n", rc); 7881 hl_fw_unmask_irq(hdev, event_type); 7882 break; 7883 7884 case GAUDI_EVENT_STATUS_NIC0_ENG0 ... GAUDI_EVENT_STATUS_NIC4_ENG1: 7885 break; 7886 7887 case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E: 7888 gaudi_print_clk_change_info(hdev, event_type, &event_mask); 7889 hl_fw_unmask_irq(hdev, event_type); 7890 break; 7891 7892 case GAUDI_EVENT_PSOC_GPIO_U16_0: 7893 cause = le64_to_cpu(eq_entry->data[0]) & 0xFF; 7894 dev_err(hdev->dev, 7895 "Received high temp H/W interrupt %d (cause %d)\n", 7896 event_type, cause); 7897 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 7898 break; 7899 7900 case GAUDI_EVENT_DEV_RESET_REQ: 7901 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7902 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7903 goto reset_device; 7904 7905 case GAUDI_EVENT_PKT_QUEUE_OUT_SYNC: 7906 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7907 gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); 7908 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7909 goto reset_device; 7910 7911 case GAUDI_EVENT_FW_ALIVE_S: 7912 gaudi_print_irq_info(hdev, event_type, false, &event_mask); 7913 gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive); 7914 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 7915 goto reset_device; 7916 7917 default: 7918 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", 7919 event_type); 7920 break; 7921 } 7922 7923 if (event_mask) 7924 hl_notifier_event_send_all(hdev, event_mask); 7925 7926 return; 7927 7928 reset_device: 7929 reset_required = true; 7930 7931 if (hdev->asic_prop.fw_security_enabled && !reset_direct) { 7932 flags = HL_DRV_RESET_HARD | HL_DRV_RESET_BYPASS_REQ_TO_FW | fw_fatal_err_flag; 7933 7934 /* notify on device unavailable while the reset triggered by fw */ 7935 event_mask |= (HL_NOTIFIER_EVENT_DEVICE_RESET | 7936 HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE); 7937 } else if (hdev->hard_reset_on_fw_events) { 7938 flags = HL_DRV_RESET_HARD | HL_DRV_RESET_DELAY | fw_fatal_err_flag; 7939 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 7940 } else { 7941 reset_required = false; 7942 } 7943 7944 if (reset_required) { 7945 hl_device_cond_reset(hdev, flags, event_mask); 7946 } else { 7947 hl_fw_unmask_irq(hdev, event_type); 7948 /* Notification on occurred event needs to be sent although reset is not executed */ 7949 if (event_mask) 7950 hl_notifier_event_send_all(hdev, event_mask); 7951 } 7952 } 7953 7954 static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 7955 { 7956 struct gaudi_device *gaudi = hdev->asic_specific; 7957 7958 if (aggregate) { 7959 *size = (u32) sizeof(gaudi->events_stat_aggregate); 7960 return gaudi->events_stat_aggregate; 7961 } 7962 7963 *size = (u32) sizeof(gaudi->events_stat); 7964 return gaudi->events_stat; 7965 } 7966 7967 static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) 7968 { 7969 struct gaudi_device *gaudi = hdev->asic_specific; 7970 u32 status, timeout_usec; 7971 int rc; 7972 7973 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) || 7974 hdev->reset_info.hard_reset_pending) 7975 return 0; 7976 7977 if (hdev->pldm) 7978 timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC; 7979 else 7980 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 7981 7982 /* L0 & L1 invalidation */ 7983 WREG32(mmSTLB_INV_PS, 3); 7984 WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++); 7985 WREG32(mmSTLB_INV_PS, 2); 7986 7987 rc = hl_poll_timeout( 7988 hdev, 7989 mmSTLB_INV_PS, 7990 status, 7991 !status, 7992 1000, 7993 timeout_usec); 7994 7995 WREG32(mmSTLB_INV_SET, 0); 7996 7997 return rc; 7998 } 7999 8000 static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, 8001 bool is_hard, u32 flags, 8002 u32 asid, u64 va, u64 size) 8003 { 8004 /* Treat as invalidate all because there is no range invalidation 8005 * in Gaudi 8006 */ 8007 return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); 8008 } 8009 8010 static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, u64 phys_addr) 8011 { 8012 u32 status, timeout_usec; 8013 int rc; 8014 8015 if (hdev->pldm) 8016 timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC; 8017 else 8018 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 8019 8020 WREG32(MMU_ASID, asid); 8021 WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 8022 WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT); 8023 WREG32(MMU_BUSY, 0x80000000); 8024 8025 rc = hl_poll_timeout( 8026 hdev, 8027 MMU_BUSY, 8028 status, 8029 !(status & 0x80000000), 8030 1000, 8031 timeout_usec); 8032 8033 if (rc) { 8034 dev_err(hdev->dev, 8035 "Timeout during MMU hop0 config of asid %d\n", asid); 8036 return rc; 8037 } 8038 8039 return 0; 8040 } 8041 8042 static int gaudi_send_heartbeat(struct hl_device *hdev) 8043 { 8044 struct gaudi_device *gaudi = hdev->asic_specific; 8045 8046 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) 8047 return 0; 8048 8049 return hl_fw_send_heartbeat(hdev); 8050 } 8051 8052 static int gaudi_cpucp_info_get(struct hl_device *hdev) 8053 { 8054 struct gaudi_device *gaudi = hdev->asic_specific; 8055 struct asic_fixed_properties *prop = &hdev->asic_prop; 8056 int rc; 8057 8058 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) 8059 return 0; 8060 8061 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, 8062 mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 8063 mmCPU_BOOT_ERR1); 8064 if (rc) 8065 return rc; 8066 8067 if (!strlen(prop->cpucp_info.card_name)) 8068 strncpy(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME, 8069 CARD_NAME_MAX_LEN); 8070 8071 hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type); 8072 8073 set_default_power_values(hdev); 8074 8075 return 0; 8076 } 8077 8078 static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 8079 struct engines_data *e) 8080 { 8081 struct gaudi_device *gaudi = hdev->asic_specific; 8082 const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n"; 8083 const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n"; 8084 const char *nic_fmt = "%-5d%-9s%#-14x%#x\n"; 8085 unsigned long *mask = (unsigned long *)mask_arr; 8086 u32 qm_glbl_sts0, qm_cgm_sts, dma_core_sts0, tpc_cfg_sts, mme_arch_sts; 8087 bool is_idle = true, is_eng_idle, is_slave; 8088 u64 offset; 8089 int i, dma_id, port; 8090 8091 if (e) 8092 hl_engine_data_sprintf(e, 8093 "\nDMA is_idle QM_GLBL_STS0 QM_CGM_STS DMA_CORE_STS0\n" 8094 "--- ------- ------------ ---------- -------------\n"); 8095 8096 for (i = 0 ; i < DMA_NUMBER_OF_CHNLS ; i++) { 8097 dma_id = gaudi_dma_assignment[i]; 8098 offset = dma_id * DMA_QMAN_OFFSET; 8099 8100 qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + offset); 8101 qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + offset); 8102 dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + offset); 8103 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) && 8104 IS_DMA_IDLE(dma_core_sts0); 8105 is_idle &= is_eng_idle; 8106 8107 if (mask && !is_eng_idle) 8108 set_bit(GAUDI_ENGINE_ID_DMA_0 + dma_id, mask); 8109 if (e) 8110 hl_engine_data_sprintf(e, fmt, dma_id, 8111 is_eng_idle ? "Y" : "N", qm_glbl_sts0, 8112 qm_cgm_sts, dma_core_sts0); 8113 } 8114 8115 if (e) 8116 hl_engine_data_sprintf(e, 8117 "\nTPC is_idle QM_GLBL_STS0 QM_CGM_STS CFG_STATUS\n" 8118 "--- ------- ------------ ---------- ----------\n"); 8119 8120 for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) { 8121 offset = i * TPC_QMAN_OFFSET; 8122 qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + offset); 8123 qm_cgm_sts = RREG32(mmTPC0_QM_CGM_STS + offset); 8124 tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + offset); 8125 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) && 8126 IS_TPC_IDLE(tpc_cfg_sts); 8127 is_idle &= is_eng_idle; 8128 8129 if (mask && !is_eng_idle) 8130 set_bit(GAUDI_ENGINE_ID_TPC_0 + i, mask); 8131 if (e) 8132 hl_engine_data_sprintf(e, fmt, i, 8133 is_eng_idle ? "Y" : "N", 8134 qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); 8135 } 8136 8137 if (e) 8138 hl_engine_data_sprintf(e, 8139 "\nMME is_idle QM_GLBL_STS0 QM_CGM_STS ARCH_STATUS\n" 8140 "--- ------- ------------ ---------- -----------\n"); 8141 8142 for (i = 0 ; i < MME_NUMBER_OF_ENGINES ; i++) { 8143 offset = i * MME_QMAN_OFFSET; 8144 mme_arch_sts = RREG32(mmMME0_CTRL_ARCH_STATUS + offset); 8145 is_eng_idle = IS_MME_IDLE(mme_arch_sts); 8146 8147 /* MME 1 & 3 are slaves, no need to check their QMANs */ 8148 is_slave = i % 2; 8149 if (!is_slave) { 8150 qm_glbl_sts0 = RREG32(mmMME0_QM_GLBL_STS0 + offset); 8151 qm_cgm_sts = RREG32(mmMME0_QM_CGM_STS + offset); 8152 is_eng_idle &= IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); 8153 } 8154 8155 is_idle &= is_eng_idle; 8156 8157 if (mask && !is_eng_idle) 8158 set_bit(GAUDI_ENGINE_ID_MME_0 + i, mask); 8159 if (e) { 8160 if (!is_slave) 8161 hl_engine_data_sprintf(e, fmt, i, 8162 is_eng_idle ? "Y" : "N", 8163 qm_glbl_sts0, qm_cgm_sts, mme_arch_sts); 8164 else 8165 hl_engine_data_sprintf(e, mme_slave_fmt, i, 8166 is_eng_idle ? "Y" : "N", "-", 8167 "-", mme_arch_sts); 8168 } 8169 } 8170 8171 if (e) 8172 hl_engine_data_sprintf(e, 8173 "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" 8174 "--- ------- ------------ ----------\n"); 8175 8176 for (i = 0 ; i < (NIC_NUMBER_OF_ENGINES / 2) ; i++) { 8177 offset = i * NIC_MACRO_QMAN_OFFSET; 8178 port = 2 * i; 8179 if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { 8180 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); 8181 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); 8182 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); 8183 is_idle &= is_eng_idle; 8184 8185 if (mask && !is_eng_idle) 8186 set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); 8187 if (e) 8188 hl_engine_data_sprintf(e, nic_fmt, port, 8189 is_eng_idle ? "Y" : "N", 8190 qm_glbl_sts0, qm_cgm_sts); 8191 } 8192 8193 port = 2 * i + 1; 8194 if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { 8195 qm_glbl_sts0 = RREG32(mmNIC0_QM1_GLBL_STS0 + offset); 8196 qm_cgm_sts = RREG32(mmNIC0_QM1_CGM_STS + offset); 8197 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); 8198 is_idle &= is_eng_idle; 8199 8200 if (mask && !is_eng_idle) 8201 set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); 8202 if (e) 8203 hl_engine_data_sprintf(e, nic_fmt, port, 8204 is_eng_idle ? "Y" : "N", 8205 qm_glbl_sts0, qm_cgm_sts); 8206 } 8207 } 8208 8209 if (e) 8210 hl_engine_data_sprintf(e, "\n"); 8211 8212 return is_idle; 8213 } 8214 8215 static void gaudi_hw_queues_lock(struct hl_device *hdev) 8216 __acquires(&gaudi->hw_queues_lock) 8217 { 8218 struct gaudi_device *gaudi = hdev->asic_specific; 8219 8220 spin_lock(&gaudi->hw_queues_lock); 8221 } 8222 8223 static void gaudi_hw_queues_unlock(struct hl_device *hdev) 8224 __releases(&gaudi->hw_queues_lock) 8225 { 8226 struct gaudi_device *gaudi = hdev->asic_specific; 8227 8228 spin_unlock(&gaudi->hw_queues_lock); 8229 } 8230 8231 static u32 gaudi_get_pci_id(struct hl_device *hdev) 8232 { 8233 return hdev->pdev->device; 8234 } 8235 8236 static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data, 8237 size_t max_size) 8238 { 8239 struct gaudi_device *gaudi = hdev->asic_specific; 8240 8241 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) 8242 return 0; 8243 8244 return hl_fw_get_eeprom_data(hdev, data, max_size); 8245 } 8246 8247 static int gaudi_get_monitor_dump(struct hl_device *hdev, void *data) 8248 { 8249 struct gaudi_device *gaudi = hdev->asic_specific; 8250 8251 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) 8252 return 0; 8253 8254 return hl_fw_get_monitor_dump(hdev, data); 8255 } 8256 8257 /* 8258 * this function should be used only during initialization and/or after reset, 8259 * when there are no active users. 8260 */ 8261 static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, u32 tpc_id) 8262 { 8263 u64 kernel_timeout; 8264 u32 status, offset; 8265 int rc; 8266 8267 offset = tpc_id * (mmTPC1_CFG_STATUS - mmTPC0_CFG_STATUS); 8268 8269 if (hdev->pldm) 8270 kernel_timeout = GAUDI_PLDM_TPC_KERNEL_WAIT_USEC; 8271 else 8272 kernel_timeout = HL_DEVICE_TIMEOUT_USEC; 8273 8274 WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW + offset, 8275 lower_32_bits(tpc_kernel)); 8276 WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH + offset, 8277 upper_32_bits(tpc_kernel)); 8278 8279 WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_LOW + offset, 8280 lower_32_bits(tpc_kernel)); 8281 WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH + offset, 8282 upper_32_bits(tpc_kernel)); 8283 /* set a valid LUT pointer, content is of no significance */ 8284 WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_LO + offset, 8285 lower_32_bits(tpc_kernel)); 8286 WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_HI + offset, 8287 upper_32_bits(tpc_kernel)); 8288 8289 WREG32(mmTPC0_CFG_QM_SYNC_OBJECT_ADDR + offset, 8290 lower_32_bits(CFG_BASE + 8291 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0)); 8292 8293 WREG32(mmTPC0_CFG_TPC_CMD + offset, 8294 (1 << TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_SHIFT | 8295 1 << TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_SHIFT)); 8296 /* wait a bit for the engine to start executing */ 8297 usleep_range(1000, 1500); 8298 8299 /* wait until engine has finished executing */ 8300 rc = hl_poll_timeout( 8301 hdev, 8302 mmTPC0_CFG_STATUS + offset, 8303 status, 8304 (status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) == 8305 TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK, 8306 1000, 8307 kernel_timeout); 8308 8309 if (rc) { 8310 dev_err(hdev->dev, 8311 "Timeout while waiting for TPC%d icache prefetch\n", 8312 tpc_id); 8313 return -EIO; 8314 } 8315 8316 WREG32(mmTPC0_CFG_TPC_EXECUTE + offset, 8317 1 << TPC0_CFG_TPC_EXECUTE_V_SHIFT); 8318 8319 /* wait a bit for the engine to start executing */ 8320 usleep_range(1000, 1500); 8321 8322 /* wait until engine has finished executing */ 8323 rc = hl_poll_timeout( 8324 hdev, 8325 mmTPC0_CFG_STATUS + offset, 8326 status, 8327 (status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) == 8328 TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK, 8329 1000, 8330 kernel_timeout); 8331 8332 if (rc) { 8333 dev_err(hdev->dev, 8334 "Timeout while waiting for TPC%d vector pipe\n", 8335 tpc_id); 8336 return -EIO; 8337 } 8338 8339 rc = hl_poll_timeout( 8340 hdev, 8341 mmTPC0_CFG_WQ_INFLIGHT_CNTR + offset, 8342 status, 8343 (status == 0), 8344 1000, 8345 kernel_timeout); 8346 8347 if (rc) { 8348 dev_err(hdev->dev, 8349 "Timeout while waiting for TPC%d kernel to execute\n", 8350 tpc_id); 8351 return -EIO; 8352 } 8353 8354 return 0; 8355 } 8356 8357 static int gaudi_internal_cb_pool_init(struct hl_device *hdev, 8358 struct hl_ctx *ctx) 8359 { 8360 struct gaudi_device *gaudi = hdev->asic_specific; 8361 int min_alloc_order, rc, collective_cb_size; 8362 8363 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) 8364 return 0; 8365 8366 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, 8367 HOST_SPACE_INTERNAL_CB_SZ, 8368 &hdev->internal_cb_pool_dma_addr, 8369 GFP_KERNEL | __GFP_ZERO); 8370 8371 if (!hdev->internal_cb_pool_virt_addr) 8372 return -ENOMEM; 8373 8374 collective_cb_size = sizeof(struct packet_msg_short) * 5 + 8375 sizeof(struct packet_fence); 8376 min_alloc_order = ilog2(collective_cb_size); 8377 8378 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); 8379 if (!hdev->internal_cb_pool) { 8380 dev_err(hdev->dev, 8381 "Failed to create internal CB pool\n"); 8382 rc = -ENOMEM; 8383 goto free_internal_cb_pool; 8384 } 8385 8386 rc = gen_pool_add(hdev->internal_cb_pool, 8387 (uintptr_t) hdev->internal_cb_pool_virt_addr, 8388 HOST_SPACE_INTERNAL_CB_SZ, -1); 8389 if (rc) { 8390 dev_err(hdev->dev, 8391 "Failed to add memory to internal CB pool\n"); 8392 rc = -EFAULT; 8393 goto destroy_internal_cb_pool; 8394 } 8395 8396 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, 8397 HL_VA_RANGE_TYPE_HOST, HOST_SPACE_INTERNAL_CB_SZ, 8398 HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 8399 8400 if (!hdev->internal_cb_va_base) { 8401 rc = -ENOMEM; 8402 goto destroy_internal_cb_pool; 8403 } 8404 8405 mutex_lock(&hdev->mmu_lock); 8406 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, 8407 hdev->internal_cb_pool_dma_addr, 8408 HOST_SPACE_INTERNAL_CB_SZ); 8409 8410 hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); 8411 mutex_unlock(&hdev->mmu_lock); 8412 8413 if (rc) 8414 goto unreserve_internal_cb_pool; 8415 8416 return 0; 8417 8418 unreserve_internal_cb_pool: 8419 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, 8420 HOST_SPACE_INTERNAL_CB_SZ); 8421 destroy_internal_cb_pool: 8422 gen_pool_destroy(hdev->internal_cb_pool); 8423 free_internal_cb_pool: 8424 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 8425 hdev->internal_cb_pool_dma_addr); 8426 8427 return rc; 8428 } 8429 8430 static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, 8431 struct hl_ctx *ctx) 8432 { 8433 struct gaudi_device *gaudi = hdev->asic_specific; 8434 8435 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) 8436 return; 8437 8438 mutex_lock(&hdev->mmu_lock); 8439 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, 8440 HOST_SPACE_INTERNAL_CB_SZ); 8441 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, 8442 HOST_SPACE_INTERNAL_CB_SZ); 8443 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 8444 mutex_unlock(&hdev->mmu_lock); 8445 8446 gen_pool_destroy(hdev->internal_cb_pool); 8447 8448 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 8449 hdev->internal_cb_pool_dma_addr); 8450 } 8451 8452 static int gaudi_ctx_init(struct hl_ctx *ctx) 8453 { 8454 int rc; 8455 8456 if (ctx->asid == HL_KERNEL_ASID_ID) 8457 return 0; 8458 8459 rc = gaudi_internal_cb_pool_init(ctx->hdev, ctx); 8460 if (rc) 8461 return rc; 8462 8463 rc = gaudi_restore_user_registers(ctx->hdev); 8464 if (rc) 8465 gaudi_internal_cb_pool_fini(ctx->hdev, ctx); 8466 8467 return rc; 8468 } 8469 8470 static void gaudi_ctx_fini(struct hl_ctx *ctx) 8471 { 8472 if (ctx->asid == HL_KERNEL_ASID_ID) 8473 return; 8474 8475 gaudi_internal_cb_pool_fini(ctx->hdev, ctx); 8476 } 8477 8478 static int gaudi_pre_schedule_cs(struct hl_cs *cs) 8479 { 8480 return 0; 8481 } 8482 8483 static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 8484 { 8485 return gaudi_cq_assignment[cq_idx]; 8486 } 8487 8488 static u32 gaudi_get_signal_cb_size(struct hl_device *hdev) 8489 { 8490 return sizeof(struct packet_msg_short) + 8491 sizeof(struct packet_msg_prot) * 2; 8492 } 8493 8494 static u32 gaudi_get_wait_cb_size(struct hl_device *hdev) 8495 { 8496 return sizeof(struct packet_msg_short) * 4 + 8497 sizeof(struct packet_fence) + 8498 sizeof(struct packet_msg_prot) * 2; 8499 } 8500 8501 static u32 gaudi_get_sob_addr(struct hl_device *hdev, u32 sob_id) 8502 { 8503 return mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + (sob_id * 4); 8504 } 8505 8506 static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, 8507 u32 size, bool eb) 8508 { 8509 struct hl_cb *cb = (struct hl_cb *) data; 8510 struct packet_msg_short *pkt; 8511 u32 value, ctl, pkt_size = sizeof(*pkt); 8512 8513 pkt = cb->kernel_address + size; 8514 memset(pkt, 0, pkt_size); 8515 8516 /* Inc by 1, Mode ADD */ 8517 value = FIELD_PREP(GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); 8518 value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_SOB_MOD_MASK, 1); 8519 8520 ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); 8521 ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */ 8522 ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 3); /* W_S SOB base */ 8523 ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 8524 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, eb); 8525 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 8526 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 8527 8528 pkt->value = cpu_to_le32(value); 8529 pkt->ctl = cpu_to_le32(ctl); 8530 8531 return size + pkt_size; 8532 } 8533 8534 static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, 8535 u16 addr) 8536 { 8537 u32 ctl, pkt_size = sizeof(*pkt); 8538 8539 memset(pkt, 0, pkt_size); 8540 8541 ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, addr); 8542 ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */ 8543 ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 8544 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); 8545 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 8546 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 0); /* last pkt MB */ 8547 8548 pkt->value = cpu_to_le32(value); 8549 pkt->ctl = cpu_to_le32(ctl); 8550 8551 return pkt_size; 8552 } 8553 8554 static u32 gaudi_add_arm_monitor_pkt(struct hl_device *hdev, 8555 struct packet_msg_short *pkt, u16 sob_base, u8 sob_mask, 8556 u16 sob_val, u16 mon_id) 8557 { 8558 u64 monitor_base; 8559 u32 ctl, value, pkt_size = sizeof(*pkt); 8560 u16 msg_addr_offset; 8561 u8 mask; 8562 8563 if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) { 8564 dev_err(hdev->dev, 8565 "sob_base %u (mask %#x) is not valid\n", 8566 sob_base, sob_mask); 8567 return 0; 8568 } 8569 8570 /* 8571 * monitor_base should be the content of the base0 address registers, 8572 * so it will be added to the msg short offsets 8573 */ 8574 monitor_base = mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 8575 8576 msg_addr_offset = 8577 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0 + mon_id * 4) - 8578 monitor_base; 8579 8580 memset(pkt, 0, pkt_size); 8581 8582 /* Monitor config packet: bind the monitor to a sync object */ 8583 value = FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8); 8584 value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val); 8585 value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_MODE_MASK, 8586 0); /* GREATER OR EQUAL*/ 8587 value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_MASK_MASK, mask); 8588 8589 ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, msg_addr_offset); 8590 ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */ 8591 ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */ 8592 ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 8593 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); 8594 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 8595 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 8596 8597 pkt->value = cpu_to_le32(value); 8598 pkt->ctl = cpu_to_le32(ctl); 8599 8600 return pkt_size; 8601 } 8602 8603 static u32 gaudi_add_fence_pkt(struct packet_fence *pkt) 8604 { 8605 u32 ctl, cfg, pkt_size = sizeof(*pkt); 8606 8607 memset(pkt, 0, pkt_size); 8608 8609 cfg = FIELD_PREP(GAUDI_PKT_FENCE_CFG_DEC_VAL_MASK, 1); 8610 cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); 8611 cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_ID_MASK, 2); 8612 8613 ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_FENCE); 8614 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); 8615 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); 8616 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); 8617 8618 pkt->cfg = cpu_to_le32(cfg); 8619 pkt->ctl = cpu_to_le32(ctl); 8620 8621 return pkt_size; 8622 } 8623 8624 static int gaudi_get_fence_addr(struct hl_device *hdev, u32 queue_id, u64 *addr) 8625 { 8626 u32 offset, nic_index; 8627 8628 switch (queue_id) { 8629 case GAUDI_QUEUE_ID_DMA_0_0: 8630 offset = mmDMA0_QM_CP_FENCE2_RDATA_0; 8631 break; 8632 case GAUDI_QUEUE_ID_DMA_0_1: 8633 offset = mmDMA0_QM_CP_FENCE2_RDATA_1; 8634 break; 8635 case GAUDI_QUEUE_ID_DMA_0_2: 8636 offset = mmDMA0_QM_CP_FENCE2_RDATA_2; 8637 break; 8638 case GAUDI_QUEUE_ID_DMA_0_3: 8639 offset = mmDMA0_QM_CP_FENCE2_RDATA_3; 8640 break; 8641 case GAUDI_QUEUE_ID_DMA_1_0: 8642 offset = mmDMA1_QM_CP_FENCE2_RDATA_0; 8643 break; 8644 case GAUDI_QUEUE_ID_DMA_1_1: 8645 offset = mmDMA1_QM_CP_FENCE2_RDATA_1; 8646 break; 8647 case GAUDI_QUEUE_ID_DMA_1_2: 8648 offset = mmDMA1_QM_CP_FENCE2_RDATA_2; 8649 break; 8650 case GAUDI_QUEUE_ID_DMA_1_3: 8651 offset = mmDMA1_QM_CP_FENCE2_RDATA_3; 8652 break; 8653 case GAUDI_QUEUE_ID_DMA_5_0: 8654 offset = mmDMA5_QM_CP_FENCE2_RDATA_0; 8655 break; 8656 case GAUDI_QUEUE_ID_DMA_5_1: 8657 offset = mmDMA5_QM_CP_FENCE2_RDATA_1; 8658 break; 8659 case GAUDI_QUEUE_ID_DMA_5_2: 8660 offset = mmDMA5_QM_CP_FENCE2_RDATA_2; 8661 break; 8662 case GAUDI_QUEUE_ID_DMA_5_3: 8663 offset = mmDMA5_QM_CP_FENCE2_RDATA_3; 8664 break; 8665 case GAUDI_QUEUE_ID_TPC_7_0: 8666 offset = mmTPC7_QM_CP_FENCE2_RDATA_0; 8667 break; 8668 case GAUDI_QUEUE_ID_TPC_7_1: 8669 offset = mmTPC7_QM_CP_FENCE2_RDATA_1; 8670 break; 8671 case GAUDI_QUEUE_ID_TPC_7_2: 8672 offset = mmTPC7_QM_CP_FENCE2_RDATA_2; 8673 break; 8674 case GAUDI_QUEUE_ID_TPC_7_3: 8675 offset = mmTPC7_QM_CP_FENCE2_RDATA_3; 8676 break; 8677 case GAUDI_QUEUE_ID_NIC_0_0: 8678 case GAUDI_QUEUE_ID_NIC_1_0: 8679 case GAUDI_QUEUE_ID_NIC_2_0: 8680 case GAUDI_QUEUE_ID_NIC_3_0: 8681 case GAUDI_QUEUE_ID_NIC_4_0: 8682 case GAUDI_QUEUE_ID_NIC_5_0: 8683 case GAUDI_QUEUE_ID_NIC_6_0: 8684 case GAUDI_QUEUE_ID_NIC_7_0: 8685 case GAUDI_QUEUE_ID_NIC_8_0: 8686 case GAUDI_QUEUE_ID_NIC_9_0: 8687 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_0) >> 2; 8688 offset = mmNIC0_QM0_CP_FENCE2_RDATA_0 + 8689 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + 8690 (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; 8691 break; 8692 case GAUDI_QUEUE_ID_NIC_0_1: 8693 case GAUDI_QUEUE_ID_NIC_1_1: 8694 case GAUDI_QUEUE_ID_NIC_2_1: 8695 case GAUDI_QUEUE_ID_NIC_3_1: 8696 case GAUDI_QUEUE_ID_NIC_4_1: 8697 case GAUDI_QUEUE_ID_NIC_5_1: 8698 case GAUDI_QUEUE_ID_NIC_6_1: 8699 case GAUDI_QUEUE_ID_NIC_7_1: 8700 case GAUDI_QUEUE_ID_NIC_8_1: 8701 case GAUDI_QUEUE_ID_NIC_9_1: 8702 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_1) >> 2; 8703 offset = mmNIC0_QM0_CP_FENCE2_RDATA_1 + 8704 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + 8705 (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; 8706 break; 8707 case GAUDI_QUEUE_ID_NIC_0_2: 8708 case GAUDI_QUEUE_ID_NIC_1_2: 8709 case GAUDI_QUEUE_ID_NIC_2_2: 8710 case GAUDI_QUEUE_ID_NIC_3_2: 8711 case GAUDI_QUEUE_ID_NIC_4_2: 8712 case GAUDI_QUEUE_ID_NIC_5_2: 8713 case GAUDI_QUEUE_ID_NIC_6_2: 8714 case GAUDI_QUEUE_ID_NIC_7_2: 8715 case GAUDI_QUEUE_ID_NIC_8_2: 8716 case GAUDI_QUEUE_ID_NIC_9_2: 8717 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_2) >> 2; 8718 offset = mmNIC0_QM0_CP_FENCE2_RDATA_2 + 8719 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + 8720 (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; 8721 break; 8722 case GAUDI_QUEUE_ID_NIC_0_3: 8723 case GAUDI_QUEUE_ID_NIC_1_3: 8724 case GAUDI_QUEUE_ID_NIC_2_3: 8725 case GAUDI_QUEUE_ID_NIC_3_3: 8726 case GAUDI_QUEUE_ID_NIC_4_3: 8727 case GAUDI_QUEUE_ID_NIC_5_3: 8728 case GAUDI_QUEUE_ID_NIC_6_3: 8729 case GAUDI_QUEUE_ID_NIC_7_3: 8730 case GAUDI_QUEUE_ID_NIC_8_3: 8731 case GAUDI_QUEUE_ID_NIC_9_3: 8732 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_3) >> 2; 8733 offset = mmNIC0_QM0_CP_FENCE2_RDATA_3 + 8734 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + 8735 (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; 8736 break; 8737 default: 8738 return -EINVAL; 8739 } 8740 8741 *addr = CFG_BASE + offset; 8742 8743 return 0; 8744 } 8745 8746 static u32 gaudi_add_mon_pkts(void *buf, u16 mon_id, u64 fence_addr) 8747 { 8748 u64 monitor_base; 8749 u32 size = 0; 8750 u16 msg_addr_offset; 8751 8752 /* 8753 * monitor_base should be the content of the base0 address registers, 8754 * so it will be added to the msg short offsets 8755 */ 8756 monitor_base = mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 8757 8758 /* First monitor config packet: low address of the sync */ 8759 msg_addr_offset = 8760 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_id * 4) - 8761 monitor_base; 8762 8763 size += gaudi_add_mon_msg_short(buf + size, (u32) fence_addr, 8764 msg_addr_offset); 8765 8766 /* Second monitor config packet: high address of the sync */ 8767 msg_addr_offset = 8768 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_id * 4) - 8769 monitor_base; 8770 8771 size += gaudi_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), 8772 msg_addr_offset); 8773 8774 /* 8775 * Third monitor config packet: the payload, i.e. what to write when the 8776 * sync triggers 8777 */ 8778 msg_addr_offset = 8779 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_id * 4) - 8780 monitor_base; 8781 8782 size += gaudi_add_mon_msg_short(buf + size, 1, msg_addr_offset); 8783 8784 return size; 8785 } 8786 8787 static u32 gaudi_gen_wait_cb(struct hl_device *hdev, 8788 struct hl_gen_wait_properties *prop) 8789 { 8790 struct hl_cb *cb = (struct hl_cb *) prop->data; 8791 void *buf = cb->kernel_address; 8792 u64 fence_addr = 0; 8793 u32 size = prop->size; 8794 8795 if (gaudi_get_fence_addr(hdev, prop->q_idx, &fence_addr)) { 8796 dev_crit(hdev->dev, "wrong queue id %d for wait packet\n", 8797 prop->q_idx); 8798 return 0; 8799 } 8800 8801 size += gaudi_add_mon_pkts(buf + size, prop->mon_id, fence_addr); 8802 size += gaudi_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, 8803 prop->sob_mask, prop->sob_val, prop->mon_id); 8804 size += gaudi_add_fence_pkt(buf + size); 8805 8806 return size; 8807 } 8808 8809 static void gaudi_reset_sob(struct hl_device *hdev, void *data) 8810 { 8811 struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data; 8812 8813 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, 8814 hw_sob->sob_id); 8815 8816 WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + 8817 hw_sob->sob_id * 4, 0); 8818 8819 kref_init(&hw_sob->kref); 8820 } 8821 8822 static u64 gaudi_get_device_time(struct hl_device *hdev) 8823 { 8824 u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 8825 8826 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 8827 } 8828 8829 static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 8830 u32 *block_size, u32 *block_id) 8831 { 8832 return -EPERM; 8833 } 8834 8835 static int gaudi_block_mmap(struct hl_device *hdev, 8836 struct vm_area_struct *vma, 8837 u32 block_id, u32 block_size) 8838 { 8839 return -EPERM; 8840 } 8841 8842 static void gaudi_enable_events_from_fw(struct hl_device *hdev) 8843 { 8844 struct cpu_dyn_regs *dyn_regs = 8845 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 8846 u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? 8847 mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : 8848 le32_to_cpu(dyn_regs->gic_host_ints_irq); 8849 8850 WREG32(irq_handler_offset, 8851 gaudi_irq_map_table[GAUDI_EVENT_INTS_REGISTER].cpu_id); 8852 } 8853 8854 static int gaudi_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 8855 { 8856 return -EINVAL; 8857 } 8858 8859 static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx) 8860 { 8861 switch (pll_idx) { 8862 case HL_GAUDI_CPU_PLL: return CPU_PLL; 8863 case HL_GAUDI_PCI_PLL: return PCI_PLL; 8864 case HL_GAUDI_NIC_PLL: return NIC_PLL; 8865 case HL_GAUDI_DMA_PLL: return DMA_PLL; 8866 case HL_GAUDI_MESH_PLL: return MESH_PLL; 8867 case HL_GAUDI_MME_PLL: return MME_PLL; 8868 case HL_GAUDI_TPC_PLL: return TPC_PLL; 8869 case HL_GAUDI_IF_PLL: return IF_PLL; 8870 case HL_GAUDI_SRAM_PLL: return SRAM_PLL; 8871 case HL_GAUDI_HBM_PLL: return HBM_PLL; 8872 default: return -EINVAL; 8873 } 8874 } 8875 8876 static int gaudi_add_sync_to_engine_map_entry( 8877 struct hl_sync_to_engine_map *map, u32 reg_value, 8878 enum hl_sync_engine_type engine_type, u32 engine_id) 8879 { 8880 struct hl_sync_to_engine_map_entry *entry; 8881 8882 /* Reg value represents a partial address of sync object, 8883 * it is used as unique identifier. For this we need to 8884 * clear the cutoff cfg base bits from the value. 8885 */ 8886 if (reg_value == 0 || reg_value == 0xffffffff) 8887 return 0; 8888 reg_value -= lower_32_bits(CFG_BASE); 8889 8890 /* create a new hash entry */ 8891 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 8892 if (!entry) 8893 return -ENOMEM; 8894 entry->engine_type = engine_type; 8895 entry->engine_id = engine_id; 8896 entry->sync_id = reg_value; 8897 hash_add(map->tb, &entry->node, reg_value); 8898 8899 return 0; 8900 } 8901 8902 static int gaudi_gen_sync_to_engine_map(struct hl_device *hdev, 8903 struct hl_sync_to_engine_map *map) 8904 { 8905 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 8906 int i, j, rc; 8907 u32 reg_value; 8908 8909 /* Iterate over TPC engines */ 8910 for (i = 0; i < sds->props[SP_NUM_OF_TPC_ENGINES]; ++i) { 8911 8912 reg_value = RREG32(sds->props[SP_TPC0_CFG_SO] + 8913 sds->props[SP_NEXT_TPC] * i); 8914 8915 rc = gaudi_add_sync_to_engine_map_entry(map, reg_value, 8916 ENGINE_TPC, i); 8917 if (rc) 8918 goto free_sync_to_engine_map; 8919 } 8920 8921 /* Iterate over MME engines */ 8922 for (i = 0; i < sds->props[SP_NUM_OF_MME_ENGINES]; ++i) { 8923 for (j = 0; j < sds->props[SP_SUB_MME_ENG_NUM]; ++j) { 8924 8925 reg_value = RREG32(sds->props[SP_MME_CFG_SO] + 8926 sds->props[SP_NEXT_MME] * i + 8927 j * sizeof(u32)); 8928 8929 rc = gaudi_add_sync_to_engine_map_entry( 8930 map, reg_value, ENGINE_MME, 8931 i * sds->props[SP_SUB_MME_ENG_NUM] + j); 8932 if (rc) 8933 goto free_sync_to_engine_map; 8934 } 8935 } 8936 8937 /* Iterate over DMA engines */ 8938 for (i = 0; i < sds->props[SP_NUM_OF_DMA_ENGINES]; ++i) { 8939 reg_value = RREG32(sds->props[SP_DMA_CFG_SO] + 8940 sds->props[SP_DMA_QUEUES_OFFSET] * i); 8941 rc = gaudi_add_sync_to_engine_map_entry(map, reg_value, 8942 ENGINE_DMA, i); 8943 if (rc) 8944 goto free_sync_to_engine_map; 8945 } 8946 8947 return 0; 8948 8949 free_sync_to_engine_map: 8950 hl_state_dump_free_sync_to_engine_map(map); 8951 8952 return rc; 8953 } 8954 8955 static int gaudi_monitor_valid(struct hl_mon_state_dump *mon) 8956 { 8957 return FIELD_GET( 8958 SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_MASK, 8959 mon->status); 8960 } 8961 8962 static void gaudi_fill_sobs_from_mon(char *sobs, struct hl_mon_state_dump *mon) 8963 { 8964 const size_t max_write = 10; 8965 u32 gid, mask, sob; 8966 int i, offset; 8967 8968 /* Sync object ID is calculated as follows: 8969 * (8 * group_id + cleared bits in mask) 8970 */ 8971 gid = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK, 8972 mon->arm_data); 8973 mask = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK, 8974 mon->arm_data); 8975 8976 for (i = 0, offset = 0; mask && offset < MONITOR_SOB_STRING_SIZE - 8977 max_write; mask >>= 1, i++) { 8978 if (!(mask & 1)) { 8979 sob = gid * MONITOR_MAX_SOBS + i; 8980 8981 if (offset > 0) 8982 offset += snprintf(sobs + offset, max_write, 8983 ", "); 8984 8985 offset += snprintf(sobs + offset, max_write, "%u", sob); 8986 } 8987 } 8988 } 8989 8990 static int gaudi_print_single_monitor(char **buf, size_t *size, size_t *offset, 8991 struct hl_device *hdev, 8992 struct hl_mon_state_dump *mon) 8993 { 8994 const char *name; 8995 char scratch_buf1[BIN_REG_STRING_SIZE], 8996 scratch_buf2[BIN_REG_STRING_SIZE]; 8997 char monitored_sobs[MONITOR_SOB_STRING_SIZE] = {0}; 8998 8999 name = hl_state_dump_get_monitor_name(hdev, mon); 9000 if (!name) 9001 name = ""; 9002 9003 gaudi_fill_sobs_from_mon(monitored_sobs, mon); 9004 9005 return hl_snprintf_resize( 9006 buf, size, offset, 9007 "Mon id: %u%s, wait for group id: %u mask %s to reach val: %u and write %u to address 0x%llx. Pending: %s. Means sync objects [%s] are being monitored.", 9008 mon->id, name, 9009 FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK, 9010 mon->arm_data), 9011 hl_format_as_binary( 9012 scratch_buf1, sizeof(scratch_buf1), 9013 FIELD_GET( 9014 SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK, 9015 mon->arm_data)), 9016 FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_MASK, 9017 mon->arm_data), 9018 mon->wr_data, 9019 (((u64)mon->wr_addr_high) << 32) | mon->wr_addr_low, 9020 hl_format_as_binary( 9021 scratch_buf2, sizeof(scratch_buf2), 9022 FIELD_GET( 9023 SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_MASK, 9024 mon->status)), 9025 monitored_sobs); 9026 } 9027 9028 9029 static int gaudi_print_fences_single_engine( 9030 struct hl_device *hdev, u64 base_offset, u64 status_base_offset, 9031 enum hl_sync_engine_type engine_type, u32 engine_id, char **buf, 9032 size_t *size, size_t *offset) 9033 { 9034 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 9035 int rc = -ENOMEM, i; 9036 u32 *statuses, *fences; 9037 9038 statuses = kcalloc(sds->props[SP_ENGINE_NUM_OF_QUEUES], 9039 sizeof(*statuses), GFP_KERNEL); 9040 if (!statuses) 9041 goto out; 9042 9043 fences = kcalloc(sds->props[SP_ENGINE_NUM_OF_FENCES] * 9044 sds->props[SP_ENGINE_NUM_OF_QUEUES], 9045 sizeof(*fences), GFP_KERNEL); 9046 if (!fences) 9047 goto free_status; 9048 9049 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES]; ++i) 9050 statuses[i] = RREG32(status_base_offset + i * sizeof(u32)); 9051 9052 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES] * 9053 sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) 9054 fences[i] = RREG32(base_offset + i * sizeof(u32)); 9055 9056 /* The actual print */ 9057 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) { 9058 u32 fence_id; 9059 u64 fence_cnt, fence_rdata; 9060 const char *engine_name; 9061 9062 if (!FIELD_GET(TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_MASK, 9063 statuses[i])) 9064 continue; 9065 9066 fence_id = 9067 FIELD_GET(TPC0_QM_CP_STS_0_FENCE_ID_MASK, statuses[i]); 9068 fence_cnt = base_offset + CFG_BASE + 9069 sizeof(u32) * 9070 (i + fence_id * sds->props[SP_ENGINE_NUM_OF_QUEUES]); 9071 fence_rdata = fence_cnt - sds->props[SP_FENCE0_CNT_OFFSET] + 9072 sds->props[SP_FENCE0_RDATA_OFFSET]; 9073 engine_name = hl_sync_engine_to_string(engine_type); 9074 9075 rc = hl_snprintf_resize( 9076 buf, size, offset, 9077 "%s%u, stream %u: fence id %u cnt = 0x%llx (%s%u_QM.CP_FENCE%u_CNT_%u) rdata = 0x%llx (%s%u_QM.CP_FENCE%u_RDATA_%u) value = %u, cp_status = %u\n", 9078 engine_name, engine_id, 9079 i, fence_id, 9080 fence_cnt, engine_name, engine_id, fence_id, i, 9081 fence_rdata, engine_name, engine_id, fence_id, i, 9082 fences[fence_id], 9083 statuses[i]); 9084 if (rc) 9085 goto free_fences; 9086 } 9087 9088 rc = 0; 9089 9090 free_fences: 9091 kfree(fences); 9092 free_status: 9093 kfree(statuses); 9094 out: 9095 return rc; 9096 } 9097 9098 9099 static struct hl_state_dump_specs_funcs gaudi_state_dump_funcs = { 9100 .monitor_valid = gaudi_monitor_valid, 9101 .print_single_monitor = gaudi_print_single_monitor, 9102 .gen_sync_to_engine_map = gaudi_gen_sync_to_engine_map, 9103 .print_fences_single_engine = gaudi_print_fences_single_engine, 9104 }; 9105 9106 static void gaudi_state_dump_init(struct hl_device *hdev) 9107 { 9108 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 9109 int i; 9110 9111 for (i = 0; i < ARRAY_SIZE(gaudi_so_id_to_str); ++i) 9112 hash_add(sds->so_id_to_str_tb, 9113 &gaudi_so_id_to_str[i].node, 9114 gaudi_so_id_to_str[i].id); 9115 9116 for (i = 0; i < ARRAY_SIZE(gaudi_monitor_id_to_str); ++i) 9117 hash_add(sds->monitor_id_to_str_tb, 9118 &gaudi_monitor_id_to_str[i].node, 9119 gaudi_monitor_id_to_str[i].id); 9120 9121 sds->props = gaudi_state_dump_specs_props; 9122 9123 sds->sync_namager_names = gaudi_sync_manager_names; 9124 9125 sds->funcs = gaudi_state_dump_funcs; 9126 } 9127 9128 static u32 *gaudi_get_stream_master_qid_arr(void) 9129 { 9130 return gaudi_stream_master; 9131 } 9132 9133 static int gaudi_set_dram_properties(struct hl_device *hdev) 9134 { 9135 return 0; 9136 } 9137 9138 static int gaudi_set_binning_masks(struct hl_device *hdev) 9139 { 9140 return 0; 9141 } 9142 9143 static void gaudi_check_if_razwi_happened(struct hl_device *hdev) 9144 { 9145 } 9146 9147 static ssize_t infineon_ver_show(struct device *dev, struct device_attribute *attr, char *buf) 9148 { 9149 struct hl_device *hdev = dev_get_drvdata(dev); 9150 struct cpucp_info *cpucp_info; 9151 9152 cpucp_info = &hdev->asic_prop.cpucp_info; 9153 9154 return sprintf(buf, "%#04x\n", le32_to_cpu(cpucp_info->infineon_version)); 9155 } 9156 9157 static DEVICE_ATTR_RO(infineon_ver); 9158 9159 static struct attribute *gaudi_vrm_dev_attrs[] = { 9160 &dev_attr_infineon_ver.attr, 9161 NULL, 9162 }; 9163 9164 static void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 9165 struct attribute_group *dev_vrm_attr_grp) 9166 { 9167 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp); 9168 dev_vrm_attr_grp->attrs = gaudi_vrm_dev_attrs; 9169 } 9170 9171 static int gaudi_send_device_activity(struct hl_device *hdev, bool open) 9172 { 9173 return 0; 9174 } 9175 9176 static const struct hl_asic_funcs gaudi_funcs = { 9177 .early_init = gaudi_early_init, 9178 .early_fini = gaudi_early_fini, 9179 .late_init = gaudi_late_init, 9180 .late_fini = gaudi_late_fini, 9181 .sw_init = gaudi_sw_init, 9182 .sw_fini = gaudi_sw_fini, 9183 .hw_init = gaudi_hw_init, 9184 .hw_fini = gaudi_hw_fini, 9185 .halt_engines = gaudi_halt_engines, 9186 .suspend = gaudi_suspend, 9187 .resume = gaudi_resume, 9188 .mmap = gaudi_mmap, 9189 .ring_doorbell = gaudi_ring_doorbell, 9190 .pqe_write = gaudi_pqe_write, 9191 .asic_dma_alloc_coherent = gaudi_dma_alloc_coherent, 9192 .asic_dma_free_coherent = gaudi_dma_free_coherent, 9193 .scrub_device_mem = gaudi_scrub_device_mem, 9194 .scrub_device_dram = gaudi_scrub_device_dram, 9195 .get_int_queue_base = gaudi_get_int_queue_base, 9196 .test_queues = gaudi_test_queues, 9197 .asic_dma_pool_zalloc = gaudi_dma_pool_zalloc, 9198 .asic_dma_pool_free = gaudi_dma_pool_free, 9199 .cpu_accessible_dma_pool_alloc = gaudi_cpu_accessible_dma_pool_alloc, 9200 .cpu_accessible_dma_pool_free = gaudi_cpu_accessible_dma_pool_free, 9201 .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable, 9202 .cs_parser = gaudi_cs_parser, 9203 .asic_dma_map_sgtable = hl_dma_map_sgtable, 9204 .add_end_of_cb_packets = gaudi_add_end_of_cb_packets, 9205 .update_eq_ci = gaudi_update_eq_ci, 9206 .context_switch = gaudi_context_switch, 9207 .restore_phase_topology = gaudi_restore_phase_topology, 9208 .debugfs_read_dma = gaudi_debugfs_read_dma, 9209 .add_device_attr = gaudi_add_device_attr, 9210 .handle_eqe = gaudi_handle_eqe, 9211 .get_events_stat = gaudi_get_events_stat, 9212 .read_pte = gaudi_read_pte, 9213 .write_pte = gaudi_write_pte, 9214 .mmu_invalidate_cache = gaudi_mmu_invalidate_cache, 9215 .mmu_invalidate_cache_range = gaudi_mmu_invalidate_cache_range, 9216 .mmu_prefetch_cache_range = NULL, 9217 .send_heartbeat = gaudi_send_heartbeat, 9218 .debug_coresight = gaudi_debug_coresight, 9219 .is_device_idle = gaudi_is_device_idle, 9220 .compute_reset_late_init = gaudi_compute_reset_late_init, 9221 .hw_queues_lock = gaudi_hw_queues_lock, 9222 .hw_queues_unlock = gaudi_hw_queues_unlock, 9223 .get_pci_id = gaudi_get_pci_id, 9224 .get_eeprom_data = gaudi_get_eeprom_data, 9225 .get_monitor_dump = gaudi_get_monitor_dump, 9226 .send_cpu_message = gaudi_send_cpu_message, 9227 .pci_bars_map = gaudi_pci_bars_map, 9228 .init_iatu = gaudi_init_iatu, 9229 .rreg = hl_rreg, 9230 .wreg = hl_wreg, 9231 .halt_coresight = gaudi_halt_coresight, 9232 .ctx_init = gaudi_ctx_init, 9233 .ctx_fini = gaudi_ctx_fini, 9234 .pre_schedule_cs = gaudi_pre_schedule_cs, 9235 .get_queue_id_for_cq = gaudi_get_queue_id_for_cq, 9236 .load_firmware_to_device = gaudi_load_firmware_to_device, 9237 .load_boot_fit_to_device = gaudi_load_boot_fit_to_device, 9238 .get_signal_cb_size = gaudi_get_signal_cb_size, 9239 .get_wait_cb_size = gaudi_get_wait_cb_size, 9240 .gen_signal_cb = gaudi_gen_signal_cb, 9241 .gen_wait_cb = gaudi_gen_wait_cb, 9242 .reset_sob = gaudi_reset_sob, 9243 .reset_sob_group = gaudi_reset_sob_group, 9244 .get_device_time = gaudi_get_device_time, 9245 .pb_print_security_errors = NULL, 9246 .collective_wait_init_cs = gaudi_collective_wait_init_cs, 9247 .collective_wait_create_jobs = gaudi_collective_wait_create_jobs, 9248 .get_dec_base_addr = NULL, 9249 .scramble_addr = hl_mmu_scramble_addr, 9250 .descramble_addr = hl_mmu_descramble_addr, 9251 .ack_protection_bits_errors = gaudi_ack_protection_bits_errors, 9252 .get_hw_block_id = gaudi_get_hw_block_id, 9253 .hw_block_mmap = gaudi_block_mmap, 9254 .enable_events_from_fw = gaudi_enable_events_from_fw, 9255 .ack_mmu_errors = gaudi_ack_mmu_page_fault_or_access_error, 9256 .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx, 9257 .init_firmware_preload_params = gaudi_init_firmware_preload_params, 9258 .init_firmware_loader = gaudi_init_firmware_loader, 9259 .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm, 9260 .state_dump_init = gaudi_state_dump_init, 9261 .get_sob_addr = gaudi_get_sob_addr, 9262 .set_pci_memory_regions = gaudi_set_pci_memory_regions, 9263 .get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr, 9264 .check_if_razwi_happened = gaudi_check_if_razwi_happened, 9265 .mmu_get_real_page_size = hl_mmu_get_real_page_size, 9266 .access_dev_mem = hl_access_dev_mem, 9267 .set_dram_bar_base = gaudi_set_hbm_bar_base, 9268 .send_device_activity = gaudi_send_device_activity, 9269 .set_dram_properties = gaudi_set_dram_properties, 9270 .set_binning_masks = gaudi_set_binning_masks, 9271 }; 9272 9273 /** 9274 * gaudi_set_asic_funcs - set GAUDI function pointers 9275 * 9276 * @hdev: pointer to hl_device structure 9277 * 9278 */ 9279 void gaudi_set_asic_funcs(struct hl_device *hdev) 9280 { 9281 hdev->asic_funcs = &gaudi_funcs; 9282 } 9283