1e65e175bSOded Gabbay // SPDX-License-Identifier: GPL-2.0 2e65e175bSOded Gabbay 3e65e175bSOded Gabbay /* 4e65e175bSOded Gabbay * Copyright 2016-2022 HabanaLabs, Ltd. 5e65e175bSOded Gabbay * All Rights Reserved. 6e65e175bSOded Gabbay */ 7e65e175bSOded Gabbay 8e65e175bSOded Gabbay #include "goyaP.h" 9e65e175bSOded Gabbay #include "../include/hw_ip/mmu/mmu_general.h" 10e65e175bSOded Gabbay #include "../include/hw_ip/mmu/mmu_v1_0.h" 11e65e175bSOded Gabbay #include "../include/goya/asic_reg/goya_masks.h" 12e65e175bSOded Gabbay #include "../include/goya/goya_reg_map.h" 13e65e175bSOded Gabbay 14e65e175bSOded Gabbay #include <linux/pci.h> 15e65e175bSOded Gabbay #include <linux/hwmon.h> 16e65e175bSOded Gabbay #include <linux/iommu.h> 17e65e175bSOded Gabbay #include <linux/seq_file.h> 18e65e175bSOded Gabbay 19e65e175bSOded Gabbay /* 20e65e175bSOded Gabbay * GOYA security scheme: 21e65e175bSOded Gabbay * 22e65e175bSOded Gabbay * 1. Host is protected by: 23e65e175bSOded Gabbay * - Range registers (When MMU is enabled, DMA RR does NOT protect host) 24e65e175bSOded Gabbay * - MMU 25e65e175bSOded Gabbay * 26e65e175bSOded Gabbay * 2. DRAM is protected by: 27e65e175bSOded Gabbay * - Range registers (protect the first 512MB) 28e65e175bSOded Gabbay * - MMU (isolation between users) 29e65e175bSOded Gabbay * 30e65e175bSOded Gabbay * 3. Configuration is protected by: 31e65e175bSOded Gabbay * - Range registers 32e65e175bSOded Gabbay * - Protection bits 33e65e175bSOded Gabbay * 34e65e175bSOded Gabbay * When MMU is disabled: 35e65e175bSOded Gabbay * 36e65e175bSOded Gabbay * QMAN DMA: PQ, CQ, CP, DMA are secured. 37e65e175bSOded Gabbay * PQ, CB and the data are on the host. 38e65e175bSOded Gabbay * 39e65e175bSOded Gabbay * QMAN TPC/MME: 40e65e175bSOded Gabbay * PQ, CQ and CP are not secured. 41e65e175bSOded Gabbay * PQ, CB and the data are on the SRAM/DRAM. 42e65e175bSOded Gabbay * 43e65e175bSOded Gabbay * Since QMAN DMA is secured, the driver is parsing the DMA CB: 44e65e175bSOded Gabbay * - checks DMA pointer 45e65e175bSOded Gabbay * - WREG, MSG_PROT are not allowed. 46e65e175bSOded Gabbay * - MSG_LONG/SHORT are allowed. 47e65e175bSOded Gabbay * 48e65e175bSOded Gabbay * A read/write transaction by the QMAN to a protected area will succeed if 49e65e175bSOded Gabbay * and only if the QMAN's CP is secured and MSG_PROT is used 50e65e175bSOded Gabbay * 51e65e175bSOded Gabbay * 52e65e175bSOded Gabbay * When MMU is enabled: 53e65e175bSOded Gabbay * 54e65e175bSOded Gabbay * QMAN DMA: PQ, CQ and CP are secured. 55e65e175bSOded Gabbay * MMU is set to bypass on the Secure props register of the QMAN. 56e65e175bSOded Gabbay * The reasons we don't enable MMU for PQ, CQ and CP are: 57e65e175bSOded Gabbay * - PQ entry is in kernel address space and the driver doesn't map it. 58e65e175bSOded Gabbay * - CP writes to MSIX register and to kernel address space (completion 59e65e175bSOded Gabbay * queue). 60e65e175bSOded Gabbay * 61e65e175bSOded Gabbay * DMA is not secured but because CP is secured, the driver still needs to parse 62e65e175bSOded Gabbay * the CB, but doesn't need to check the DMA addresses. 63e65e175bSOded Gabbay * 64e65e175bSOded Gabbay * For QMAN DMA 0, DMA is also secured because only the driver uses this DMA and 65e65e175bSOded Gabbay * the driver doesn't map memory in MMU. 66e65e175bSOded Gabbay * 67e65e175bSOded Gabbay * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode) 68e65e175bSOded Gabbay * 69e65e175bSOded Gabbay * DMA RR does NOT protect host because DMA is not secured 70e65e175bSOded Gabbay * 71e65e175bSOded Gabbay */ 72e65e175bSOded Gabbay 73e65e175bSOded Gabbay #define GOYA_BOOT_FIT_FILE "habanalabs/goya/goya-boot-fit.itb" 74e65e175bSOded Gabbay #define GOYA_LINUX_FW_FILE "habanalabs/goya/goya-fit.itb" 75e65e175bSOded Gabbay 76e65e175bSOded Gabbay #define GOYA_MMU_REGS_NUM 63 77e65e175bSOded Gabbay 78e65e175bSOded Gabbay #define GOYA_DMA_POOL_BLK_SIZE 0x100 /* 256 bytes */ 79e65e175bSOded Gabbay 80e65e175bSOded Gabbay #define GOYA_RESET_TIMEOUT_MSEC 500 /* 500ms */ 81e65e175bSOded Gabbay #define GOYA_PLDM_RESET_TIMEOUT_MSEC 20000 /* 20s */ 82e65e175bSOded Gabbay #define GOYA_RESET_WAIT_MSEC 1 /* 1ms */ 83e65e175bSOded Gabbay #define GOYA_CPU_RESET_WAIT_MSEC 100 /* 100ms */ 84e65e175bSOded Gabbay #define GOYA_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 85e65e175bSOded Gabbay #define GOYA_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 86e65e175bSOded Gabbay #define GOYA_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100) 87e65e175bSOded Gabbay #define GOYA_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) 88e65e175bSOded Gabbay #define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */ 89e65e175bSOded Gabbay #define GOYA_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 90e65e175bSOded Gabbay #define GOYA_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */ 91e65e175bSOded Gabbay 92e65e175bSOded Gabbay #define GOYA_QMAN0_FENCE_VAL 0xD169B243 93e65e175bSOded Gabbay 94e65e175bSOded Gabbay #define GOYA_MAX_STRING_LEN 20 95e65e175bSOded Gabbay 96e65e175bSOded Gabbay #define GOYA_CB_POOL_CB_CNT 512 97e65e175bSOded Gabbay #define GOYA_CB_POOL_CB_SIZE 0x20000 /* 128KB */ 98e65e175bSOded Gabbay 99e65e175bSOded Gabbay #define IS_QM_IDLE(engine, qm_glbl_sts0) \ 100e65e175bSOded Gabbay (((qm_glbl_sts0) & engine##_QM_IDLE_MASK) == engine##_QM_IDLE_MASK) 101e65e175bSOded Gabbay #define IS_DMA_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(DMA, qm_glbl_sts0) 102e65e175bSOded Gabbay #define IS_TPC_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(TPC, qm_glbl_sts0) 103e65e175bSOded Gabbay #define IS_MME_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(MME, qm_glbl_sts0) 104e65e175bSOded Gabbay 105e65e175bSOded Gabbay #define IS_CMDQ_IDLE(engine, cmdq_glbl_sts0) \ 106e65e175bSOded Gabbay (((cmdq_glbl_sts0) & engine##_CMDQ_IDLE_MASK) == \ 107e65e175bSOded Gabbay engine##_CMDQ_IDLE_MASK) 108e65e175bSOded Gabbay #define IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) \ 109e65e175bSOded Gabbay IS_CMDQ_IDLE(TPC, cmdq_glbl_sts0) 110e65e175bSOded Gabbay #define IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) \ 111e65e175bSOded Gabbay IS_CMDQ_IDLE(MME, cmdq_glbl_sts0) 112e65e175bSOded Gabbay 113e65e175bSOded Gabbay #define IS_DMA_IDLE(dma_core_sts0) \ 114e65e175bSOded Gabbay !((dma_core_sts0) & DMA_CH_0_STS0_DMA_BUSY_MASK) 115e65e175bSOded Gabbay 116e65e175bSOded Gabbay #define IS_TPC_IDLE(tpc_cfg_sts) \ 117e65e175bSOded Gabbay (((tpc_cfg_sts) & TPC_CFG_IDLE_MASK) == TPC_CFG_IDLE_MASK) 118e65e175bSOded Gabbay 119e65e175bSOded Gabbay #define IS_MME_IDLE(mme_arch_sts) \ 120e65e175bSOded Gabbay (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) 121e65e175bSOded Gabbay 122e65e175bSOded Gabbay static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = { 123e65e175bSOded Gabbay "goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3", 124e65e175bSOded Gabbay "goya cq 4", "goya cpu eq" 125e65e175bSOded Gabbay }; 126e65e175bSOded Gabbay 127e65e175bSOded Gabbay static u16 goya_packet_sizes[MAX_PACKET_ID] = { 128e65e175bSOded Gabbay [PACKET_WREG_32] = sizeof(struct packet_wreg32), 129e65e175bSOded Gabbay [PACKET_WREG_BULK] = sizeof(struct packet_wreg_bulk), 130e65e175bSOded Gabbay [PACKET_MSG_LONG] = sizeof(struct packet_msg_long), 131e65e175bSOded Gabbay [PACKET_MSG_SHORT] = sizeof(struct packet_msg_short), 132e65e175bSOded Gabbay [PACKET_CP_DMA] = sizeof(struct packet_cp_dma), 133e65e175bSOded Gabbay [PACKET_MSG_PROT] = sizeof(struct packet_msg_prot), 134e65e175bSOded Gabbay [PACKET_FENCE] = sizeof(struct packet_fence), 135e65e175bSOded Gabbay [PACKET_LIN_DMA] = sizeof(struct packet_lin_dma), 136e65e175bSOded Gabbay [PACKET_NOP] = sizeof(struct packet_nop), 137e65e175bSOded Gabbay [PACKET_STOP] = sizeof(struct packet_stop) 138e65e175bSOded Gabbay }; 139e65e175bSOded Gabbay 140e65e175bSOded Gabbay static inline bool validate_packet_id(enum packet_id id) 141e65e175bSOded Gabbay { 142e65e175bSOded Gabbay switch (id) { 143e65e175bSOded Gabbay case PACKET_WREG_32: 144e65e175bSOded Gabbay case PACKET_WREG_BULK: 145e65e175bSOded Gabbay case PACKET_MSG_LONG: 146e65e175bSOded Gabbay case PACKET_MSG_SHORT: 147e65e175bSOded Gabbay case PACKET_CP_DMA: 148e65e175bSOded Gabbay case PACKET_MSG_PROT: 149e65e175bSOded Gabbay case PACKET_FENCE: 150e65e175bSOded Gabbay case PACKET_LIN_DMA: 151e65e175bSOded Gabbay case PACKET_NOP: 152e65e175bSOded Gabbay case PACKET_STOP: 153e65e175bSOded Gabbay return true; 154e65e175bSOded Gabbay default: 155e65e175bSOded Gabbay return false; 156e65e175bSOded Gabbay } 157e65e175bSOded Gabbay } 158e65e175bSOded Gabbay 159e65e175bSOded Gabbay static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = { 160e65e175bSOded Gabbay mmDMA_QM_0_GLBL_NON_SECURE_PROPS, 161e65e175bSOded Gabbay mmDMA_QM_1_GLBL_NON_SECURE_PROPS, 162e65e175bSOded Gabbay mmDMA_QM_2_GLBL_NON_SECURE_PROPS, 163e65e175bSOded Gabbay mmDMA_QM_3_GLBL_NON_SECURE_PROPS, 164e65e175bSOded Gabbay mmDMA_QM_4_GLBL_NON_SECURE_PROPS, 165e65e175bSOded Gabbay mmTPC0_QM_GLBL_SECURE_PROPS, 166e65e175bSOded Gabbay mmTPC0_QM_GLBL_NON_SECURE_PROPS, 167e65e175bSOded Gabbay mmTPC0_CMDQ_GLBL_SECURE_PROPS, 168e65e175bSOded Gabbay mmTPC0_CMDQ_GLBL_NON_SECURE_PROPS, 169e65e175bSOded Gabbay mmTPC0_CFG_ARUSER, 170e65e175bSOded Gabbay mmTPC0_CFG_AWUSER, 171e65e175bSOded Gabbay mmTPC1_QM_GLBL_SECURE_PROPS, 172e65e175bSOded Gabbay mmTPC1_QM_GLBL_NON_SECURE_PROPS, 173e65e175bSOded Gabbay mmTPC1_CMDQ_GLBL_SECURE_PROPS, 174e65e175bSOded Gabbay mmTPC1_CMDQ_GLBL_NON_SECURE_PROPS, 175e65e175bSOded Gabbay mmTPC1_CFG_ARUSER, 176e65e175bSOded Gabbay mmTPC1_CFG_AWUSER, 177e65e175bSOded Gabbay mmTPC2_QM_GLBL_SECURE_PROPS, 178e65e175bSOded Gabbay mmTPC2_QM_GLBL_NON_SECURE_PROPS, 179e65e175bSOded Gabbay mmTPC2_CMDQ_GLBL_SECURE_PROPS, 180e65e175bSOded Gabbay mmTPC2_CMDQ_GLBL_NON_SECURE_PROPS, 181e65e175bSOded Gabbay mmTPC2_CFG_ARUSER, 182e65e175bSOded Gabbay mmTPC2_CFG_AWUSER, 183e65e175bSOded Gabbay mmTPC3_QM_GLBL_SECURE_PROPS, 184e65e175bSOded Gabbay mmTPC3_QM_GLBL_NON_SECURE_PROPS, 185e65e175bSOded Gabbay mmTPC3_CMDQ_GLBL_SECURE_PROPS, 186e65e175bSOded Gabbay mmTPC3_CMDQ_GLBL_NON_SECURE_PROPS, 187e65e175bSOded Gabbay mmTPC3_CFG_ARUSER, 188e65e175bSOded Gabbay mmTPC3_CFG_AWUSER, 189e65e175bSOded Gabbay mmTPC4_QM_GLBL_SECURE_PROPS, 190e65e175bSOded Gabbay mmTPC4_QM_GLBL_NON_SECURE_PROPS, 191e65e175bSOded Gabbay mmTPC4_CMDQ_GLBL_SECURE_PROPS, 192e65e175bSOded Gabbay mmTPC4_CMDQ_GLBL_NON_SECURE_PROPS, 193e65e175bSOded Gabbay mmTPC4_CFG_ARUSER, 194e65e175bSOded Gabbay mmTPC4_CFG_AWUSER, 195e65e175bSOded Gabbay mmTPC5_QM_GLBL_SECURE_PROPS, 196e65e175bSOded Gabbay mmTPC5_QM_GLBL_NON_SECURE_PROPS, 197e65e175bSOded Gabbay mmTPC5_CMDQ_GLBL_SECURE_PROPS, 198e65e175bSOded Gabbay mmTPC5_CMDQ_GLBL_NON_SECURE_PROPS, 199e65e175bSOded Gabbay mmTPC5_CFG_ARUSER, 200e65e175bSOded Gabbay mmTPC5_CFG_AWUSER, 201e65e175bSOded Gabbay mmTPC6_QM_GLBL_SECURE_PROPS, 202e65e175bSOded Gabbay mmTPC6_QM_GLBL_NON_SECURE_PROPS, 203e65e175bSOded Gabbay mmTPC6_CMDQ_GLBL_SECURE_PROPS, 204e65e175bSOded Gabbay mmTPC6_CMDQ_GLBL_NON_SECURE_PROPS, 205e65e175bSOded Gabbay mmTPC6_CFG_ARUSER, 206e65e175bSOded Gabbay mmTPC6_CFG_AWUSER, 207e65e175bSOded Gabbay mmTPC7_QM_GLBL_SECURE_PROPS, 208e65e175bSOded Gabbay mmTPC7_QM_GLBL_NON_SECURE_PROPS, 209e65e175bSOded Gabbay mmTPC7_CMDQ_GLBL_SECURE_PROPS, 210e65e175bSOded Gabbay mmTPC7_CMDQ_GLBL_NON_SECURE_PROPS, 211e65e175bSOded Gabbay mmTPC7_CFG_ARUSER, 212e65e175bSOded Gabbay mmTPC7_CFG_AWUSER, 213e65e175bSOded Gabbay mmMME_QM_GLBL_SECURE_PROPS, 214e65e175bSOded Gabbay mmMME_QM_GLBL_NON_SECURE_PROPS, 215e65e175bSOded Gabbay mmMME_CMDQ_GLBL_SECURE_PROPS, 216e65e175bSOded Gabbay mmMME_CMDQ_GLBL_NON_SECURE_PROPS, 217e65e175bSOded Gabbay mmMME_SBA_CONTROL_DATA, 218e65e175bSOded Gabbay mmMME_SBB_CONTROL_DATA, 219e65e175bSOded Gabbay mmMME_SBC_CONTROL_DATA, 220e65e175bSOded Gabbay mmMME_WBC_CONTROL_DATA, 221e65e175bSOded Gabbay mmPCIE_WRAP_PSOC_ARUSER, 222e65e175bSOded Gabbay mmPCIE_WRAP_PSOC_AWUSER 223e65e175bSOded Gabbay }; 224e65e175bSOded Gabbay 225e65e175bSOded Gabbay static u32 goya_all_events[] = { 226e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PCIE_IF, 227e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_ECC, 228e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC1_ECC, 229e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC2_ECC, 230e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC3_ECC, 231e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC4_ECC, 232e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC5_ECC, 233e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC6_ECC, 234e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC7_ECC, 235e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MME_ECC, 236e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MME_ECC_EXT, 237e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MMU_ECC, 238e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_MACRO, 239e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_ECC, 240e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_CPU_IF_ECC, 241e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PSOC_MEM, 242e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT, 243e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM0, 244e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM1, 245e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM2, 246e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM3, 247e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM4, 248e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM5, 249e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM6, 250e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM7, 251e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM8, 252e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM9, 253e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM10, 254e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM11, 255e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM12, 256e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM13, 257e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM14, 258e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM15, 259e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM16, 260e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM17, 261e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM18, 262e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM19, 263e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM20, 264e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM21, 265e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM22, 266e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM23, 267e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM24, 268e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM25, 269e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM26, 270e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM27, 271e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM28, 272e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SRAM29, 273e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_GIC500, 274e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PLL0, 275e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PLL1, 276e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PLL3, 277e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PLL4, 278e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PLL5, 279e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PLL6, 280e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_AXI_ECC, 281e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_L2_RAM_ECC, 282e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET, 283e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT, 284e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PCIE_DEC, 285e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_DEC, 286e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC1_DEC, 287e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC2_DEC, 288e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC3_DEC, 289e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC4_DEC, 290e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC5_DEC, 291e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC6_DEC, 292e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC7_DEC, 293e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MME_WACS, 294e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MME_WACSD, 295e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER, 296e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC, 297e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PSOC, 298e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR, 299e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR, 300e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR, 301e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR, 302e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR, 303e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR, 304e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR, 305e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR, 306e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_CMDQ, 307e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC1_CMDQ, 308e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC2_CMDQ, 309e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC3_CMDQ, 310e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC4_CMDQ, 311e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC5_CMDQ, 312e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC6_CMDQ, 313e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC7_CMDQ, 314e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_QM, 315e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC1_QM, 316e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC2_QM, 317e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC3_QM, 318e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC4_QM, 319e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC5_QM, 320e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC6_QM, 321e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC7_QM, 322e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MME_QM, 323e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_MME_CMDQ, 324e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA0_QM, 325e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA1_QM, 326e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA2_QM, 327e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA3_QM, 328e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA4_QM, 329e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA0_CH, 330e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA1_CH, 331e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA2_CH, 332e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA3_CH, 333e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA4_CH, 334e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU, 335e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU, 336e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU, 337e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU, 338e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU, 339e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU, 340e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU, 341e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU, 342e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_BM_CH0, 343e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_BM_CH1, 344e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_BM_CH2, 345e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_BM_CH3, 346e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA_BM_CH4, 347e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S, 348e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E, 349e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S, 350e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E 351e65e175bSOded Gabbay }; 352e65e175bSOded Gabbay 353e65e175bSOded Gabbay static s64 goya_state_dump_specs_props[SP_MAX] = {0}; 354e65e175bSOded Gabbay 355e65e175bSOded Gabbay static int goya_mmu_clear_pgt_range(struct hl_device *hdev); 356e65e175bSOded Gabbay static int goya_mmu_set_dram_default_page(struct hl_device *hdev); 357e65e175bSOded Gabbay static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev); 358e65e175bSOded Gabbay static void goya_mmu_prepare(struct hl_device *hdev, u32 asid); 359e65e175bSOded Gabbay 360e65e175bSOded Gabbay int goya_set_fixed_properties(struct hl_device *hdev) 361e65e175bSOded Gabbay { 362e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 363e65e175bSOded Gabbay int i; 364e65e175bSOded Gabbay 365e65e175bSOded Gabbay prop->max_queues = GOYA_QUEUE_ID_SIZE; 366e65e175bSOded Gabbay prop->hw_queues_props = kcalloc(prop->max_queues, 367e65e175bSOded Gabbay sizeof(struct hw_queue_properties), 368e65e175bSOded Gabbay GFP_KERNEL); 369e65e175bSOded Gabbay 370e65e175bSOded Gabbay if (!prop->hw_queues_props) 371e65e175bSOded Gabbay return -ENOMEM; 372e65e175bSOded Gabbay 373e65e175bSOded Gabbay for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) { 374e65e175bSOded Gabbay prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; 375e65e175bSOded Gabbay prop->hw_queues_props[i].driver_only = 0; 376e65e175bSOded Gabbay prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; 377e65e175bSOded Gabbay } 378e65e175bSOded Gabbay 379e65e175bSOded Gabbay for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) { 380e65e175bSOded Gabbay prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; 381e65e175bSOded Gabbay prop->hw_queues_props[i].driver_only = 1; 382e65e175bSOded Gabbay prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; 383e65e175bSOded Gabbay } 384e65e175bSOded Gabbay 385e65e175bSOded Gabbay for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES + 386e65e175bSOded Gabbay NUMBER_OF_INT_HW_QUEUES; i++) { 387e65e175bSOded Gabbay prop->hw_queues_props[i].type = QUEUE_TYPE_INT; 388e65e175bSOded Gabbay prop->hw_queues_props[i].driver_only = 0; 389e65e175bSOded Gabbay prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER; 390e65e175bSOded Gabbay } 391e65e175bSOded Gabbay 392e65e175bSOded Gabbay prop->cfg_base_address = CFG_BASE; 393e65e175bSOded Gabbay prop->device_dma_offset_for_host_access = HOST_PHYS_BASE; 394e65e175bSOded Gabbay prop->host_base_address = HOST_PHYS_BASE; 395e65e175bSOded Gabbay prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE; 396e65e175bSOded Gabbay prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; 397e65e175bSOded Gabbay prop->completion_mode = HL_COMPLETION_MODE_JOB; 398e65e175bSOded Gabbay prop->dram_base_address = DRAM_PHYS_BASE; 399e65e175bSOded Gabbay prop->dram_size = DRAM_PHYS_DEFAULT_SIZE; 400e65e175bSOded Gabbay prop->dram_end_address = prop->dram_base_address + prop->dram_size; 401e65e175bSOded Gabbay prop->dram_user_base_address = DRAM_BASE_ADDR_USER; 402e65e175bSOded Gabbay 403e65e175bSOded Gabbay prop->sram_base_address = SRAM_BASE_ADDR; 404e65e175bSOded Gabbay prop->sram_size = SRAM_SIZE; 405e65e175bSOded Gabbay prop->sram_end_address = prop->sram_base_address + prop->sram_size; 406e65e175bSOded Gabbay prop->sram_user_base_address = prop->sram_base_address + 407e65e175bSOded Gabbay SRAM_USER_BASE_OFFSET; 408e65e175bSOded Gabbay 409e65e175bSOded Gabbay prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR; 410e65e175bSOded Gabbay prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR; 411e65e175bSOded Gabbay if (hdev->pldm) 412e65e175bSOded Gabbay prop->mmu_pgt_size = 0x800000; /* 8MB */ 413e65e175bSOded Gabbay else 414e65e175bSOded Gabbay prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE; 415e65e175bSOded Gabbay prop->mmu_pte_size = HL_PTE_SIZE; 416e65e175bSOded Gabbay prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; 417e65e175bSOded Gabbay prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; 418e65e175bSOded Gabbay prop->dram_page_size = PAGE_SIZE_2MB; 419e65e175bSOded Gabbay prop->device_mem_alloc_default_page_size = prop->dram_page_size; 420e65e175bSOded Gabbay prop->dram_supports_virtual_memory = true; 421e65e175bSOded Gabbay 422e65e175bSOded Gabbay prop->dmmu.hop_shifts[MMU_HOP0] = MMU_V1_0_HOP0_SHIFT; 423e65e175bSOded Gabbay prop->dmmu.hop_shifts[MMU_HOP1] = MMU_V1_0_HOP1_SHIFT; 424e65e175bSOded Gabbay prop->dmmu.hop_shifts[MMU_HOP2] = MMU_V1_0_HOP2_SHIFT; 425e65e175bSOded Gabbay prop->dmmu.hop_shifts[MMU_HOP3] = MMU_V1_0_HOP3_SHIFT; 426e65e175bSOded Gabbay prop->dmmu.hop_shifts[MMU_HOP4] = MMU_V1_0_HOP4_SHIFT; 427e65e175bSOded Gabbay prop->dmmu.hop_masks[MMU_HOP0] = MMU_V1_0_HOP0_MASK; 428e65e175bSOded Gabbay prop->dmmu.hop_masks[MMU_HOP1] = MMU_V1_0_HOP1_MASK; 429e65e175bSOded Gabbay prop->dmmu.hop_masks[MMU_HOP2] = MMU_V1_0_HOP2_MASK; 430e65e175bSOded Gabbay prop->dmmu.hop_masks[MMU_HOP3] = MMU_V1_0_HOP3_MASK; 431e65e175bSOded Gabbay prop->dmmu.hop_masks[MMU_HOP4] = MMU_V1_0_HOP4_MASK; 432e65e175bSOded Gabbay prop->dmmu.start_addr = VA_DDR_SPACE_START; 433e65e175bSOded Gabbay prop->dmmu.end_addr = VA_DDR_SPACE_END; 434e65e175bSOded Gabbay prop->dmmu.page_size = PAGE_SIZE_2MB; 435e65e175bSOded Gabbay prop->dmmu.num_hops = MMU_ARCH_5_HOPS; 436e65e175bSOded Gabbay prop->dmmu.last_mask = LAST_MASK; 437e65e175bSOded Gabbay /* TODO: will be duplicated until implementing per-MMU props */ 438e65e175bSOded Gabbay prop->dmmu.hop_table_size = prop->mmu_hop_table_size; 439e65e175bSOded Gabbay prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 440e65e175bSOded Gabbay 441e65e175bSOded Gabbay /* shifts and masks are the same in PMMU and DMMU */ 442e65e175bSOded Gabbay memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu)); 443e65e175bSOded Gabbay prop->pmmu.start_addr = VA_HOST_SPACE_START; 444e65e175bSOded Gabbay prop->pmmu.end_addr = VA_HOST_SPACE_END; 445e65e175bSOded Gabbay prop->pmmu.page_size = PAGE_SIZE_4KB; 446e65e175bSOded Gabbay prop->pmmu.num_hops = MMU_ARCH_5_HOPS; 447e65e175bSOded Gabbay prop->pmmu.last_mask = LAST_MASK; 448e65e175bSOded Gabbay /* TODO: will be duplicated until implementing per-MMU props */ 449e65e175bSOded Gabbay prop->pmmu.hop_table_size = prop->mmu_hop_table_size; 450e65e175bSOded Gabbay prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 451e65e175bSOded Gabbay 452e65e175bSOded Gabbay /* PMMU and HPMMU are the same except of page size */ 453e65e175bSOded Gabbay memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 454e65e175bSOded Gabbay prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 455e65e175bSOded Gabbay 456e65e175bSOded Gabbay prop->dram_size_for_default_page_mapping = VA_DDR_SPACE_END; 457e65e175bSOded Gabbay prop->cfg_size = CFG_SIZE; 458e65e175bSOded Gabbay prop->max_asid = MAX_ASID; 459e65e175bSOded Gabbay prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE; 460e65e175bSOded Gabbay prop->high_pll = PLL_HIGH_DEFAULT; 461e65e175bSOded Gabbay prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT; 462e65e175bSOded Gabbay prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE; 463e65e175bSOded Gabbay prop->max_power_default = MAX_POWER_DEFAULT; 464e65e175bSOded Gabbay prop->dc_power_default = DC_POWER_DEFAULT; 465e65e175bSOded Gabbay prop->tpc_enabled_mask = TPC_ENABLED_MASK; 466e65e175bSOded Gabbay prop->pcie_dbi_base_address = mmPCIE_DBI_BASE; 467e65e175bSOded Gabbay prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 468e65e175bSOded Gabbay 469e65e175bSOded Gabbay strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, 470e65e175bSOded Gabbay CARD_NAME_MAX_LEN); 471e65e175bSOded Gabbay 472e65e175bSOded Gabbay prop->max_pending_cs = GOYA_MAX_PENDING_CS; 473e65e175bSOded Gabbay 474e65e175bSOded Gabbay prop->first_available_user_interrupt = USHRT_MAX; 475e65e175bSOded Gabbay 476e65e175bSOded Gabbay for (i = 0 ; i < HL_MAX_DCORES ; i++) 477e65e175bSOded Gabbay prop->first_available_cq[i] = USHRT_MAX; 478e65e175bSOded Gabbay 479e65e175bSOded Gabbay prop->fw_cpu_boot_dev_sts0_valid = false; 480e65e175bSOded Gabbay prop->fw_cpu_boot_dev_sts1_valid = false; 481e65e175bSOded Gabbay prop->hard_reset_done_by_fw = false; 482e65e175bSOded Gabbay prop->gic_interrupts_enable = true; 483e65e175bSOded Gabbay 484e65e175bSOded Gabbay prop->server_type = HL_SERVER_TYPE_UNKNOWN; 485e65e175bSOded Gabbay 486e65e175bSOded Gabbay prop->clk_pll_index = HL_GOYA_MME_PLL; 487e65e175bSOded Gabbay 488e65e175bSOded Gabbay prop->use_get_power_for_reset_history = true; 489e65e175bSOded Gabbay 490e65e175bSOded Gabbay prop->configurable_stop_on_err = true; 491e65e175bSOded Gabbay 492e65e175bSOded Gabbay prop->set_max_power_on_device_init = true; 493e65e175bSOded Gabbay 494e65e175bSOded Gabbay prop->dma_mask = 48; 495e65e175bSOded Gabbay 496e65e175bSOded Gabbay return 0; 497e65e175bSOded Gabbay } 498e65e175bSOded Gabbay 499e65e175bSOded Gabbay /* 500e65e175bSOded Gabbay * goya_pci_bars_map - Map PCI BARS of Goya device 501e65e175bSOded Gabbay * 502e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 503e65e175bSOded Gabbay * 504e65e175bSOded Gabbay * Request PCI regions and map them to kernel virtual addresses. 505e65e175bSOded Gabbay * Returns 0 on success 506e65e175bSOded Gabbay * 507e65e175bSOded Gabbay */ 508e65e175bSOded Gabbay static int goya_pci_bars_map(struct hl_device *hdev) 509e65e175bSOded Gabbay { 510e65e175bSOded Gabbay static const char * const name[] = {"SRAM_CFG", "MSIX", "DDR"}; 511e65e175bSOded Gabbay bool is_wc[3] = {false, false, true}; 512e65e175bSOded Gabbay int rc; 513e65e175bSOded Gabbay 514e65e175bSOded Gabbay rc = hl_pci_bars_map(hdev, name, is_wc); 515e65e175bSOded Gabbay if (rc) 516e65e175bSOded Gabbay return rc; 517e65e175bSOded Gabbay 518e65e175bSOded Gabbay hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + 519e65e175bSOded Gabbay (CFG_BASE - SRAM_BASE_ADDR); 520e65e175bSOded Gabbay 521e65e175bSOded Gabbay return 0; 522e65e175bSOded Gabbay } 523e65e175bSOded Gabbay 524e65e175bSOded Gabbay static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr) 525e65e175bSOded Gabbay { 526e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 527e65e175bSOded Gabbay struct hl_inbound_pci_region pci_region; 528e65e175bSOded Gabbay u64 old_addr = addr; 529e65e175bSOded Gabbay int rc; 530e65e175bSOded Gabbay 531e65e175bSOded Gabbay if ((goya) && (goya->ddr_bar_cur_addr == addr)) 532e65e175bSOded Gabbay return old_addr; 533e65e175bSOded Gabbay 534e65e175bSOded Gabbay /* Inbound Region 1 - Bar 4 - Point to DDR */ 535e65e175bSOded Gabbay pci_region.mode = PCI_BAR_MATCH_MODE; 536e65e175bSOded Gabbay pci_region.bar = DDR_BAR_ID; 537e65e175bSOded Gabbay pci_region.addr = addr; 538e65e175bSOded Gabbay rc = hl_pci_set_inbound_region(hdev, 1, &pci_region); 539e65e175bSOded Gabbay if (rc) 540e65e175bSOded Gabbay return U64_MAX; 541e65e175bSOded Gabbay 542e65e175bSOded Gabbay if (goya) { 543e65e175bSOded Gabbay old_addr = goya->ddr_bar_cur_addr; 544e65e175bSOded Gabbay goya->ddr_bar_cur_addr = addr; 545e65e175bSOded Gabbay } 546e65e175bSOded Gabbay 547e65e175bSOded Gabbay return old_addr; 548e65e175bSOded Gabbay } 549e65e175bSOded Gabbay 550e65e175bSOded Gabbay /* 551e65e175bSOded Gabbay * goya_init_iatu - Initialize the iATU unit inside the PCI controller 552e65e175bSOded Gabbay * 553e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 554e65e175bSOded Gabbay * 555e65e175bSOded Gabbay * This is needed in case the firmware doesn't initialize the iATU 556e65e175bSOded Gabbay * 557e65e175bSOded Gabbay */ 558e65e175bSOded Gabbay static int goya_init_iatu(struct hl_device *hdev) 559e65e175bSOded Gabbay { 560e65e175bSOded Gabbay struct hl_inbound_pci_region inbound_region; 561e65e175bSOded Gabbay struct hl_outbound_pci_region outbound_region; 562e65e175bSOded Gabbay int rc; 563e65e175bSOded Gabbay 564e65e175bSOded Gabbay if (hdev->asic_prop.iatu_done_by_fw) 565e65e175bSOded Gabbay return 0; 566e65e175bSOded Gabbay 567e65e175bSOded Gabbay /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */ 568e65e175bSOded Gabbay inbound_region.mode = PCI_BAR_MATCH_MODE; 569e65e175bSOded Gabbay inbound_region.bar = SRAM_CFG_BAR_ID; 570e65e175bSOded Gabbay inbound_region.addr = SRAM_BASE_ADDR; 571e65e175bSOded Gabbay rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 572e65e175bSOded Gabbay if (rc) 573e65e175bSOded Gabbay goto done; 574e65e175bSOded Gabbay 575e65e175bSOded Gabbay /* Inbound Region 1 - Bar 4 - Point to DDR */ 576e65e175bSOded Gabbay inbound_region.mode = PCI_BAR_MATCH_MODE; 577e65e175bSOded Gabbay inbound_region.bar = DDR_BAR_ID; 578e65e175bSOded Gabbay inbound_region.addr = DRAM_PHYS_BASE; 579e65e175bSOded Gabbay rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 580e65e175bSOded Gabbay if (rc) 581e65e175bSOded Gabbay goto done; 582e65e175bSOded Gabbay 583e65e175bSOded Gabbay /* Outbound Region 0 - Point to Host */ 584e65e175bSOded Gabbay outbound_region.addr = HOST_PHYS_BASE; 585e65e175bSOded Gabbay outbound_region.size = HOST_PHYS_SIZE; 586e65e175bSOded Gabbay rc = hl_pci_set_outbound_region(hdev, &outbound_region); 587e65e175bSOded Gabbay 588e65e175bSOded Gabbay done: 589e65e175bSOded Gabbay return rc; 590e65e175bSOded Gabbay } 591e65e175bSOded Gabbay 592e65e175bSOded Gabbay static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev) 593e65e175bSOded Gabbay { 594e65e175bSOded Gabbay return RREG32(mmHW_STATE); 595e65e175bSOded Gabbay } 596e65e175bSOded Gabbay 597e65e175bSOded Gabbay /* 598e65e175bSOded Gabbay * goya_early_init - GOYA early initialization code 599e65e175bSOded Gabbay * 600e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 601e65e175bSOded Gabbay * 602e65e175bSOded Gabbay * Verify PCI bars 603e65e175bSOded Gabbay * Set DMA masks 604e65e175bSOded Gabbay * PCI controller initialization 605e65e175bSOded Gabbay * Map PCI bars 606e65e175bSOded Gabbay * 607e65e175bSOded Gabbay */ 608e65e175bSOded Gabbay static int goya_early_init(struct hl_device *hdev) 609e65e175bSOded Gabbay { 610e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 611e65e175bSOded Gabbay struct pci_dev *pdev = hdev->pdev; 612e65e175bSOded Gabbay resource_size_t pci_bar_size; 613e65e175bSOded Gabbay u32 fw_boot_status, val; 614e65e175bSOded Gabbay int rc; 615e65e175bSOded Gabbay 616e65e175bSOded Gabbay rc = goya_set_fixed_properties(hdev); 617e65e175bSOded Gabbay if (rc) { 618e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to get fixed properties\n"); 619e65e175bSOded Gabbay return rc; 620e65e175bSOded Gabbay } 621e65e175bSOded Gabbay 622e65e175bSOded Gabbay /* Check BAR sizes */ 623e65e175bSOded Gabbay pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID); 624e65e175bSOded Gabbay 625e65e175bSOded Gabbay if (pci_bar_size != CFG_BAR_SIZE) { 626e65e175bSOded Gabbay dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 627e65e175bSOded Gabbay SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 628e65e175bSOded Gabbay rc = -ENODEV; 629e65e175bSOded Gabbay goto free_queue_props; 630e65e175bSOded Gabbay } 631e65e175bSOded Gabbay 632e65e175bSOded Gabbay pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID); 633e65e175bSOded Gabbay 634e65e175bSOded Gabbay if (pci_bar_size != MSIX_BAR_SIZE) { 635e65e175bSOded Gabbay dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 636e65e175bSOded Gabbay MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE); 637e65e175bSOded Gabbay rc = -ENODEV; 638e65e175bSOded Gabbay goto free_queue_props; 639e65e175bSOded Gabbay } 640e65e175bSOded Gabbay 641e65e175bSOded Gabbay prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); 642e65e175bSOded Gabbay hdev->dram_pci_bar_start = pci_resource_start(pdev, DDR_BAR_ID); 643e65e175bSOded Gabbay 644e65e175bSOded Gabbay /* If FW security is enabled at this point it means no access to ELBI */ 645e65e175bSOded Gabbay if (hdev->asic_prop.fw_security_enabled) { 646e65e175bSOded Gabbay hdev->asic_prop.iatu_done_by_fw = true; 647e65e175bSOded Gabbay goto pci_init; 648e65e175bSOded Gabbay } 649e65e175bSOded Gabbay 650e65e175bSOded Gabbay rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0, 651e65e175bSOded Gabbay &fw_boot_status); 652e65e175bSOded Gabbay if (rc) 653e65e175bSOded Gabbay goto free_queue_props; 654e65e175bSOded Gabbay 655e65e175bSOded Gabbay /* Check whether FW is configuring iATU */ 656e65e175bSOded Gabbay if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) && 657e65e175bSOded Gabbay (fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN)) 658e65e175bSOded Gabbay hdev->asic_prop.iatu_done_by_fw = true; 659e65e175bSOded Gabbay 660e65e175bSOded Gabbay pci_init: 661e65e175bSOded Gabbay rc = hl_pci_init(hdev); 662e65e175bSOded Gabbay if (rc) 663e65e175bSOded Gabbay goto free_queue_props; 664e65e175bSOded Gabbay 665e65e175bSOded Gabbay /* Before continuing in the initialization, we need to read the preboot 666e65e175bSOded Gabbay * version to determine whether we run with a security-enabled firmware 667e65e175bSOded Gabbay */ 668e65e175bSOded Gabbay rc = hl_fw_read_preboot_status(hdev); 669e65e175bSOded Gabbay if (rc) { 670e65e175bSOded Gabbay if (hdev->reset_on_preboot_fail) 671e65e175bSOded Gabbay hdev->asic_funcs->hw_fini(hdev, true, false); 672e65e175bSOded Gabbay goto pci_fini; 673e65e175bSOded Gabbay } 674e65e175bSOded Gabbay 675e65e175bSOded Gabbay if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 676e65e175bSOded Gabbay dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 677e65e175bSOded Gabbay hdev->asic_funcs->hw_fini(hdev, true, false); 678e65e175bSOded Gabbay } 679e65e175bSOded Gabbay 680e65e175bSOded Gabbay if (!hdev->pldm) { 681e65e175bSOded Gabbay val = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS); 682e65e175bSOded Gabbay if (val & PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_SRIOV_EN_MASK) 683e65e175bSOded Gabbay dev_warn(hdev->dev, 684e65e175bSOded Gabbay "PCI strap is not configured correctly, PCI bus errors may occur\n"); 685e65e175bSOded Gabbay } 686e65e175bSOded Gabbay 687e65e175bSOded Gabbay return 0; 688e65e175bSOded Gabbay 689e65e175bSOded Gabbay pci_fini: 690e65e175bSOded Gabbay hl_pci_fini(hdev); 691e65e175bSOded Gabbay free_queue_props: 692e65e175bSOded Gabbay kfree(hdev->asic_prop.hw_queues_props); 693e65e175bSOded Gabbay return rc; 694e65e175bSOded Gabbay } 695e65e175bSOded Gabbay 696e65e175bSOded Gabbay /* 697e65e175bSOded Gabbay * goya_early_fini - GOYA early finalization code 698e65e175bSOded Gabbay * 699e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 700e65e175bSOded Gabbay * 701e65e175bSOded Gabbay * Unmap PCI bars 702e65e175bSOded Gabbay * 703e65e175bSOded Gabbay */ 704e65e175bSOded Gabbay static int goya_early_fini(struct hl_device *hdev) 705e65e175bSOded Gabbay { 706e65e175bSOded Gabbay kfree(hdev->asic_prop.hw_queues_props); 707e65e175bSOded Gabbay hl_pci_fini(hdev); 708e65e175bSOded Gabbay 709e65e175bSOded Gabbay return 0; 710e65e175bSOded Gabbay } 711e65e175bSOded Gabbay 712e65e175bSOded Gabbay static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) 713e65e175bSOded Gabbay { 714e65e175bSOded Gabbay /* mask to zero the MMBP and ASID bits */ 715e65e175bSOded Gabbay WREG32_AND(reg, ~0x7FF); 716e65e175bSOded Gabbay WREG32_OR(reg, asid); 717e65e175bSOded Gabbay } 718e65e175bSOded Gabbay 719e65e175bSOded Gabbay static void goya_qman0_set_security(struct hl_device *hdev, bool secure) 720e65e175bSOded Gabbay { 721e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 722e65e175bSOded Gabbay 723e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 724e65e175bSOded Gabbay return; 725e65e175bSOded Gabbay 726e65e175bSOded Gabbay if (secure) 727e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED); 728e65e175bSOded Gabbay else 729e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED); 730e65e175bSOded Gabbay 731e65e175bSOded Gabbay RREG32(mmDMA_QM_0_GLBL_PROT); 732e65e175bSOded Gabbay } 733e65e175bSOded Gabbay 734e65e175bSOded Gabbay /* 735e65e175bSOded Gabbay * goya_fetch_psoc_frequency - Fetch PSOC frequency values 736e65e175bSOded Gabbay * 737e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 738e65e175bSOded Gabbay * 739e65e175bSOded Gabbay */ 740e65e175bSOded Gabbay static void goya_fetch_psoc_frequency(struct hl_device *hdev) 741e65e175bSOded Gabbay { 742e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 743e65e175bSOded Gabbay u32 nr = 0, nf = 0, od = 0, div_fctr = 0, pll_clk, div_sel; 744e65e175bSOded Gabbay u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; 745e65e175bSOded Gabbay int rc; 746e65e175bSOded Gabbay 747e65e175bSOded Gabbay if (hdev->asic_prop.fw_security_enabled) { 748e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 749e65e175bSOded Gabbay 750e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) 751e65e175bSOded Gabbay return; 752e65e175bSOded Gabbay 753e65e175bSOded Gabbay rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL, 754e65e175bSOded Gabbay pll_freq_arr); 755e65e175bSOded Gabbay 756e65e175bSOded Gabbay if (rc) 757e65e175bSOded Gabbay return; 758e65e175bSOded Gabbay 759e65e175bSOded Gabbay freq = pll_freq_arr[1]; 760e65e175bSOded Gabbay } else { 761e65e175bSOded Gabbay div_fctr = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1); 762e65e175bSOded Gabbay div_sel = RREG32(mmPSOC_PCI_PLL_DIV_SEL_1); 763e65e175bSOded Gabbay nr = RREG32(mmPSOC_PCI_PLL_NR); 764e65e175bSOded Gabbay nf = RREG32(mmPSOC_PCI_PLL_NF); 765e65e175bSOded Gabbay od = RREG32(mmPSOC_PCI_PLL_OD); 766e65e175bSOded Gabbay 767e65e175bSOded Gabbay if (div_sel == DIV_SEL_REF_CLK || 768e65e175bSOded Gabbay div_sel == DIV_SEL_DIVIDED_REF) { 769e65e175bSOded Gabbay if (div_sel == DIV_SEL_REF_CLK) 770e65e175bSOded Gabbay freq = PLL_REF_CLK; 771e65e175bSOded Gabbay else 772e65e175bSOded Gabbay freq = PLL_REF_CLK / (div_fctr + 1); 773e65e175bSOded Gabbay } else if (div_sel == DIV_SEL_PLL_CLK || 774e65e175bSOded Gabbay div_sel == DIV_SEL_DIVIDED_PLL) { 775e65e175bSOded Gabbay pll_clk = PLL_REF_CLK * (nf + 1) / 776e65e175bSOded Gabbay ((nr + 1) * (od + 1)); 777e65e175bSOded Gabbay if (div_sel == DIV_SEL_PLL_CLK) 778e65e175bSOded Gabbay freq = pll_clk; 779e65e175bSOded Gabbay else 780e65e175bSOded Gabbay freq = pll_clk / (div_fctr + 1); 781e65e175bSOded Gabbay } else { 782e65e175bSOded Gabbay dev_warn(hdev->dev, 783e65e175bSOded Gabbay "Received invalid div select value: %d", 784e65e175bSOded Gabbay div_sel); 785e65e175bSOded Gabbay freq = 0; 786e65e175bSOded Gabbay } 787e65e175bSOded Gabbay } 788e65e175bSOded Gabbay 789e65e175bSOded Gabbay prop->psoc_timestamp_frequency = freq; 790e65e175bSOded Gabbay prop->psoc_pci_pll_nr = nr; 791e65e175bSOded Gabbay prop->psoc_pci_pll_nf = nf; 792e65e175bSOded Gabbay prop->psoc_pci_pll_od = od; 793e65e175bSOded Gabbay prop->psoc_pci_pll_div_factor = div_fctr; 794e65e175bSOded Gabbay } 795e65e175bSOded Gabbay 796e65e175bSOded Gabbay /* 797e65e175bSOded Gabbay * goya_set_frequency - set the frequency of the device 798e65e175bSOded Gabbay * 799e65e175bSOded Gabbay * @hdev: pointer to habanalabs device structure 800e65e175bSOded Gabbay * @freq: the new frequency value 801e65e175bSOded Gabbay * 802e65e175bSOded Gabbay * Change the frequency if needed. This function has no protection against 803e65e175bSOded Gabbay * concurrency, therefore it is assumed that the calling function has protected 804e65e175bSOded Gabbay * itself against the case of calling this function from multiple threads with 805e65e175bSOded Gabbay * different values 806e65e175bSOded Gabbay * 807e65e175bSOded Gabbay * Returns 0 if no change was done, otherwise returns 1 808e65e175bSOded Gabbay */ 809e65e175bSOded Gabbay int goya_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq) 810e65e175bSOded Gabbay { 811e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 812e65e175bSOded Gabbay 813e65e175bSOded Gabbay if ((goya->pm_mng_profile == PM_MANUAL) || 814e65e175bSOded Gabbay (goya->curr_pll_profile == freq)) 815e65e175bSOded Gabbay return 0; 816e65e175bSOded Gabbay 817e65e175bSOded Gabbay dev_dbg(hdev->dev, "Changing device frequency to %s\n", 818e65e175bSOded Gabbay freq == PLL_HIGH ? "high" : "low"); 819e65e175bSOded Gabbay 820e65e175bSOded Gabbay goya_set_pll_profile(hdev, freq); 821e65e175bSOded Gabbay 822e65e175bSOded Gabbay goya->curr_pll_profile = freq; 823e65e175bSOded Gabbay 824e65e175bSOded Gabbay return 1; 825e65e175bSOded Gabbay } 826e65e175bSOded Gabbay 827e65e175bSOded Gabbay static void goya_set_freq_to_low_job(struct work_struct *work) 828e65e175bSOded Gabbay { 829e65e175bSOded Gabbay struct goya_work_freq *goya_work = container_of(work, 830e65e175bSOded Gabbay struct goya_work_freq, 831e65e175bSOded Gabbay work_freq.work); 832e65e175bSOded Gabbay struct hl_device *hdev = goya_work->hdev; 833e65e175bSOded Gabbay 834e65e175bSOded Gabbay mutex_lock(&hdev->fpriv_list_lock); 835e65e175bSOded Gabbay 836e65e175bSOded Gabbay if (!hdev->is_compute_ctx_active) 837e65e175bSOded Gabbay goya_set_frequency(hdev, PLL_LOW); 838e65e175bSOded Gabbay 839e65e175bSOded Gabbay mutex_unlock(&hdev->fpriv_list_lock); 840e65e175bSOded Gabbay 841e65e175bSOded Gabbay schedule_delayed_work(&goya_work->work_freq, 842e65e175bSOded Gabbay usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC)); 843e65e175bSOded Gabbay } 844e65e175bSOded Gabbay 845e65e175bSOded Gabbay int goya_late_init(struct hl_device *hdev) 846e65e175bSOded Gabbay { 847e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 848e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 849e65e175bSOded Gabbay int rc; 850e65e175bSOded Gabbay 851e65e175bSOded Gabbay goya_fetch_psoc_frequency(hdev); 852e65e175bSOded Gabbay 853e65e175bSOded Gabbay rc = goya_mmu_clear_pgt_range(hdev); 854e65e175bSOded Gabbay if (rc) { 855e65e175bSOded Gabbay dev_err(hdev->dev, 856e65e175bSOded Gabbay "Failed to clear MMU page tables range %d\n", rc); 857e65e175bSOded Gabbay return rc; 858e65e175bSOded Gabbay } 859e65e175bSOded Gabbay 860e65e175bSOded Gabbay rc = goya_mmu_set_dram_default_page(hdev); 861e65e175bSOded Gabbay if (rc) { 862e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); 863e65e175bSOded Gabbay return rc; 864e65e175bSOded Gabbay } 865e65e175bSOded Gabbay 866e65e175bSOded Gabbay rc = goya_mmu_add_mappings_for_device_cpu(hdev); 867e65e175bSOded Gabbay if (rc) 868e65e175bSOded Gabbay return rc; 869e65e175bSOded Gabbay 870e65e175bSOded Gabbay rc = goya_init_cpu_queues(hdev); 871e65e175bSOded Gabbay if (rc) 872e65e175bSOded Gabbay return rc; 873e65e175bSOded Gabbay 874e65e175bSOded Gabbay rc = goya_test_cpu_queue(hdev); 875e65e175bSOded Gabbay if (rc) 876e65e175bSOded Gabbay return rc; 877e65e175bSOded Gabbay 878e65e175bSOded Gabbay rc = goya_cpucp_info_get(hdev); 879e65e175bSOded Gabbay if (rc) { 880e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc); 881e65e175bSOded Gabbay return rc; 882e65e175bSOded Gabbay } 883e65e175bSOded Gabbay 884e65e175bSOded Gabbay /* Now that we have the DRAM size in ASIC prop, we need to check 885e65e175bSOded Gabbay * its size and configure the DMA_IF DDR wrap protection (which is in 886e65e175bSOded Gabbay * the MMU block) accordingly. The value is the log2 of the DRAM size 887e65e175bSOded Gabbay */ 888e65e175bSOded Gabbay WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size)); 889e65e175bSOded Gabbay 890e65e175bSOded Gabbay rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 0x0); 891e65e175bSOded Gabbay if (rc) { 892e65e175bSOded Gabbay dev_err(hdev->dev, 893e65e175bSOded Gabbay "Failed to enable PCI access from CPU %d\n", rc); 894e65e175bSOded Gabbay return rc; 895e65e175bSOded Gabbay } 896e65e175bSOded Gabbay 897e65e175bSOded Gabbay /* force setting to low frequency */ 898e65e175bSOded Gabbay goya->curr_pll_profile = PLL_LOW; 899e65e175bSOded Gabbay 900e65e175bSOded Gabbay goya->pm_mng_profile = PM_AUTO; 901e65e175bSOded Gabbay 902e65e175bSOded Gabbay goya_set_pll_profile(hdev, PLL_LOW); 903e65e175bSOded Gabbay 904e65e175bSOded Gabbay schedule_delayed_work(&goya->goya_work->work_freq, 905e65e175bSOded Gabbay usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC)); 906e65e175bSOded Gabbay 907e65e175bSOded Gabbay return 0; 908e65e175bSOded Gabbay } 909e65e175bSOded Gabbay 910e65e175bSOded Gabbay /* 911e65e175bSOded Gabbay * goya_late_fini - GOYA late tear-down code 912e65e175bSOded Gabbay * 913e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 914e65e175bSOded Gabbay * 915e65e175bSOded Gabbay * Free sensors allocated structures 916e65e175bSOded Gabbay */ 917e65e175bSOded Gabbay void goya_late_fini(struct hl_device *hdev) 918e65e175bSOded Gabbay { 919e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 920e65e175bSOded Gabbay 921e65e175bSOded Gabbay cancel_delayed_work_sync(&goya->goya_work->work_freq); 922e65e175bSOded Gabbay 923e65e175bSOded Gabbay hl_hwmon_release_resources(hdev); 924e65e175bSOded Gabbay } 925e65e175bSOded Gabbay 926e65e175bSOded Gabbay static void goya_set_pci_memory_regions(struct hl_device *hdev) 927e65e175bSOded Gabbay { 928e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 929e65e175bSOded Gabbay struct pci_mem_region *region; 930e65e175bSOded Gabbay 931e65e175bSOded Gabbay /* CFG */ 932e65e175bSOded Gabbay region = &hdev->pci_mem_region[PCI_REGION_CFG]; 933e65e175bSOded Gabbay region->region_base = CFG_BASE; 934e65e175bSOded Gabbay region->region_size = CFG_SIZE; 935e65e175bSOded Gabbay region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR; 936e65e175bSOded Gabbay region->bar_size = CFG_BAR_SIZE; 937e65e175bSOded Gabbay region->bar_id = SRAM_CFG_BAR_ID; 938e65e175bSOded Gabbay region->used = 1; 939e65e175bSOded Gabbay 940e65e175bSOded Gabbay /* SRAM */ 941e65e175bSOded Gabbay region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 942e65e175bSOded Gabbay region->region_base = SRAM_BASE_ADDR; 943e65e175bSOded Gabbay region->region_size = SRAM_SIZE; 944e65e175bSOded Gabbay region->offset_in_bar = 0; 945e65e175bSOded Gabbay region->bar_size = CFG_BAR_SIZE; 946e65e175bSOded Gabbay region->bar_id = SRAM_CFG_BAR_ID; 947e65e175bSOded Gabbay region->used = 1; 948e65e175bSOded Gabbay 949e65e175bSOded Gabbay /* DRAM */ 950e65e175bSOded Gabbay region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 951e65e175bSOded Gabbay region->region_base = DRAM_PHYS_BASE; 952e65e175bSOded Gabbay region->region_size = hdev->asic_prop.dram_size; 953e65e175bSOded Gabbay region->offset_in_bar = 0; 954e65e175bSOded Gabbay region->bar_size = prop->dram_pci_bar_size; 955e65e175bSOded Gabbay region->bar_id = DDR_BAR_ID; 956e65e175bSOded Gabbay region->used = 1; 957e65e175bSOded Gabbay } 958e65e175bSOded Gabbay 959e65e175bSOded Gabbay /* 960e65e175bSOded Gabbay * goya_sw_init - Goya software initialization code 961e65e175bSOded Gabbay * 962e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 963e65e175bSOded Gabbay * 964e65e175bSOded Gabbay */ 965e65e175bSOded Gabbay static int goya_sw_init(struct hl_device *hdev) 966e65e175bSOded Gabbay { 967e65e175bSOded Gabbay struct goya_device *goya; 968e65e175bSOded Gabbay int rc; 969e65e175bSOded Gabbay 970e65e175bSOded Gabbay /* Allocate device structure */ 971e65e175bSOded Gabbay goya = kzalloc(sizeof(*goya), GFP_KERNEL); 972e65e175bSOded Gabbay if (!goya) 973e65e175bSOded Gabbay return -ENOMEM; 974e65e175bSOded Gabbay 975e65e175bSOded Gabbay /* according to goya_init_iatu */ 976e65e175bSOded Gabbay goya->ddr_bar_cur_addr = DRAM_PHYS_BASE; 977e65e175bSOded Gabbay 978e65e175bSOded Gabbay goya->mme_clk = GOYA_PLL_FREQ_LOW; 979e65e175bSOded Gabbay goya->tpc_clk = GOYA_PLL_FREQ_LOW; 980e65e175bSOded Gabbay goya->ic_clk = GOYA_PLL_FREQ_LOW; 981e65e175bSOded Gabbay 982e65e175bSOded Gabbay hdev->asic_specific = goya; 983e65e175bSOded Gabbay 984e65e175bSOded Gabbay /* Create DMA pool for small allocations */ 985e65e175bSOded Gabbay hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), 986e65e175bSOded Gabbay &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); 987e65e175bSOded Gabbay if (!hdev->dma_pool) { 988e65e175bSOded Gabbay dev_err(hdev->dev, "failed to create DMA pool\n"); 989e65e175bSOded Gabbay rc = -ENOMEM; 990e65e175bSOded Gabbay goto free_goya_device; 991e65e175bSOded Gabbay } 992e65e175bSOded Gabbay 993e65e175bSOded Gabbay hdev->cpu_accessible_dma_mem = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 994e65e175bSOded Gabbay &hdev->cpu_accessible_dma_address, 995e65e175bSOded Gabbay GFP_KERNEL | __GFP_ZERO); 996e65e175bSOded Gabbay 997e65e175bSOded Gabbay if (!hdev->cpu_accessible_dma_mem) { 998e65e175bSOded Gabbay rc = -ENOMEM; 999e65e175bSOded Gabbay goto free_dma_pool; 1000e65e175bSOded Gabbay } 1001e65e175bSOded Gabbay 1002e65e175bSOded Gabbay dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", 1003e65e175bSOded Gabbay &hdev->cpu_accessible_dma_address); 1004e65e175bSOded Gabbay 1005e65e175bSOded Gabbay hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 1006e65e175bSOded Gabbay if (!hdev->cpu_accessible_dma_pool) { 1007e65e175bSOded Gabbay dev_err(hdev->dev, 1008e65e175bSOded Gabbay "Failed to create CPU accessible DMA pool\n"); 1009e65e175bSOded Gabbay rc = -ENOMEM; 1010e65e175bSOded Gabbay goto free_cpu_dma_mem; 1011e65e175bSOded Gabbay } 1012e65e175bSOded Gabbay 1013e65e175bSOded Gabbay rc = gen_pool_add(hdev->cpu_accessible_dma_pool, 1014e65e175bSOded Gabbay (uintptr_t) hdev->cpu_accessible_dma_mem, 1015e65e175bSOded Gabbay HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 1016e65e175bSOded Gabbay if (rc) { 1017e65e175bSOded Gabbay dev_err(hdev->dev, 1018e65e175bSOded Gabbay "Failed to add memory to CPU accessible DMA pool\n"); 1019e65e175bSOded Gabbay rc = -EFAULT; 1020e65e175bSOded Gabbay goto free_cpu_accessible_dma_pool; 1021e65e175bSOded Gabbay } 1022e65e175bSOded Gabbay 1023e65e175bSOded Gabbay spin_lock_init(&goya->hw_queues_lock); 1024e65e175bSOded Gabbay hdev->supports_coresight = true; 1025e65e175bSOded Gabbay hdev->asic_prop.supports_compute_reset = true; 1026e65e175bSOded Gabbay hdev->asic_prop.allow_inference_soft_reset = true; 1027e65e175bSOded Gabbay hdev->supports_wait_for_multi_cs = false; 1028e65e175bSOded Gabbay hdev->supports_ctx_switch = true; 1029e65e175bSOded Gabbay 1030e65e175bSOded Gabbay hdev->asic_funcs->set_pci_memory_regions(hdev); 1031e65e175bSOded Gabbay 1032e65e175bSOded Gabbay goya->goya_work = kmalloc(sizeof(struct goya_work_freq), GFP_KERNEL); 1033e65e175bSOded Gabbay if (!goya->goya_work) { 1034e65e175bSOded Gabbay rc = -ENOMEM; 1035e65e175bSOded Gabbay goto free_cpu_accessible_dma_pool; 1036e65e175bSOded Gabbay } 1037e65e175bSOded Gabbay 1038e65e175bSOded Gabbay goya->goya_work->hdev = hdev; 1039e65e175bSOded Gabbay INIT_DELAYED_WORK(&goya->goya_work->work_freq, goya_set_freq_to_low_job); 1040e65e175bSOded Gabbay 1041e65e175bSOded Gabbay return 0; 1042e65e175bSOded Gabbay 1043e65e175bSOded Gabbay free_cpu_accessible_dma_pool: 1044e65e175bSOded Gabbay gen_pool_destroy(hdev->cpu_accessible_dma_pool); 1045e65e175bSOded Gabbay free_cpu_dma_mem: 1046e65e175bSOded Gabbay hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 1047e65e175bSOded Gabbay hdev->cpu_accessible_dma_address); 1048e65e175bSOded Gabbay free_dma_pool: 1049e65e175bSOded Gabbay dma_pool_destroy(hdev->dma_pool); 1050e65e175bSOded Gabbay free_goya_device: 1051e65e175bSOded Gabbay kfree(goya); 1052e65e175bSOded Gabbay 1053e65e175bSOded Gabbay return rc; 1054e65e175bSOded Gabbay } 1055e65e175bSOded Gabbay 1056e65e175bSOded Gabbay /* 1057e65e175bSOded Gabbay * goya_sw_fini - Goya software tear-down code 1058e65e175bSOded Gabbay * 1059e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 1060e65e175bSOded Gabbay * 1061e65e175bSOded Gabbay */ 1062e65e175bSOded Gabbay static int goya_sw_fini(struct hl_device *hdev) 1063e65e175bSOded Gabbay { 1064e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1065e65e175bSOded Gabbay 1066e65e175bSOded Gabbay gen_pool_destroy(hdev->cpu_accessible_dma_pool); 1067e65e175bSOded Gabbay 1068e65e175bSOded Gabbay hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 1069e65e175bSOded Gabbay hdev->cpu_accessible_dma_address); 1070e65e175bSOded Gabbay 1071e65e175bSOded Gabbay dma_pool_destroy(hdev->dma_pool); 1072e65e175bSOded Gabbay 1073e65e175bSOded Gabbay kfree(goya->goya_work); 1074e65e175bSOded Gabbay kfree(goya); 1075e65e175bSOded Gabbay 1076e65e175bSOded Gabbay return 0; 1077e65e175bSOded Gabbay } 1078e65e175bSOded Gabbay 1079e65e175bSOded Gabbay static void goya_init_dma_qman(struct hl_device *hdev, int dma_id, 1080e65e175bSOded Gabbay dma_addr_t bus_address) 1081e65e175bSOded Gabbay { 1082e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1083e65e175bSOded Gabbay u32 mtr_base_lo, mtr_base_hi; 1084e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 1085e65e175bSOded Gabbay u32 gic_base_lo, gic_base_hi; 1086e65e175bSOded Gabbay u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI); 1087e65e175bSOded Gabbay u32 dma_err_cfg = QMAN_DMA_ERR_MSG_EN; 1088e65e175bSOded Gabbay 1089e65e175bSOded Gabbay mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1090e65e175bSOded Gabbay mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1091e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1092e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1093e65e175bSOded Gabbay 1094e65e175bSOded Gabbay gic_base_lo = 1095e65e175bSOded Gabbay lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1096e65e175bSOded Gabbay gic_base_hi = 1097e65e175bSOded Gabbay upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1098e65e175bSOded Gabbay 1099e65e175bSOded Gabbay WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address)); 1100e65e175bSOded Gabbay WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address)); 1101e65e175bSOded Gabbay 1102e65e175bSOded Gabbay WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH)); 1103e65e175bSOded Gabbay WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0); 1104e65e175bSOded Gabbay WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0); 1105e65e175bSOded Gabbay 1106e65e175bSOded Gabbay WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo); 1107e65e175bSOded Gabbay WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi); 1108e65e175bSOded Gabbay WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo); 1109e65e175bSOded Gabbay WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi); 1110e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo); 1111e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi); 1112e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off, 1113e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id); 1114e65e175bSOded Gabbay 1115e65e175bSOded Gabbay /* PQ has buffer of 2 cache lines, while CQ has 8 lines */ 1116e65e175bSOded Gabbay WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002); 1117e65e175bSOded Gabbay WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008); 1118e65e175bSOded Gabbay 1119e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MMU) 1120e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_PARTLY_TRUSTED); 1121e65e175bSOded Gabbay else 1122e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED); 1123e65e175bSOded Gabbay 1124e65e175bSOded Gabbay if (hdev->stop_on_err) 1125e65e175bSOded Gabbay dma_err_cfg |= 1 << DMA_QM_0_GLBL_ERR_CFG_DMA_STOP_ON_ERR_SHIFT; 1126e65e175bSOded Gabbay 1127e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, dma_err_cfg); 1128e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE); 1129e65e175bSOded Gabbay } 1130e65e175bSOded Gabbay 1131e65e175bSOded Gabbay static void goya_init_dma_ch(struct hl_device *hdev, int dma_id) 1132e65e175bSOded Gabbay { 1133e65e175bSOded Gabbay u32 gic_base_lo, gic_base_hi; 1134e65e175bSOded Gabbay u64 sob_addr; 1135e65e175bSOded Gabbay u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1); 1136e65e175bSOded Gabbay 1137e65e175bSOded Gabbay gic_base_lo = 1138e65e175bSOded Gabbay lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1139e65e175bSOded Gabbay gic_base_hi = 1140e65e175bSOded Gabbay upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1141e65e175bSOded Gabbay 1142e65e175bSOded Gabbay WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo); 1143e65e175bSOded Gabbay WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi); 1144e65e175bSOded Gabbay WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off, 1145e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id); 1146e65e175bSOded Gabbay 1147e65e175bSOded Gabbay if (dma_id) 1148e65e175bSOded Gabbay sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 + 1149e65e175bSOded Gabbay (dma_id - 1) * 4; 1150e65e175bSOded Gabbay else 1151e65e175bSOded Gabbay sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007; 1152e65e175bSOded Gabbay 1153e65e175bSOded Gabbay WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off, upper_32_bits(sob_addr)); 1154e65e175bSOded Gabbay WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001); 1155e65e175bSOded Gabbay } 1156e65e175bSOded Gabbay 1157e65e175bSOded Gabbay /* 1158e65e175bSOded Gabbay * goya_init_dma_qmans - Initialize QMAN DMA registers 1159e65e175bSOded Gabbay * 1160e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 1161e65e175bSOded Gabbay * 1162e65e175bSOded Gabbay * Initialize the H/W registers of the QMAN DMA channels 1163e65e175bSOded Gabbay * 1164e65e175bSOded Gabbay */ 1165e65e175bSOded Gabbay void goya_init_dma_qmans(struct hl_device *hdev) 1166e65e175bSOded Gabbay { 1167e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1168e65e175bSOded Gabbay struct hl_hw_queue *q; 1169e65e175bSOded Gabbay int i; 1170e65e175bSOded Gabbay 1171e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_DMA) 1172e65e175bSOded Gabbay return; 1173e65e175bSOded Gabbay 1174e65e175bSOded Gabbay q = &hdev->kernel_queues[0]; 1175e65e175bSOded Gabbay 1176e65e175bSOded Gabbay for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) { 1177e65e175bSOded Gabbay q->cq_id = q->msi_vec = i; 1178e65e175bSOded Gabbay goya_init_dma_qman(hdev, i, q->bus_address); 1179e65e175bSOded Gabbay goya_init_dma_ch(hdev, i); 1180e65e175bSOded Gabbay } 1181e65e175bSOded Gabbay 1182e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_DMA; 1183e65e175bSOded Gabbay } 1184e65e175bSOded Gabbay 1185e65e175bSOded Gabbay /* 1186e65e175bSOded Gabbay * goya_disable_external_queues - Disable external queues 1187e65e175bSOded Gabbay * 1188e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 1189e65e175bSOded Gabbay * 1190e65e175bSOded Gabbay */ 1191e65e175bSOded Gabbay static void goya_disable_external_queues(struct hl_device *hdev) 1192e65e175bSOded Gabbay { 1193e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1194e65e175bSOded Gabbay 1195e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_DMA)) 1196e65e175bSOded Gabbay return; 1197e65e175bSOded Gabbay 1198e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_CFG0, 0); 1199e65e175bSOded Gabbay WREG32(mmDMA_QM_1_GLBL_CFG0, 0); 1200e65e175bSOded Gabbay WREG32(mmDMA_QM_2_GLBL_CFG0, 0); 1201e65e175bSOded Gabbay WREG32(mmDMA_QM_3_GLBL_CFG0, 0); 1202e65e175bSOded Gabbay WREG32(mmDMA_QM_4_GLBL_CFG0, 0); 1203e65e175bSOded Gabbay } 1204e65e175bSOded Gabbay 1205e65e175bSOded Gabbay static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg, 1206e65e175bSOded Gabbay u32 cp_sts_reg, u32 glbl_sts0_reg) 1207e65e175bSOded Gabbay { 1208e65e175bSOded Gabbay int rc; 1209e65e175bSOded Gabbay u32 status; 1210e65e175bSOded Gabbay 1211e65e175bSOded Gabbay /* use the values of TPC0 as they are all the same*/ 1212e65e175bSOded Gabbay 1213e65e175bSOded Gabbay WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT); 1214e65e175bSOded Gabbay 1215e65e175bSOded Gabbay status = RREG32(cp_sts_reg); 1216e65e175bSOded Gabbay if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) { 1217e65e175bSOded Gabbay rc = hl_poll_timeout( 1218e65e175bSOded Gabbay hdev, 1219e65e175bSOded Gabbay cp_sts_reg, 1220e65e175bSOded Gabbay status, 1221e65e175bSOded Gabbay !(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK), 1222e65e175bSOded Gabbay 1000, 1223e65e175bSOded Gabbay QMAN_FENCE_TIMEOUT_USEC); 1224e65e175bSOded Gabbay 1225e65e175bSOded Gabbay /* if QMAN is stuck in fence no need to check for stop */ 1226e65e175bSOded Gabbay if (rc) 1227e65e175bSOded Gabbay return 0; 1228e65e175bSOded Gabbay } 1229e65e175bSOded Gabbay 1230e65e175bSOded Gabbay rc = hl_poll_timeout( 1231e65e175bSOded Gabbay hdev, 1232e65e175bSOded Gabbay glbl_sts0_reg, 1233e65e175bSOded Gabbay status, 1234e65e175bSOded Gabbay (status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK), 1235e65e175bSOded Gabbay 1000, 1236e65e175bSOded Gabbay QMAN_STOP_TIMEOUT_USEC); 1237e65e175bSOded Gabbay 1238e65e175bSOded Gabbay if (rc) { 1239e65e175bSOded Gabbay dev_err(hdev->dev, 1240e65e175bSOded Gabbay "Timeout while waiting for QMAN to stop\n"); 1241e65e175bSOded Gabbay return -EINVAL; 1242e65e175bSOded Gabbay } 1243e65e175bSOded Gabbay 1244e65e175bSOded Gabbay return 0; 1245e65e175bSOded Gabbay } 1246e65e175bSOded Gabbay 1247e65e175bSOded Gabbay /* 1248e65e175bSOded Gabbay * goya_stop_external_queues - Stop external queues 1249e65e175bSOded Gabbay * 1250e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 1251e65e175bSOded Gabbay * 1252e65e175bSOded Gabbay * Returns 0 on success 1253e65e175bSOded Gabbay * 1254e65e175bSOded Gabbay */ 1255e65e175bSOded Gabbay static int goya_stop_external_queues(struct hl_device *hdev) 1256e65e175bSOded Gabbay { 1257e65e175bSOded Gabbay int rc, retval = 0; 1258e65e175bSOded Gabbay 1259e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1260e65e175bSOded Gabbay 1261e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_DMA)) 1262e65e175bSOded Gabbay return retval; 1263e65e175bSOded Gabbay 1264e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 1265e65e175bSOded Gabbay mmDMA_QM_0_GLBL_CFG1, 1266e65e175bSOded Gabbay mmDMA_QM_0_CP_STS, 1267e65e175bSOded Gabbay mmDMA_QM_0_GLBL_STS0); 1268e65e175bSOded Gabbay 1269e65e175bSOded Gabbay if (rc) { 1270e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); 1271e65e175bSOded Gabbay retval = -EIO; 1272e65e175bSOded Gabbay } 1273e65e175bSOded Gabbay 1274e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 1275e65e175bSOded Gabbay mmDMA_QM_1_GLBL_CFG1, 1276e65e175bSOded Gabbay mmDMA_QM_1_CP_STS, 1277e65e175bSOded Gabbay mmDMA_QM_1_GLBL_STS0); 1278e65e175bSOded Gabbay 1279e65e175bSOded Gabbay if (rc) { 1280e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); 1281e65e175bSOded Gabbay retval = -EIO; 1282e65e175bSOded Gabbay } 1283e65e175bSOded Gabbay 1284e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 1285e65e175bSOded Gabbay mmDMA_QM_2_GLBL_CFG1, 1286e65e175bSOded Gabbay mmDMA_QM_2_CP_STS, 1287e65e175bSOded Gabbay mmDMA_QM_2_GLBL_STS0); 1288e65e175bSOded Gabbay 1289e65e175bSOded Gabbay if (rc) { 1290e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); 1291e65e175bSOded Gabbay retval = -EIO; 1292e65e175bSOded Gabbay } 1293e65e175bSOded Gabbay 1294e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 1295e65e175bSOded Gabbay mmDMA_QM_3_GLBL_CFG1, 1296e65e175bSOded Gabbay mmDMA_QM_3_CP_STS, 1297e65e175bSOded Gabbay mmDMA_QM_3_GLBL_STS0); 1298e65e175bSOded Gabbay 1299e65e175bSOded Gabbay if (rc) { 1300e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); 1301e65e175bSOded Gabbay retval = -EIO; 1302e65e175bSOded Gabbay } 1303e65e175bSOded Gabbay 1304e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 1305e65e175bSOded Gabbay mmDMA_QM_4_GLBL_CFG1, 1306e65e175bSOded Gabbay mmDMA_QM_4_CP_STS, 1307e65e175bSOded Gabbay mmDMA_QM_4_GLBL_STS0); 1308e65e175bSOded Gabbay 1309e65e175bSOded Gabbay if (rc) { 1310e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); 1311e65e175bSOded Gabbay retval = -EIO; 1312e65e175bSOded Gabbay } 1313e65e175bSOded Gabbay 1314e65e175bSOded Gabbay return retval; 1315e65e175bSOded Gabbay } 1316e65e175bSOded Gabbay 1317e65e175bSOded Gabbay /* 1318e65e175bSOded Gabbay * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU 1319e65e175bSOded Gabbay * 1320e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 1321e65e175bSOded Gabbay * 1322e65e175bSOded Gabbay * Returns 0 on success 1323e65e175bSOded Gabbay * 1324e65e175bSOded Gabbay */ 1325e65e175bSOded Gabbay int goya_init_cpu_queues(struct hl_device *hdev) 1326e65e175bSOded Gabbay { 1327e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1328e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 1329e65e175bSOded Gabbay struct hl_eq *eq; 1330e65e175bSOded Gabbay u32 status; 1331e65e175bSOded Gabbay struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; 1332e65e175bSOded Gabbay int err; 1333e65e175bSOded Gabbay 1334e65e175bSOded Gabbay if (!hdev->cpu_queues_enable) 1335e65e175bSOded Gabbay return 0; 1336e65e175bSOded Gabbay 1337e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_CPU_Q) 1338e65e175bSOded Gabbay return 0; 1339e65e175bSOded Gabbay 1340e65e175bSOded Gabbay eq = &hdev->event_queue; 1341e65e175bSOded Gabbay 1342e65e175bSOded Gabbay WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 1343e65e175bSOded Gabbay WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 1344e65e175bSOded Gabbay 1345e65e175bSOded Gabbay WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 1346e65e175bSOded Gabbay WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 1347e65e175bSOded Gabbay 1348e65e175bSOded Gabbay WREG32(mmCPU_CQ_BASE_ADDR_LOW, 1349e65e175bSOded Gabbay lower_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR)); 1350e65e175bSOded Gabbay WREG32(mmCPU_CQ_BASE_ADDR_HIGH, 1351e65e175bSOded Gabbay upper_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR)); 1352e65e175bSOded Gabbay 1353e65e175bSOded Gabbay WREG32(mmCPU_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 1354e65e175bSOded Gabbay WREG32(mmCPU_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 1355e65e175bSOded Gabbay WREG32(mmCPU_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 1356e65e175bSOded Gabbay 1357e65e175bSOded Gabbay /* Used for EQ CI */ 1358e65e175bSOded Gabbay WREG32(mmCPU_EQ_CI, 0); 1359e65e175bSOded Gabbay 1360e65e175bSOded Gabbay WREG32(mmCPU_IF_PF_PQ_PI, 0); 1361e65e175bSOded Gabbay 1362e65e175bSOded Gabbay WREG32(mmCPU_PQ_INIT_STATUS, PQ_INIT_STATUS_READY_FOR_CP); 1363e65e175bSOded Gabbay 1364e65e175bSOded Gabbay WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, 1365e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PI_UPDATE); 1366e65e175bSOded Gabbay 1367e65e175bSOded Gabbay err = hl_poll_timeout( 1368e65e175bSOded Gabbay hdev, 1369e65e175bSOded Gabbay mmCPU_PQ_INIT_STATUS, 1370e65e175bSOded Gabbay status, 1371e65e175bSOded Gabbay (status == PQ_INIT_STATUS_READY_FOR_HOST), 1372e65e175bSOded Gabbay 1000, 1373e65e175bSOded Gabbay GOYA_CPU_TIMEOUT_USEC); 1374e65e175bSOded Gabbay 1375e65e175bSOded Gabbay if (err) { 1376e65e175bSOded Gabbay dev_err(hdev->dev, 1377e65e175bSOded Gabbay "Failed to setup communication with device CPU\n"); 1378e65e175bSOded Gabbay return -EIO; 1379e65e175bSOded Gabbay } 1380e65e175bSOded Gabbay 1381e65e175bSOded Gabbay /* update FW application security bits */ 1382e65e175bSOded Gabbay if (prop->fw_cpu_boot_dev_sts0_valid) 1383e65e175bSOded Gabbay prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 1384e65e175bSOded Gabbay 1385e65e175bSOded Gabbay if (prop->fw_cpu_boot_dev_sts1_valid) 1386e65e175bSOded Gabbay prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 1387e65e175bSOded Gabbay 1388e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_CPU_Q; 1389e65e175bSOded Gabbay return 0; 1390e65e175bSOded Gabbay } 1391e65e175bSOded Gabbay 1392e65e175bSOded Gabbay static void goya_set_pll_refclk(struct hl_device *hdev) 1393e65e175bSOded Gabbay { 1394e65e175bSOded Gabbay WREG32(mmCPU_PLL_DIV_SEL_0, 0x0); 1395e65e175bSOded Gabbay WREG32(mmCPU_PLL_DIV_SEL_1, 0x0); 1396e65e175bSOded Gabbay WREG32(mmCPU_PLL_DIV_SEL_2, 0x0); 1397e65e175bSOded Gabbay WREG32(mmCPU_PLL_DIV_SEL_3, 0x0); 1398e65e175bSOded Gabbay 1399e65e175bSOded Gabbay WREG32(mmIC_PLL_DIV_SEL_0, 0x0); 1400e65e175bSOded Gabbay WREG32(mmIC_PLL_DIV_SEL_1, 0x0); 1401e65e175bSOded Gabbay WREG32(mmIC_PLL_DIV_SEL_2, 0x0); 1402e65e175bSOded Gabbay WREG32(mmIC_PLL_DIV_SEL_3, 0x0); 1403e65e175bSOded Gabbay 1404e65e175bSOded Gabbay WREG32(mmMC_PLL_DIV_SEL_0, 0x0); 1405e65e175bSOded Gabbay WREG32(mmMC_PLL_DIV_SEL_1, 0x0); 1406e65e175bSOded Gabbay WREG32(mmMC_PLL_DIV_SEL_2, 0x0); 1407e65e175bSOded Gabbay WREG32(mmMC_PLL_DIV_SEL_3, 0x0); 1408e65e175bSOded Gabbay 1409e65e175bSOded Gabbay WREG32(mmPSOC_MME_PLL_DIV_SEL_0, 0x0); 1410e65e175bSOded Gabbay WREG32(mmPSOC_MME_PLL_DIV_SEL_1, 0x0); 1411e65e175bSOded Gabbay WREG32(mmPSOC_MME_PLL_DIV_SEL_2, 0x0); 1412e65e175bSOded Gabbay WREG32(mmPSOC_MME_PLL_DIV_SEL_3, 0x0); 1413e65e175bSOded Gabbay 1414e65e175bSOded Gabbay WREG32(mmPSOC_PCI_PLL_DIV_SEL_0, 0x0); 1415e65e175bSOded Gabbay WREG32(mmPSOC_PCI_PLL_DIV_SEL_1, 0x0); 1416e65e175bSOded Gabbay WREG32(mmPSOC_PCI_PLL_DIV_SEL_2, 0x0); 1417e65e175bSOded Gabbay WREG32(mmPSOC_PCI_PLL_DIV_SEL_3, 0x0); 1418e65e175bSOded Gabbay 1419e65e175bSOded Gabbay WREG32(mmPSOC_EMMC_PLL_DIV_SEL_0, 0x0); 1420e65e175bSOded Gabbay WREG32(mmPSOC_EMMC_PLL_DIV_SEL_1, 0x0); 1421e65e175bSOded Gabbay WREG32(mmPSOC_EMMC_PLL_DIV_SEL_2, 0x0); 1422e65e175bSOded Gabbay WREG32(mmPSOC_EMMC_PLL_DIV_SEL_3, 0x0); 1423e65e175bSOded Gabbay 1424e65e175bSOded Gabbay WREG32(mmTPC_PLL_DIV_SEL_0, 0x0); 1425e65e175bSOded Gabbay WREG32(mmTPC_PLL_DIV_SEL_1, 0x0); 1426e65e175bSOded Gabbay WREG32(mmTPC_PLL_DIV_SEL_2, 0x0); 1427e65e175bSOded Gabbay WREG32(mmTPC_PLL_DIV_SEL_3, 0x0); 1428e65e175bSOded Gabbay } 1429e65e175bSOded Gabbay 1430e65e175bSOded Gabbay static void goya_disable_clk_rlx(struct hl_device *hdev) 1431e65e175bSOded Gabbay { 1432e65e175bSOded Gabbay WREG32(mmPSOC_MME_PLL_CLK_RLX_0, 0x100010); 1433e65e175bSOded Gabbay WREG32(mmIC_PLL_CLK_RLX_0, 0x100010); 1434e65e175bSOded Gabbay } 1435e65e175bSOded Gabbay 1436e65e175bSOded Gabbay static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id) 1437e65e175bSOded Gabbay { 1438e65e175bSOded Gabbay u64 tpc_eml_address; 1439e65e175bSOded Gabbay u32 val, tpc_offset, tpc_eml_offset, tpc_slm_offset; 1440e65e175bSOded Gabbay int err, slm_index; 1441e65e175bSOded Gabbay 1442e65e175bSOded Gabbay tpc_offset = tpc_id * 0x40000; 1443e65e175bSOded Gabbay tpc_eml_offset = tpc_id * 0x200000; 1444e65e175bSOded Gabbay tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE); 1445e65e175bSOded Gabbay tpc_slm_offset = tpc_eml_address + 0x100000; 1446e65e175bSOded Gabbay 1447e65e175bSOded Gabbay /* 1448e65e175bSOded Gabbay * Workaround for Bug H2 #2443 : 1449e65e175bSOded Gabbay * "TPC SB is not initialized on chip reset" 1450e65e175bSOded Gabbay */ 1451e65e175bSOded Gabbay 1452e65e175bSOded Gabbay val = RREG32(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset); 1453e65e175bSOded Gabbay if (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_MASK) 1454e65e175bSOded Gabbay dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", 1455e65e175bSOded Gabbay tpc_id); 1456e65e175bSOded Gabbay 1457e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_PAT + tpc_offset, val & 0xFFFFF000); 1458e65e175bSOded Gabbay 1459e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_0 + tpc_offset, 0x37FF); 1460e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_1 + tpc_offset, 0x303F); 1461e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_2 + tpc_offset, 0x71FF); 1462e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_3 + tpc_offset, 0x71FF); 1463e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_4 + tpc_offset, 0x70FF); 1464e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_5 + tpc_offset, 0x70FF); 1465e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_6 + tpc_offset, 0x70FF); 1466e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_7 + tpc_offset, 0x70FF); 1467e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_8 + tpc_offset, 0x70FF); 1468e65e175bSOded Gabbay WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_9 + tpc_offset, 0x70FF); 1469e65e175bSOded Gabbay 1470e65e175bSOded Gabbay WREG32_OR(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset, 1471e65e175bSOded Gabbay 1 << TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_SHIFT); 1472e65e175bSOded Gabbay 1473e65e175bSOded Gabbay err = hl_poll_timeout( 1474e65e175bSOded Gabbay hdev, 1475e65e175bSOded Gabbay mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset, 1476e65e175bSOded Gabbay val, 1477e65e175bSOded Gabbay (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_MASK), 1478e65e175bSOded Gabbay 1000, 1479e65e175bSOded Gabbay HL_DEVICE_TIMEOUT_USEC); 1480e65e175bSOded Gabbay 1481e65e175bSOded Gabbay if (err) 1482e65e175bSOded Gabbay dev_err(hdev->dev, 1483e65e175bSOded Gabbay "Timeout while waiting for TPC%d MBIST DONE\n", tpc_id); 1484e65e175bSOded Gabbay 1485e65e175bSOded Gabbay WREG32_OR(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset, 1486e65e175bSOded Gabbay 1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT); 1487e65e175bSOded Gabbay 1488e65e175bSOded Gabbay msleep(GOYA_RESET_WAIT_MSEC); 1489e65e175bSOded Gabbay 1490e65e175bSOded Gabbay WREG32_AND(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset, 1491e65e175bSOded Gabbay ~(1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT)); 1492e65e175bSOded Gabbay 1493e65e175bSOded Gabbay msleep(GOYA_RESET_WAIT_MSEC); 1494e65e175bSOded Gabbay 1495e65e175bSOded Gabbay for (slm_index = 0 ; slm_index < 256 ; slm_index++) 1496e65e175bSOded Gabbay WREG32(tpc_slm_offset + (slm_index << 2), 0); 1497e65e175bSOded Gabbay 1498e65e175bSOded Gabbay val = RREG32(tpc_slm_offset); 1499e65e175bSOded Gabbay } 1500e65e175bSOded Gabbay 1501e65e175bSOded Gabbay static void goya_tpc_mbist_workaround(struct hl_device *hdev) 1502e65e175bSOded Gabbay { 1503e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1504e65e175bSOded Gabbay int i; 1505e65e175bSOded Gabbay 1506e65e175bSOded Gabbay if (hdev->pldm) 1507e65e175bSOded Gabbay return; 1508e65e175bSOded Gabbay 1509e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST) 1510e65e175bSOded Gabbay return; 1511e65e175bSOded Gabbay 1512e65e175bSOded Gabbay /* Workaround for H2 #2443 */ 1513e65e175bSOded Gabbay 1514e65e175bSOded Gabbay for (i = 0 ; i < TPC_MAX_NUM ; i++) 1515e65e175bSOded Gabbay _goya_tpc_mbist_workaround(hdev, i); 1516e65e175bSOded Gabbay 1517e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_TPC_MBIST; 1518e65e175bSOded Gabbay } 1519e65e175bSOded Gabbay 1520e65e175bSOded Gabbay /* 1521e65e175bSOded Gabbay * goya_init_golden_registers - Initialize golden registers 1522e65e175bSOded Gabbay * 1523e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 1524e65e175bSOded Gabbay * 1525e65e175bSOded Gabbay * Initialize the H/W registers of the device 1526e65e175bSOded Gabbay * 1527e65e175bSOded Gabbay */ 1528e65e175bSOded Gabbay static void goya_init_golden_registers(struct hl_device *hdev) 1529e65e175bSOded Gabbay { 1530e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1531e65e175bSOded Gabbay u32 polynom[10], tpc_intr_mask, offset; 1532e65e175bSOded Gabbay int i; 1533e65e175bSOded Gabbay 1534e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_GOLDEN) 1535e65e175bSOded Gabbay return; 1536e65e175bSOded Gabbay 1537e65e175bSOded Gabbay polynom[0] = 0x00020080; 1538e65e175bSOded Gabbay polynom[1] = 0x00401000; 1539e65e175bSOded Gabbay polynom[2] = 0x00200800; 1540e65e175bSOded Gabbay polynom[3] = 0x00002000; 1541e65e175bSOded Gabbay polynom[4] = 0x00080200; 1542e65e175bSOded Gabbay polynom[5] = 0x00040100; 1543e65e175bSOded Gabbay polynom[6] = 0x00100400; 1544e65e175bSOded Gabbay polynom[7] = 0x00004000; 1545e65e175bSOded Gabbay polynom[8] = 0x00010000; 1546e65e175bSOded Gabbay polynom[9] = 0x00008000; 1547e65e175bSOded Gabbay 1548e65e175bSOded Gabbay /* Mask all arithmetic interrupts from TPC */ 1549e65e175bSOded Gabbay tpc_intr_mask = 0x7FFF; 1550e65e175bSOded Gabbay 1551e65e175bSOded Gabbay for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x20000) { 1552e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X0_RTR_HBW_RD_RQ_L_ARB + offset, 0x302); 1553e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X1_RTR_HBW_RD_RQ_L_ARB + offset, 0x302); 1554e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X2_RTR_HBW_RD_RQ_L_ARB + offset, 0x302); 1555e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X3_RTR_HBW_RD_RQ_L_ARB + offset, 0x302); 1556e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X4_RTR_HBW_RD_RQ_L_ARB + offset, 0x302); 1557e65e175bSOded Gabbay 1558e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_L_ARB + offset, 0x204); 1559e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_L_ARB + offset, 0x204); 1560e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_L_ARB + offset, 0x204); 1561e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_L_ARB + offset, 0x204); 1562e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_L_ARB + offset, 0x204); 1563e65e175bSOded Gabbay 1564e65e175bSOded Gabbay 1565e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_E_ARB + offset, 0x206); 1566e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_E_ARB + offset, 0x206); 1567e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_E_ARB + offset, 0x206); 1568e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_E_ARB + offset, 0x207); 1569e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_E_ARB + offset, 0x207); 1570e65e175bSOded Gabbay 1571e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_W_ARB + offset, 0x207); 1572e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_W_ARB + offset, 0x207); 1573e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_W_ARB + offset, 0x206); 1574e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_W_ARB + offset, 0x206); 1575e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_W_ARB + offset, 0x206); 1576e65e175bSOded Gabbay 1577e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_E_ARB + offset, 0x101); 1578e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_E_ARB + offset, 0x102); 1579e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_E_ARB + offset, 0x103); 1580e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_E_ARB + offset, 0x104); 1581e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_E_ARB + offset, 0x105); 1582e65e175bSOded Gabbay 1583e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_W_ARB + offset, 0x105); 1584e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_W_ARB + offset, 0x104); 1585e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_W_ARB + offset, 0x103); 1586e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_W_ARB + offset, 0x102); 1587e65e175bSOded Gabbay WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_W_ARB + offset, 0x101); 1588e65e175bSOded Gabbay } 1589e65e175bSOded Gabbay 1590e65e175bSOded Gabbay WREG32(mmMME_STORE_MAX_CREDIT, 0x21); 1591e65e175bSOded Gabbay WREG32(mmMME_AGU, 0x0f0f0f10); 1592e65e175bSOded Gabbay WREG32(mmMME_SEI_MASK, ~0x0); 1593e65e175bSOded Gabbay 1594e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RQ_N_ARB, 0x01010101); 1595e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RQ_N_ARB, 0x01040101); 1596e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RQ_N_ARB, 0x01030101); 1597e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RQ_N_ARB, 0x01020101); 1598e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RQ_N_ARB, 0x01010101); 1599e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RQ_N_ARB, 0x07010701); 1600e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RQ_S_ARB, 0x04010401); 1601e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RQ_S_ARB, 0x04050401); 1602e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RQ_S_ARB, 0x03070301); 1603e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RQ_S_ARB, 0x01030101); 1604e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RQ_S_ARB, 0x01040101); 1605e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RQ_S_ARB, 0x01050105); 1606e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RQ_W_ARB, 0x01010501); 1607e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RQ_W_ARB, 0x01010501); 1608e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RQ_W_ARB, 0x01040301); 1609e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RQ_W_ARB, 0x01030401); 1610e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RQ_W_ARB, 0x01040101); 1611e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RQ_W_ARB, 0x01050101); 1612e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RQ_N_ARB, 0x02020202); 1613e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RQ_N_ARB, 0x01070101); 1614e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RQ_N_ARB, 0x02020201); 1615e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RQ_N_ARB, 0x07020701); 1616e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RQ_N_ARB, 0x01020101); 1617e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01010101); 1618e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RQ_S_ARB, 0x01070101); 1619e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RQ_S_ARB, 0x01070101); 1620e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RQ_S_ARB, 0x07020701); 1621e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RQ_S_ARB, 0x02020201); 1622e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RQ_S_ARB, 0x01070101); 1623e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01020102); 1624e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RQ_W_ARB, 0x01020701); 1625e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RQ_W_ARB, 0x01020701); 1626e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RQ_W_ARB, 0x07020707); 1627e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RQ_W_ARB, 0x01020201); 1628e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RQ_W_ARB, 0x01070201); 1629e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RQ_W_ARB, 0x01070201); 1630e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RS_N_ARB, 0x01070102); 1631e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RS_N_ARB, 0x01070102); 1632e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RS_N_ARB, 0x01060102); 1633e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RS_N_ARB, 0x01040102); 1634e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RS_N_ARB, 0x01020102); 1635e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RS_N_ARB, 0x01020107); 1636e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RS_S_ARB, 0x01020106); 1637e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RS_S_ARB, 0x01020102); 1638e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RS_S_ARB, 0x01040102); 1639e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RS_S_ARB, 0x01060102); 1640e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RS_S_ARB, 0x01070102); 1641e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RS_S_ARB, 0x01070102); 1642e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RS_E_ARB, 0x01020702); 1643e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RS_E_ARB, 0x01020702); 1644e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RS_E_ARB, 0x01040602); 1645e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RS_E_ARB, 0x01060402); 1646e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RS_E_ARB, 0x01070202); 1647e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RS_E_ARB, 0x01070102); 1648e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_RD_RS_W_ARB, 0x01060401); 1649e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_RD_RS_W_ARB, 0x01060401); 1650e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_RD_RS_W_ARB, 0x01060401); 1651e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_RD_RS_W_ARB, 0x01060401); 1652e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_RD_RS_W_ARB, 0x01060401); 1653e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_RD_RS_W_ARB, 0x01060401); 1654e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RS_N_ARB, 0x01050101); 1655e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RS_N_ARB, 0x01040101); 1656e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RS_N_ARB, 0x01030101); 1657e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RS_N_ARB, 0x01020101); 1658e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RS_N_ARB, 0x01010101); 1659e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RS_N_ARB, 0x01010107); 1660e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RS_S_ARB, 0x01010107); 1661e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RS_S_ARB, 0x01010101); 1662e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RS_S_ARB, 0x01020101); 1663e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RS_S_ARB, 0x01030101); 1664e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RS_S_ARB, 0x01040101); 1665e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RS_S_ARB, 0x01050101); 1666e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RS_E_ARB, 0x01010501); 1667e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RS_E_ARB, 0x01010501); 1668e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RS_E_ARB, 0x01040301); 1669e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RS_E_ARB, 0x01030401); 1670e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RS_E_ARB, 0x01040101); 1671e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RS_E_ARB, 0x01050101); 1672e65e175bSOded Gabbay WREG32(mmMME6_RTR_HBW_WR_RS_W_ARB, 0x01010101); 1673e65e175bSOded Gabbay WREG32(mmMME5_RTR_HBW_WR_RS_W_ARB, 0x01010101); 1674e65e175bSOded Gabbay WREG32(mmMME4_RTR_HBW_WR_RS_W_ARB, 0x01010101); 1675e65e175bSOded Gabbay WREG32(mmMME3_RTR_HBW_WR_RS_W_ARB, 0x01010101); 1676e65e175bSOded Gabbay WREG32(mmMME2_RTR_HBW_WR_RS_W_ARB, 0x01010101); 1677e65e175bSOded Gabbay WREG32(mmMME1_RTR_HBW_WR_RS_W_ARB, 0x01010101); 1678e65e175bSOded Gabbay 1679e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_RD_RQ_N_ARB, 0x01010101); 1680e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_RD_RQ_S_ARB, 0x01010101); 1681e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_RD_RQ_E_ARB, 0x01060101); 1682e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_WR_RQ_N_ARB, 0x02020102); 1683e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_WR_RQ_S_ARB, 0x01010101); 1684e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_WR_RQ_E_ARB, 0x02070202); 1685e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_RD_RS_N_ARB, 0x01020201); 1686e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_RD_RS_S_ARB, 0x01070201); 1687e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_RD_RS_W_ARB, 0x01070202); 1688e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_WR_RS_N_ARB, 0x01010101); 1689e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_WR_RS_S_ARB, 0x01050101); 1690e65e175bSOded Gabbay WREG32(mmTPC1_RTR_HBW_WR_RS_W_ARB, 0x01050101); 1691e65e175bSOded Gabbay 1692e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_RD_RQ_N_ARB, 0x01020101); 1693e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_RD_RQ_S_ARB, 0x01050101); 1694e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_RD_RQ_E_ARB, 0x01010201); 1695e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_WR_RQ_N_ARB, 0x02040102); 1696e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_WR_RQ_S_ARB, 0x01050101); 1697e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_WR_RQ_E_ARB, 0x02060202); 1698e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_RD_RS_N_ARB, 0x01020201); 1699e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_RD_RS_S_ARB, 0x01070201); 1700e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_RD_RS_W_ARB, 0x01070202); 1701e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_WR_RS_N_ARB, 0x01010101); 1702e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_WR_RS_S_ARB, 0x01040101); 1703e65e175bSOded Gabbay WREG32(mmTPC2_RTR_HBW_WR_RS_W_ARB, 0x01040101); 1704e65e175bSOded Gabbay 1705e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_RD_RQ_N_ARB, 0x01030101); 1706e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_RD_RQ_S_ARB, 0x01040101); 1707e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_RD_RQ_E_ARB, 0x01040301); 1708e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_WR_RQ_N_ARB, 0x02060102); 1709e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_WR_RQ_S_ARB, 0x01040101); 1710e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_WR_RQ_E_ARB, 0x01040301); 1711e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_RD_RS_N_ARB, 0x01040201); 1712e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_RD_RS_S_ARB, 0x01060201); 1713e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_RD_RS_W_ARB, 0x01060402); 1714e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_WR_RS_N_ARB, 0x01020101); 1715e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_WR_RS_S_ARB, 0x01030101); 1716e65e175bSOded Gabbay WREG32(mmTPC3_RTR_HBW_WR_RS_W_ARB, 0x01030401); 1717e65e175bSOded Gabbay 1718e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_RD_RQ_N_ARB, 0x01040101); 1719e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_RD_RQ_S_ARB, 0x01030101); 1720e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_RD_RQ_E_ARB, 0x01030401); 1721e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_WR_RQ_N_ARB, 0x02070102); 1722e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_WR_RQ_S_ARB, 0x01030101); 1723e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_WR_RQ_E_ARB, 0x02060702); 1724e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_RD_RS_N_ARB, 0x01060201); 1725e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_RD_RS_S_ARB, 0x01040201); 1726e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_RD_RS_W_ARB, 0x01040602); 1727e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_WR_RS_N_ARB, 0x01030101); 1728e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_WR_RS_S_ARB, 0x01020101); 1729e65e175bSOded Gabbay WREG32(mmTPC4_RTR_HBW_WR_RS_W_ARB, 0x01040301); 1730e65e175bSOded Gabbay 1731e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_RD_RQ_N_ARB, 0x01050101); 1732e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_RD_RQ_S_ARB, 0x01020101); 1733e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_RD_RQ_E_ARB, 0x01200501); 1734e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_WR_RQ_N_ARB, 0x02070102); 1735e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_WR_RQ_S_ARB, 0x01020101); 1736e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_WR_RQ_E_ARB, 0x02020602); 1737e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_RD_RS_N_ARB, 0x01070201); 1738e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_RD_RS_S_ARB, 0x01020201); 1739e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_RD_RS_W_ARB, 0x01020702); 1740e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_WR_RS_N_ARB, 0x01040101); 1741e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_WR_RS_S_ARB, 0x01010101); 1742e65e175bSOded Gabbay WREG32(mmTPC5_RTR_HBW_WR_RS_W_ARB, 0x01010501); 1743e65e175bSOded Gabbay 1744e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_RD_RQ_N_ARB, 0x01010101); 1745e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_RD_RQ_S_ARB, 0x01010101); 1746e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_RD_RQ_E_ARB, 0x01010601); 1747e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_WR_RQ_N_ARB, 0x01010101); 1748e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_WR_RQ_S_ARB, 0x01010101); 1749e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_WR_RQ_E_ARB, 0x02020702); 1750e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_RD_RS_N_ARB, 0x01010101); 1751e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_RD_RS_S_ARB, 0x01010101); 1752e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_RD_RS_W_ARB, 0x01020702); 1753e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_WR_RS_N_ARB, 0x01050101); 1754e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_WR_RS_S_ARB, 0x01010101); 1755e65e175bSOded Gabbay WREG32(mmTPC6_RTR_HBW_WR_RS_W_ARB, 0x01010501); 1756e65e175bSOded Gabbay 1757e65e175bSOded Gabbay for (i = 0, offset = 0 ; i < 10 ; i++, offset += 4) { 1758e65e175bSOded Gabbay WREG32(mmMME1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1759e65e175bSOded Gabbay WREG32(mmMME2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1760e65e175bSOded Gabbay WREG32(mmMME3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1761e65e175bSOded Gabbay WREG32(mmMME4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1762e65e175bSOded Gabbay WREG32(mmMME5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1763e65e175bSOded Gabbay WREG32(mmMME6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1764e65e175bSOded Gabbay 1765e65e175bSOded Gabbay WREG32(mmTPC0_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1766e65e175bSOded Gabbay WREG32(mmTPC1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1767e65e175bSOded Gabbay WREG32(mmTPC2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1768e65e175bSOded Gabbay WREG32(mmTPC3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1769e65e175bSOded Gabbay WREG32(mmTPC4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1770e65e175bSOded Gabbay WREG32(mmTPC5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1771e65e175bSOded Gabbay WREG32(mmTPC6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1772e65e175bSOded Gabbay WREG32(mmTPC7_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1773e65e175bSOded Gabbay 1774e65e175bSOded Gabbay WREG32(mmPCI_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1775e65e175bSOded Gabbay WREG32(mmDMA_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7); 1776e65e175bSOded Gabbay } 1777e65e175bSOded Gabbay 1778e65e175bSOded Gabbay for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x40000) { 1779e65e175bSOded Gabbay WREG32(mmMME1_RTR_SCRAMB_EN + offset, 1780e65e175bSOded Gabbay 1 << MME1_RTR_SCRAMB_EN_VAL_SHIFT); 1781e65e175bSOded Gabbay WREG32(mmMME1_RTR_NON_LIN_SCRAMB + offset, 1782e65e175bSOded Gabbay 1 << MME1_RTR_NON_LIN_SCRAMB_EN_SHIFT); 1783e65e175bSOded Gabbay } 1784e65e175bSOded Gabbay 1785e65e175bSOded Gabbay for (i = 0, offset = 0 ; i < 8 ; i++, offset += 0x40000) { 1786e65e175bSOded Gabbay /* 1787e65e175bSOded Gabbay * Workaround for Bug H2 #2441 : 1788e65e175bSOded Gabbay * "ST.NOP set trace event illegal opcode" 1789e65e175bSOded Gabbay */ 1790e65e175bSOded Gabbay WREG32(mmTPC0_CFG_TPC_INTR_MASK + offset, tpc_intr_mask); 1791e65e175bSOded Gabbay 1792e65e175bSOded Gabbay WREG32(mmTPC0_NRTR_SCRAMB_EN + offset, 1793e65e175bSOded Gabbay 1 << TPC0_NRTR_SCRAMB_EN_VAL_SHIFT); 1794e65e175bSOded Gabbay WREG32(mmTPC0_NRTR_NON_LIN_SCRAMB + offset, 1795e65e175bSOded Gabbay 1 << TPC0_NRTR_NON_LIN_SCRAMB_EN_SHIFT); 1796e65e175bSOded Gabbay 1797e65e175bSOded Gabbay WREG32_FIELD(TPC0_CFG_MSS_CONFIG, offset, 1798e65e175bSOded Gabbay ICACHE_FETCH_LINE_NUM, 2); 1799e65e175bSOded Gabbay } 1800e65e175bSOded Gabbay 1801e65e175bSOded Gabbay WREG32(mmDMA_NRTR_SCRAMB_EN, 1 << DMA_NRTR_SCRAMB_EN_VAL_SHIFT); 1802e65e175bSOded Gabbay WREG32(mmDMA_NRTR_NON_LIN_SCRAMB, 1803e65e175bSOded Gabbay 1 << DMA_NRTR_NON_LIN_SCRAMB_EN_SHIFT); 1804e65e175bSOded Gabbay 1805e65e175bSOded Gabbay WREG32(mmPCI_NRTR_SCRAMB_EN, 1 << PCI_NRTR_SCRAMB_EN_VAL_SHIFT); 1806e65e175bSOded Gabbay WREG32(mmPCI_NRTR_NON_LIN_SCRAMB, 1807e65e175bSOded Gabbay 1 << PCI_NRTR_NON_LIN_SCRAMB_EN_SHIFT); 1808e65e175bSOded Gabbay 1809e65e175bSOded Gabbay /* 1810e65e175bSOded Gabbay * Workaround for H2 #HW-23 bug 1811e65e175bSOded Gabbay * Set DMA max outstanding read requests to 240 on DMA CH 1. 1812e65e175bSOded Gabbay * This limitation is still large enough to not affect Gen4 bandwidth. 1813e65e175bSOded Gabbay * We need to only limit that DMA channel because the user can only read 1814e65e175bSOded Gabbay * from Host using DMA CH 1 1815e65e175bSOded Gabbay */ 1816e65e175bSOded Gabbay WREG32(mmDMA_CH_1_CFG0, 0x0fff00F0); 1817e65e175bSOded Gabbay 1818e65e175bSOded Gabbay WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020); 1819e65e175bSOded Gabbay 1820e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_GOLDEN; 1821e65e175bSOded Gabbay } 1822e65e175bSOded Gabbay 1823e65e175bSOded Gabbay static void goya_init_mme_qman(struct hl_device *hdev) 1824e65e175bSOded Gabbay { 1825e65e175bSOded Gabbay u32 mtr_base_lo, mtr_base_hi; 1826e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 1827e65e175bSOded Gabbay u32 gic_base_lo, gic_base_hi; 1828e65e175bSOded Gabbay u64 qman_base_addr; 1829e65e175bSOded Gabbay 1830e65e175bSOded Gabbay mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1831e65e175bSOded Gabbay mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1832e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1833e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1834e65e175bSOded Gabbay 1835e65e175bSOded Gabbay gic_base_lo = 1836e65e175bSOded Gabbay lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1837e65e175bSOded Gabbay gic_base_hi = 1838e65e175bSOded Gabbay upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1839e65e175bSOded Gabbay 1840e65e175bSOded Gabbay qman_base_addr = hdev->asic_prop.sram_base_address + 1841e65e175bSOded Gabbay MME_QMAN_BASE_OFFSET; 1842e65e175bSOded Gabbay 1843e65e175bSOded Gabbay WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr)); 1844e65e175bSOded Gabbay WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr)); 1845e65e175bSOded Gabbay WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH)); 1846e65e175bSOded Gabbay WREG32(mmMME_QM_PQ_PI, 0); 1847e65e175bSOded Gabbay WREG32(mmMME_QM_PQ_CI, 0); 1848e65e175bSOded Gabbay WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0); 1849e65e175bSOded Gabbay WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4); 1850e65e175bSOded Gabbay WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8); 1851e65e175bSOded Gabbay WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC); 1852e65e175bSOded Gabbay 1853e65e175bSOded Gabbay WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo); 1854e65e175bSOded Gabbay WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi); 1855e65e175bSOded Gabbay WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo); 1856e65e175bSOded Gabbay WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi); 1857e65e175bSOded Gabbay 1858e65e175bSOded Gabbay /* QMAN CQ has 8 cache lines */ 1859e65e175bSOded Gabbay WREG32(mmMME_QM_CQ_CFG1, 0x00080008); 1860e65e175bSOded Gabbay 1861e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo); 1862e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi); 1863e65e175bSOded Gabbay 1864e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM); 1865e65e175bSOded Gabbay 1866e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN); 1867e65e175bSOded Gabbay 1868e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT); 1869e65e175bSOded Gabbay 1870e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE); 1871e65e175bSOded Gabbay } 1872e65e175bSOded Gabbay 1873e65e175bSOded Gabbay static void goya_init_mme_cmdq(struct hl_device *hdev) 1874e65e175bSOded Gabbay { 1875e65e175bSOded Gabbay u32 mtr_base_lo, mtr_base_hi; 1876e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 1877e65e175bSOded Gabbay u32 gic_base_lo, gic_base_hi; 1878e65e175bSOded Gabbay 1879e65e175bSOded Gabbay mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1880e65e175bSOded Gabbay mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1881e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1882e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1883e65e175bSOded Gabbay 1884e65e175bSOded Gabbay gic_base_lo = 1885e65e175bSOded Gabbay lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1886e65e175bSOded Gabbay gic_base_hi = 1887e65e175bSOded Gabbay upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1888e65e175bSOded Gabbay 1889e65e175bSOded Gabbay WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo); 1890e65e175bSOded Gabbay WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi); 1891e65e175bSOded Gabbay WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo); 1892e65e175bSOded Gabbay WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi); 1893e65e175bSOded Gabbay 1894e65e175bSOded Gabbay /* CMDQ CQ has 20 cache lines */ 1895e65e175bSOded Gabbay WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014); 1896e65e175bSOded Gabbay 1897e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo); 1898e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi); 1899e65e175bSOded Gabbay 1900e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ); 1901e65e175bSOded Gabbay 1902e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN); 1903e65e175bSOded Gabbay 1904e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT); 1905e65e175bSOded Gabbay 1906e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE); 1907e65e175bSOded Gabbay } 1908e65e175bSOded Gabbay 1909e65e175bSOded Gabbay void goya_init_mme_qmans(struct hl_device *hdev) 1910e65e175bSOded Gabbay { 1911e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 1912e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 1913e65e175bSOded Gabbay 1914e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MME) 1915e65e175bSOded Gabbay return; 1916e65e175bSOded Gabbay 1917e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1918e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1919e65e175bSOded Gabbay 1920e65e175bSOded Gabbay WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo); 1921e65e175bSOded Gabbay WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi); 1922e65e175bSOded Gabbay 1923e65e175bSOded Gabbay goya_init_mme_qman(hdev); 1924e65e175bSOded Gabbay goya_init_mme_cmdq(hdev); 1925e65e175bSOded Gabbay 1926e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_MME; 1927e65e175bSOded Gabbay } 1928e65e175bSOded Gabbay 1929e65e175bSOded Gabbay static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id) 1930e65e175bSOded Gabbay { 1931e65e175bSOded Gabbay u32 mtr_base_lo, mtr_base_hi; 1932e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 1933e65e175bSOded Gabbay u32 gic_base_lo, gic_base_hi; 1934e65e175bSOded Gabbay u64 qman_base_addr; 1935e65e175bSOded Gabbay u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI); 1936e65e175bSOded Gabbay 1937e65e175bSOded Gabbay mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1938e65e175bSOded Gabbay mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1939e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1940e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1941e65e175bSOded Gabbay 1942e65e175bSOded Gabbay gic_base_lo = 1943e65e175bSOded Gabbay lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1944e65e175bSOded Gabbay gic_base_hi = 1945e65e175bSOded Gabbay upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1946e65e175bSOded Gabbay 1947e65e175bSOded Gabbay qman_base_addr = hdev->asic_prop.sram_base_address + base_off; 1948e65e175bSOded Gabbay 1949e65e175bSOded Gabbay WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr)); 1950e65e175bSOded Gabbay WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr)); 1951e65e175bSOded Gabbay WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH)); 1952e65e175bSOded Gabbay WREG32(mmTPC0_QM_PQ_PI + reg_off, 0); 1953e65e175bSOded Gabbay WREG32(mmTPC0_QM_PQ_CI + reg_off, 0); 1954e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0); 1955e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4); 1956e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8); 1957e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC); 1958e65e175bSOded Gabbay 1959e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo); 1960e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi); 1961e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo); 1962e65e175bSOded Gabbay WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi); 1963e65e175bSOded Gabbay 1964e65e175bSOded Gabbay WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008); 1965e65e175bSOded Gabbay 1966e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo); 1967e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi); 1968e65e175bSOded Gabbay 1969e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off, 1970e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id); 1971e65e175bSOded Gabbay 1972e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN); 1973e65e175bSOded Gabbay 1974e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT); 1975e65e175bSOded Gabbay 1976e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE); 1977e65e175bSOded Gabbay } 1978e65e175bSOded Gabbay 1979e65e175bSOded Gabbay static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id) 1980e65e175bSOded Gabbay { 1981e65e175bSOded Gabbay u32 mtr_base_lo, mtr_base_hi; 1982e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 1983e65e175bSOded Gabbay u32 gic_base_lo, gic_base_hi; 1984e65e175bSOded Gabbay u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1); 1985e65e175bSOded Gabbay 1986e65e175bSOded Gabbay mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1987e65e175bSOded Gabbay mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0); 1988e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1989e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 1990e65e175bSOded Gabbay 1991e65e175bSOded Gabbay gic_base_lo = 1992e65e175bSOded Gabbay lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1993e65e175bSOded Gabbay gic_base_hi = 1994e65e175bSOded Gabbay upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR); 1995e65e175bSOded Gabbay 1996e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo); 1997e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi); 1998e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo); 1999e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi); 2000e65e175bSOded Gabbay 2001e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014); 2002e65e175bSOded Gabbay 2003e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo); 2004e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi); 2005e65e175bSOded Gabbay 2006e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off, 2007e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id); 2008e65e175bSOded Gabbay 2009e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN); 2010e65e175bSOded Gabbay 2011e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT); 2012e65e175bSOded Gabbay 2013e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE); 2014e65e175bSOded Gabbay } 2015e65e175bSOded Gabbay 2016e65e175bSOded Gabbay void goya_init_tpc_qmans(struct hl_device *hdev) 2017e65e175bSOded Gabbay { 2018e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2019e65e175bSOded Gabbay u32 so_base_lo, so_base_hi; 2020e65e175bSOded Gabbay u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW - 2021e65e175bSOded Gabbay mmTPC0_CFG_SM_BASE_ADDRESS_LOW; 2022e65e175bSOded Gabbay int i; 2023e65e175bSOded Gabbay 2024e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_TPC) 2025e65e175bSOded Gabbay return; 2026e65e175bSOded Gabbay 2027e65e175bSOded Gabbay so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 2028e65e175bSOded Gabbay so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 2029e65e175bSOded Gabbay 2030e65e175bSOded Gabbay for (i = 0 ; i < TPC_MAX_NUM ; i++) { 2031e65e175bSOded Gabbay WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off, 2032e65e175bSOded Gabbay so_base_lo); 2033e65e175bSOded Gabbay WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off, 2034e65e175bSOded Gabbay so_base_hi); 2035e65e175bSOded Gabbay } 2036e65e175bSOded Gabbay 2037e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0); 2038e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1); 2039e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2); 2040e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3); 2041e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4); 2042e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5); 2043e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6); 2044e65e175bSOded Gabbay goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7); 2045e65e175bSOded Gabbay 2046e65e175bSOded Gabbay for (i = 0 ; i < TPC_MAX_NUM ; i++) 2047e65e175bSOded Gabbay goya_init_tpc_cmdq(hdev, i); 2048e65e175bSOded Gabbay 2049e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_TPC; 2050e65e175bSOded Gabbay } 2051e65e175bSOded Gabbay 2052e65e175bSOded Gabbay /* 2053e65e175bSOded Gabbay * goya_disable_internal_queues - Disable internal queues 2054e65e175bSOded Gabbay * 2055e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 2056e65e175bSOded Gabbay * 2057e65e175bSOded Gabbay */ 2058e65e175bSOded Gabbay static void goya_disable_internal_queues(struct hl_device *hdev) 2059e65e175bSOded Gabbay { 2060e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2061e65e175bSOded Gabbay 2062e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MME)) 2063e65e175bSOded Gabbay goto disable_tpc; 2064e65e175bSOded Gabbay 2065e65e175bSOded Gabbay WREG32(mmMME_QM_GLBL_CFG0, 0); 2066e65e175bSOded Gabbay WREG32(mmMME_CMDQ_GLBL_CFG0, 0); 2067e65e175bSOded Gabbay 2068e65e175bSOded Gabbay disable_tpc: 2069e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_TPC)) 2070e65e175bSOded Gabbay return; 2071e65e175bSOded Gabbay 2072e65e175bSOded Gabbay WREG32(mmTPC0_QM_GLBL_CFG0, 0); 2073e65e175bSOded Gabbay WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0); 2074e65e175bSOded Gabbay 2075e65e175bSOded Gabbay WREG32(mmTPC1_QM_GLBL_CFG0, 0); 2076e65e175bSOded Gabbay WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0); 2077e65e175bSOded Gabbay 2078e65e175bSOded Gabbay WREG32(mmTPC2_QM_GLBL_CFG0, 0); 2079e65e175bSOded Gabbay WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0); 2080e65e175bSOded Gabbay 2081e65e175bSOded Gabbay WREG32(mmTPC3_QM_GLBL_CFG0, 0); 2082e65e175bSOded Gabbay WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0); 2083e65e175bSOded Gabbay 2084e65e175bSOded Gabbay WREG32(mmTPC4_QM_GLBL_CFG0, 0); 2085e65e175bSOded Gabbay WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0); 2086e65e175bSOded Gabbay 2087e65e175bSOded Gabbay WREG32(mmTPC5_QM_GLBL_CFG0, 0); 2088e65e175bSOded Gabbay WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0); 2089e65e175bSOded Gabbay 2090e65e175bSOded Gabbay WREG32(mmTPC6_QM_GLBL_CFG0, 0); 2091e65e175bSOded Gabbay WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0); 2092e65e175bSOded Gabbay 2093e65e175bSOded Gabbay WREG32(mmTPC7_QM_GLBL_CFG0, 0); 2094e65e175bSOded Gabbay WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0); 2095e65e175bSOded Gabbay } 2096e65e175bSOded Gabbay 2097e65e175bSOded Gabbay /* 2098e65e175bSOded Gabbay * goya_stop_internal_queues - Stop internal queues 2099e65e175bSOded Gabbay * 2100e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 2101e65e175bSOded Gabbay * 2102e65e175bSOded Gabbay * Returns 0 on success 2103e65e175bSOded Gabbay * 2104e65e175bSOded Gabbay */ 2105e65e175bSOded Gabbay static int goya_stop_internal_queues(struct hl_device *hdev) 2106e65e175bSOded Gabbay { 2107e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2108e65e175bSOded Gabbay int rc, retval = 0; 2109e65e175bSOded Gabbay 2110e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MME)) 2111e65e175bSOded Gabbay goto stop_tpc; 2112e65e175bSOded Gabbay 2113e65e175bSOded Gabbay /* 2114e65e175bSOded Gabbay * Each queue (QMAN) is a separate H/W logic. That means that each 2115e65e175bSOded Gabbay * QMAN can be stopped independently and failure to stop one does NOT 2116e65e175bSOded Gabbay * mandate we should not try to stop other QMANs 2117e65e175bSOded Gabbay */ 2118e65e175bSOded Gabbay 2119e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2120e65e175bSOded Gabbay mmMME_QM_GLBL_CFG1, 2121e65e175bSOded Gabbay mmMME_QM_CP_STS, 2122e65e175bSOded Gabbay mmMME_QM_GLBL_STS0); 2123e65e175bSOded Gabbay 2124e65e175bSOded Gabbay if (rc) { 2125e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop MME QMAN\n"); 2126e65e175bSOded Gabbay retval = -EIO; 2127e65e175bSOded Gabbay } 2128e65e175bSOded Gabbay 2129e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2130e65e175bSOded Gabbay mmMME_CMDQ_GLBL_CFG1, 2131e65e175bSOded Gabbay mmMME_CMDQ_CP_STS, 2132e65e175bSOded Gabbay mmMME_CMDQ_GLBL_STS0); 2133e65e175bSOded Gabbay 2134e65e175bSOded Gabbay if (rc) { 2135e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop MME CMDQ\n"); 2136e65e175bSOded Gabbay retval = -EIO; 2137e65e175bSOded Gabbay } 2138e65e175bSOded Gabbay 2139e65e175bSOded Gabbay stop_tpc: 2140e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_TPC)) 2141e65e175bSOded Gabbay return retval; 2142e65e175bSOded Gabbay 2143e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2144e65e175bSOded Gabbay mmTPC0_QM_GLBL_CFG1, 2145e65e175bSOded Gabbay mmTPC0_QM_CP_STS, 2146e65e175bSOded Gabbay mmTPC0_QM_GLBL_STS0); 2147e65e175bSOded Gabbay 2148e65e175bSOded Gabbay if (rc) { 2149e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); 2150e65e175bSOded Gabbay retval = -EIO; 2151e65e175bSOded Gabbay } 2152e65e175bSOded Gabbay 2153e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2154e65e175bSOded Gabbay mmTPC0_CMDQ_GLBL_CFG1, 2155e65e175bSOded Gabbay mmTPC0_CMDQ_CP_STS, 2156e65e175bSOded Gabbay mmTPC0_CMDQ_GLBL_STS0); 2157e65e175bSOded Gabbay 2158e65e175bSOded Gabbay if (rc) { 2159e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); 2160e65e175bSOded Gabbay retval = -EIO; 2161e65e175bSOded Gabbay } 2162e65e175bSOded Gabbay 2163e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2164e65e175bSOded Gabbay mmTPC1_QM_GLBL_CFG1, 2165e65e175bSOded Gabbay mmTPC1_QM_CP_STS, 2166e65e175bSOded Gabbay mmTPC1_QM_GLBL_STS0); 2167e65e175bSOded Gabbay 2168e65e175bSOded Gabbay if (rc) { 2169e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); 2170e65e175bSOded Gabbay retval = -EIO; 2171e65e175bSOded Gabbay } 2172e65e175bSOded Gabbay 2173e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2174e65e175bSOded Gabbay mmTPC1_CMDQ_GLBL_CFG1, 2175e65e175bSOded Gabbay mmTPC1_CMDQ_CP_STS, 2176e65e175bSOded Gabbay mmTPC1_CMDQ_GLBL_STS0); 2177e65e175bSOded Gabbay 2178e65e175bSOded Gabbay if (rc) { 2179e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); 2180e65e175bSOded Gabbay retval = -EIO; 2181e65e175bSOded Gabbay } 2182e65e175bSOded Gabbay 2183e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2184e65e175bSOded Gabbay mmTPC2_QM_GLBL_CFG1, 2185e65e175bSOded Gabbay mmTPC2_QM_CP_STS, 2186e65e175bSOded Gabbay mmTPC2_QM_GLBL_STS0); 2187e65e175bSOded Gabbay 2188e65e175bSOded Gabbay if (rc) { 2189e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); 2190e65e175bSOded Gabbay retval = -EIO; 2191e65e175bSOded Gabbay } 2192e65e175bSOded Gabbay 2193e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2194e65e175bSOded Gabbay mmTPC2_CMDQ_GLBL_CFG1, 2195e65e175bSOded Gabbay mmTPC2_CMDQ_CP_STS, 2196e65e175bSOded Gabbay mmTPC2_CMDQ_GLBL_STS0); 2197e65e175bSOded Gabbay 2198e65e175bSOded Gabbay if (rc) { 2199e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); 2200e65e175bSOded Gabbay retval = -EIO; 2201e65e175bSOded Gabbay } 2202e65e175bSOded Gabbay 2203e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2204e65e175bSOded Gabbay mmTPC3_QM_GLBL_CFG1, 2205e65e175bSOded Gabbay mmTPC3_QM_CP_STS, 2206e65e175bSOded Gabbay mmTPC3_QM_GLBL_STS0); 2207e65e175bSOded Gabbay 2208e65e175bSOded Gabbay if (rc) { 2209e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); 2210e65e175bSOded Gabbay retval = -EIO; 2211e65e175bSOded Gabbay } 2212e65e175bSOded Gabbay 2213e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2214e65e175bSOded Gabbay mmTPC3_CMDQ_GLBL_CFG1, 2215e65e175bSOded Gabbay mmTPC3_CMDQ_CP_STS, 2216e65e175bSOded Gabbay mmTPC3_CMDQ_GLBL_STS0); 2217e65e175bSOded Gabbay 2218e65e175bSOded Gabbay if (rc) { 2219e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); 2220e65e175bSOded Gabbay retval = -EIO; 2221e65e175bSOded Gabbay } 2222e65e175bSOded Gabbay 2223e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2224e65e175bSOded Gabbay mmTPC4_QM_GLBL_CFG1, 2225e65e175bSOded Gabbay mmTPC4_QM_CP_STS, 2226e65e175bSOded Gabbay mmTPC4_QM_GLBL_STS0); 2227e65e175bSOded Gabbay 2228e65e175bSOded Gabbay if (rc) { 2229e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); 2230e65e175bSOded Gabbay retval = -EIO; 2231e65e175bSOded Gabbay } 2232e65e175bSOded Gabbay 2233e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2234e65e175bSOded Gabbay mmTPC4_CMDQ_GLBL_CFG1, 2235e65e175bSOded Gabbay mmTPC4_CMDQ_CP_STS, 2236e65e175bSOded Gabbay mmTPC4_CMDQ_GLBL_STS0); 2237e65e175bSOded Gabbay 2238e65e175bSOded Gabbay if (rc) { 2239e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); 2240e65e175bSOded Gabbay retval = -EIO; 2241e65e175bSOded Gabbay } 2242e65e175bSOded Gabbay 2243e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2244e65e175bSOded Gabbay mmTPC5_QM_GLBL_CFG1, 2245e65e175bSOded Gabbay mmTPC5_QM_CP_STS, 2246e65e175bSOded Gabbay mmTPC5_QM_GLBL_STS0); 2247e65e175bSOded Gabbay 2248e65e175bSOded Gabbay if (rc) { 2249e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); 2250e65e175bSOded Gabbay retval = -EIO; 2251e65e175bSOded Gabbay } 2252e65e175bSOded Gabbay 2253e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2254e65e175bSOded Gabbay mmTPC5_CMDQ_GLBL_CFG1, 2255e65e175bSOded Gabbay mmTPC5_CMDQ_CP_STS, 2256e65e175bSOded Gabbay mmTPC5_CMDQ_GLBL_STS0); 2257e65e175bSOded Gabbay 2258e65e175bSOded Gabbay if (rc) { 2259e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); 2260e65e175bSOded Gabbay retval = -EIO; 2261e65e175bSOded Gabbay } 2262e65e175bSOded Gabbay 2263e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2264e65e175bSOded Gabbay mmTPC6_QM_GLBL_CFG1, 2265e65e175bSOded Gabbay mmTPC6_QM_CP_STS, 2266e65e175bSOded Gabbay mmTPC6_QM_GLBL_STS0); 2267e65e175bSOded Gabbay 2268e65e175bSOded Gabbay if (rc) { 2269e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); 2270e65e175bSOded Gabbay retval = -EIO; 2271e65e175bSOded Gabbay } 2272e65e175bSOded Gabbay 2273e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2274e65e175bSOded Gabbay mmTPC6_CMDQ_GLBL_CFG1, 2275e65e175bSOded Gabbay mmTPC6_CMDQ_CP_STS, 2276e65e175bSOded Gabbay mmTPC6_CMDQ_GLBL_STS0); 2277e65e175bSOded Gabbay 2278e65e175bSOded Gabbay if (rc) { 2279e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); 2280e65e175bSOded Gabbay retval = -EIO; 2281e65e175bSOded Gabbay } 2282e65e175bSOded Gabbay 2283e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2284e65e175bSOded Gabbay mmTPC7_QM_GLBL_CFG1, 2285e65e175bSOded Gabbay mmTPC7_QM_CP_STS, 2286e65e175bSOded Gabbay mmTPC7_QM_GLBL_STS0); 2287e65e175bSOded Gabbay 2288e65e175bSOded Gabbay if (rc) { 2289e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); 2290e65e175bSOded Gabbay retval = -EIO; 2291e65e175bSOded Gabbay } 2292e65e175bSOded Gabbay 2293e65e175bSOded Gabbay rc = goya_stop_queue(hdev, 2294e65e175bSOded Gabbay mmTPC7_CMDQ_GLBL_CFG1, 2295e65e175bSOded Gabbay mmTPC7_CMDQ_CP_STS, 2296e65e175bSOded Gabbay mmTPC7_CMDQ_GLBL_STS0); 2297e65e175bSOded Gabbay 2298e65e175bSOded Gabbay if (rc) { 2299e65e175bSOded Gabbay dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); 2300e65e175bSOded Gabbay retval = -EIO; 2301e65e175bSOded Gabbay } 2302e65e175bSOded Gabbay 2303e65e175bSOded Gabbay return retval; 2304e65e175bSOded Gabbay } 2305e65e175bSOded Gabbay 2306e65e175bSOded Gabbay static void goya_dma_stall(struct hl_device *hdev) 2307e65e175bSOded Gabbay { 2308e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2309e65e175bSOded Gabbay 2310e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_DMA)) 2311e65e175bSOded Gabbay return; 2312e65e175bSOded Gabbay 2313e65e175bSOded Gabbay WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT); 2314e65e175bSOded Gabbay WREG32(mmDMA_QM_1_GLBL_CFG1, 1 << DMA_QM_1_GLBL_CFG1_DMA_STOP_SHIFT); 2315e65e175bSOded Gabbay WREG32(mmDMA_QM_2_GLBL_CFG1, 1 << DMA_QM_2_GLBL_CFG1_DMA_STOP_SHIFT); 2316e65e175bSOded Gabbay WREG32(mmDMA_QM_3_GLBL_CFG1, 1 << DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT); 2317e65e175bSOded Gabbay WREG32(mmDMA_QM_4_GLBL_CFG1, 1 << DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT); 2318e65e175bSOded Gabbay } 2319e65e175bSOded Gabbay 2320e65e175bSOded Gabbay static void goya_tpc_stall(struct hl_device *hdev) 2321e65e175bSOded Gabbay { 2322e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2323e65e175bSOded Gabbay 2324e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_TPC)) 2325e65e175bSOded Gabbay return; 2326e65e175bSOded Gabbay 2327e65e175bSOded Gabbay WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); 2328e65e175bSOded Gabbay WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC1_CFG_TPC_STALL_V_SHIFT); 2329e65e175bSOded Gabbay WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC2_CFG_TPC_STALL_V_SHIFT); 2330e65e175bSOded Gabbay WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC3_CFG_TPC_STALL_V_SHIFT); 2331e65e175bSOded Gabbay WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC4_CFG_TPC_STALL_V_SHIFT); 2332e65e175bSOded Gabbay WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC5_CFG_TPC_STALL_V_SHIFT); 2333e65e175bSOded Gabbay WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC6_CFG_TPC_STALL_V_SHIFT); 2334e65e175bSOded Gabbay WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC7_CFG_TPC_STALL_V_SHIFT); 2335e65e175bSOded Gabbay } 2336e65e175bSOded Gabbay 2337e65e175bSOded Gabbay static void goya_mme_stall(struct hl_device *hdev) 2338e65e175bSOded Gabbay { 2339e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2340e65e175bSOded Gabbay 2341e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MME)) 2342e65e175bSOded Gabbay return; 2343e65e175bSOded Gabbay 2344e65e175bSOded Gabbay WREG32(mmMME_STALL, 0xFFFFFFFF); 2345e65e175bSOded Gabbay } 2346e65e175bSOded Gabbay 2347e65e175bSOded Gabbay static int goya_enable_msix(struct hl_device *hdev) 2348e65e175bSOded Gabbay { 2349e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2350e65e175bSOded Gabbay int cq_cnt = hdev->asic_prop.completion_queues_count; 2351e65e175bSOded Gabbay int rc, i, irq_cnt_init, irq; 2352e65e175bSOded Gabbay 2353e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MSIX) 2354e65e175bSOded Gabbay return 0; 2355e65e175bSOded Gabbay 2356e65e175bSOded Gabbay rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, 2357e65e175bSOded Gabbay GOYA_MSIX_ENTRIES, PCI_IRQ_MSIX); 2358e65e175bSOded Gabbay if (rc < 0) { 2359e65e175bSOded Gabbay dev_err(hdev->dev, 2360e65e175bSOded Gabbay "MSI-X: Failed to enable support -- %d/%d\n", 2361e65e175bSOded Gabbay GOYA_MSIX_ENTRIES, rc); 2362e65e175bSOded Gabbay return rc; 2363e65e175bSOded Gabbay } 2364e65e175bSOded Gabbay 2365e65e175bSOded Gabbay for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) { 2366e65e175bSOded Gabbay irq = pci_irq_vector(hdev->pdev, i); 2367e65e175bSOded Gabbay rc = request_irq(irq, hl_irq_handler_cq, 0, goya_irq_name[i], 2368e65e175bSOded Gabbay &hdev->completion_queue[i]); 2369e65e175bSOded Gabbay if (rc) { 2370e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to request IRQ %d", irq); 2371e65e175bSOded Gabbay goto free_irqs; 2372e65e175bSOded Gabbay } 2373e65e175bSOded Gabbay } 2374e65e175bSOded Gabbay 2375e65e175bSOded Gabbay irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); 2376e65e175bSOded Gabbay 2377e65e175bSOded Gabbay rc = request_irq(irq, hl_irq_handler_eq, 0, 2378e65e175bSOded Gabbay goya_irq_name[GOYA_EVENT_QUEUE_MSIX_IDX], 2379e65e175bSOded Gabbay &hdev->event_queue); 2380e65e175bSOded Gabbay if (rc) { 2381e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to request IRQ %d", irq); 2382e65e175bSOded Gabbay goto free_irqs; 2383e65e175bSOded Gabbay } 2384e65e175bSOded Gabbay 2385e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_MSIX; 2386e65e175bSOded Gabbay return 0; 2387e65e175bSOded Gabbay 2388e65e175bSOded Gabbay free_irqs: 2389e65e175bSOded Gabbay for (i = 0 ; i < irq_cnt_init ; i++) 2390e65e175bSOded Gabbay free_irq(pci_irq_vector(hdev->pdev, i), 2391e65e175bSOded Gabbay &hdev->completion_queue[i]); 2392e65e175bSOded Gabbay 2393e65e175bSOded Gabbay pci_free_irq_vectors(hdev->pdev); 2394e65e175bSOded Gabbay return rc; 2395e65e175bSOded Gabbay } 2396e65e175bSOded Gabbay 2397e65e175bSOded Gabbay static void goya_sync_irqs(struct hl_device *hdev) 2398e65e175bSOded Gabbay { 2399e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2400e65e175bSOded Gabbay int i; 2401e65e175bSOded Gabbay 2402e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) 2403e65e175bSOded Gabbay return; 2404e65e175bSOded Gabbay 2405e65e175bSOded Gabbay /* Wait for all pending IRQs to be finished */ 2406e65e175bSOded Gabbay for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) 2407e65e175bSOded Gabbay synchronize_irq(pci_irq_vector(hdev->pdev, i)); 2408e65e175bSOded Gabbay 2409e65e175bSOded Gabbay synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); 2410e65e175bSOded Gabbay } 2411e65e175bSOded Gabbay 2412e65e175bSOded Gabbay static void goya_disable_msix(struct hl_device *hdev) 2413e65e175bSOded Gabbay { 2414e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2415e65e175bSOded Gabbay int i, irq; 2416e65e175bSOded Gabbay 2417e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) 2418e65e175bSOded Gabbay return; 2419e65e175bSOded Gabbay 2420e65e175bSOded Gabbay goya_sync_irqs(hdev); 2421e65e175bSOded Gabbay 2422e65e175bSOded Gabbay irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); 2423e65e175bSOded Gabbay free_irq(irq, &hdev->event_queue); 2424e65e175bSOded Gabbay 2425e65e175bSOded Gabbay for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { 2426e65e175bSOded Gabbay irq = pci_irq_vector(hdev->pdev, i); 2427e65e175bSOded Gabbay free_irq(irq, &hdev->completion_queue[i]); 2428e65e175bSOded Gabbay } 2429e65e175bSOded Gabbay 2430e65e175bSOded Gabbay pci_free_irq_vectors(hdev->pdev); 2431e65e175bSOded Gabbay 2432e65e175bSOded Gabbay goya->hw_cap_initialized &= ~HW_CAP_MSIX; 2433e65e175bSOded Gabbay } 2434e65e175bSOded Gabbay 2435e65e175bSOded Gabbay static void goya_enable_timestamp(struct hl_device *hdev) 2436e65e175bSOded Gabbay { 2437e65e175bSOded Gabbay /* Disable the timestamp counter */ 2438e65e175bSOded Gabbay WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); 2439e65e175bSOded Gabbay 2440e65e175bSOded Gabbay /* Zero the lower/upper parts of the 64-bit counter */ 2441e65e175bSOded Gabbay WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0); 2442e65e175bSOded Gabbay WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0); 2443e65e175bSOded Gabbay 2444e65e175bSOded Gabbay /* Enable the counter */ 2445e65e175bSOded Gabbay WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1); 2446e65e175bSOded Gabbay } 2447e65e175bSOded Gabbay 2448e65e175bSOded Gabbay static void goya_disable_timestamp(struct hl_device *hdev) 2449e65e175bSOded Gabbay { 2450e65e175bSOded Gabbay /* Disable the timestamp counter */ 2451e65e175bSOded Gabbay WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); 2452e65e175bSOded Gabbay } 2453e65e175bSOded Gabbay 2454e65e175bSOded Gabbay static void goya_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 2455e65e175bSOded Gabbay { 2456e65e175bSOded Gabbay u32 wait_timeout_ms; 2457e65e175bSOded Gabbay 2458e65e175bSOded Gabbay if (hdev->pldm) 2459e65e175bSOded Gabbay wait_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC; 2460e65e175bSOded Gabbay else 2461e65e175bSOded Gabbay wait_timeout_ms = GOYA_RESET_WAIT_MSEC; 2462e65e175bSOded Gabbay 2463e65e175bSOded Gabbay goya_stop_external_queues(hdev); 2464e65e175bSOded Gabbay goya_stop_internal_queues(hdev); 2465e65e175bSOded Gabbay 2466e65e175bSOded Gabbay msleep(wait_timeout_ms); 2467e65e175bSOded Gabbay 2468e65e175bSOded Gabbay goya_dma_stall(hdev); 2469e65e175bSOded Gabbay goya_tpc_stall(hdev); 2470e65e175bSOded Gabbay goya_mme_stall(hdev); 2471e65e175bSOded Gabbay 2472e65e175bSOded Gabbay msleep(wait_timeout_ms); 2473e65e175bSOded Gabbay 2474e65e175bSOded Gabbay goya_disable_external_queues(hdev); 2475e65e175bSOded Gabbay goya_disable_internal_queues(hdev); 2476e65e175bSOded Gabbay 2477e65e175bSOded Gabbay goya_disable_timestamp(hdev); 2478e65e175bSOded Gabbay 2479e65e175bSOded Gabbay if (hard_reset) { 2480e65e175bSOded Gabbay goya_disable_msix(hdev); 2481e65e175bSOded Gabbay goya_mmu_remove_device_cpu_mappings(hdev); 2482e65e175bSOded Gabbay } else { 2483e65e175bSOded Gabbay goya_sync_irqs(hdev); 2484e65e175bSOded Gabbay } 2485e65e175bSOded Gabbay } 2486e65e175bSOded Gabbay 2487e65e175bSOded Gabbay /* 2488e65e175bSOded Gabbay * goya_load_firmware_to_device() - Load LINUX FW code to device. 2489e65e175bSOded Gabbay * @hdev: Pointer to hl_device structure. 2490e65e175bSOded Gabbay * 2491e65e175bSOded Gabbay * Copy LINUX fw code from firmware file to HBM BAR. 2492e65e175bSOded Gabbay * 2493e65e175bSOded Gabbay * Return: 0 on success, non-zero for failure. 2494e65e175bSOded Gabbay */ 2495e65e175bSOded Gabbay static int goya_load_firmware_to_device(struct hl_device *hdev) 2496e65e175bSOded Gabbay { 2497e65e175bSOded Gabbay void __iomem *dst; 2498e65e175bSOded Gabbay 2499e65e175bSOded Gabbay dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; 2500e65e175bSOded Gabbay 2501e65e175bSOded Gabbay return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst, 0, 0); 2502e65e175bSOded Gabbay } 2503e65e175bSOded Gabbay 2504e65e175bSOded Gabbay /* 2505e65e175bSOded Gabbay * goya_load_boot_fit_to_device() - Load boot fit to device. 2506e65e175bSOded Gabbay * @hdev: Pointer to hl_device structure. 2507e65e175bSOded Gabbay * 2508e65e175bSOded Gabbay * Copy boot fit file to SRAM BAR. 2509e65e175bSOded Gabbay * 2510e65e175bSOded Gabbay * Return: 0 on success, non-zero for failure. 2511e65e175bSOded Gabbay */ 2512e65e175bSOded Gabbay static int goya_load_boot_fit_to_device(struct hl_device *hdev) 2513e65e175bSOded Gabbay { 2514e65e175bSOded Gabbay void __iomem *dst; 2515e65e175bSOded Gabbay 2516e65e175bSOded Gabbay dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET; 2517e65e175bSOded Gabbay 2518e65e175bSOded Gabbay return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst, 0, 0); 2519e65e175bSOded Gabbay } 2520e65e175bSOded Gabbay 2521e65e175bSOded Gabbay static void goya_init_dynamic_firmware_loader(struct hl_device *hdev) 2522e65e175bSOded Gabbay { 2523e65e175bSOded Gabbay struct dynamic_fw_load_mgr *dynamic_loader; 2524e65e175bSOded Gabbay struct cpu_dyn_regs *dyn_regs; 2525e65e175bSOded Gabbay 2526e65e175bSOded Gabbay dynamic_loader = &hdev->fw_loader.dynamic_loader; 2527e65e175bSOded Gabbay 2528e65e175bSOded Gabbay /* 2529e65e175bSOded Gabbay * here we update initial values for few specific dynamic regs (as 2530e65e175bSOded Gabbay * before reading the first descriptor from FW those value has to be 2531e65e175bSOded Gabbay * hard-coded) in later stages of the protocol those values will be 2532e65e175bSOded Gabbay * updated automatically by reading the FW descriptor so data there 2533e65e175bSOded Gabbay * will always be up-to-date 2534e65e175bSOded Gabbay */ 2535e65e175bSOded Gabbay dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 2536e65e175bSOded Gabbay dyn_regs->kmd_msg_to_cpu = 2537e65e175bSOded Gabbay cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 2538e65e175bSOded Gabbay dyn_regs->cpu_cmd_status_to_host = 2539e65e175bSOded Gabbay cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 2540e65e175bSOded Gabbay 2541e65e175bSOded Gabbay dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC; 2542e65e175bSOded Gabbay } 2543e65e175bSOded Gabbay 2544e65e175bSOded Gabbay static void goya_init_static_firmware_loader(struct hl_device *hdev) 2545e65e175bSOded Gabbay { 2546e65e175bSOded Gabbay struct static_fw_load_mgr *static_loader; 2547e65e175bSOded Gabbay 2548e65e175bSOded Gabbay static_loader = &hdev->fw_loader.static_loader; 2549e65e175bSOded Gabbay 2550e65e175bSOded Gabbay static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; 2551e65e175bSOded Gabbay static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; 2552e65e175bSOded Gabbay static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; 2553e65e175bSOded Gabbay static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; 2554e65e175bSOded Gabbay static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 2555e65e175bSOded Gabbay static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; 2556e65e175bSOded Gabbay static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; 2557e65e175bSOded Gabbay static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; 2558e65e175bSOded Gabbay static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; 2559e65e175bSOded Gabbay static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; 2560e65e175bSOded Gabbay static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; 2561e65e175bSOded Gabbay static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); 2562e65e175bSOded Gabbay } 2563e65e175bSOded Gabbay 2564e65e175bSOded Gabbay static void goya_init_firmware_preload_params(struct hl_device *hdev) 2565e65e175bSOded Gabbay { 2566e65e175bSOded Gabbay struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 2567e65e175bSOded Gabbay 2568e65e175bSOded Gabbay pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 2569e65e175bSOded Gabbay pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 2570e65e175bSOded Gabbay pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 2571e65e175bSOded Gabbay pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 2572e65e175bSOded Gabbay pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 2573e65e175bSOded Gabbay pre_fw_load->wait_for_preboot_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; 2574e65e175bSOded Gabbay } 2575e65e175bSOded Gabbay 2576e65e175bSOded Gabbay static void goya_init_firmware_loader(struct hl_device *hdev) 2577e65e175bSOded Gabbay { 2578e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 2579e65e175bSOded Gabbay struct fw_load_mgr *fw_loader = &hdev->fw_loader; 2580e65e175bSOded Gabbay 2581e65e175bSOded Gabbay /* fill common fields */ 2582e65e175bSOded Gabbay fw_loader->fw_comp_loaded = FW_TYPE_NONE; 2583e65e175bSOded Gabbay fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE; 2584e65e175bSOded Gabbay fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE; 2585e65e175bSOded Gabbay fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC; 2586e65e175bSOded Gabbay fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; 2587e65e175bSOded Gabbay fw_loader->skip_bmc = false; 2588e65e175bSOded Gabbay fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; 2589e65e175bSOded Gabbay fw_loader->dram_bar_id = DDR_BAR_ID; 2590e65e175bSOded Gabbay 2591e65e175bSOded Gabbay if (prop->dynamic_fw_load) 2592e65e175bSOded Gabbay goya_init_dynamic_firmware_loader(hdev); 2593e65e175bSOded Gabbay else 2594e65e175bSOded Gabbay goya_init_static_firmware_loader(hdev); 2595e65e175bSOded Gabbay } 2596e65e175bSOded Gabbay 2597e65e175bSOded Gabbay static int goya_init_cpu(struct hl_device *hdev) 2598e65e175bSOded Gabbay { 2599e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2600e65e175bSOded Gabbay int rc; 2601e65e175bSOded Gabbay 2602e65e175bSOded Gabbay if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 2603e65e175bSOded Gabbay return 0; 2604e65e175bSOded Gabbay 2605e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_CPU) 2606e65e175bSOded Gabbay return 0; 2607e65e175bSOded Gabbay 2608e65e175bSOded Gabbay /* 2609e65e175bSOded Gabbay * Before pushing u-boot/linux to device, need to set the ddr bar to 2610e65e175bSOded Gabbay * base address of dram 2611e65e175bSOded Gabbay */ 2612e65e175bSOded Gabbay if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 2613e65e175bSOded Gabbay dev_err(hdev->dev, 2614e65e175bSOded Gabbay "failed to map DDR bar to DRAM base address\n"); 2615e65e175bSOded Gabbay return -EIO; 2616e65e175bSOded Gabbay } 2617e65e175bSOded Gabbay 2618e65e175bSOded Gabbay rc = hl_fw_init_cpu(hdev); 2619e65e175bSOded Gabbay 2620e65e175bSOded Gabbay if (rc) 2621e65e175bSOded Gabbay return rc; 2622e65e175bSOded Gabbay 2623e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_CPU; 2624e65e175bSOded Gabbay 2625e65e175bSOded Gabbay return 0; 2626e65e175bSOded Gabbay } 2627e65e175bSOded Gabbay 2628e65e175bSOded Gabbay static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, 2629e65e175bSOded Gabbay u64 phys_addr) 2630e65e175bSOded Gabbay { 2631e65e175bSOded Gabbay u32 status, timeout_usec; 2632e65e175bSOded Gabbay int rc; 2633e65e175bSOded Gabbay 2634e65e175bSOded Gabbay if (hdev->pldm) 2635e65e175bSOded Gabbay timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC; 2636e65e175bSOded Gabbay else 2637e65e175bSOded Gabbay timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 2638e65e175bSOded Gabbay 2639e65e175bSOded Gabbay WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 2640e65e175bSOded Gabbay WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT); 2641e65e175bSOded Gabbay WREG32(MMU_ASID_BUSY, 0x80000000 | asid); 2642e65e175bSOded Gabbay 2643e65e175bSOded Gabbay rc = hl_poll_timeout( 2644e65e175bSOded Gabbay hdev, 2645e65e175bSOded Gabbay MMU_ASID_BUSY, 2646e65e175bSOded Gabbay status, 2647e65e175bSOded Gabbay !(status & 0x80000000), 2648e65e175bSOded Gabbay 1000, 2649e65e175bSOded Gabbay timeout_usec); 2650e65e175bSOded Gabbay 2651e65e175bSOded Gabbay if (rc) { 2652e65e175bSOded Gabbay dev_err(hdev->dev, 2653e65e175bSOded Gabbay "Timeout during MMU hop0 config of asid %d\n", asid); 2654e65e175bSOded Gabbay return rc; 2655e65e175bSOded Gabbay } 2656e65e175bSOded Gabbay 2657e65e175bSOded Gabbay return 0; 2658e65e175bSOded Gabbay } 2659e65e175bSOded Gabbay 2660e65e175bSOded Gabbay int goya_mmu_init(struct hl_device *hdev) 2661e65e175bSOded Gabbay { 2662e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 2663e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2664e65e175bSOded Gabbay u64 hop0_addr; 2665e65e175bSOded Gabbay int rc, i; 2666e65e175bSOded Gabbay 2667e65e175bSOded Gabbay if (!hdev->mmu_enable) 2668e65e175bSOded Gabbay return 0; 2669e65e175bSOded Gabbay 2670e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MMU) 2671e65e175bSOded Gabbay return 0; 2672e65e175bSOded Gabbay 2673e65e175bSOded Gabbay hdev->dram_default_page_mapping = true; 2674e65e175bSOded Gabbay 2675e65e175bSOded Gabbay for (i = 0 ; i < prop->max_asid ; i++) { 2676e65e175bSOded Gabbay hop0_addr = prop->mmu_pgt_addr + 2677e65e175bSOded Gabbay (i * prop->mmu_hop_table_size); 2678e65e175bSOded Gabbay 2679e65e175bSOded Gabbay rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr); 2680e65e175bSOded Gabbay if (rc) { 2681e65e175bSOded Gabbay dev_err(hdev->dev, 2682e65e175bSOded Gabbay "failed to set hop0 addr for asid %d\n", i); 2683e65e175bSOded Gabbay goto err; 2684e65e175bSOded Gabbay } 2685e65e175bSOded Gabbay } 2686e65e175bSOded Gabbay 2687e65e175bSOded Gabbay goya->hw_cap_initialized |= HW_CAP_MMU; 2688e65e175bSOded Gabbay 2689e65e175bSOded Gabbay /* init MMU cache manage page */ 2690e65e175bSOded Gabbay WREG32(mmSTLB_CACHE_INV_BASE_39_8, 2691e65e175bSOded Gabbay lower_32_bits(MMU_CACHE_MNG_ADDR >> 8)); 2692e65e175bSOded Gabbay WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40); 2693e65e175bSOded Gabbay 2694e65e175bSOded Gabbay /* Remove follower feature due to performance bug */ 2695e65e175bSOded Gabbay WREG32_AND(mmSTLB_STLB_FEATURE_EN, 2696e65e175bSOded Gabbay (~STLB_STLB_FEATURE_EN_FOLLOWER_EN_MASK)); 2697e65e175bSOded Gabbay 2698e65e175bSOded Gabbay hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR | MMU_OP_PHYS_PACK); 2699e65e175bSOded Gabbay 2700e65e175bSOded Gabbay WREG32(mmMMU_MMU_ENABLE, 1); 2701e65e175bSOded Gabbay WREG32(mmMMU_SPI_MASK, 0xF); 2702e65e175bSOded Gabbay 2703e65e175bSOded Gabbay return 0; 2704e65e175bSOded Gabbay 2705e65e175bSOded Gabbay err: 2706e65e175bSOded Gabbay return rc; 2707e65e175bSOded Gabbay } 2708e65e175bSOded Gabbay 2709e65e175bSOded Gabbay /* 2710e65e175bSOded Gabbay * goya_hw_init - Goya hardware initialization code 2711e65e175bSOded Gabbay * 2712e65e175bSOded Gabbay * @hdev: pointer to hl_device structure 2713e65e175bSOded Gabbay * 2714e65e175bSOded Gabbay * Returns 0 on success 2715e65e175bSOded Gabbay * 2716e65e175bSOded Gabbay */ 2717e65e175bSOded Gabbay static int goya_hw_init(struct hl_device *hdev) 2718e65e175bSOded Gabbay { 2719e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 2720e65e175bSOded Gabbay int rc; 2721e65e175bSOded Gabbay 2722e65e175bSOded Gabbay /* Perform read from the device to make sure device is up */ 2723e65e175bSOded Gabbay RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG); 2724e65e175bSOded Gabbay 2725e65e175bSOded Gabbay /* 2726e65e175bSOded Gabbay * Let's mark in the H/W that we have reached this point. We check 2727e65e175bSOded Gabbay * this value in the reset_before_init function to understand whether 2728e65e175bSOded Gabbay * we need to reset the chip before doing H/W init. This register is 2729e65e175bSOded Gabbay * cleared by the H/W upon H/W reset 2730e65e175bSOded Gabbay */ 2731e65e175bSOded Gabbay WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 2732e65e175bSOded Gabbay 2733e65e175bSOded Gabbay rc = goya_init_cpu(hdev); 2734e65e175bSOded Gabbay if (rc) { 2735e65e175bSOded Gabbay dev_err(hdev->dev, "failed to initialize CPU\n"); 2736e65e175bSOded Gabbay return rc; 2737e65e175bSOded Gabbay } 2738e65e175bSOded Gabbay 2739e65e175bSOded Gabbay goya_tpc_mbist_workaround(hdev); 2740e65e175bSOded Gabbay 2741e65e175bSOded Gabbay goya_init_golden_registers(hdev); 2742e65e175bSOded Gabbay 2743e65e175bSOded Gabbay /* 2744e65e175bSOded Gabbay * After CPU initialization is finished, change DDR bar mapping inside 2745e65e175bSOded Gabbay * iATU to point to the start address of the MMU page tables 2746e65e175bSOded Gabbay */ 2747e65e175bSOded Gabbay if (goya_set_ddr_bar_base(hdev, (MMU_PAGE_TABLES_ADDR & 2748e65e175bSOded Gabbay ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) { 2749e65e175bSOded Gabbay dev_err(hdev->dev, 2750e65e175bSOded Gabbay "failed to map DDR bar to MMU page tables\n"); 2751e65e175bSOded Gabbay return -EIO; 2752e65e175bSOded Gabbay } 2753e65e175bSOded Gabbay 2754e65e175bSOded Gabbay rc = goya_mmu_init(hdev); 2755e65e175bSOded Gabbay if (rc) 2756e65e175bSOded Gabbay return rc; 2757e65e175bSOded Gabbay 2758e65e175bSOded Gabbay goya_init_security(hdev); 2759e65e175bSOded Gabbay 2760e65e175bSOded Gabbay goya_init_dma_qmans(hdev); 2761e65e175bSOded Gabbay 2762e65e175bSOded Gabbay goya_init_mme_qmans(hdev); 2763e65e175bSOded Gabbay 2764e65e175bSOded Gabbay goya_init_tpc_qmans(hdev); 2765e65e175bSOded Gabbay 2766e65e175bSOded Gabbay goya_enable_timestamp(hdev); 2767e65e175bSOded Gabbay 2768e65e175bSOded Gabbay /* MSI-X must be enabled before CPU queues are initialized */ 2769e65e175bSOded Gabbay rc = goya_enable_msix(hdev); 2770e65e175bSOded Gabbay if (rc) 2771e65e175bSOded Gabbay goto disable_queues; 2772e65e175bSOded Gabbay 2773e65e175bSOded Gabbay /* Perform read from the device to flush all MSI-X configuration */ 2774e65e175bSOded Gabbay RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG); 2775e65e175bSOded Gabbay 2776e65e175bSOded Gabbay return 0; 2777e65e175bSOded Gabbay 2778e65e175bSOded Gabbay disable_queues: 2779e65e175bSOded Gabbay goya_disable_internal_queues(hdev); 2780e65e175bSOded Gabbay goya_disable_external_queues(hdev); 2781e65e175bSOded Gabbay 2782e65e175bSOded Gabbay return rc; 2783e65e175bSOded Gabbay } 2784e65e175bSOded Gabbay 2785e65e175bSOded Gabbay static void goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 2786e65e175bSOded Gabbay { 2787e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 2788e65e175bSOded Gabbay u32 reset_timeout_ms, cpu_timeout_ms, status; 2789e65e175bSOded Gabbay 2790e65e175bSOded Gabbay if (hdev->pldm) { 2791e65e175bSOded Gabbay reset_timeout_ms = GOYA_PLDM_RESET_TIMEOUT_MSEC; 2792e65e175bSOded Gabbay cpu_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC; 2793e65e175bSOded Gabbay } else { 2794e65e175bSOded Gabbay reset_timeout_ms = GOYA_RESET_TIMEOUT_MSEC; 2795e65e175bSOded Gabbay cpu_timeout_ms = GOYA_CPU_RESET_WAIT_MSEC; 2796e65e175bSOded Gabbay } 2797e65e175bSOded Gabbay 2798e65e175bSOded Gabbay if (hard_reset) { 2799e65e175bSOded Gabbay /* I don't know what is the state of the CPU so make sure it is 2800e65e175bSOded Gabbay * stopped in any means necessary 2801e65e175bSOded Gabbay */ 2802e65e175bSOded Gabbay WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_GOTO_WFE); 2803e65e175bSOded Gabbay WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, 2804e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_HALT_MACHINE); 2805e65e175bSOded Gabbay 2806e65e175bSOded Gabbay msleep(cpu_timeout_ms); 2807e65e175bSOded Gabbay 2808e65e175bSOded Gabbay goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE); 2809e65e175bSOded Gabbay goya_disable_clk_rlx(hdev); 2810e65e175bSOded Gabbay goya_set_pll_refclk(hdev); 2811e65e175bSOded Gabbay 2812e65e175bSOded Gabbay WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, RESET_ALL); 2813e65e175bSOded Gabbay dev_dbg(hdev->dev, 2814e65e175bSOded Gabbay "Issued HARD reset command, going to wait %dms\n", 2815e65e175bSOded Gabbay reset_timeout_ms); 2816e65e175bSOded Gabbay } else { 2817e65e175bSOded Gabbay WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, DMA_MME_TPC_RESET); 2818e65e175bSOded Gabbay dev_dbg(hdev->dev, 2819e65e175bSOded Gabbay "Issued SOFT reset command, going to wait %dms\n", 2820e65e175bSOded Gabbay reset_timeout_ms); 2821e65e175bSOded Gabbay } 2822e65e175bSOded Gabbay 2823e65e175bSOded Gabbay /* 2824e65e175bSOded Gabbay * After hard reset, we can't poll the BTM_FSM register because the PSOC 2825e65e175bSOded Gabbay * itself is in reset. In either reset we need to wait until the reset 2826e65e175bSOded Gabbay * is deasserted 2827e65e175bSOded Gabbay */ 2828e65e175bSOded Gabbay msleep(reset_timeout_ms); 2829e65e175bSOded Gabbay 2830e65e175bSOded Gabbay status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM); 2831e65e175bSOded Gabbay if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK) 2832e65e175bSOded Gabbay dev_err(hdev->dev, 2833e65e175bSOded Gabbay "Timeout while waiting for device to reset 0x%x\n", 2834e65e175bSOded Gabbay status); 2835e65e175bSOded Gabbay 2836e65e175bSOded Gabbay if (!hard_reset && goya) { 2837e65e175bSOded Gabbay goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME | 2838e65e175bSOded Gabbay HW_CAP_GOLDEN | HW_CAP_TPC); 2839e65e175bSOded Gabbay WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, 2840e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_SOFT_RESET); 2841e65e175bSOded Gabbay return; 2842e65e175bSOded Gabbay } 2843e65e175bSOded Gabbay 2844e65e175bSOded Gabbay /* Chicken bit to re-initiate boot sequencer flow */ 2845e65e175bSOded Gabbay WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 2846e65e175bSOded Gabbay 1 << PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_SHIFT); 2847e65e175bSOded Gabbay /* Move boot manager FSM to pre boot sequencer init state */ 2848e65e175bSOded Gabbay WREG32(mmPSOC_GLOBAL_CONF_SW_BTM_FSM, 2849e65e175bSOded Gabbay 0xA << PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_SHIFT); 2850e65e175bSOded Gabbay 2851e65e175bSOded Gabbay if (goya) { 2852e65e175bSOded Gabbay goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | 2853e65e175bSOded Gabbay HW_CAP_DDR_0 | HW_CAP_DDR_1 | 2854e65e175bSOded Gabbay HW_CAP_DMA | HW_CAP_MME | 2855e65e175bSOded Gabbay HW_CAP_MMU | HW_CAP_TPC_MBIST | 2856e65e175bSOded Gabbay HW_CAP_GOLDEN | HW_CAP_TPC); 2857e65e175bSOded Gabbay 2858e65e175bSOded Gabbay memset(goya->events_stat, 0, sizeof(goya->events_stat)); 2859e65e175bSOded Gabbay } 2860e65e175bSOded Gabbay } 2861e65e175bSOded Gabbay 2862e65e175bSOded Gabbay int goya_suspend(struct hl_device *hdev) 2863e65e175bSOded Gabbay { 2864e65e175bSOded Gabbay int rc; 2865e65e175bSOded Gabbay 2866e65e175bSOded Gabbay rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 2867e65e175bSOded Gabbay if (rc) 2868e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); 2869e65e175bSOded Gabbay 2870e65e175bSOded Gabbay return rc; 2871e65e175bSOded Gabbay } 2872e65e175bSOded Gabbay 2873e65e175bSOded Gabbay int goya_resume(struct hl_device *hdev) 2874e65e175bSOded Gabbay { 2875e65e175bSOded Gabbay return goya_init_iatu(hdev); 2876e65e175bSOded Gabbay } 2877e65e175bSOded Gabbay 2878e65e175bSOded Gabbay static int goya_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 2879e65e175bSOded Gabbay void *cpu_addr, dma_addr_t dma_addr, size_t size) 2880e65e175bSOded Gabbay { 2881e65e175bSOded Gabbay int rc; 2882e65e175bSOded Gabbay 2883*3822a7c4SLinus Torvalds vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 2884*3822a7c4SLinus Torvalds VM_DONTCOPY | VM_NORESERVE); 2885e65e175bSOded Gabbay 2886e65e175bSOded Gabbay rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, 2887e65e175bSOded Gabbay (dma_addr - HOST_PHYS_BASE), size); 2888e65e175bSOded Gabbay if (rc) 2889e65e175bSOded Gabbay dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 2890e65e175bSOded Gabbay 2891e65e175bSOded Gabbay return rc; 2892e65e175bSOded Gabbay } 2893e65e175bSOded Gabbay 2894e65e175bSOded Gabbay void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 2895e65e175bSOded Gabbay { 2896e65e175bSOded Gabbay u32 db_reg_offset, db_value; 2897e65e175bSOded Gabbay 2898e65e175bSOded Gabbay switch (hw_queue_id) { 2899e65e175bSOded Gabbay case GOYA_QUEUE_ID_DMA_0: 2900e65e175bSOded Gabbay db_reg_offset = mmDMA_QM_0_PQ_PI; 2901e65e175bSOded Gabbay break; 2902e65e175bSOded Gabbay 2903e65e175bSOded Gabbay case GOYA_QUEUE_ID_DMA_1: 2904e65e175bSOded Gabbay db_reg_offset = mmDMA_QM_1_PQ_PI; 2905e65e175bSOded Gabbay break; 2906e65e175bSOded Gabbay 2907e65e175bSOded Gabbay case GOYA_QUEUE_ID_DMA_2: 2908e65e175bSOded Gabbay db_reg_offset = mmDMA_QM_2_PQ_PI; 2909e65e175bSOded Gabbay break; 2910e65e175bSOded Gabbay 2911e65e175bSOded Gabbay case GOYA_QUEUE_ID_DMA_3: 2912e65e175bSOded Gabbay db_reg_offset = mmDMA_QM_3_PQ_PI; 2913e65e175bSOded Gabbay break; 2914e65e175bSOded Gabbay 2915e65e175bSOded Gabbay case GOYA_QUEUE_ID_DMA_4: 2916e65e175bSOded Gabbay db_reg_offset = mmDMA_QM_4_PQ_PI; 2917e65e175bSOded Gabbay break; 2918e65e175bSOded Gabbay 2919e65e175bSOded Gabbay case GOYA_QUEUE_ID_CPU_PQ: 2920e65e175bSOded Gabbay db_reg_offset = mmCPU_IF_PF_PQ_PI; 2921e65e175bSOded Gabbay break; 2922e65e175bSOded Gabbay 2923e65e175bSOded Gabbay case GOYA_QUEUE_ID_MME: 2924e65e175bSOded Gabbay db_reg_offset = mmMME_QM_PQ_PI; 2925e65e175bSOded Gabbay break; 2926e65e175bSOded Gabbay 2927e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC0: 2928e65e175bSOded Gabbay db_reg_offset = mmTPC0_QM_PQ_PI; 2929e65e175bSOded Gabbay break; 2930e65e175bSOded Gabbay 2931e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC1: 2932e65e175bSOded Gabbay db_reg_offset = mmTPC1_QM_PQ_PI; 2933e65e175bSOded Gabbay break; 2934e65e175bSOded Gabbay 2935e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC2: 2936e65e175bSOded Gabbay db_reg_offset = mmTPC2_QM_PQ_PI; 2937e65e175bSOded Gabbay break; 2938e65e175bSOded Gabbay 2939e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC3: 2940e65e175bSOded Gabbay db_reg_offset = mmTPC3_QM_PQ_PI; 2941e65e175bSOded Gabbay break; 2942e65e175bSOded Gabbay 2943e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC4: 2944e65e175bSOded Gabbay db_reg_offset = mmTPC4_QM_PQ_PI; 2945e65e175bSOded Gabbay break; 2946e65e175bSOded Gabbay 2947e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC5: 2948e65e175bSOded Gabbay db_reg_offset = mmTPC5_QM_PQ_PI; 2949e65e175bSOded Gabbay break; 2950e65e175bSOded Gabbay 2951e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC6: 2952e65e175bSOded Gabbay db_reg_offset = mmTPC6_QM_PQ_PI; 2953e65e175bSOded Gabbay break; 2954e65e175bSOded Gabbay 2955e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC7: 2956e65e175bSOded Gabbay db_reg_offset = mmTPC7_QM_PQ_PI; 2957e65e175bSOded Gabbay break; 2958e65e175bSOded Gabbay 2959e65e175bSOded Gabbay default: 2960e65e175bSOded Gabbay /* Should never get here */ 2961e65e175bSOded Gabbay dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", 2962e65e175bSOded Gabbay hw_queue_id); 2963e65e175bSOded Gabbay return; 2964e65e175bSOded Gabbay } 2965e65e175bSOded Gabbay 2966e65e175bSOded Gabbay db_value = pi; 2967e65e175bSOded Gabbay 2968e65e175bSOded Gabbay /* ring the doorbell */ 2969e65e175bSOded Gabbay WREG32(db_reg_offset, db_value); 2970e65e175bSOded Gabbay 2971e65e175bSOded Gabbay if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ) { 2972e65e175bSOded Gabbay /* make sure device CPU will read latest data from host */ 2973e65e175bSOded Gabbay mb(); 2974e65e175bSOded Gabbay WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, 2975e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_PI_UPDATE); 2976e65e175bSOded Gabbay } 2977e65e175bSOded Gabbay } 2978e65e175bSOded Gabbay 2979e65e175bSOded Gabbay void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) 2980e65e175bSOded Gabbay { 2981e65e175bSOded Gabbay /* The QMANs are on the SRAM so need to copy to IO space */ 2982e65e175bSOded Gabbay memcpy_toio((void __iomem *) pqe, bd, sizeof(struct hl_bd)); 2983e65e175bSOded Gabbay } 2984e65e175bSOded Gabbay 2985e65e175bSOded Gabbay static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size, 2986e65e175bSOded Gabbay dma_addr_t *dma_handle, gfp_t flags) 2987e65e175bSOded Gabbay { 2988e65e175bSOded Gabbay void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, 2989e65e175bSOded Gabbay dma_handle, flags); 2990e65e175bSOded Gabbay 2991e65e175bSOded Gabbay /* Shift to the device's base physical address of host memory */ 2992e65e175bSOded Gabbay if (kernel_addr) 2993e65e175bSOded Gabbay *dma_handle += HOST_PHYS_BASE; 2994e65e175bSOded Gabbay 2995e65e175bSOded Gabbay return kernel_addr; 2996e65e175bSOded Gabbay } 2997e65e175bSOded Gabbay 2998e65e175bSOded Gabbay static void goya_dma_free_coherent(struct hl_device *hdev, size_t size, 2999e65e175bSOded Gabbay void *cpu_addr, dma_addr_t dma_handle) 3000e65e175bSOded Gabbay { 3001e65e175bSOded Gabbay /* Cancel the device's base physical address of host memory */ 3002e65e175bSOded Gabbay dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE; 3003e65e175bSOded Gabbay 3004e65e175bSOded Gabbay dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); 3005e65e175bSOded Gabbay } 3006e65e175bSOded Gabbay 3007e65e175bSOded Gabbay int goya_scrub_device_mem(struct hl_device *hdev) 3008e65e175bSOded Gabbay { 3009e65e175bSOded Gabbay return 0; 3010e65e175bSOded Gabbay } 3011e65e175bSOded Gabbay 3012e65e175bSOded Gabbay void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, 3013e65e175bSOded Gabbay dma_addr_t *dma_handle, u16 *queue_len) 3014e65e175bSOded Gabbay { 3015e65e175bSOded Gabbay void *base; 3016e65e175bSOded Gabbay u32 offset; 3017e65e175bSOded Gabbay 3018e65e175bSOded Gabbay *dma_handle = hdev->asic_prop.sram_base_address; 3019e65e175bSOded Gabbay 3020e65e175bSOded Gabbay base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; 3021e65e175bSOded Gabbay 3022e65e175bSOded Gabbay switch (queue_id) { 3023e65e175bSOded Gabbay case GOYA_QUEUE_ID_MME: 3024e65e175bSOded Gabbay offset = MME_QMAN_BASE_OFFSET; 3025e65e175bSOded Gabbay *queue_len = MME_QMAN_LENGTH; 3026e65e175bSOded Gabbay break; 3027e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC0: 3028e65e175bSOded Gabbay offset = TPC0_QMAN_BASE_OFFSET; 3029e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3030e65e175bSOded Gabbay break; 3031e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC1: 3032e65e175bSOded Gabbay offset = TPC1_QMAN_BASE_OFFSET; 3033e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3034e65e175bSOded Gabbay break; 3035e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC2: 3036e65e175bSOded Gabbay offset = TPC2_QMAN_BASE_OFFSET; 3037e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3038e65e175bSOded Gabbay break; 3039e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC3: 3040e65e175bSOded Gabbay offset = TPC3_QMAN_BASE_OFFSET; 3041e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3042e65e175bSOded Gabbay break; 3043e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC4: 3044e65e175bSOded Gabbay offset = TPC4_QMAN_BASE_OFFSET; 3045e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3046e65e175bSOded Gabbay break; 3047e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC5: 3048e65e175bSOded Gabbay offset = TPC5_QMAN_BASE_OFFSET; 3049e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3050e65e175bSOded Gabbay break; 3051e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC6: 3052e65e175bSOded Gabbay offset = TPC6_QMAN_BASE_OFFSET; 3053e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3054e65e175bSOded Gabbay break; 3055e65e175bSOded Gabbay case GOYA_QUEUE_ID_TPC7: 3056e65e175bSOded Gabbay offset = TPC7_QMAN_BASE_OFFSET; 3057e65e175bSOded Gabbay *queue_len = TPC_QMAN_LENGTH; 3058e65e175bSOded Gabbay break; 3059e65e175bSOded Gabbay default: 3060e65e175bSOded Gabbay dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); 3061e65e175bSOded Gabbay return NULL; 3062e65e175bSOded Gabbay } 3063e65e175bSOded Gabbay 3064e65e175bSOded Gabbay base += offset; 3065e65e175bSOded Gabbay *dma_handle += offset; 3066e65e175bSOded Gabbay 3067e65e175bSOded Gabbay return base; 3068e65e175bSOded Gabbay } 3069e65e175bSOded Gabbay 3070e65e175bSOded Gabbay static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) 3071e65e175bSOded Gabbay { 3072e65e175bSOded Gabbay struct packet_msg_prot *fence_pkt; 3073e65e175bSOded Gabbay u32 *fence_ptr; 3074e65e175bSOded Gabbay dma_addr_t fence_dma_addr; 3075e65e175bSOded Gabbay struct hl_cb *cb; 3076e65e175bSOded Gabbay u32 tmp, timeout; 3077e65e175bSOded Gabbay int rc; 3078e65e175bSOded Gabbay 3079e65e175bSOded Gabbay if (hdev->pldm) 3080e65e175bSOded Gabbay timeout = GOYA_PLDM_QMAN0_TIMEOUT_USEC; 3081e65e175bSOded Gabbay else 3082e65e175bSOded Gabbay timeout = HL_DEVICE_TIMEOUT_USEC; 3083e65e175bSOded Gabbay 3084e65e175bSOded Gabbay if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { 3085e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, 3086e65e175bSOded Gabbay "Can't send driver job on QMAN0 because the device is not idle\n"); 3087e65e175bSOded Gabbay return -EBUSY; 3088e65e175bSOded Gabbay } 3089e65e175bSOded Gabbay 3090e65e175bSOded Gabbay fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr); 3091e65e175bSOded Gabbay if (!fence_ptr) { 3092e65e175bSOded Gabbay dev_err(hdev->dev, 3093e65e175bSOded Gabbay "Failed to allocate fence memory for QMAN0\n"); 3094e65e175bSOded Gabbay return -ENOMEM; 3095e65e175bSOded Gabbay } 3096e65e175bSOded Gabbay 3097e65e175bSOded Gabbay goya_qman0_set_security(hdev, true); 3098e65e175bSOded Gabbay 3099e65e175bSOded Gabbay cb = job->patched_cb; 3100e65e175bSOded Gabbay 3101e65e175bSOded Gabbay fence_pkt = cb->kernel_address + 3102e65e175bSOded Gabbay job->job_cb_size - sizeof(struct packet_msg_prot); 3103e65e175bSOded Gabbay 3104e65e175bSOded Gabbay tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) | 3105e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_EB_SHIFT) | 3106e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_MB_SHIFT); 3107e65e175bSOded Gabbay fence_pkt->ctl = cpu_to_le32(tmp); 3108e65e175bSOded Gabbay fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL); 3109e65e175bSOded Gabbay fence_pkt->addr = cpu_to_le64(fence_dma_addr); 3110e65e175bSOded Gabbay 3111e65e175bSOded Gabbay rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0, 3112e65e175bSOded Gabbay job->job_cb_size, cb->bus_address); 3113e65e175bSOded Gabbay if (rc) { 3114e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); 3115e65e175bSOded Gabbay goto free_fence_ptr; 3116e65e175bSOded Gabbay } 3117e65e175bSOded Gabbay 3118e65e175bSOded Gabbay rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, 3119e65e175bSOded Gabbay (tmp == GOYA_QMAN0_FENCE_VAL), 1000, 3120e65e175bSOded Gabbay timeout, true); 3121e65e175bSOded Gabbay 3122e65e175bSOded Gabbay hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0); 3123e65e175bSOded Gabbay 3124e65e175bSOded Gabbay if (rc == -ETIMEDOUT) { 3125e65e175bSOded Gabbay dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); 3126e65e175bSOded Gabbay goto free_fence_ptr; 3127e65e175bSOded Gabbay } 3128e65e175bSOded Gabbay 3129e65e175bSOded Gabbay free_fence_ptr: 3130e65e175bSOded Gabbay hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr); 3131e65e175bSOded Gabbay 3132e65e175bSOded Gabbay goya_qman0_set_security(hdev, false); 3133e65e175bSOded Gabbay 3134e65e175bSOded Gabbay return rc; 3135e65e175bSOded Gabbay } 3136e65e175bSOded Gabbay 3137e65e175bSOded Gabbay int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, 3138e65e175bSOded Gabbay u32 timeout, u64 *result) 3139e65e175bSOded Gabbay { 3140e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 3141e65e175bSOded Gabbay 3142e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) { 3143e65e175bSOded Gabbay if (result) 3144e65e175bSOded Gabbay *result = 0; 3145e65e175bSOded Gabbay return 0; 3146e65e175bSOded Gabbay } 3147e65e175bSOded Gabbay 3148e65e175bSOded Gabbay if (!timeout) 3149e65e175bSOded Gabbay timeout = GOYA_MSG_TO_CPU_TIMEOUT_USEC; 3150e65e175bSOded Gabbay 3151e65e175bSOded Gabbay return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len, 3152e65e175bSOded Gabbay timeout, result); 3153e65e175bSOded Gabbay } 3154e65e175bSOded Gabbay 3155e65e175bSOded Gabbay int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id) 3156e65e175bSOded Gabbay { 3157e65e175bSOded Gabbay struct packet_msg_prot *fence_pkt; 3158e65e175bSOded Gabbay dma_addr_t pkt_dma_addr; 3159e65e175bSOded Gabbay u32 fence_val, tmp; 3160e65e175bSOded Gabbay dma_addr_t fence_dma_addr; 3161e65e175bSOded Gabbay u32 *fence_ptr; 3162e65e175bSOded Gabbay int rc; 3163e65e175bSOded Gabbay 3164e65e175bSOded Gabbay fence_val = GOYA_QMAN0_FENCE_VAL; 3165e65e175bSOded Gabbay 3166e65e175bSOded Gabbay fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr); 3167e65e175bSOded Gabbay if (!fence_ptr) { 3168e65e175bSOded Gabbay dev_err(hdev->dev, 3169e65e175bSOded Gabbay "Failed to allocate memory for H/W queue %d testing\n", 3170e65e175bSOded Gabbay hw_queue_id); 3171e65e175bSOded Gabbay return -ENOMEM; 3172e65e175bSOded Gabbay } 3173e65e175bSOded Gabbay 3174e65e175bSOded Gabbay *fence_ptr = 0; 3175e65e175bSOded Gabbay 3176e65e175bSOded Gabbay fence_pkt = hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_prot), GFP_KERNEL, 3177e65e175bSOded Gabbay &pkt_dma_addr); 3178e65e175bSOded Gabbay if (!fence_pkt) { 3179e65e175bSOded Gabbay dev_err(hdev->dev, 3180e65e175bSOded Gabbay "Failed to allocate packet for H/W queue %d testing\n", 3181e65e175bSOded Gabbay hw_queue_id); 3182e65e175bSOded Gabbay rc = -ENOMEM; 3183e65e175bSOded Gabbay goto free_fence_ptr; 3184e65e175bSOded Gabbay } 3185e65e175bSOded Gabbay 3186e65e175bSOded Gabbay tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) | 3187e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_EB_SHIFT) | 3188e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_MB_SHIFT); 3189e65e175bSOded Gabbay fence_pkt->ctl = cpu_to_le32(tmp); 3190e65e175bSOded Gabbay fence_pkt->value = cpu_to_le32(fence_val); 3191e65e175bSOded Gabbay fence_pkt->addr = cpu_to_le64(fence_dma_addr); 3192e65e175bSOded Gabbay 3193e65e175bSOded Gabbay rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, 3194e65e175bSOded Gabbay sizeof(struct packet_msg_prot), 3195e65e175bSOded Gabbay pkt_dma_addr); 3196e65e175bSOded Gabbay if (rc) { 3197e65e175bSOded Gabbay dev_err(hdev->dev, 3198e65e175bSOded Gabbay "Failed to send fence packet to H/W queue %d\n", 3199e65e175bSOded Gabbay hw_queue_id); 3200e65e175bSOded Gabbay goto free_pkt; 3201e65e175bSOded Gabbay } 3202e65e175bSOded Gabbay 3203e65e175bSOded Gabbay rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val), 3204e65e175bSOded Gabbay 1000, GOYA_TEST_QUEUE_WAIT_USEC, true); 3205e65e175bSOded Gabbay 3206e65e175bSOded Gabbay hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); 3207e65e175bSOded Gabbay 3208e65e175bSOded Gabbay if (rc == -ETIMEDOUT) { 3209e65e175bSOded Gabbay dev_err(hdev->dev, 3210e65e175bSOded Gabbay "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n", 3211e65e175bSOded Gabbay hw_queue_id, (unsigned long long) fence_dma_addr, tmp); 3212e65e175bSOded Gabbay rc = -EIO; 3213e65e175bSOded Gabbay } 3214e65e175bSOded Gabbay 3215e65e175bSOded Gabbay free_pkt: 3216e65e175bSOded Gabbay hl_asic_dma_pool_free(hdev, (void *) fence_pkt, pkt_dma_addr); 3217e65e175bSOded Gabbay free_fence_ptr: 3218e65e175bSOded Gabbay hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr); 3219e65e175bSOded Gabbay return rc; 3220e65e175bSOded Gabbay } 3221e65e175bSOded Gabbay 3222e65e175bSOded Gabbay int goya_test_cpu_queue(struct hl_device *hdev) 3223e65e175bSOded Gabbay { 3224e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 3225e65e175bSOded Gabbay 3226e65e175bSOded Gabbay /* 3227e65e175bSOded Gabbay * check capability here as send_cpu_message() won't update the result 3228e65e175bSOded Gabbay * value if no capability 3229e65e175bSOded Gabbay */ 3230e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) 3231e65e175bSOded Gabbay return 0; 3232e65e175bSOded Gabbay 3233e65e175bSOded Gabbay return hl_fw_test_cpu_queue(hdev); 3234e65e175bSOded Gabbay } 3235e65e175bSOded Gabbay 3236e65e175bSOded Gabbay int goya_test_queues(struct hl_device *hdev) 3237e65e175bSOded Gabbay { 3238e65e175bSOded Gabbay int i, rc, ret_val = 0; 3239e65e175bSOded Gabbay 3240e65e175bSOded Gabbay for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) { 3241e65e175bSOded Gabbay rc = goya_test_queue(hdev, i); 3242e65e175bSOded Gabbay if (rc) 3243e65e175bSOded Gabbay ret_val = -EINVAL; 3244e65e175bSOded Gabbay } 3245e65e175bSOded Gabbay 3246e65e175bSOded Gabbay return ret_val; 3247e65e175bSOded Gabbay } 3248e65e175bSOded Gabbay 3249e65e175bSOded Gabbay static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size, 3250e65e175bSOded Gabbay gfp_t mem_flags, dma_addr_t *dma_handle) 3251e65e175bSOded Gabbay { 3252e65e175bSOded Gabbay void *kernel_addr; 3253e65e175bSOded Gabbay 3254e65e175bSOded Gabbay if (size > GOYA_DMA_POOL_BLK_SIZE) 3255e65e175bSOded Gabbay return NULL; 3256e65e175bSOded Gabbay 3257e65e175bSOded Gabbay kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 3258e65e175bSOded Gabbay 3259e65e175bSOded Gabbay /* Shift to the device's base physical address of host memory */ 3260e65e175bSOded Gabbay if (kernel_addr) 3261e65e175bSOded Gabbay *dma_handle += HOST_PHYS_BASE; 3262e65e175bSOded Gabbay 3263e65e175bSOded Gabbay return kernel_addr; 3264e65e175bSOded Gabbay } 3265e65e175bSOded Gabbay 3266e65e175bSOded Gabbay static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr, 3267e65e175bSOded Gabbay dma_addr_t dma_addr) 3268e65e175bSOded Gabbay { 3269e65e175bSOded Gabbay /* Cancel the device's base physical address of host memory */ 3270e65e175bSOded Gabbay dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE; 3271e65e175bSOded Gabbay 3272e65e175bSOded Gabbay dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); 3273e65e175bSOded Gabbay } 3274e65e175bSOded Gabbay 3275e65e175bSOded Gabbay void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 3276e65e175bSOded Gabbay dma_addr_t *dma_handle) 3277e65e175bSOded Gabbay { 3278e65e175bSOded Gabbay void *vaddr; 3279e65e175bSOded Gabbay 3280e65e175bSOded Gabbay vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 3281e65e175bSOded Gabbay *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + 3282e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR; 3283e65e175bSOded Gabbay 3284e65e175bSOded Gabbay return vaddr; 3285e65e175bSOded Gabbay } 3286e65e175bSOded Gabbay 3287e65e175bSOded Gabbay void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, 3288e65e175bSOded Gabbay void *vaddr) 3289e65e175bSOded Gabbay { 3290e65e175bSOded Gabbay hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 3291e65e175bSOded Gabbay } 3292e65e175bSOded Gabbay 3293e65e175bSOded Gabbay u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) 3294e65e175bSOded Gabbay { 3295e65e175bSOded Gabbay struct scatterlist *sg, *sg_next_iter; 3296e65e175bSOded Gabbay u32 count, dma_desc_cnt; 3297e65e175bSOded Gabbay u64 len, len_next; 3298e65e175bSOded Gabbay dma_addr_t addr, addr_next; 3299e65e175bSOded Gabbay 3300e65e175bSOded Gabbay dma_desc_cnt = 0; 3301e65e175bSOded Gabbay 3302e65e175bSOded Gabbay for_each_sgtable_dma_sg(sgt, sg, count) { 3303e65e175bSOded Gabbay len = sg_dma_len(sg); 3304e65e175bSOded Gabbay addr = sg_dma_address(sg); 3305e65e175bSOded Gabbay 3306e65e175bSOded Gabbay if (len == 0) 3307e65e175bSOded Gabbay break; 3308e65e175bSOded Gabbay 3309e65e175bSOded Gabbay while ((count + 1) < sgt->nents) { 3310e65e175bSOded Gabbay sg_next_iter = sg_next(sg); 3311e65e175bSOded Gabbay len_next = sg_dma_len(sg_next_iter); 3312e65e175bSOded Gabbay addr_next = sg_dma_address(sg_next_iter); 3313e65e175bSOded Gabbay 3314e65e175bSOded Gabbay if (len_next == 0) 3315e65e175bSOded Gabbay break; 3316e65e175bSOded Gabbay 3317e65e175bSOded Gabbay if ((addr + len == addr_next) && 3318e65e175bSOded Gabbay (len + len_next <= DMA_MAX_TRANSFER_SIZE)) { 3319e65e175bSOded Gabbay len += len_next; 3320e65e175bSOded Gabbay count++; 3321e65e175bSOded Gabbay sg = sg_next_iter; 3322e65e175bSOded Gabbay } else { 3323e65e175bSOded Gabbay break; 3324e65e175bSOded Gabbay } 3325e65e175bSOded Gabbay } 3326e65e175bSOded Gabbay 3327e65e175bSOded Gabbay dma_desc_cnt++; 3328e65e175bSOded Gabbay } 3329e65e175bSOded Gabbay 3330e65e175bSOded Gabbay return dma_desc_cnt * sizeof(struct packet_lin_dma); 3331e65e175bSOded Gabbay } 3332e65e175bSOded Gabbay 3333e65e175bSOded Gabbay static int goya_pin_memory_before_cs(struct hl_device *hdev, 3334e65e175bSOded Gabbay struct hl_cs_parser *parser, 3335e65e175bSOded Gabbay struct packet_lin_dma *user_dma_pkt, 3336e65e175bSOded Gabbay u64 addr, enum dma_data_direction dir) 3337e65e175bSOded Gabbay { 3338e65e175bSOded Gabbay struct hl_userptr *userptr; 3339e65e175bSOded Gabbay int rc; 3340e65e175bSOded Gabbay 3341e65e175bSOded Gabbay if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), 3342e65e175bSOded Gabbay parser->job_userptr_list, &userptr)) 3343e65e175bSOded Gabbay goto already_pinned; 3344e65e175bSOded Gabbay 3345e65e175bSOded Gabbay userptr = kzalloc(sizeof(*userptr), GFP_KERNEL); 3346e65e175bSOded Gabbay if (!userptr) 3347e65e175bSOded Gabbay return -ENOMEM; 3348e65e175bSOded Gabbay 3349e65e175bSOded Gabbay rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), 3350e65e175bSOded Gabbay userptr); 3351e65e175bSOded Gabbay if (rc) 3352e65e175bSOded Gabbay goto free_userptr; 3353e65e175bSOded Gabbay 3354e65e175bSOded Gabbay list_add_tail(&userptr->job_node, parser->job_userptr_list); 3355e65e175bSOded Gabbay 3356e65e175bSOded Gabbay rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, dir); 3357e65e175bSOded Gabbay if (rc) { 3358e65e175bSOded Gabbay dev_err(hdev->dev, "failed to map sgt with DMA region\n"); 3359e65e175bSOded Gabbay goto unpin_memory; 3360e65e175bSOded Gabbay } 3361e65e175bSOded Gabbay 3362e65e175bSOded Gabbay userptr->dma_mapped = true; 3363e65e175bSOded Gabbay userptr->dir = dir; 3364e65e175bSOded Gabbay 3365e65e175bSOded Gabbay already_pinned: 3366e65e175bSOded Gabbay parser->patched_cb_size += 3367e65e175bSOded Gabbay goya_get_dma_desc_list_size(hdev, userptr->sgt); 3368e65e175bSOded Gabbay 3369e65e175bSOded Gabbay return 0; 3370e65e175bSOded Gabbay 3371e65e175bSOded Gabbay unpin_memory: 3372e65e175bSOded Gabbay list_del(&userptr->job_node); 3373e65e175bSOded Gabbay hl_unpin_host_memory(hdev, userptr); 3374e65e175bSOded Gabbay free_userptr: 3375e65e175bSOded Gabbay kfree(userptr); 3376e65e175bSOded Gabbay return rc; 3377e65e175bSOded Gabbay } 3378e65e175bSOded Gabbay 3379e65e175bSOded Gabbay static int goya_validate_dma_pkt_host(struct hl_device *hdev, 3380e65e175bSOded Gabbay struct hl_cs_parser *parser, 3381e65e175bSOded Gabbay struct packet_lin_dma *user_dma_pkt) 3382e65e175bSOded Gabbay { 3383e65e175bSOded Gabbay u64 device_memory_addr, addr; 3384e65e175bSOded Gabbay enum dma_data_direction dir; 3385e65e175bSOded Gabbay enum hl_goya_dma_direction user_dir; 3386e65e175bSOded Gabbay bool sram_addr = true; 3387e65e175bSOded Gabbay bool skip_host_mem_pin = false; 3388e65e175bSOded Gabbay bool user_memset; 3389e65e175bSOded Gabbay u32 ctl; 3390e65e175bSOded Gabbay int rc = 0; 3391e65e175bSOded Gabbay 3392e65e175bSOded Gabbay ctl = le32_to_cpu(user_dma_pkt->ctl); 3393e65e175bSOded Gabbay 3394e65e175bSOded Gabbay user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >> 3395e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; 3396e65e175bSOded Gabbay 3397e65e175bSOded Gabbay user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >> 3398e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT; 3399e65e175bSOded Gabbay 3400e65e175bSOded Gabbay switch (user_dir) { 3401e65e175bSOded Gabbay case HL_DMA_HOST_TO_DRAM: 3402e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); 3403e65e175bSOded Gabbay dir = DMA_TO_DEVICE; 3404e65e175bSOded Gabbay sram_addr = false; 3405e65e175bSOded Gabbay addr = le64_to_cpu(user_dma_pkt->src_addr); 3406e65e175bSOded Gabbay device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); 3407e65e175bSOded Gabbay if (user_memset) 3408e65e175bSOded Gabbay skip_host_mem_pin = true; 3409e65e175bSOded Gabbay break; 3410e65e175bSOded Gabbay 3411e65e175bSOded Gabbay case HL_DMA_DRAM_TO_HOST: 3412e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); 3413e65e175bSOded Gabbay dir = DMA_FROM_DEVICE; 3414e65e175bSOded Gabbay sram_addr = false; 3415e65e175bSOded Gabbay addr = le64_to_cpu(user_dma_pkt->dst_addr); 3416e65e175bSOded Gabbay device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); 3417e65e175bSOded Gabbay break; 3418e65e175bSOded Gabbay 3419e65e175bSOded Gabbay case HL_DMA_HOST_TO_SRAM: 3420e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); 3421e65e175bSOded Gabbay dir = DMA_TO_DEVICE; 3422e65e175bSOded Gabbay addr = le64_to_cpu(user_dma_pkt->src_addr); 3423e65e175bSOded Gabbay device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); 3424e65e175bSOded Gabbay if (user_memset) 3425e65e175bSOded Gabbay skip_host_mem_pin = true; 3426e65e175bSOded Gabbay break; 3427e65e175bSOded Gabbay 3428e65e175bSOded Gabbay case HL_DMA_SRAM_TO_HOST: 3429e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); 3430e65e175bSOded Gabbay dir = DMA_FROM_DEVICE; 3431e65e175bSOded Gabbay addr = le64_to_cpu(user_dma_pkt->dst_addr); 3432e65e175bSOded Gabbay device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); 3433e65e175bSOded Gabbay break; 3434e65e175bSOded Gabbay default: 3435e65e175bSOded Gabbay dev_err(hdev->dev, "DMA direction %d is unsupported/undefined\n", user_dir); 3436e65e175bSOded Gabbay return -EFAULT; 3437e65e175bSOded Gabbay } 3438e65e175bSOded Gabbay 3439e65e175bSOded Gabbay if (sram_addr) { 3440e65e175bSOded Gabbay if (!hl_mem_area_inside_range(device_memory_addr, 3441e65e175bSOded Gabbay le32_to_cpu(user_dma_pkt->tsize), 3442e65e175bSOded Gabbay hdev->asic_prop.sram_user_base_address, 3443e65e175bSOded Gabbay hdev->asic_prop.sram_end_address)) { 3444e65e175bSOded Gabbay 3445e65e175bSOded Gabbay dev_err(hdev->dev, 3446e65e175bSOded Gabbay "SRAM address 0x%llx + 0x%x is invalid\n", 3447e65e175bSOded Gabbay device_memory_addr, 3448e65e175bSOded Gabbay user_dma_pkt->tsize); 3449e65e175bSOded Gabbay return -EFAULT; 3450e65e175bSOded Gabbay } 3451e65e175bSOded Gabbay } else { 3452e65e175bSOded Gabbay if (!hl_mem_area_inside_range(device_memory_addr, 3453e65e175bSOded Gabbay le32_to_cpu(user_dma_pkt->tsize), 3454e65e175bSOded Gabbay hdev->asic_prop.dram_user_base_address, 3455e65e175bSOded Gabbay hdev->asic_prop.dram_end_address)) { 3456e65e175bSOded Gabbay 3457e65e175bSOded Gabbay dev_err(hdev->dev, 3458e65e175bSOded Gabbay "DRAM address 0x%llx + 0x%x is invalid\n", 3459e65e175bSOded Gabbay device_memory_addr, 3460e65e175bSOded Gabbay user_dma_pkt->tsize); 3461e65e175bSOded Gabbay return -EFAULT; 3462e65e175bSOded Gabbay } 3463e65e175bSOded Gabbay } 3464e65e175bSOded Gabbay 3465e65e175bSOded Gabbay if (skip_host_mem_pin) 3466e65e175bSOded Gabbay parser->patched_cb_size += sizeof(*user_dma_pkt); 3467e65e175bSOded Gabbay else { 3468e65e175bSOded Gabbay if ((dir == DMA_TO_DEVICE) && 3469e65e175bSOded Gabbay (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) { 3470e65e175bSOded Gabbay dev_err(hdev->dev, 3471e65e175bSOded Gabbay "Can't DMA from host on queue other then 1\n"); 3472e65e175bSOded Gabbay return -EFAULT; 3473e65e175bSOded Gabbay } 3474e65e175bSOded Gabbay 3475e65e175bSOded Gabbay rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt, 3476e65e175bSOded Gabbay addr, dir); 3477e65e175bSOded Gabbay } 3478e65e175bSOded Gabbay 3479e65e175bSOded Gabbay return rc; 3480e65e175bSOded Gabbay } 3481e65e175bSOded Gabbay 3482e65e175bSOded Gabbay static int goya_validate_dma_pkt_no_host(struct hl_device *hdev, 3483e65e175bSOded Gabbay struct hl_cs_parser *parser, 3484e65e175bSOded Gabbay struct packet_lin_dma *user_dma_pkt) 3485e65e175bSOded Gabbay { 3486e65e175bSOded Gabbay u64 sram_memory_addr, dram_memory_addr; 3487e65e175bSOded Gabbay enum hl_goya_dma_direction user_dir; 3488e65e175bSOded Gabbay u32 ctl; 3489e65e175bSOded Gabbay 3490e65e175bSOded Gabbay ctl = le32_to_cpu(user_dma_pkt->ctl); 3491e65e175bSOded Gabbay user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >> 3492e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; 3493e65e175bSOded Gabbay 3494e65e175bSOded Gabbay if (user_dir == HL_DMA_DRAM_TO_SRAM) { 3495e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); 3496e65e175bSOded Gabbay dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); 3497e65e175bSOded Gabbay sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); 3498e65e175bSOded Gabbay } else { 3499e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); 3500e65e175bSOded Gabbay sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); 3501e65e175bSOded Gabbay dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); 3502e65e175bSOded Gabbay } 3503e65e175bSOded Gabbay 3504e65e175bSOded Gabbay if (!hl_mem_area_inside_range(sram_memory_addr, 3505e65e175bSOded Gabbay le32_to_cpu(user_dma_pkt->tsize), 3506e65e175bSOded Gabbay hdev->asic_prop.sram_user_base_address, 3507e65e175bSOded Gabbay hdev->asic_prop.sram_end_address)) { 3508e65e175bSOded Gabbay dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", 3509e65e175bSOded Gabbay sram_memory_addr, user_dma_pkt->tsize); 3510e65e175bSOded Gabbay return -EFAULT; 3511e65e175bSOded Gabbay } 3512e65e175bSOded Gabbay 3513e65e175bSOded Gabbay if (!hl_mem_area_inside_range(dram_memory_addr, 3514e65e175bSOded Gabbay le32_to_cpu(user_dma_pkt->tsize), 3515e65e175bSOded Gabbay hdev->asic_prop.dram_user_base_address, 3516e65e175bSOded Gabbay hdev->asic_prop.dram_end_address)) { 3517e65e175bSOded Gabbay dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", 3518e65e175bSOded Gabbay dram_memory_addr, user_dma_pkt->tsize); 3519e65e175bSOded Gabbay return -EFAULT; 3520e65e175bSOded Gabbay } 3521e65e175bSOded Gabbay 3522e65e175bSOded Gabbay parser->patched_cb_size += sizeof(*user_dma_pkt); 3523e65e175bSOded Gabbay 3524e65e175bSOded Gabbay return 0; 3525e65e175bSOded Gabbay } 3526e65e175bSOded Gabbay 3527e65e175bSOded Gabbay static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev, 3528e65e175bSOded Gabbay struct hl_cs_parser *parser, 3529e65e175bSOded Gabbay struct packet_lin_dma *user_dma_pkt) 3530e65e175bSOded Gabbay { 3531e65e175bSOded Gabbay enum hl_goya_dma_direction user_dir; 3532e65e175bSOded Gabbay u32 ctl; 3533e65e175bSOded Gabbay int rc; 3534e65e175bSOded Gabbay 3535e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA packet details:\n"); 3536e65e175bSOded Gabbay dev_dbg(hdev->dev, "source == 0x%llx\n", 3537e65e175bSOded Gabbay le64_to_cpu(user_dma_pkt->src_addr)); 3538e65e175bSOded Gabbay dev_dbg(hdev->dev, "destination == 0x%llx\n", 3539e65e175bSOded Gabbay le64_to_cpu(user_dma_pkt->dst_addr)); 3540e65e175bSOded Gabbay dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); 3541e65e175bSOded Gabbay 3542e65e175bSOded Gabbay ctl = le32_to_cpu(user_dma_pkt->ctl); 3543e65e175bSOded Gabbay user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >> 3544e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; 3545e65e175bSOded Gabbay 3546e65e175bSOded Gabbay /* 3547e65e175bSOded Gabbay * Special handling for DMA with size 0. The H/W has a bug where 3548e65e175bSOded Gabbay * this can cause the QMAN DMA to get stuck, so block it here. 3549e65e175bSOded Gabbay */ 3550e65e175bSOded Gabbay if (user_dma_pkt->tsize == 0) { 3551e65e175bSOded Gabbay dev_err(hdev->dev, 3552e65e175bSOded Gabbay "Got DMA with size 0, might reset the device\n"); 3553e65e175bSOded Gabbay return -EINVAL; 3554e65e175bSOded Gabbay } 3555e65e175bSOded Gabbay 3556e65e175bSOded Gabbay if ((user_dir == HL_DMA_DRAM_TO_SRAM) || (user_dir == HL_DMA_SRAM_TO_DRAM)) 3557e65e175bSOded Gabbay rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt); 3558e65e175bSOded Gabbay else 3559e65e175bSOded Gabbay rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt); 3560e65e175bSOded Gabbay 3561e65e175bSOded Gabbay return rc; 3562e65e175bSOded Gabbay } 3563e65e175bSOded Gabbay 3564e65e175bSOded Gabbay static int goya_validate_dma_pkt_mmu(struct hl_device *hdev, 3565e65e175bSOded Gabbay struct hl_cs_parser *parser, 3566e65e175bSOded Gabbay struct packet_lin_dma *user_dma_pkt) 3567e65e175bSOded Gabbay { 3568e65e175bSOded Gabbay dev_dbg(hdev->dev, "DMA packet details:\n"); 3569e65e175bSOded Gabbay dev_dbg(hdev->dev, "source == 0x%llx\n", 3570e65e175bSOded Gabbay le64_to_cpu(user_dma_pkt->src_addr)); 3571e65e175bSOded Gabbay dev_dbg(hdev->dev, "destination == 0x%llx\n", 3572e65e175bSOded Gabbay le64_to_cpu(user_dma_pkt->dst_addr)); 3573e65e175bSOded Gabbay dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); 3574e65e175bSOded Gabbay 3575e65e175bSOded Gabbay /* 3576e65e175bSOded Gabbay * WA for HW-23. 3577e65e175bSOded Gabbay * We can't allow user to read from Host using QMANs other than 1. 3578e65e175bSOded Gabbay * PMMU and HPMMU addresses are equal, check only one of them. 3579e65e175bSOded Gabbay */ 3580e65e175bSOded Gabbay if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 && 3581e65e175bSOded Gabbay hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr), 3582e65e175bSOded Gabbay le32_to_cpu(user_dma_pkt->tsize), 3583e65e175bSOded Gabbay hdev->asic_prop.pmmu.start_addr, 3584e65e175bSOded Gabbay hdev->asic_prop.pmmu.end_addr)) { 3585e65e175bSOded Gabbay dev_err(hdev->dev, 3586e65e175bSOded Gabbay "Can't DMA from host on queue other then 1\n"); 3587e65e175bSOded Gabbay return -EFAULT; 3588e65e175bSOded Gabbay } 3589e65e175bSOded Gabbay 3590e65e175bSOded Gabbay if (user_dma_pkt->tsize == 0) { 3591e65e175bSOded Gabbay dev_err(hdev->dev, 3592e65e175bSOded Gabbay "Got DMA with size 0, might reset the device\n"); 3593e65e175bSOded Gabbay return -EINVAL; 3594e65e175bSOded Gabbay } 3595e65e175bSOded Gabbay 3596e65e175bSOded Gabbay parser->patched_cb_size += sizeof(*user_dma_pkt); 3597e65e175bSOded Gabbay 3598e65e175bSOded Gabbay return 0; 3599e65e175bSOded Gabbay } 3600e65e175bSOded Gabbay 3601e65e175bSOded Gabbay static int goya_validate_wreg32(struct hl_device *hdev, 3602e65e175bSOded Gabbay struct hl_cs_parser *parser, 3603e65e175bSOded Gabbay struct packet_wreg32 *wreg_pkt) 3604e65e175bSOded Gabbay { 3605e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 3606e65e175bSOded Gabbay u32 sob_start_addr, sob_end_addr; 3607e65e175bSOded Gabbay u16 reg_offset; 3608e65e175bSOded Gabbay 3609e65e175bSOded Gabbay reg_offset = le32_to_cpu(wreg_pkt->ctl) & 3610e65e175bSOded Gabbay GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK; 3611e65e175bSOded Gabbay 3612e65e175bSOded Gabbay dev_dbg(hdev->dev, "WREG32 packet details:\n"); 3613e65e175bSOded Gabbay dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); 3614e65e175bSOded Gabbay dev_dbg(hdev->dev, "value == 0x%x\n", 3615e65e175bSOded Gabbay le32_to_cpu(wreg_pkt->value)); 3616e65e175bSOded Gabbay 3617e65e175bSOded Gabbay if (reg_offset != (mmDMA_CH_0_WR_COMP_ADDR_LO & 0x1FFF)) { 3618e65e175bSOded Gabbay dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", 3619e65e175bSOded Gabbay reg_offset); 3620e65e175bSOded Gabbay return -EPERM; 3621e65e175bSOded Gabbay } 3622e65e175bSOded Gabbay 3623e65e175bSOded Gabbay /* 3624e65e175bSOded Gabbay * With MMU, DMA channels are not secured, so it doesn't matter where 3625e65e175bSOded Gabbay * the WR COMP will be written to because it will go out with 3626e65e175bSOded Gabbay * non-secured property 3627e65e175bSOded Gabbay */ 3628e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MMU) 3629e65e175bSOded Gabbay return 0; 3630e65e175bSOded Gabbay 3631e65e175bSOded Gabbay sob_start_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0); 3632e65e175bSOded Gabbay sob_end_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1023); 3633e65e175bSOded Gabbay 3634e65e175bSOded Gabbay if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) || 3635e65e175bSOded Gabbay (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) { 3636e65e175bSOded Gabbay 3637e65e175bSOded Gabbay dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", 3638e65e175bSOded Gabbay wreg_pkt->value); 3639e65e175bSOded Gabbay return -EPERM; 3640e65e175bSOded Gabbay } 3641e65e175bSOded Gabbay 3642e65e175bSOded Gabbay return 0; 3643e65e175bSOded Gabbay } 3644e65e175bSOded Gabbay 3645e65e175bSOded Gabbay static int goya_validate_cb(struct hl_device *hdev, 3646e65e175bSOded Gabbay struct hl_cs_parser *parser, bool is_mmu) 3647e65e175bSOded Gabbay { 3648e65e175bSOded Gabbay u32 cb_parsed_length = 0; 3649e65e175bSOded Gabbay int rc = 0; 3650e65e175bSOded Gabbay 3651e65e175bSOded Gabbay parser->patched_cb_size = 0; 3652e65e175bSOded Gabbay 3653e65e175bSOded Gabbay /* cb_user_size is more than 0 so loop will always be executed */ 3654e65e175bSOded Gabbay while (cb_parsed_length < parser->user_cb_size) { 3655e65e175bSOded Gabbay enum packet_id pkt_id; 3656e65e175bSOded Gabbay u16 pkt_size; 3657e65e175bSOded Gabbay struct goya_packet *user_pkt; 3658e65e175bSOded Gabbay 3659e65e175bSOded Gabbay user_pkt = parser->user_cb->kernel_address + cb_parsed_length; 3660e65e175bSOded Gabbay 3661e65e175bSOded Gabbay pkt_id = (enum packet_id) ( 3662e65e175bSOded Gabbay (le64_to_cpu(user_pkt->header) & 3663e65e175bSOded Gabbay PACKET_HEADER_PACKET_ID_MASK) >> 3664e65e175bSOded Gabbay PACKET_HEADER_PACKET_ID_SHIFT); 3665e65e175bSOded Gabbay 3666e65e175bSOded Gabbay if (!validate_packet_id(pkt_id)) { 3667e65e175bSOded Gabbay dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); 3668e65e175bSOded Gabbay rc = -EINVAL; 3669e65e175bSOded Gabbay break; 3670e65e175bSOded Gabbay } 3671e65e175bSOded Gabbay 3672e65e175bSOded Gabbay pkt_size = goya_packet_sizes[pkt_id]; 3673e65e175bSOded Gabbay cb_parsed_length += pkt_size; 3674e65e175bSOded Gabbay if (cb_parsed_length > parser->user_cb_size) { 3675e65e175bSOded Gabbay dev_err(hdev->dev, 3676e65e175bSOded Gabbay "packet 0x%x is out of CB boundary\n", pkt_id); 3677e65e175bSOded Gabbay rc = -EINVAL; 3678e65e175bSOded Gabbay break; 3679e65e175bSOded Gabbay } 3680e65e175bSOded Gabbay 3681e65e175bSOded Gabbay switch (pkt_id) { 3682e65e175bSOded Gabbay case PACKET_WREG_32: 3683e65e175bSOded Gabbay /* 3684e65e175bSOded Gabbay * Although it is validated after copy in patch_cb(), 3685e65e175bSOded Gabbay * need to validate here as well because patch_cb() is 3686e65e175bSOded Gabbay * not called in MMU path while this function is called 3687e65e175bSOded Gabbay */ 3688e65e175bSOded Gabbay rc = goya_validate_wreg32(hdev, 3689e65e175bSOded Gabbay parser, (struct packet_wreg32 *) user_pkt); 3690e65e175bSOded Gabbay parser->patched_cb_size += pkt_size; 3691e65e175bSOded Gabbay break; 3692e65e175bSOded Gabbay 3693e65e175bSOded Gabbay case PACKET_WREG_BULK: 3694e65e175bSOded Gabbay dev_err(hdev->dev, 3695e65e175bSOded Gabbay "User not allowed to use WREG_BULK\n"); 3696e65e175bSOded Gabbay rc = -EPERM; 3697e65e175bSOded Gabbay break; 3698e65e175bSOded Gabbay 3699e65e175bSOded Gabbay case PACKET_MSG_PROT: 3700e65e175bSOded Gabbay dev_err(hdev->dev, 3701e65e175bSOded Gabbay "User not allowed to use MSG_PROT\n"); 3702e65e175bSOded Gabbay rc = -EPERM; 3703e65e175bSOded Gabbay break; 3704e65e175bSOded Gabbay 3705e65e175bSOded Gabbay case PACKET_CP_DMA: 3706e65e175bSOded Gabbay dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); 3707e65e175bSOded Gabbay rc = -EPERM; 3708e65e175bSOded Gabbay break; 3709e65e175bSOded Gabbay 3710e65e175bSOded Gabbay case PACKET_STOP: 3711e65e175bSOded Gabbay dev_err(hdev->dev, "User not allowed to use STOP\n"); 3712e65e175bSOded Gabbay rc = -EPERM; 3713e65e175bSOded Gabbay break; 3714e65e175bSOded Gabbay 3715e65e175bSOded Gabbay case PACKET_LIN_DMA: 3716e65e175bSOded Gabbay if (is_mmu) 3717e65e175bSOded Gabbay rc = goya_validate_dma_pkt_mmu(hdev, parser, 3718e65e175bSOded Gabbay (struct packet_lin_dma *) user_pkt); 3719e65e175bSOded Gabbay else 3720e65e175bSOded Gabbay rc = goya_validate_dma_pkt_no_mmu(hdev, parser, 3721e65e175bSOded Gabbay (struct packet_lin_dma *) user_pkt); 3722e65e175bSOded Gabbay break; 3723e65e175bSOded Gabbay 3724e65e175bSOded Gabbay case PACKET_MSG_LONG: 3725e65e175bSOded Gabbay case PACKET_MSG_SHORT: 3726e65e175bSOded Gabbay case PACKET_FENCE: 3727e65e175bSOded Gabbay case PACKET_NOP: 3728e65e175bSOded Gabbay parser->patched_cb_size += pkt_size; 3729e65e175bSOded Gabbay break; 3730e65e175bSOded Gabbay 3731e65e175bSOded Gabbay default: 3732e65e175bSOded Gabbay dev_err(hdev->dev, "Invalid packet header 0x%x\n", 3733e65e175bSOded Gabbay pkt_id); 3734e65e175bSOded Gabbay rc = -EINVAL; 3735e65e175bSOded Gabbay break; 3736e65e175bSOded Gabbay } 3737e65e175bSOded Gabbay 3738e65e175bSOded Gabbay if (rc) 3739e65e175bSOded Gabbay break; 3740e65e175bSOded Gabbay } 3741e65e175bSOded Gabbay 3742e65e175bSOded Gabbay /* 3743e65e175bSOded Gabbay * The new CB should have space at the end for two MSG_PROT packets: 3744e65e175bSOded Gabbay * 1. A packet that will act as a completion packet 3745e65e175bSOded Gabbay * 2. A packet that will generate MSI-X interrupt 3746e65e175bSOded Gabbay */ 3747e65e175bSOded Gabbay parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; 3748e65e175bSOded Gabbay 3749e65e175bSOded Gabbay return rc; 3750e65e175bSOded Gabbay } 3751e65e175bSOded Gabbay 3752e65e175bSOded Gabbay static int goya_patch_dma_packet(struct hl_device *hdev, 3753e65e175bSOded Gabbay struct hl_cs_parser *parser, 3754e65e175bSOded Gabbay struct packet_lin_dma *user_dma_pkt, 3755e65e175bSOded Gabbay struct packet_lin_dma *new_dma_pkt, 3756e65e175bSOded Gabbay u32 *new_dma_pkt_size) 3757e65e175bSOded Gabbay { 3758e65e175bSOded Gabbay struct hl_userptr *userptr; 3759e65e175bSOded Gabbay struct scatterlist *sg, *sg_next_iter; 3760e65e175bSOded Gabbay u32 count, dma_desc_cnt; 3761e65e175bSOded Gabbay u64 len, len_next; 3762e65e175bSOded Gabbay dma_addr_t dma_addr, dma_addr_next; 3763e65e175bSOded Gabbay enum hl_goya_dma_direction user_dir; 3764e65e175bSOded Gabbay u64 device_memory_addr, addr; 3765e65e175bSOded Gabbay enum dma_data_direction dir; 3766e65e175bSOded Gabbay struct sg_table *sgt; 3767e65e175bSOded Gabbay bool skip_host_mem_pin = false; 3768e65e175bSOded Gabbay bool user_memset; 3769e65e175bSOded Gabbay u32 user_rdcomp_mask, user_wrcomp_mask, ctl; 3770e65e175bSOded Gabbay 3771e65e175bSOded Gabbay ctl = le32_to_cpu(user_dma_pkt->ctl); 3772e65e175bSOded Gabbay 3773e65e175bSOded Gabbay user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >> 3774e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; 3775e65e175bSOded Gabbay 3776e65e175bSOded Gabbay user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >> 3777e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT; 3778e65e175bSOded Gabbay 3779e65e175bSOded Gabbay if ((user_dir == HL_DMA_DRAM_TO_SRAM) || (user_dir == HL_DMA_SRAM_TO_DRAM) || 3780e65e175bSOded Gabbay (user_dma_pkt->tsize == 0)) { 3781e65e175bSOded Gabbay memcpy(new_dma_pkt, user_dma_pkt, sizeof(*new_dma_pkt)); 3782e65e175bSOded Gabbay *new_dma_pkt_size = sizeof(*new_dma_pkt); 3783e65e175bSOded Gabbay return 0; 3784e65e175bSOded Gabbay } 3785e65e175bSOded Gabbay 3786e65e175bSOded Gabbay if ((user_dir == HL_DMA_HOST_TO_DRAM) || (user_dir == HL_DMA_HOST_TO_SRAM)) { 3787e65e175bSOded Gabbay addr = le64_to_cpu(user_dma_pkt->src_addr); 3788e65e175bSOded Gabbay device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); 3789e65e175bSOded Gabbay dir = DMA_TO_DEVICE; 3790e65e175bSOded Gabbay if (user_memset) 3791e65e175bSOded Gabbay skip_host_mem_pin = true; 3792e65e175bSOded Gabbay } else { 3793e65e175bSOded Gabbay addr = le64_to_cpu(user_dma_pkt->dst_addr); 3794e65e175bSOded Gabbay device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); 3795e65e175bSOded Gabbay dir = DMA_FROM_DEVICE; 3796e65e175bSOded Gabbay } 3797e65e175bSOded Gabbay 3798e65e175bSOded Gabbay if ((!skip_host_mem_pin) && 3799e65e175bSOded Gabbay (hl_userptr_is_pinned(hdev, addr, 3800e65e175bSOded Gabbay le32_to_cpu(user_dma_pkt->tsize), 3801e65e175bSOded Gabbay parser->job_userptr_list, &userptr) == false)) { 3802e65e175bSOded Gabbay dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", 3803e65e175bSOded Gabbay addr, user_dma_pkt->tsize); 3804e65e175bSOded Gabbay return -EFAULT; 3805e65e175bSOded Gabbay } 3806e65e175bSOded Gabbay 3807e65e175bSOded Gabbay if ((user_memset) && (dir == DMA_TO_DEVICE)) { 3808e65e175bSOded Gabbay memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt)); 3809e65e175bSOded Gabbay *new_dma_pkt_size = sizeof(*user_dma_pkt); 3810e65e175bSOded Gabbay return 0; 3811e65e175bSOded Gabbay } 3812e65e175bSOded Gabbay 3813e65e175bSOded Gabbay user_rdcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK; 3814e65e175bSOded Gabbay 3815e65e175bSOded Gabbay user_wrcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK; 3816e65e175bSOded Gabbay 3817e65e175bSOded Gabbay sgt = userptr->sgt; 3818e65e175bSOded Gabbay dma_desc_cnt = 0; 3819e65e175bSOded Gabbay 3820e65e175bSOded Gabbay for_each_sgtable_dma_sg(sgt, sg, count) { 3821e65e175bSOded Gabbay len = sg_dma_len(sg); 3822e65e175bSOded Gabbay dma_addr = sg_dma_address(sg); 3823e65e175bSOded Gabbay 3824e65e175bSOded Gabbay if (len == 0) 3825e65e175bSOded Gabbay break; 3826e65e175bSOded Gabbay 3827e65e175bSOded Gabbay while ((count + 1) < sgt->nents) { 3828e65e175bSOded Gabbay sg_next_iter = sg_next(sg); 3829e65e175bSOded Gabbay len_next = sg_dma_len(sg_next_iter); 3830e65e175bSOded Gabbay dma_addr_next = sg_dma_address(sg_next_iter); 3831e65e175bSOded Gabbay 3832e65e175bSOded Gabbay if (len_next == 0) 3833e65e175bSOded Gabbay break; 3834e65e175bSOded Gabbay 3835e65e175bSOded Gabbay if ((dma_addr + len == dma_addr_next) && 3836e65e175bSOded Gabbay (len + len_next <= DMA_MAX_TRANSFER_SIZE)) { 3837e65e175bSOded Gabbay len += len_next; 3838e65e175bSOded Gabbay count++; 3839e65e175bSOded Gabbay sg = sg_next_iter; 3840e65e175bSOded Gabbay } else { 3841e65e175bSOded Gabbay break; 3842e65e175bSOded Gabbay } 3843e65e175bSOded Gabbay } 3844e65e175bSOded Gabbay 3845e65e175bSOded Gabbay ctl = le32_to_cpu(user_dma_pkt->ctl); 3846e65e175bSOded Gabbay if (likely(dma_desc_cnt)) 3847e65e175bSOded Gabbay ctl &= ~GOYA_PKT_CTL_EB_MASK; 3848e65e175bSOded Gabbay ctl &= ~(GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK | 3849e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK); 3850e65e175bSOded Gabbay new_dma_pkt->ctl = cpu_to_le32(ctl); 3851e65e175bSOded Gabbay new_dma_pkt->tsize = cpu_to_le32((u32) len); 3852e65e175bSOded Gabbay 3853e65e175bSOded Gabbay if (dir == DMA_TO_DEVICE) { 3854e65e175bSOded Gabbay new_dma_pkt->src_addr = cpu_to_le64(dma_addr); 3855e65e175bSOded Gabbay new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr); 3856e65e175bSOded Gabbay } else { 3857e65e175bSOded Gabbay new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr); 3858e65e175bSOded Gabbay new_dma_pkt->dst_addr = cpu_to_le64(dma_addr); 3859e65e175bSOded Gabbay } 3860e65e175bSOded Gabbay 3861e65e175bSOded Gabbay if (!user_memset) 3862e65e175bSOded Gabbay device_memory_addr += len; 3863e65e175bSOded Gabbay dma_desc_cnt++; 3864e65e175bSOded Gabbay new_dma_pkt++; 3865e65e175bSOded Gabbay } 3866e65e175bSOded Gabbay 3867e65e175bSOded Gabbay if (!dma_desc_cnt) { 3868e65e175bSOded Gabbay dev_err(hdev->dev, 3869e65e175bSOded Gabbay "Error of 0 SG entries when patching DMA packet\n"); 3870e65e175bSOded Gabbay return -EFAULT; 3871e65e175bSOded Gabbay } 3872e65e175bSOded Gabbay 3873e65e175bSOded Gabbay /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */ 3874e65e175bSOded Gabbay new_dma_pkt--; 3875e65e175bSOded Gabbay new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask); 3876e65e175bSOded Gabbay 3877e65e175bSOded Gabbay *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma); 3878e65e175bSOded Gabbay 3879e65e175bSOded Gabbay return 0; 3880e65e175bSOded Gabbay } 3881e65e175bSOded Gabbay 3882e65e175bSOded Gabbay static int goya_patch_cb(struct hl_device *hdev, 3883e65e175bSOded Gabbay struct hl_cs_parser *parser) 3884e65e175bSOded Gabbay { 3885e65e175bSOded Gabbay u32 cb_parsed_length = 0; 3886e65e175bSOded Gabbay u32 cb_patched_cur_length = 0; 3887e65e175bSOded Gabbay int rc = 0; 3888e65e175bSOded Gabbay 3889e65e175bSOded Gabbay /* cb_user_size is more than 0 so loop will always be executed */ 3890e65e175bSOded Gabbay while (cb_parsed_length < parser->user_cb_size) { 3891e65e175bSOded Gabbay enum packet_id pkt_id; 3892e65e175bSOded Gabbay u16 pkt_size; 3893e65e175bSOded Gabbay u32 new_pkt_size = 0; 3894e65e175bSOded Gabbay struct goya_packet *user_pkt, *kernel_pkt; 3895e65e175bSOded Gabbay 3896e65e175bSOded Gabbay user_pkt = parser->user_cb->kernel_address + cb_parsed_length; 3897e65e175bSOded Gabbay kernel_pkt = parser->patched_cb->kernel_address + 3898e65e175bSOded Gabbay cb_patched_cur_length; 3899e65e175bSOded Gabbay 3900e65e175bSOded Gabbay pkt_id = (enum packet_id) ( 3901e65e175bSOded Gabbay (le64_to_cpu(user_pkt->header) & 3902e65e175bSOded Gabbay PACKET_HEADER_PACKET_ID_MASK) >> 3903e65e175bSOded Gabbay PACKET_HEADER_PACKET_ID_SHIFT); 3904e65e175bSOded Gabbay 3905e65e175bSOded Gabbay if (!validate_packet_id(pkt_id)) { 3906e65e175bSOded Gabbay dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); 3907e65e175bSOded Gabbay rc = -EINVAL; 3908e65e175bSOded Gabbay break; 3909e65e175bSOded Gabbay } 3910e65e175bSOded Gabbay 3911e65e175bSOded Gabbay pkt_size = goya_packet_sizes[pkt_id]; 3912e65e175bSOded Gabbay cb_parsed_length += pkt_size; 3913e65e175bSOded Gabbay if (cb_parsed_length > parser->user_cb_size) { 3914e65e175bSOded Gabbay dev_err(hdev->dev, 3915e65e175bSOded Gabbay "packet 0x%x is out of CB boundary\n", pkt_id); 3916e65e175bSOded Gabbay rc = -EINVAL; 3917e65e175bSOded Gabbay break; 3918e65e175bSOded Gabbay } 3919e65e175bSOded Gabbay 3920e65e175bSOded Gabbay switch (pkt_id) { 3921e65e175bSOded Gabbay case PACKET_LIN_DMA: 3922e65e175bSOded Gabbay rc = goya_patch_dma_packet(hdev, parser, 3923e65e175bSOded Gabbay (struct packet_lin_dma *) user_pkt, 3924e65e175bSOded Gabbay (struct packet_lin_dma *) kernel_pkt, 3925e65e175bSOded Gabbay &new_pkt_size); 3926e65e175bSOded Gabbay cb_patched_cur_length += new_pkt_size; 3927e65e175bSOded Gabbay break; 3928e65e175bSOded Gabbay 3929e65e175bSOded Gabbay case PACKET_WREG_32: 3930e65e175bSOded Gabbay memcpy(kernel_pkt, user_pkt, pkt_size); 3931e65e175bSOded Gabbay cb_patched_cur_length += pkt_size; 3932e65e175bSOded Gabbay rc = goya_validate_wreg32(hdev, parser, 3933e65e175bSOded Gabbay (struct packet_wreg32 *) kernel_pkt); 3934e65e175bSOded Gabbay break; 3935e65e175bSOded Gabbay 3936e65e175bSOded Gabbay case PACKET_WREG_BULK: 3937e65e175bSOded Gabbay dev_err(hdev->dev, 3938e65e175bSOded Gabbay "User not allowed to use WREG_BULK\n"); 3939e65e175bSOded Gabbay rc = -EPERM; 3940e65e175bSOded Gabbay break; 3941e65e175bSOded Gabbay 3942e65e175bSOded Gabbay case PACKET_MSG_PROT: 3943e65e175bSOded Gabbay dev_err(hdev->dev, 3944e65e175bSOded Gabbay "User not allowed to use MSG_PROT\n"); 3945e65e175bSOded Gabbay rc = -EPERM; 3946e65e175bSOded Gabbay break; 3947e65e175bSOded Gabbay 3948e65e175bSOded Gabbay case PACKET_CP_DMA: 3949e65e175bSOded Gabbay dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); 3950e65e175bSOded Gabbay rc = -EPERM; 3951e65e175bSOded Gabbay break; 3952e65e175bSOded Gabbay 3953e65e175bSOded Gabbay case PACKET_STOP: 3954e65e175bSOded Gabbay dev_err(hdev->dev, "User not allowed to use STOP\n"); 3955e65e175bSOded Gabbay rc = -EPERM; 3956e65e175bSOded Gabbay break; 3957e65e175bSOded Gabbay 3958e65e175bSOded Gabbay case PACKET_MSG_LONG: 3959e65e175bSOded Gabbay case PACKET_MSG_SHORT: 3960e65e175bSOded Gabbay case PACKET_FENCE: 3961e65e175bSOded Gabbay case PACKET_NOP: 3962e65e175bSOded Gabbay memcpy(kernel_pkt, user_pkt, pkt_size); 3963e65e175bSOded Gabbay cb_patched_cur_length += pkt_size; 3964e65e175bSOded Gabbay break; 3965e65e175bSOded Gabbay 3966e65e175bSOded Gabbay default: 3967e65e175bSOded Gabbay dev_err(hdev->dev, "Invalid packet header 0x%x\n", 3968e65e175bSOded Gabbay pkt_id); 3969e65e175bSOded Gabbay rc = -EINVAL; 3970e65e175bSOded Gabbay break; 3971e65e175bSOded Gabbay } 3972e65e175bSOded Gabbay 3973e65e175bSOded Gabbay if (rc) 3974e65e175bSOded Gabbay break; 3975e65e175bSOded Gabbay } 3976e65e175bSOded Gabbay 3977e65e175bSOded Gabbay return rc; 3978e65e175bSOded Gabbay } 3979e65e175bSOded Gabbay 3980e65e175bSOded Gabbay static int goya_parse_cb_mmu(struct hl_device *hdev, 3981e65e175bSOded Gabbay struct hl_cs_parser *parser) 3982e65e175bSOded Gabbay { 3983e65e175bSOded Gabbay u64 handle; 3984e65e175bSOded Gabbay u32 patched_cb_size; 3985e65e175bSOded Gabbay struct hl_cb *user_cb; 3986e65e175bSOded Gabbay int rc; 3987e65e175bSOded Gabbay 3988e65e175bSOded Gabbay /* 3989e65e175bSOded Gabbay * The new CB should have space at the end for two MSG_PROT pkt: 3990e65e175bSOded Gabbay * 1. A packet that will act as a completion packet 3991e65e175bSOded Gabbay * 2. A packet that will generate MSI-X interrupt 3992e65e175bSOded Gabbay */ 3993e65e175bSOded Gabbay parser->patched_cb_size = parser->user_cb_size + 3994e65e175bSOded Gabbay sizeof(struct packet_msg_prot) * 2; 3995e65e175bSOded Gabbay 3996e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, 3997e65e175bSOded Gabbay parser->patched_cb_size, false, false, 3998e65e175bSOded Gabbay &handle); 3999e65e175bSOded Gabbay 4000e65e175bSOded Gabbay if (rc) { 4001e65e175bSOded Gabbay dev_err(hdev->dev, 4002e65e175bSOded Gabbay "Failed to allocate patched CB for DMA CS %d\n", 4003e65e175bSOded Gabbay rc); 4004e65e175bSOded Gabbay return rc; 4005e65e175bSOded Gabbay } 4006e65e175bSOded Gabbay 4007e65e175bSOded Gabbay parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); 4008e65e175bSOded Gabbay /* hl_cb_get should never fail here */ 4009e65e175bSOded Gabbay if (!parser->patched_cb) { 4010e65e175bSOded Gabbay dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); 4011e65e175bSOded Gabbay rc = -EFAULT; 4012e65e175bSOded Gabbay goto out; 4013e65e175bSOded Gabbay } 4014e65e175bSOded Gabbay 4015e65e175bSOded Gabbay /* 4016e65e175bSOded Gabbay * The check that parser->user_cb_size <= parser->user_cb->size was done 4017e65e175bSOded Gabbay * in validate_queue_index(). 4018e65e175bSOded Gabbay */ 4019e65e175bSOded Gabbay memcpy(parser->patched_cb->kernel_address, 4020e65e175bSOded Gabbay parser->user_cb->kernel_address, 4021e65e175bSOded Gabbay parser->user_cb_size); 4022e65e175bSOded Gabbay 4023e65e175bSOded Gabbay patched_cb_size = parser->patched_cb_size; 4024e65e175bSOded Gabbay 4025e65e175bSOded Gabbay /* validate patched CB instead of user CB */ 4026e65e175bSOded Gabbay user_cb = parser->user_cb; 4027e65e175bSOded Gabbay parser->user_cb = parser->patched_cb; 4028e65e175bSOded Gabbay rc = goya_validate_cb(hdev, parser, true); 4029e65e175bSOded Gabbay parser->user_cb = user_cb; 4030e65e175bSOded Gabbay 4031e65e175bSOded Gabbay if (rc) { 4032e65e175bSOded Gabbay hl_cb_put(parser->patched_cb); 4033e65e175bSOded Gabbay goto out; 4034e65e175bSOded Gabbay } 4035e65e175bSOded Gabbay 4036e65e175bSOded Gabbay if (patched_cb_size != parser->patched_cb_size) { 4037e65e175bSOded Gabbay dev_err(hdev->dev, "user CB size mismatch\n"); 4038e65e175bSOded Gabbay hl_cb_put(parser->patched_cb); 4039e65e175bSOded Gabbay rc = -EINVAL; 4040e65e175bSOded Gabbay goto out; 4041e65e175bSOded Gabbay } 4042e65e175bSOded Gabbay 4043e65e175bSOded Gabbay out: 4044e65e175bSOded Gabbay /* 4045e65e175bSOded Gabbay * Always call cb destroy here because we still have 1 reference 4046e65e175bSOded Gabbay * to it by calling cb_get earlier. After the job will be completed, 4047e65e175bSOded Gabbay * cb_put will release it, but here we want to remove it from the 4048e65e175bSOded Gabbay * idr 4049e65e175bSOded Gabbay */ 4050e65e175bSOded Gabbay hl_cb_destroy(&hdev->kernel_mem_mgr, handle); 4051e65e175bSOded Gabbay 4052e65e175bSOded Gabbay return rc; 4053e65e175bSOded Gabbay } 4054e65e175bSOded Gabbay 4055e65e175bSOded Gabbay static int goya_parse_cb_no_mmu(struct hl_device *hdev, 4056e65e175bSOded Gabbay struct hl_cs_parser *parser) 4057e65e175bSOded Gabbay { 4058e65e175bSOded Gabbay u64 handle; 4059e65e175bSOded Gabbay int rc; 4060e65e175bSOded Gabbay 4061e65e175bSOded Gabbay rc = goya_validate_cb(hdev, parser, false); 4062e65e175bSOded Gabbay 4063e65e175bSOded Gabbay if (rc) 4064e65e175bSOded Gabbay goto free_userptr; 4065e65e175bSOded Gabbay 4066e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, 4067e65e175bSOded Gabbay parser->patched_cb_size, false, false, 4068e65e175bSOded Gabbay &handle); 4069e65e175bSOded Gabbay if (rc) { 4070e65e175bSOded Gabbay dev_err(hdev->dev, 4071e65e175bSOded Gabbay "Failed to allocate patched CB for DMA CS %d\n", rc); 4072e65e175bSOded Gabbay goto free_userptr; 4073e65e175bSOded Gabbay } 4074e65e175bSOded Gabbay 4075e65e175bSOded Gabbay parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); 4076e65e175bSOded Gabbay /* hl_cb_get should never fail here */ 4077e65e175bSOded Gabbay if (!parser->patched_cb) { 4078e65e175bSOded Gabbay dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); 4079e65e175bSOded Gabbay rc = -EFAULT; 4080e65e175bSOded Gabbay goto out; 4081e65e175bSOded Gabbay } 4082e65e175bSOded Gabbay 4083e65e175bSOded Gabbay rc = goya_patch_cb(hdev, parser); 4084e65e175bSOded Gabbay 4085e65e175bSOded Gabbay if (rc) 4086e65e175bSOded Gabbay hl_cb_put(parser->patched_cb); 4087e65e175bSOded Gabbay 4088e65e175bSOded Gabbay out: 4089e65e175bSOded Gabbay /* 4090e65e175bSOded Gabbay * Always call cb destroy here because we still have 1 reference 4091e65e175bSOded Gabbay * to it by calling cb_get earlier. After the job will be completed, 4092e65e175bSOded Gabbay * cb_put will release it, but here we want to remove it from the 4093e65e175bSOded Gabbay * idr 4094e65e175bSOded Gabbay */ 4095e65e175bSOded Gabbay hl_cb_destroy(&hdev->kernel_mem_mgr, handle); 4096e65e175bSOded Gabbay 4097e65e175bSOded Gabbay free_userptr: 4098e65e175bSOded Gabbay if (rc) 4099e65e175bSOded Gabbay hl_userptr_delete_list(hdev, parser->job_userptr_list); 4100e65e175bSOded Gabbay return rc; 4101e65e175bSOded Gabbay } 4102e65e175bSOded Gabbay 4103e65e175bSOded Gabbay static int goya_parse_cb_no_ext_queue(struct hl_device *hdev, 4104e65e175bSOded Gabbay struct hl_cs_parser *parser) 4105e65e175bSOded Gabbay { 4106e65e175bSOded Gabbay struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 4107e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4108e65e175bSOded Gabbay 4109e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MMU) 4110e65e175bSOded Gabbay return 0; 4111e65e175bSOded Gabbay 4112e65e175bSOded Gabbay /* For internal queue jobs, just check if CB address is valid */ 4113e65e175bSOded Gabbay if (hl_mem_area_inside_range( 4114e65e175bSOded Gabbay (u64) (uintptr_t) parser->user_cb, 4115e65e175bSOded Gabbay parser->user_cb_size, 4116e65e175bSOded Gabbay asic_prop->sram_user_base_address, 4117e65e175bSOded Gabbay asic_prop->sram_end_address)) 4118e65e175bSOded Gabbay return 0; 4119e65e175bSOded Gabbay 4120e65e175bSOded Gabbay if (hl_mem_area_inside_range( 4121e65e175bSOded Gabbay (u64) (uintptr_t) parser->user_cb, 4122e65e175bSOded Gabbay parser->user_cb_size, 4123e65e175bSOded Gabbay asic_prop->dram_user_base_address, 4124e65e175bSOded Gabbay asic_prop->dram_end_address)) 4125e65e175bSOded Gabbay return 0; 4126e65e175bSOded Gabbay 4127e65e175bSOded Gabbay dev_err(hdev->dev, 4128e65e175bSOded Gabbay "Internal CB address 0x%px + 0x%x is not in SRAM nor in DRAM\n", 4129e65e175bSOded Gabbay parser->user_cb, parser->user_cb_size); 4130e65e175bSOded Gabbay 4131e65e175bSOded Gabbay return -EFAULT; 4132e65e175bSOded Gabbay } 4133e65e175bSOded Gabbay 4134e65e175bSOded Gabbay int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 4135e65e175bSOded Gabbay { 4136e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4137e65e175bSOded Gabbay 4138e65e175bSOded Gabbay if (parser->queue_type == QUEUE_TYPE_INT) 4139e65e175bSOded Gabbay return goya_parse_cb_no_ext_queue(hdev, parser); 4140e65e175bSOded Gabbay 4141e65e175bSOded Gabbay if (goya->hw_cap_initialized & HW_CAP_MMU) 4142e65e175bSOded Gabbay return goya_parse_cb_mmu(hdev, parser); 4143e65e175bSOded Gabbay else 4144e65e175bSOded Gabbay return goya_parse_cb_no_mmu(hdev, parser); 4145e65e175bSOded Gabbay } 4146e65e175bSOded Gabbay 4147e65e175bSOded Gabbay void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address, 4148e65e175bSOded Gabbay u32 len, u32 original_len, u64 cq_addr, u32 cq_val, 4149e65e175bSOded Gabbay u32 msix_vec, bool eb) 4150e65e175bSOded Gabbay { 4151e65e175bSOded Gabbay struct packet_msg_prot *cq_pkt; 4152e65e175bSOded Gabbay u32 tmp; 4153e65e175bSOded Gabbay 4154e65e175bSOded Gabbay cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2); 4155e65e175bSOded Gabbay 4156e65e175bSOded Gabbay tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) | 4157e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_EB_SHIFT) | 4158e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_MB_SHIFT); 4159e65e175bSOded Gabbay cq_pkt->ctl = cpu_to_le32(tmp); 4160e65e175bSOded Gabbay cq_pkt->value = cpu_to_le32(cq_val); 4161e65e175bSOded Gabbay cq_pkt->addr = cpu_to_le64(cq_addr); 4162e65e175bSOded Gabbay 4163e65e175bSOded Gabbay cq_pkt++; 4164e65e175bSOded Gabbay 4165e65e175bSOded Gabbay tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) | 4166e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_MB_SHIFT); 4167e65e175bSOded Gabbay cq_pkt->ctl = cpu_to_le32(tmp); 4168e65e175bSOded Gabbay cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF); 4169e65e175bSOded Gabbay cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF); 4170e65e175bSOded Gabbay } 4171e65e175bSOded Gabbay 4172e65e175bSOded Gabbay void goya_update_eq_ci(struct hl_device *hdev, u32 val) 4173e65e175bSOded Gabbay { 4174e65e175bSOded Gabbay WREG32(mmCPU_EQ_CI, val); 4175e65e175bSOded Gabbay } 4176e65e175bSOded Gabbay 4177e65e175bSOded Gabbay void goya_restore_phase_topology(struct hl_device *hdev) 4178e65e175bSOded Gabbay { 4179e65e175bSOded Gabbay 4180e65e175bSOded Gabbay } 4181e65e175bSOded Gabbay 4182e65e175bSOded Gabbay static void goya_clear_sm_regs(struct hl_device *hdev) 4183e65e175bSOded Gabbay { 4184e65e175bSOded Gabbay int i, num_of_sob_in_longs, num_of_mon_in_longs; 4185e65e175bSOded Gabbay 4186e65e175bSOded Gabbay num_of_sob_in_longs = 4187e65e175bSOded Gabbay ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4); 4188e65e175bSOded Gabbay 4189e65e175bSOded Gabbay num_of_mon_in_longs = 4190e65e175bSOded Gabbay ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4); 4191e65e175bSOded Gabbay 4192e65e175bSOded Gabbay for (i = 0 ; i < num_of_sob_in_longs ; i += 4) 4193e65e175bSOded Gabbay WREG32(mmSYNC_MNGR_SOB_OBJ_0 + i, 0); 4194e65e175bSOded Gabbay 4195e65e175bSOded Gabbay for (i = 0 ; i < num_of_mon_in_longs ; i += 4) 4196e65e175bSOded Gabbay WREG32(mmSYNC_MNGR_MON_STATUS_0 + i, 0); 4197e65e175bSOded Gabbay 4198e65e175bSOded Gabbay /* Flush all WREG to prevent race */ 4199e65e175bSOded Gabbay i = RREG32(mmSYNC_MNGR_SOB_OBJ_0); 4200e65e175bSOded Gabbay } 4201e65e175bSOded Gabbay 4202e65e175bSOded Gabbay static int goya_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) 4203e65e175bSOded Gabbay { 4204e65e175bSOded Gabbay dev_err(hdev->dev, "Reading via DMA is unimplemented yet\n"); 4205e65e175bSOded Gabbay return -EPERM; 4206e65e175bSOded Gabbay } 4207e65e175bSOded Gabbay 4208e65e175bSOded Gabbay static u64 goya_read_pte(struct hl_device *hdev, u64 addr) 4209e65e175bSOded Gabbay { 4210e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4211e65e175bSOded Gabbay 4212e65e175bSOded Gabbay if (hdev->reset_info.hard_reset_pending) 4213e65e175bSOded Gabbay return U64_MAX; 4214e65e175bSOded Gabbay 4215e65e175bSOded Gabbay return readq(hdev->pcie_bar[DDR_BAR_ID] + 4216e65e175bSOded Gabbay (addr - goya->ddr_bar_cur_addr)); 4217e65e175bSOded Gabbay } 4218e65e175bSOded Gabbay 4219e65e175bSOded Gabbay static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val) 4220e65e175bSOded Gabbay { 4221e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4222e65e175bSOded Gabbay 4223e65e175bSOded Gabbay if (hdev->reset_info.hard_reset_pending) 4224e65e175bSOded Gabbay return; 4225e65e175bSOded Gabbay 4226e65e175bSOded Gabbay writeq(val, hdev->pcie_bar[DDR_BAR_ID] + 4227e65e175bSOded Gabbay (addr - goya->ddr_bar_cur_addr)); 4228e65e175bSOded Gabbay } 4229e65e175bSOded Gabbay 4230e65e175bSOded Gabbay static const char *_goya_get_event_desc(u16 event_type) 4231e65e175bSOded Gabbay { 4232e65e175bSOded Gabbay switch (event_type) { 4233e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PCIE_IF: 4234e65e175bSOded Gabbay return "PCIe_if"; 4235e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_ECC: 4236e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_ECC: 4237e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_ECC: 4238e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_ECC: 4239e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_ECC: 4240e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_ECC: 4241e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_ECC: 4242e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_ECC: 4243e65e175bSOded Gabbay return "TPC%d_ecc"; 4244e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_ECC: 4245e65e175bSOded Gabbay return "MME_ecc"; 4246e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT: 4247e65e175bSOded Gabbay return "MME_ecc_ext"; 4248e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MMU_ECC: 4249e65e175bSOded Gabbay return "MMU_ecc"; 4250e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_MACRO: 4251e65e175bSOded Gabbay return "DMA_macro"; 4252e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_ECC: 4253e65e175bSOded Gabbay return "DMA_ecc"; 4254e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC: 4255e65e175bSOded Gabbay return "CPU_if_ecc"; 4256e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_MEM: 4257e65e175bSOded Gabbay return "PSOC_mem"; 4258e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT: 4259e65e175bSOded Gabbay return "PSOC_coresight"; 4260e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29: 4261e65e175bSOded Gabbay return "SRAM%d"; 4262e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_GIC500: 4263e65e175bSOded Gabbay return "GIC500"; 4264e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6: 4265e65e175bSOded Gabbay return "PLL%d"; 4266e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_AXI_ECC: 4267e65e175bSOded Gabbay return "AXI_ecc"; 4268e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC: 4269e65e175bSOded Gabbay return "L2_ram_ecc"; 4270e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET: 4271e65e175bSOded Gabbay return "PSOC_gpio_05_sw_reset"; 4272e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT: 4273e65e175bSOded Gabbay return "PSOC_gpio_10_vrhot_icrit"; 4274e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PCIE_DEC: 4275e65e175bSOded Gabbay return "PCIe_dec"; 4276e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_DEC: 4277e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_DEC: 4278e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_DEC: 4279e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_DEC: 4280e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_DEC: 4281e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_DEC: 4282e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_DEC: 4283e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_DEC: 4284e65e175bSOded Gabbay return "TPC%d_dec"; 4285e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_WACS: 4286e65e175bSOded Gabbay return "MME_wacs"; 4287e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_WACSD: 4288e65e175bSOded Gabbay return "MME_wacsd"; 4289e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER: 4290e65e175bSOded Gabbay return "CPU_axi_splitter"; 4291e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC: 4292e65e175bSOded Gabbay return "PSOC_axi_dec"; 4293e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC: 4294e65e175bSOded Gabbay return "PSOC"; 4295e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR: 4296e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR: 4297e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR: 4298e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR: 4299e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR: 4300e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR: 4301e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR: 4302e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR: 4303e65e175bSOded Gabbay return "TPC%d_krn_err"; 4304e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ: 4305e65e175bSOded Gabbay return "TPC%d_cq"; 4306e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM: 4307e65e175bSOded Gabbay return "TPC%d_qm"; 4308e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_QM: 4309e65e175bSOded Gabbay return "MME_qm"; 4310e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_CMDQ: 4311e65e175bSOded Gabbay return "MME_cq"; 4312e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM: 4313e65e175bSOded Gabbay return "DMA%d_qm"; 4314e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH: 4315e65e175bSOded Gabbay return "DMA%d_ch"; 4316e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU: 4317e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU: 4318e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU: 4319e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU: 4320e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU: 4321e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU: 4322e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU: 4323e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU: 4324e65e175bSOded Gabbay return "TPC%d_bmon_spmu"; 4325e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: 4326e65e175bSOded Gabbay return "DMA_bm_ch%d"; 4327e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S: 4328e65e175bSOded Gabbay return "POWER_ENV_S"; 4329e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E: 4330e65e175bSOded Gabbay return "POWER_ENV_E"; 4331e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S: 4332e65e175bSOded Gabbay return "THERMAL_ENV_S"; 4333e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E: 4334e65e175bSOded Gabbay return "THERMAL_ENV_E"; 4335e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC: 4336e65e175bSOded Gabbay return "QUEUE_OUT_OF_SYNC"; 4337e65e175bSOded Gabbay default: 4338e65e175bSOded Gabbay return "N/A"; 4339e65e175bSOded Gabbay } 4340e65e175bSOded Gabbay } 4341e65e175bSOded Gabbay 4342e65e175bSOded Gabbay static void goya_get_event_desc(u16 event_type, char *desc, size_t size) 4343e65e175bSOded Gabbay { 4344e65e175bSOded Gabbay u8 index; 4345e65e175bSOded Gabbay 4346e65e175bSOded Gabbay switch (event_type) { 4347e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_ECC: 4348e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_ECC: 4349e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_ECC: 4350e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_ECC: 4351e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_ECC: 4352e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_ECC: 4353e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_ECC: 4354e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_ECC: 4355e65e175bSOded Gabbay index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3; 4356e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4357e65e175bSOded Gabbay break; 4358e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29: 4359e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0; 4360e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4361e65e175bSOded Gabbay break; 4362e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6: 4363e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_PLL0; 4364e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4365e65e175bSOded Gabbay break; 4366e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_DEC: 4367e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_DEC: 4368e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_DEC: 4369e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_DEC: 4370e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_DEC: 4371e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_DEC: 4372e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_DEC: 4373e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_DEC: 4374e65e175bSOded Gabbay index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3; 4375e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4376e65e175bSOded Gabbay break; 4377e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR: 4378e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR: 4379e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR: 4380e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR: 4381e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR: 4382e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR: 4383e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR: 4384e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR: 4385e65e175bSOded Gabbay index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10; 4386e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4387e65e175bSOded Gabbay break; 4388e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ: 4389e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ; 4390e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4391e65e175bSOded Gabbay break; 4392e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM: 4393e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM; 4394e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4395e65e175bSOded Gabbay break; 4396e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM: 4397e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM; 4398e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4399e65e175bSOded Gabbay break; 4400e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH: 4401e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH; 4402e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4403e65e175bSOded Gabbay break; 4404e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU: 4405e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU: 4406e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU: 4407e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU: 4408e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU: 4409e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU: 4410e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU: 4411e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU: 4412e65e175bSOded Gabbay index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10; 4413e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4414e65e175bSOded Gabbay break; 4415e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: 4416e65e175bSOded Gabbay index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0; 4417e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type), index); 4418e65e175bSOded Gabbay break; 4419e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC: 4420e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type)); 4421e65e175bSOded Gabbay break; 4422e65e175bSOded Gabbay default: 4423e65e175bSOded Gabbay snprintf(desc, size, _goya_get_event_desc(event_type)); 4424e65e175bSOded Gabbay break; 4425e65e175bSOded Gabbay } 4426e65e175bSOded Gabbay } 4427e65e175bSOded Gabbay 4428e65e175bSOded Gabbay static void goya_print_razwi_info(struct hl_device *hdev) 4429e65e175bSOded Gabbay { 4430e65e175bSOded Gabbay if (RREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD)) { 4431e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n"); 4432e65e175bSOded Gabbay WREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD, 0); 4433e65e175bSOded Gabbay } 4434e65e175bSOded Gabbay 4435e65e175bSOded Gabbay if (RREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD)) { 4436e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n"); 4437e65e175bSOded Gabbay WREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD, 0); 4438e65e175bSOded Gabbay } 4439e65e175bSOded Gabbay 4440e65e175bSOded Gabbay if (RREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD)) { 4441e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n"); 4442e65e175bSOded Gabbay WREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD, 0); 4443e65e175bSOded Gabbay } 4444e65e175bSOded Gabbay 4445e65e175bSOded Gabbay if (RREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD)) { 4446e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n"); 4447e65e175bSOded Gabbay WREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD, 0); 4448e65e175bSOded Gabbay } 4449e65e175bSOded Gabbay } 4450e65e175bSOded Gabbay 4451e65e175bSOded Gabbay static void goya_print_mmu_error_info(struct hl_device *hdev) 4452e65e175bSOded Gabbay { 4453e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4454e65e175bSOded Gabbay u64 addr; 4455e65e175bSOded Gabbay u32 val; 4456e65e175bSOded Gabbay 4457e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 4458e65e175bSOded Gabbay return; 4459e65e175bSOded Gabbay 4460e65e175bSOded Gabbay val = RREG32(mmMMU_PAGE_ERROR_CAPTURE); 4461e65e175bSOded Gabbay if (val & MMU_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) { 4462e65e175bSOded Gabbay addr = val & MMU_PAGE_ERROR_CAPTURE_VA_49_32_MASK; 4463e65e175bSOded Gabbay addr <<= 32; 4464e65e175bSOded Gabbay addr |= RREG32(mmMMU_PAGE_ERROR_CAPTURE_VA); 4465e65e175bSOded Gabbay 4466e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", 4467e65e175bSOded Gabbay addr); 4468e65e175bSOded Gabbay 4469e65e175bSOded Gabbay WREG32(mmMMU_PAGE_ERROR_CAPTURE, 0); 4470e65e175bSOded Gabbay } 4471e65e175bSOded Gabbay } 4472e65e175bSOded Gabbay 4473e65e175bSOded Gabbay static void goya_print_out_of_sync_info(struct hl_device *hdev, 4474e65e175bSOded Gabbay struct cpucp_pkt_sync_err *sync_err) 4475e65e175bSOded Gabbay { 4476e65e175bSOded Gabbay struct hl_hw_queue *q = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; 4477e65e175bSOded Gabbay 4478e65e175bSOded Gabbay dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n", 4479e65e175bSOded Gabbay le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 4480e65e175bSOded Gabbay } 4481e65e175bSOded Gabbay 4482e65e175bSOded Gabbay static void goya_print_irq_info(struct hl_device *hdev, u16 event_type, 4483e65e175bSOded Gabbay bool razwi) 4484e65e175bSOded Gabbay { 4485e65e175bSOded Gabbay char desc[20] = ""; 4486e65e175bSOded Gabbay 4487e65e175bSOded Gabbay goya_get_event_desc(event_type, desc, sizeof(desc)); 4488e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", 4489e65e175bSOded Gabbay event_type, desc); 4490e65e175bSOded Gabbay 4491e65e175bSOded Gabbay if (razwi) { 4492e65e175bSOded Gabbay goya_print_razwi_info(hdev); 4493e65e175bSOded Gabbay goya_print_mmu_error_info(hdev); 4494e65e175bSOded Gabbay } 4495e65e175bSOded Gabbay } 4496e65e175bSOded Gabbay 4497e65e175bSOded Gabbay static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, 4498e65e175bSOded Gabbay size_t irq_arr_size) 4499e65e175bSOded Gabbay { 4500e65e175bSOded Gabbay struct cpucp_unmask_irq_arr_packet *pkt; 4501e65e175bSOded Gabbay size_t total_pkt_size; 4502e65e175bSOded Gabbay u64 result; 4503e65e175bSOded Gabbay int rc; 4504e65e175bSOded Gabbay int irq_num_entries, irq_arr_index; 4505e65e175bSOded Gabbay __le32 *goya_irq_arr; 4506e65e175bSOded Gabbay 4507e65e175bSOded Gabbay total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) + 4508e65e175bSOded Gabbay irq_arr_size; 4509e65e175bSOded Gabbay 4510e65e175bSOded Gabbay /* data should be aligned to 8 bytes in order to CPU-CP to copy it */ 4511e65e175bSOded Gabbay total_pkt_size = (total_pkt_size + 0x7) & ~0x7; 4512e65e175bSOded Gabbay 4513e65e175bSOded Gabbay /* total_pkt_size is casted to u16 later on */ 4514e65e175bSOded Gabbay if (total_pkt_size > USHRT_MAX) { 4515e65e175bSOded Gabbay dev_err(hdev->dev, "too many elements in IRQ array\n"); 4516e65e175bSOded Gabbay return -EINVAL; 4517e65e175bSOded Gabbay } 4518e65e175bSOded Gabbay 4519e65e175bSOded Gabbay pkt = kzalloc(total_pkt_size, GFP_KERNEL); 4520e65e175bSOded Gabbay if (!pkt) 4521e65e175bSOded Gabbay return -ENOMEM; 4522e65e175bSOded Gabbay 4523e65e175bSOded Gabbay irq_num_entries = irq_arr_size / sizeof(irq_arr[0]); 4524e65e175bSOded Gabbay pkt->length = cpu_to_le32(irq_num_entries); 4525e65e175bSOded Gabbay 4526e65e175bSOded Gabbay /* We must perform any necessary endianness conversation on the irq 4527e65e175bSOded Gabbay * array being passed to the goya hardware 4528e65e175bSOded Gabbay */ 4529e65e175bSOded Gabbay for (irq_arr_index = 0, goya_irq_arr = (__le32 *) &pkt->irqs; 4530e65e175bSOded Gabbay irq_arr_index < irq_num_entries ; irq_arr_index++) 4531e65e175bSOded Gabbay goya_irq_arr[irq_arr_index] = 4532e65e175bSOded Gabbay cpu_to_le32(irq_arr[irq_arr_index]); 4533e65e175bSOded Gabbay 4534e65e175bSOded Gabbay pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY << 4535e65e175bSOded Gabbay CPUCP_PKT_CTL_OPCODE_SHIFT); 4536e65e175bSOded Gabbay 4537e65e175bSOded Gabbay rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt, 4538e65e175bSOded Gabbay total_pkt_size, 0, &result); 4539e65e175bSOded Gabbay 4540e65e175bSOded Gabbay if (rc) 4541e65e175bSOded Gabbay dev_err(hdev->dev, "failed to unmask IRQ array\n"); 4542e65e175bSOded Gabbay 4543e65e175bSOded Gabbay kfree(pkt); 4544e65e175bSOded Gabbay 4545e65e175bSOded Gabbay return rc; 4546e65e175bSOded Gabbay } 4547e65e175bSOded Gabbay 4548e65e175bSOded Gabbay static int goya_compute_reset_late_init(struct hl_device *hdev) 4549e65e175bSOded Gabbay { 4550e65e175bSOded Gabbay /* 4551e65e175bSOded Gabbay * Unmask all IRQs since some could have been received 4552e65e175bSOded Gabbay * during the soft reset 4553e65e175bSOded Gabbay */ 4554e65e175bSOded Gabbay return goya_unmask_irq_arr(hdev, goya_all_events, 4555e65e175bSOded Gabbay sizeof(goya_all_events)); 4556e65e175bSOded Gabbay } 4557e65e175bSOded Gabbay 4558e65e175bSOded Gabbay static int goya_unmask_irq(struct hl_device *hdev, u16 event_type) 4559e65e175bSOded Gabbay { 4560e65e175bSOded Gabbay struct cpucp_packet pkt; 4561e65e175bSOded Gabbay u64 result; 4562e65e175bSOded Gabbay int rc; 4563e65e175bSOded Gabbay 4564e65e175bSOded Gabbay memset(&pkt, 0, sizeof(pkt)); 4565e65e175bSOded Gabbay 4566e65e175bSOded Gabbay pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ << 4567e65e175bSOded Gabbay CPUCP_PKT_CTL_OPCODE_SHIFT); 4568e65e175bSOded Gabbay pkt.value = cpu_to_le64(event_type); 4569e65e175bSOded Gabbay 4570e65e175bSOded Gabbay rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 4571e65e175bSOded Gabbay 0, &result); 4572e65e175bSOded Gabbay 4573e65e175bSOded Gabbay if (rc) 4574e65e175bSOded Gabbay dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); 4575e65e175bSOded Gabbay 4576e65e175bSOded Gabbay return rc; 4577e65e175bSOded Gabbay } 4578e65e175bSOded Gabbay 4579e65e175bSOded Gabbay static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type) 4580e65e175bSOded Gabbay { 4581e65e175bSOded Gabbay ktime_t zero_time = ktime_set(0, 0); 4582e65e175bSOded Gabbay 4583e65e175bSOded Gabbay mutex_lock(&hdev->clk_throttling.lock); 4584e65e175bSOded Gabbay 4585e65e175bSOded Gabbay switch (event_type) { 4586e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S: 4587e65e175bSOded Gabbay hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 4588e65e175bSOded Gabbay hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 4589e65e175bSOded Gabbay hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 4590e65e175bSOded Gabbay hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 4591e65e175bSOded Gabbay dev_info_ratelimited(hdev->dev, 4592e65e175bSOded Gabbay "Clock throttling due to power consumption\n"); 4593e65e175bSOded Gabbay break; 4594e65e175bSOded Gabbay 4595e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E: 4596e65e175bSOded Gabbay hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 4597e65e175bSOded Gabbay hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 4598e65e175bSOded Gabbay dev_info_ratelimited(hdev->dev, 4599e65e175bSOded Gabbay "Power envelop is safe, back to optimal clock\n"); 4600e65e175bSOded Gabbay break; 4601e65e175bSOded Gabbay 4602e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S: 4603e65e175bSOded Gabbay hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 4604e65e175bSOded Gabbay hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 4605e65e175bSOded Gabbay hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 4606e65e175bSOded Gabbay hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 4607e65e175bSOded Gabbay dev_info_ratelimited(hdev->dev, 4608e65e175bSOded Gabbay "Clock throttling due to overheating\n"); 4609e65e175bSOded Gabbay break; 4610e65e175bSOded Gabbay 4611e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E: 4612e65e175bSOded Gabbay hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 4613e65e175bSOded Gabbay hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 4614e65e175bSOded Gabbay dev_info_ratelimited(hdev->dev, 4615e65e175bSOded Gabbay "Thermal envelop is safe, back to optimal clock\n"); 4616e65e175bSOded Gabbay break; 4617e65e175bSOded Gabbay 4618e65e175bSOded Gabbay default: 4619e65e175bSOded Gabbay dev_err(hdev->dev, "Received invalid clock change event %d\n", 4620e65e175bSOded Gabbay event_type); 4621e65e175bSOded Gabbay break; 4622e65e175bSOded Gabbay } 4623e65e175bSOded Gabbay 4624e65e175bSOded Gabbay mutex_unlock(&hdev->clk_throttling.lock); 4625e65e175bSOded Gabbay } 4626e65e175bSOded Gabbay 4627e65e175bSOded Gabbay void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 4628e65e175bSOded Gabbay { 4629e65e175bSOded Gabbay u32 ctl = le32_to_cpu(eq_entry->hdr.ctl); 4630e65e175bSOded Gabbay u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) 4631e65e175bSOded Gabbay >> EQ_CTL_EVENT_TYPE_SHIFT); 4632e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4633e65e175bSOded Gabbay 4634e65e175bSOded Gabbay if (event_type >= GOYA_ASYNC_EVENT_ID_SIZE) { 4635e65e175bSOded Gabbay dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 4636e65e175bSOded Gabbay event_type, GOYA_ASYNC_EVENT_ID_SIZE - 1); 4637e65e175bSOded Gabbay return; 4638e65e175bSOded Gabbay } 4639e65e175bSOded Gabbay 4640e65e175bSOded Gabbay goya->events_stat[event_type]++; 4641e65e175bSOded Gabbay goya->events_stat_aggregate[event_type]++; 4642e65e175bSOded Gabbay 4643e65e175bSOded Gabbay switch (event_type) { 4644e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PCIE_IF: 4645e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_ECC: 4646e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_ECC: 4647e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_ECC: 4648e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_ECC: 4649e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_ECC: 4650e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_ECC: 4651e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_ECC: 4652e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_ECC: 4653e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_ECC: 4654e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT: 4655e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MMU_ECC: 4656e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_MACRO: 4657e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_ECC: 4658e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC: 4659e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_MEM: 4660e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT: 4661e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29: 4662e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_GIC500: 4663e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6: 4664e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_AXI_ECC: 4665e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC: 4666e65e175bSOded Gabbay goya_print_irq_info(hdev, event_type, false); 4667e65e175bSOded Gabbay if (hdev->hard_reset_on_fw_events) 4668e65e175bSOded Gabbay hl_device_reset(hdev, (HL_DRV_RESET_HARD | 4669e65e175bSOded Gabbay HL_DRV_RESET_FW_FATAL_ERR)); 4670e65e175bSOded Gabbay break; 4671e65e175bSOded Gabbay 4672e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET: 4673e65e175bSOded Gabbay goya_print_irq_info(hdev, event_type, false); 4674e65e175bSOded Gabbay if (hdev->hard_reset_on_fw_events) 4675e65e175bSOded Gabbay hl_device_reset(hdev, HL_DRV_RESET_HARD); 4676e65e175bSOded Gabbay break; 4677e65e175bSOded Gabbay 4678e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PCIE_DEC: 4679e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_DEC: 4680e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_DEC: 4681e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_DEC: 4682e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_DEC: 4683e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_DEC: 4684e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_DEC: 4685e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_DEC: 4686e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_DEC: 4687e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_WACS: 4688e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_WACSD: 4689e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER: 4690e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC: 4691e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC: 4692e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR: 4693e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR: 4694e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR: 4695e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR: 4696e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR: 4697e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR: 4698e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR: 4699e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR: 4700e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_QM: 4701e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_QM: 4702e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_MME_CMDQ: 4703e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM: 4704e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH: 4705e65e175bSOded Gabbay goya_print_irq_info(hdev, event_type, true); 4706e65e175bSOded Gabbay goya_unmask_irq(hdev, event_type); 4707e65e175bSOded Gabbay break; 4708e65e175bSOded Gabbay 4709e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT: 4710e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU: 4711e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU: 4712e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU: 4713e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU: 4714e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU: 4715e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU: 4716e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU: 4717e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU: 4718e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: 4719e65e175bSOded Gabbay goya_print_irq_info(hdev, event_type, false); 4720e65e175bSOded Gabbay goya_unmask_irq(hdev, event_type); 4721e65e175bSOded Gabbay break; 4722e65e175bSOded Gabbay 4723e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S: 4724e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E: 4725e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S: 4726e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E: 4727e65e175bSOded Gabbay goya_print_clk_change_info(hdev, event_type); 4728e65e175bSOded Gabbay goya_unmask_irq(hdev, event_type); 4729e65e175bSOded Gabbay break; 4730e65e175bSOded Gabbay 4731e65e175bSOded Gabbay case GOYA_ASYNC_EVENT_PKT_QUEUE_OUT_SYNC: 4732e65e175bSOded Gabbay goya_print_irq_info(hdev, event_type, false); 4733e65e175bSOded Gabbay goya_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); 4734e65e175bSOded Gabbay if (hdev->hard_reset_on_fw_events) 4735e65e175bSOded Gabbay hl_device_reset(hdev, HL_DRV_RESET_HARD); 4736e65e175bSOded Gabbay else 4737e65e175bSOded Gabbay hl_fw_unmask_irq(hdev, event_type); 4738e65e175bSOded Gabbay break; 4739e65e175bSOded Gabbay 4740e65e175bSOded Gabbay default: 4741e65e175bSOded Gabbay dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", 4742e65e175bSOded Gabbay event_type); 4743e65e175bSOded Gabbay break; 4744e65e175bSOded Gabbay } 4745e65e175bSOded Gabbay } 4746e65e175bSOded Gabbay 4747e65e175bSOded Gabbay void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 4748e65e175bSOded Gabbay { 4749e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4750e65e175bSOded Gabbay 4751e65e175bSOded Gabbay if (aggregate) { 4752e65e175bSOded Gabbay *size = (u32) sizeof(goya->events_stat_aggregate); 4753e65e175bSOded Gabbay return goya->events_stat_aggregate; 4754e65e175bSOded Gabbay } 4755e65e175bSOded Gabbay 4756e65e175bSOded Gabbay *size = (u32) sizeof(goya->events_stat); 4757e65e175bSOded Gabbay return goya->events_stat; 4758e65e175bSOded Gabbay } 4759e65e175bSOded Gabbay 4760e65e175bSOded Gabbay static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, 4761e65e175bSOded Gabbay u64 val, bool is_dram) 4762e65e175bSOded Gabbay { 4763e65e175bSOded Gabbay struct packet_lin_dma *lin_dma_pkt; 4764e65e175bSOded Gabbay struct hl_cs_job *job; 4765e65e175bSOded Gabbay u32 cb_size, ctl; 4766e65e175bSOded Gabbay struct hl_cb *cb; 4767e65e175bSOded Gabbay int rc, lin_dma_pkts_cnt; 4768e65e175bSOded Gabbay 4769e65e175bSOded Gabbay lin_dma_pkts_cnt = DIV_ROUND_UP_ULL(size, SZ_2G); 4770e65e175bSOded Gabbay cb_size = lin_dma_pkts_cnt * sizeof(struct packet_lin_dma) + 4771e65e175bSOded Gabbay sizeof(struct packet_msg_prot); 4772e65e175bSOded Gabbay cb = hl_cb_kernel_create(hdev, cb_size, false); 4773e65e175bSOded Gabbay if (!cb) 4774e65e175bSOded Gabbay return -ENOMEM; 4775e65e175bSOded Gabbay 4776e65e175bSOded Gabbay lin_dma_pkt = cb->kernel_address; 4777e65e175bSOded Gabbay 4778e65e175bSOded Gabbay do { 4779e65e175bSOded Gabbay memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt)); 4780e65e175bSOded Gabbay 4781e65e175bSOded Gabbay ctl = ((PACKET_LIN_DMA << GOYA_PKT_CTL_OPCODE_SHIFT) | 4782e65e175bSOded Gabbay (1 << GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT) | 4783e65e175bSOded Gabbay (1 << GOYA_PKT_LIN_DMA_CTL_WO_SHIFT) | 4784e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_RB_SHIFT) | 4785e65e175bSOded Gabbay (1 << GOYA_PKT_CTL_MB_SHIFT)); 4786e65e175bSOded Gabbay ctl |= (is_dram ? HL_DMA_HOST_TO_DRAM : HL_DMA_HOST_TO_SRAM) << 4787e65e175bSOded Gabbay GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; 4788e65e175bSOded Gabbay lin_dma_pkt->ctl = cpu_to_le32(ctl); 4789e65e175bSOded Gabbay 4790e65e175bSOded Gabbay lin_dma_pkt->src_addr = cpu_to_le64(val); 4791e65e175bSOded Gabbay lin_dma_pkt->dst_addr = cpu_to_le64(addr); 4792e65e175bSOded Gabbay if (lin_dma_pkts_cnt > 1) 4793e65e175bSOded Gabbay lin_dma_pkt->tsize = cpu_to_le32(SZ_2G); 4794e65e175bSOded Gabbay else 4795e65e175bSOded Gabbay lin_dma_pkt->tsize = cpu_to_le32(size); 4796e65e175bSOded Gabbay 4797e65e175bSOded Gabbay size -= SZ_2G; 4798e65e175bSOded Gabbay addr += SZ_2G; 4799e65e175bSOded Gabbay lin_dma_pkt++; 4800e65e175bSOded Gabbay } while (--lin_dma_pkts_cnt); 4801e65e175bSOded Gabbay 4802e65e175bSOded Gabbay job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); 4803e65e175bSOded Gabbay if (!job) { 4804e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to allocate a new job\n"); 4805e65e175bSOded Gabbay rc = -ENOMEM; 4806e65e175bSOded Gabbay goto release_cb; 4807e65e175bSOded Gabbay } 4808e65e175bSOded Gabbay 4809e65e175bSOded Gabbay job->id = 0; 4810e65e175bSOded Gabbay job->user_cb = cb; 4811e65e175bSOded Gabbay atomic_inc(&job->user_cb->cs_cnt); 4812e65e175bSOded Gabbay job->user_cb_size = cb_size; 4813e65e175bSOded Gabbay job->hw_queue_id = GOYA_QUEUE_ID_DMA_0; 4814e65e175bSOded Gabbay job->patched_cb = job->user_cb; 4815e65e175bSOded Gabbay job->job_cb_size = job->user_cb_size; 4816e65e175bSOded Gabbay 4817e65e175bSOded Gabbay hl_debugfs_add_job(hdev, job); 4818e65e175bSOded Gabbay 4819e65e175bSOded Gabbay rc = goya_send_job_on_qman0(hdev, job); 4820e65e175bSOded Gabbay 4821e65e175bSOded Gabbay hl_debugfs_remove_job(hdev, job); 4822e65e175bSOded Gabbay kfree(job); 4823e65e175bSOded Gabbay atomic_dec(&cb->cs_cnt); 4824e65e175bSOded Gabbay 4825e65e175bSOded Gabbay release_cb: 4826e65e175bSOded Gabbay hl_cb_put(cb); 4827e65e175bSOded Gabbay hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); 4828e65e175bSOded Gabbay 4829e65e175bSOded Gabbay return rc; 4830e65e175bSOded Gabbay } 4831e65e175bSOded Gabbay 4832e65e175bSOded Gabbay int goya_context_switch(struct hl_device *hdev, u32 asid) 4833e65e175bSOded Gabbay { 4834e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 4835e65e175bSOded Gabbay u64 addr = prop->sram_base_address, sob_addr; 4836e65e175bSOded Gabbay u32 size = hdev->pldm ? 0x10000 : prop->sram_size; 4837e65e175bSOded Gabbay u64 val = 0x7777777777777777ull; 4838e65e175bSOded Gabbay int rc, dma_id; 4839e65e175bSOded Gabbay u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO - 4840e65e175bSOded Gabbay mmDMA_CH_0_WR_COMP_ADDR_LO; 4841e65e175bSOded Gabbay 4842e65e175bSOded Gabbay rc = goya_memset_device_memory(hdev, addr, size, val, false); 4843e65e175bSOded Gabbay if (rc) { 4844e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); 4845e65e175bSOded Gabbay return rc; 4846e65e175bSOded Gabbay } 4847e65e175bSOded Gabbay 4848e65e175bSOded Gabbay /* we need to reset registers that the user is allowed to change */ 4849e65e175bSOded Gabbay sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007; 4850e65e175bSOded Gabbay WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO, lower_32_bits(sob_addr)); 4851e65e175bSOded Gabbay 4852e65e175bSOded Gabbay for (dma_id = 1 ; dma_id < NUMBER_OF_EXT_HW_QUEUES ; dma_id++) { 4853e65e175bSOded Gabbay sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 + 4854e65e175bSOded Gabbay (dma_id - 1) * 4; 4855e65e175bSOded Gabbay WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + channel_off * dma_id, 4856e65e175bSOded Gabbay lower_32_bits(sob_addr)); 4857e65e175bSOded Gabbay } 4858e65e175bSOded Gabbay 4859e65e175bSOded Gabbay WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020); 4860e65e175bSOded Gabbay 4861e65e175bSOded Gabbay goya_clear_sm_regs(hdev); 4862e65e175bSOded Gabbay 4863e65e175bSOded Gabbay return 0; 4864e65e175bSOded Gabbay } 4865e65e175bSOded Gabbay 4866e65e175bSOded Gabbay static int goya_mmu_clear_pgt_range(struct hl_device *hdev) 4867e65e175bSOded Gabbay { 4868e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 4869e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4870e65e175bSOded Gabbay u64 addr = prop->mmu_pgt_addr; 4871e65e175bSOded Gabbay u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE + 4872e65e175bSOded Gabbay MMU_CACHE_MNG_SIZE; 4873e65e175bSOded Gabbay 4874e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 4875e65e175bSOded Gabbay return 0; 4876e65e175bSOded Gabbay 4877e65e175bSOded Gabbay return goya_memset_device_memory(hdev, addr, size, 0, true); 4878e65e175bSOded Gabbay } 4879e65e175bSOded Gabbay 4880e65e175bSOded Gabbay static int goya_mmu_set_dram_default_page(struct hl_device *hdev) 4881e65e175bSOded Gabbay { 4882e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4883e65e175bSOded Gabbay u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; 4884e65e175bSOded Gabbay u32 size = MMU_DRAM_DEFAULT_PAGE_SIZE; 4885e65e175bSOded Gabbay u64 val = 0x9999999999999999ull; 4886e65e175bSOded Gabbay 4887e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 4888e65e175bSOded Gabbay return 0; 4889e65e175bSOded Gabbay 4890e65e175bSOded Gabbay return goya_memset_device_memory(hdev, addr, size, val, true); 4891e65e175bSOded Gabbay } 4892e65e175bSOded Gabbay 4893e65e175bSOded Gabbay static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev) 4894e65e175bSOded Gabbay { 4895e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 4896e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4897e65e175bSOded Gabbay s64 off, cpu_off; 4898e65e175bSOded Gabbay int rc; 4899e65e175bSOded Gabbay 4900e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 4901e65e175bSOded Gabbay return 0; 4902e65e175bSOded Gabbay 4903e65e175bSOded Gabbay for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) { 4904e65e175bSOded Gabbay rc = hl_mmu_map_page(hdev->kernel_ctx, 4905e65e175bSOded Gabbay prop->dram_base_address + off, 4906e65e175bSOded Gabbay prop->dram_base_address + off, PAGE_SIZE_2MB, 4907e65e175bSOded Gabbay (off + PAGE_SIZE_2MB) == CPU_FW_IMAGE_SIZE); 4908e65e175bSOded Gabbay if (rc) { 4909e65e175bSOded Gabbay dev_err(hdev->dev, "Map failed for address 0x%llx\n", 4910e65e175bSOded Gabbay prop->dram_base_address + off); 4911e65e175bSOded Gabbay goto unmap; 4912e65e175bSOded Gabbay } 4913e65e175bSOded Gabbay } 4914e65e175bSOded Gabbay 4915e65e175bSOded Gabbay if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { 4916e65e175bSOded Gabbay rc = hl_mmu_map_page(hdev->kernel_ctx, 4917e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR, 4918e65e175bSOded Gabbay hdev->cpu_accessible_dma_address, 4919e65e175bSOded Gabbay PAGE_SIZE_2MB, true); 4920e65e175bSOded Gabbay 4921e65e175bSOded Gabbay if (rc) { 4922e65e175bSOded Gabbay dev_err(hdev->dev, 4923e65e175bSOded Gabbay "Map failed for CPU accessible memory\n"); 4924e65e175bSOded Gabbay off -= PAGE_SIZE_2MB; 4925e65e175bSOded Gabbay goto unmap; 4926e65e175bSOded Gabbay } 4927e65e175bSOded Gabbay } else { 4928e65e175bSOded Gabbay for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) { 4929e65e175bSOded Gabbay rc = hl_mmu_map_page(hdev->kernel_ctx, 4930e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off, 4931e65e175bSOded Gabbay hdev->cpu_accessible_dma_address + cpu_off, 4932e65e175bSOded Gabbay PAGE_SIZE_4KB, true); 4933e65e175bSOded Gabbay if (rc) { 4934e65e175bSOded Gabbay dev_err(hdev->dev, 4935e65e175bSOded Gabbay "Map failed for CPU accessible memory\n"); 4936e65e175bSOded Gabbay cpu_off -= PAGE_SIZE_4KB; 4937e65e175bSOded Gabbay goto unmap_cpu; 4938e65e175bSOded Gabbay } 4939e65e175bSOded Gabbay } 4940e65e175bSOded Gabbay } 4941e65e175bSOded Gabbay 4942e65e175bSOded Gabbay goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID); 4943e65e175bSOded Gabbay goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID); 4944e65e175bSOded Gabbay WREG32(mmCPU_IF_ARUSER_OVR_EN, 0x7FF); 4945e65e175bSOded Gabbay WREG32(mmCPU_IF_AWUSER_OVR_EN, 0x7FF); 4946e65e175bSOded Gabbay 4947e65e175bSOded Gabbay /* Make sure configuration is flushed to device */ 4948e65e175bSOded Gabbay RREG32(mmCPU_IF_AWUSER_OVR_EN); 4949e65e175bSOded Gabbay 4950e65e175bSOded Gabbay goya->device_cpu_mmu_mappings_done = true; 4951e65e175bSOded Gabbay 4952e65e175bSOded Gabbay return 0; 4953e65e175bSOded Gabbay 4954e65e175bSOded Gabbay unmap_cpu: 4955e65e175bSOded Gabbay for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB) 4956e65e175bSOded Gabbay if (hl_mmu_unmap_page(hdev->kernel_ctx, 4957e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off, 4958e65e175bSOded Gabbay PAGE_SIZE_4KB, true)) 4959e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev, 4960e65e175bSOded Gabbay "failed to unmap address 0x%llx\n", 4961e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off); 4962e65e175bSOded Gabbay unmap: 4963e65e175bSOded Gabbay for (; off >= 0 ; off -= PAGE_SIZE_2MB) 4964e65e175bSOded Gabbay if (hl_mmu_unmap_page(hdev->kernel_ctx, 4965e65e175bSOded Gabbay prop->dram_base_address + off, PAGE_SIZE_2MB, 4966e65e175bSOded Gabbay true)) 4967e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev, 4968e65e175bSOded Gabbay "failed to unmap address 0x%llx\n", 4969e65e175bSOded Gabbay prop->dram_base_address + off); 4970e65e175bSOded Gabbay 4971e65e175bSOded Gabbay return rc; 4972e65e175bSOded Gabbay } 4973e65e175bSOded Gabbay 4974e65e175bSOded Gabbay void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev) 4975e65e175bSOded Gabbay { 4976e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 4977e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 4978e65e175bSOded Gabbay u32 off, cpu_off; 4979e65e175bSOded Gabbay 4980e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 4981e65e175bSOded Gabbay return; 4982e65e175bSOded Gabbay 4983e65e175bSOded Gabbay if (!goya->device_cpu_mmu_mappings_done) 4984e65e175bSOded Gabbay return; 4985e65e175bSOded Gabbay 4986e65e175bSOded Gabbay WREG32(mmCPU_IF_ARUSER_OVR_EN, 0); 4987e65e175bSOded Gabbay WREG32(mmCPU_IF_AWUSER_OVR_EN, 0); 4988e65e175bSOded Gabbay 4989e65e175bSOded Gabbay if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { 4990e65e175bSOded Gabbay if (hl_mmu_unmap_page(hdev->kernel_ctx, 4991e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR, 4992e65e175bSOded Gabbay PAGE_SIZE_2MB, true)) 4993e65e175bSOded Gabbay dev_warn(hdev->dev, 4994e65e175bSOded Gabbay "Failed to unmap CPU accessible memory\n"); 4995e65e175bSOded Gabbay } else { 4996e65e175bSOded Gabbay for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) 4997e65e175bSOded Gabbay if (hl_mmu_unmap_page(hdev->kernel_ctx, 4998e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off, 4999e65e175bSOded Gabbay PAGE_SIZE_4KB, 5000e65e175bSOded Gabbay (cpu_off + PAGE_SIZE_4KB) >= SZ_2M)) 5001e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev, 5002e65e175bSOded Gabbay "failed to unmap address 0x%llx\n", 5003e65e175bSOded Gabbay VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off); 5004e65e175bSOded Gabbay } 5005e65e175bSOded Gabbay 5006e65e175bSOded Gabbay for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) 5007e65e175bSOded Gabbay if (hl_mmu_unmap_page(hdev->kernel_ctx, 5008e65e175bSOded Gabbay prop->dram_base_address + off, PAGE_SIZE_2MB, 5009e65e175bSOded Gabbay (off + PAGE_SIZE_2MB) >= CPU_FW_IMAGE_SIZE)) 5010e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev, 5011e65e175bSOded Gabbay "Failed to unmap address 0x%llx\n", 5012e65e175bSOded Gabbay prop->dram_base_address + off); 5013e65e175bSOded Gabbay 5014e65e175bSOded Gabbay goya->device_cpu_mmu_mappings_done = false; 5015e65e175bSOded Gabbay } 5016e65e175bSOded Gabbay 5017e65e175bSOded Gabbay static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) 5018e65e175bSOded Gabbay { 5019e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5020e65e175bSOded Gabbay int i; 5021e65e175bSOded Gabbay 5022e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU)) 5023e65e175bSOded Gabbay return; 5024e65e175bSOded Gabbay 5025e65e175bSOded Gabbay if (asid & ~MME_QM_GLBL_SECURE_PROPS_ASID_MASK) { 5026e65e175bSOded Gabbay dev_crit(hdev->dev, "asid %u is too big\n", asid); 5027e65e175bSOded Gabbay return; 5028e65e175bSOded Gabbay } 5029e65e175bSOded Gabbay 5030e65e175bSOded Gabbay /* zero the MMBP and ASID bits and then set the ASID */ 5031e65e175bSOded Gabbay for (i = 0 ; i < GOYA_MMU_REGS_NUM ; i++) 5032e65e175bSOded Gabbay goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid); 5033e65e175bSOded Gabbay } 5034e65e175bSOded Gabbay 5035e65e175bSOded Gabbay static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, 5036e65e175bSOded Gabbay u32 flags) 5037e65e175bSOded Gabbay { 5038e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5039e65e175bSOded Gabbay u32 status, timeout_usec; 5040e65e175bSOded Gabbay int rc; 5041e65e175bSOded Gabbay 5042e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_MMU) || 5043e65e175bSOded Gabbay hdev->reset_info.hard_reset_pending) 5044e65e175bSOded Gabbay return 0; 5045e65e175bSOded Gabbay 5046e65e175bSOded Gabbay /* no need in L1 only invalidation in Goya */ 5047e65e175bSOded Gabbay if (!is_hard) 5048e65e175bSOded Gabbay return 0; 5049e65e175bSOded Gabbay 5050e65e175bSOded Gabbay if (hdev->pldm) 5051e65e175bSOded Gabbay timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC; 5052e65e175bSOded Gabbay else 5053e65e175bSOded Gabbay timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 5054e65e175bSOded Gabbay 5055e65e175bSOded Gabbay /* L0 & L1 invalidation */ 5056e65e175bSOded Gabbay WREG32(mmSTLB_INV_ALL_START, 1); 5057e65e175bSOded Gabbay 5058e65e175bSOded Gabbay rc = hl_poll_timeout( 5059e65e175bSOded Gabbay hdev, 5060e65e175bSOded Gabbay mmSTLB_INV_ALL_START, 5061e65e175bSOded Gabbay status, 5062e65e175bSOded Gabbay !status, 5063e65e175bSOded Gabbay 1000, 5064e65e175bSOded Gabbay timeout_usec); 5065e65e175bSOded Gabbay 5066e65e175bSOded Gabbay return rc; 5067e65e175bSOded Gabbay } 5068e65e175bSOded Gabbay 5069e65e175bSOded Gabbay static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, 5070e65e175bSOded Gabbay bool is_hard, u32 flags, 5071e65e175bSOded Gabbay u32 asid, u64 va, u64 size) 5072e65e175bSOded Gabbay { 5073e65e175bSOded Gabbay /* Treat as invalidate all because there is no range invalidation 5074e65e175bSOded Gabbay * in Goya 5075e65e175bSOded Gabbay */ 5076e65e175bSOded Gabbay return hl_mmu_invalidate_cache(hdev, is_hard, flags); 5077e65e175bSOded Gabbay } 5078e65e175bSOded Gabbay 5079e65e175bSOded Gabbay int goya_send_heartbeat(struct hl_device *hdev) 5080e65e175bSOded Gabbay { 5081e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5082e65e175bSOded Gabbay 5083e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) 5084e65e175bSOded Gabbay return 0; 5085e65e175bSOded Gabbay 5086e65e175bSOded Gabbay return hl_fw_send_heartbeat(hdev); 5087e65e175bSOded Gabbay } 5088e65e175bSOded Gabbay 5089e65e175bSOded Gabbay int goya_cpucp_info_get(struct hl_device *hdev) 5090e65e175bSOded Gabbay { 5091e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5092e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 5093e65e175bSOded Gabbay u64 dram_size; 5094e65e175bSOded Gabbay int rc; 5095e65e175bSOded Gabbay 5096e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) 5097e65e175bSOded Gabbay return 0; 5098e65e175bSOded Gabbay 5099e65e175bSOded Gabbay rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, 5100e65e175bSOded Gabbay mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 5101e65e175bSOded Gabbay mmCPU_BOOT_ERR1); 5102e65e175bSOded Gabbay if (rc) 5103e65e175bSOded Gabbay return rc; 5104e65e175bSOded Gabbay 5105e65e175bSOded Gabbay dram_size = le64_to_cpu(prop->cpucp_info.dram_size); 5106e65e175bSOded Gabbay if (dram_size) { 5107e65e175bSOded Gabbay if ((!is_power_of_2(dram_size)) || 5108e65e175bSOded Gabbay (dram_size < DRAM_PHYS_DEFAULT_SIZE)) { 5109e65e175bSOded Gabbay dev_err(hdev->dev, 5110e65e175bSOded Gabbay "F/W reported invalid DRAM size %llu. Trying to use default size\n", 5111e65e175bSOded Gabbay dram_size); 5112e65e175bSOded Gabbay dram_size = DRAM_PHYS_DEFAULT_SIZE; 5113e65e175bSOded Gabbay } 5114e65e175bSOded Gabbay 5115e65e175bSOded Gabbay prop->dram_size = dram_size; 5116e65e175bSOded Gabbay prop->dram_end_address = prop->dram_base_address + dram_size; 5117e65e175bSOded Gabbay } 5118e65e175bSOded Gabbay 5119e65e175bSOded Gabbay if (!strlen(prop->cpucp_info.card_name)) 5120e65e175bSOded Gabbay strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, 5121e65e175bSOded Gabbay CARD_NAME_MAX_LEN); 5122e65e175bSOded Gabbay 5123e65e175bSOded Gabbay return 0; 5124e65e175bSOded Gabbay } 5125e65e175bSOded Gabbay 5126e65e175bSOded Gabbay static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 5127e65e175bSOded Gabbay struct engines_data *e) 5128e65e175bSOded Gabbay { 5129e65e175bSOded Gabbay const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; 5130e65e175bSOded Gabbay const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; 5131e65e175bSOded Gabbay unsigned long *mask = (unsigned long *)mask_arr; 5132e65e175bSOded Gabbay u32 qm_glbl_sts0, cmdq_glbl_sts0, dma_core_sts0, tpc_cfg_sts, 5133e65e175bSOded Gabbay mme_arch_sts; 5134e65e175bSOded Gabbay bool is_idle = true, is_eng_idle; 5135e65e175bSOded Gabbay u64 offset; 5136e65e175bSOded Gabbay int i; 5137e65e175bSOded Gabbay 5138e65e175bSOded Gabbay if (e) 5139e65e175bSOded Gabbay hl_engine_data_sprintf(e, "\nDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0\n" 5140e65e175bSOded Gabbay "--- ------- ------------ -------------\n"); 5141e65e175bSOded Gabbay 5142e65e175bSOded Gabbay offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0; 5143e65e175bSOded Gabbay 5144e65e175bSOded Gabbay for (i = 0 ; i < DMA_MAX_NUM ; i++) { 5145e65e175bSOded Gabbay qm_glbl_sts0 = RREG32(mmDMA_QM_0_GLBL_STS0 + i * offset); 5146e65e175bSOded Gabbay dma_core_sts0 = RREG32(mmDMA_CH_0_STS0 + i * offset); 5147e65e175bSOded Gabbay is_eng_idle = IS_DMA_QM_IDLE(qm_glbl_sts0) && 5148e65e175bSOded Gabbay IS_DMA_IDLE(dma_core_sts0); 5149e65e175bSOded Gabbay is_idle &= is_eng_idle; 5150e65e175bSOded Gabbay 5151e65e175bSOded Gabbay if (mask && !is_eng_idle) 5152e65e175bSOded Gabbay set_bit(GOYA_ENGINE_ID_DMA_0 + i, mask); 5153e65e175bSOded Gabbay if (e) 5154e65e175bSOded Gabbay hl_engine_data_sprintf(e, dma_fmt, i, is_eng_idle ? "Y" : "N", 5155e65e175bSOded Gabbay qm_glbl_sts0, dma_core_sts0); 5156e65e175bSOded Gabbay } 5157e65e175bSOded Gabbay 5158e65e175bSOded Gabbay if (e) 5159e65e175bSOded Gabbay hl_engine_data_sprintf(e, 5160e65e175bSOded Gabbay "\nTPC is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 CFG_STATUS\n" 5161e65e175bSOded Gabbay "--- ------- ------------ -------------- ----------\n"); 5162e65e175bSOded Gabbay 5163e65e175bSOded Gabbay offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0; 5164e65e175bSOded Gabbay 5165e65e175bSOded Gabbay for (i = 0 ; i < TPC_MAX_NUM ; i++) { 5166e65e175bSOded Gabbay qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + i * offset); 5167e65e175bSOded Gabbay cmdq_glbl_sts0 = RREG32(mmTPC0_CMDQ_GLBL_STS0 + i * offset); 5168e65e175bSOded Gabbay tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + i * offset); 5169e65e175bSOded Gabbay is_eng_idle = IS_TPC_QM_IDLE(qm_glbl_sts0) && 5170e65e175bSOded Gabbay IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) && 5171e65e175bSOded Gabbay IS_TPC_IDLE(tpc_cfg_sts); 5172e65e175bSOded Gabbay is_idle &= is_eng_idle; 5173e65e175bSOded Gabbay 5174e65e175bSOded Gabbay if (mask && !is_eng_idle) 5175e65e175bSOded Gabbay set_bit(GOYA_ENGINE_ID_TPC_0 + i, mask); 5176e65e175bSOded Gabbay if (e) 5177e65e175bSOded Gabbay hl_engine_data_sprintf(e, fmt, i, is_eng_idle ? "Y" : "N", 5178e65e175bSOded Gabbay qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts); 5179e65e175bSOded Gabbay } 5180e65e175bSOded Gabbay 5181e65e175bSOded Gabbay if (e) 5182e65e175bSOded Gabbay hl_engine_data_sprintf(e, 5183e65e175bSOded Gabbay "\nMME is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 ARCH_STATUS\n" 5184e65e175bSOded Gabbay "--- ------- ------------ -------------- -----------\n"); 5185e65e175bSOded Gabbay 5186e65e175bSOded Gabbay qm_glbl_sts0 = RREG32(mmMME_QM_GLBL_STS0); 5187e65e175bSOded Gabbay cmdq_glbl_sts0 = RREG32(mmMME_CMDQ_GLBL_STS0); 5188e65e175bSOded Gabbay mme_arch_sts = RREG32(mmMME_ARCH_STATUS); 5189e65e175bSOded Gabbay is_eng_idle = IS_MME_QM_IDLE(qm_glbl_sts0) && 5190e65e175bSOded Gabbay IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) && 5191e65e175bSOded Gabbay IS_MME_IDLE(mme_arch_sts); 5192e65e175bSOded Gabbay is_idle &= is_eng_idle; 5193e65e175bSOded Gabbay 5194e65e175bSOded Gabbay if (mask && !is_eng_idle) 5195e65e175bSOded Gabbay set_bit(GOYA_ENGINE_ID_MME_0, mask); 5196e65e175bSOded Gabbay if (e) { 5197e65e175bSOded Gabbay hl_engine_data_sprintf(e, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, 5198e65e175bSOded Gabbay cmdq_glbl_sts0, mme_arch_sts); 5199e65e175bSOded Gabbay hl_engine_data_sprintf(e, "\n"); 5200e65e175bSOded Gabbay } 5201e65e175bSOded Gabbay 5202e65e175bSOded Gabbay return is_idle; 5203e65e175bSOded Gabbay } 5204e65e175bSOded Gabbay 5205e65e175bSOded Gabbay static void goya_hw_queues_lock(struct hl_device *hdev) 5206e65e175bSOded Gabbay __acquires(&goya->hw_queues_lock) 5207e65e175bSOded Gabbay { 5208e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5209e65e175bSOded Gabbay 5210e65e175bSOded Gabbay spin_lock(&goya->hw_queues_lock); 5211e65e175bSOded Gabbay } 5212e65e175bSOded Gabbay 5213e65e175bSOded Gabbay static void goya_hw_queues_unlock(struct hl_device *hdev) 5214e65e175bSOded Gabbay __releases(&goya->hw_queues_lock) 5215e65e175bSOded Gabbay { 5216e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5217e65e175bSOded Gabbay 5218e65e175bSOded Gabbay spin_unlock(&goya->hw_queues_lock); 5219e65e175bSOded Gabbay } 5220e65e175bSOded Gabbay 5221e65e175bSOded Gabbay static u32 goya_get_pci_id(struct hl_device *hdev) 5222e65e175bSOded Gabbay { 5223e65e175bSOded Gabbay return hdev->pdev->device; 5224e65e175bSOded Gabbay } 5225e65e175bSOded Gabbay 5226e65e175bSOded Gabbay static int goya_get_eeprom_data(struct hl_device *hdev, void *data, 5227e65e175bSOded Gabbay size_t max_size) 5228e65e175bSOded Gabbay { 5229e65e175bSOded Gabbay struct goya_device *goya = hdev->asic_specific; 5230e65e175bSOded Gabbay 5231e65e175bSOded Gabbay if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) 5232e65e175bSOded Gabbay return 0; 5233e65e175bSOded Gabbay 5234e65e175bSOded Gabbay return hl_fw_get_eeprom_data(hdev, data, max_size); 5235e65e175bSOded Gabbay } 5236e65e175bSOded Gabbay 5237e65e175bSOded Gabbay static void goya_cpu_init_scrambler_dram(struct hl_device *hdev) 5238e65e175bSOded Gabbay { 5239e65e175bSOded Gabbay 5240e65e175bSOded Gabbay } 5241e65e175bSOded Gabbay 5242e65e175bSOded Gabbay static int goya_ctx_init(struct hl_ctx *ctx) 5243e65e175bSOded Gabbay { 5244e65e175bSOded Gabbay if (ctx->asid != HL_KERNEL_ASID_ID) 5245e65e175bSOded Gabbay goya_mmu_prepare(ctx->hdev, ctx->asid); 5246e65e175bSOded Gabbay 5247e65e175bSOded Gabbay return 0; 5248e65e175bSOded Gabbay } 5249e65e175bSOded Gabbay 5250e65e175bSOded Gabbay static int goya_pre_schedule_cs(struct hl_cs *cs) 5251e65e175bSOded Gabbay { 5252e65e175bSOded Gabbay return 0; 5253e65e175bSOded Gabbay } 5254e65e175bSOded Gabbay 5255e65e175bSOded Gabbay u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 5256e65e175bSOded Gabbay { 5257e65e175bSOded Gabbay return cq_idx; 5258e65e175bSOded Gabbay } 5259e65e175bSOded Gabbay 5260e65e175bSOded Gabbay static u32 goya_get_signal_cb_size(struct hl_device *hdev) 5261e65e175bSOded Gabbay { 5262e65e175bSOded Gabbay return 0; 5263e65e175bSOded Gabbay } 5264e65e175bSOded Gabbay 5265e65e175bSOded Gabbay static u32 goya_get_wait_cb_size(struct hl_device *hdev) 5266e65e175bSOded Gabbay { 5267e65e175bSOded Gabbay return 0; 5268e65e175bSOded Gabbay } 5269e65e175bSOded Gabbay 5270e65e175bSOded Gabbay static u32 goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, 5271e65e175bSOded Gabbay u32 size, bool eb) 5272e65e175bSOded Gabbay { 5273e65e175bSOded Gabbay return 0; 5274e65e175bSOded Gabbay } 5275e65e175bSOded Gabbay 5276e65e175bSOded Gabbay static u32 goya_gen_wait_cb(struct hl_device *hdev, 5277e65e175bSOded Gabbay struct hl_gen_wait_properties *prop) 5278e65e175bSOded Gabbay { 5279e65e175bSOded Gabbay return 0; 5280e65e175bSOded Gabbay } 5281e65e175bSOded Gabbay 5282e65e175bSOded Gabbay static void goya_reset_sob(struct hl_device *hdev, void *data) 5283e65e175bSOded Gabbay { 5284e65e175bSOded Gabbay 5285e65e175bSOded Gabbay } 5286e65e175bSOded Gabbay 5287e65e175bSOded Gabbay static void goya_reset_sob_group(struct hl_device *hdev, u16 sob_group) 5288e65e175bSOded Gabbay { 5289e65e175bSOded Gabbay 5290e65e175bSOded Gabbay } 5291e65e175bSOded Gabbay 5292e65e175bSOded Gabbay u64 goya_get_device_time(struct hl_device *hdev) 5293e65e175bSOded Gabbay { 5294e65e175bSOded Gabbay u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 5295e65e175bSOded Gabbay 5296e65e175bSOded Gabbay return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 5297e65e175bSOded Gabbay } 5298e65e175bSOded Gabbay 5299e65e175bSOded Gabbay static int goya_collective_wait_init_cs(struct hl_cs *cs) 5300e65e175bSOded Gabbay { 5301e65e175bSOded Gabbay return 0; 5302e65e175bSOded Gabbay } 5303e65e175bSOded Gabbay 5304e65e175bSOded Gabbay static int goya_collective_wait_create_jobs(struct hl_device *hdev, 5305e65e175bSOded Gabbay struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, 5306e65e175bSOded Gabbay u32 collective_engine_id, u32 encaps_signal_offset) 5307e65e175bSOded Gabbay { 5308e65e175bSOded Gabbay return -EINVAL; 5309e65e175bSOded Gabbay } 5310e65e175bSOded Gabbay 5311e65e175bSOded Gabbay static void goya_ctx_fini(struct hl_ctx *ctx) 5312e65e175bSOded Gabbay { 5313e65e175bSOded Gabbay 5314e65e175bSOded Gabbay } 5315e65e175bSOded Gabbay 5316e65e175bSOded Gabbay static int goya_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 5317e65e175bSOded Gabbay u32 *block_size, u32 *block_id) 5318e65e175bSOded Gabbay { 5319e65e175bSOded Gabbay return -EPERM; 5320e65e175bSOded Gabbay } 5321e65e175bSOded Gabbay 5322e65e175bSOded Gabbay static int goya_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 5323e65e175bSOded Gabbay u32 block_id, u32 block_size) 5324e65e175bSOded Gabbay { 5325e65e175bSOded Gabbay return -EPERM; 5326e65e175bSOded Gabbay } 5327e65e175bSOded Gabbay 5328e65e175bSOded Gabbay static void goya_enable_events_from_fw(struct hl_device *hdev) 5329e65e175bSOded Gabbay { 5330e65e175bSOded Gabbay WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, 5331e65e175bSOded Gabbay GOYA_ASYNC_EVENT_ID_INTS_REGISTER); 5332e65e175bSOded Gabbay } 5333e65e175bSOded Gabbay 5334e65e175bSOded Gabbay static int goya_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 5335e65e175bSOded Gabbay { 5336e65e175bSOded Gabbay return -EINVAL; 5337e65e175bSOded Gabbay } 5338e65e175bSOded Gabbay 5339e65e175bSOded Gabbay static int goya_map_pll_idx_to_fw_idx(u32 pll_idx) 5340e65e175bSOded Gabbay { 5341e65e175bSOded Gabbay switch (pll_idx) { 5342e65e175bSOded Gabbay case HL_GOYA_CPU_PLL: return CPU_PLL; 5343e65e175bSOded Gabbay case HL_GOYA_PCI_PLL: return PCI_PLL; 5344e65e175bSOded Gabbay case HL_GOYA_MME_PLL: return MME_PLL; 5345e65e175bSOded Gabbay case HL_GOYA_TPC_PLL: return TPC_PLL; 5346e65e175bSOded Gabbay case HL_GOYA_IC_PLL: return IC_PLL; 5347e65e175bSOded Gabbay case HL_GOYA_MC_PLL: return MC_PLL; 5348e65e175bSOded Gabbay case HL_GOYA_EMMC_PLL: return EMMC_PLL; 5349e65e175bSOded Gabbay default: return -EINVAL; 5350e65e175bSOded Gabbay } 5351e65e175bSOded Gabbay } 5352e65e175bSOded Gabbay 5353e65e175bSOded Gabbay static int goya_gen_sync_to_engine_map(struct hl_device *hdev, 5354e65e175bSOded Gabbay struct hl_sync_to_engine_map *map) 5355e65e175bSOded Gabbay { 5356e65e175bSOded Gabbay /* Not implemented */ 5357e65e175bSOded Gabbay return 0; 5358e65e175bSOded Gabbay } 5359e65e175bSOded Gabbay 5360e65e175bSOded Gabbay static int goya_monitor_valid(struct hl_mon_state_dump *mon) 5361e65e175bSOded Gabbay { 5362e65e175bSOded Gabbay /* Not implemented */ 5363e65e175bSOded Gabbay return 0; 5364e65e175bSOded Gabbay } 5365e65e175bSOded Gabbay 5366e65e175bSOded Gabbay static int goya_print_single_monitor(char **buf, size_t *size, size_t *offset, 5367e65e175bSOded Gabbay struct hl_device *hdev, 5368e65e175bSOded Gabbay struct hl_mon_state_dump *mon) 5369e65e175bSOded Gabbay { 5370e65e175bSOded Gabbay /* Not implemented */ 5371e65e175bSOded Gabbay return 0; 5372e65e175bSOded Gabbay } 5373e65e175bSOded Gabbay 5374e65e175bSOded Gabbay 5375e65e175bSOded Gabbay static int goya_print_fences_single_engine( 5376e65e175bSOded Gabbay struct hl_device *hdev, u64 base_offset, u64 status_base_offset, 5377e65e175bSOded Gabbay enum hl_sync_engine_type engine_type, u32 engine_id, char **buf, 5378e65e175bSOded Gabbay size_t *size, size_t *offset) 5379e65e175bSOded Gabbay { 5380e65e175bSOded Gabbay /* Not implemented */ 5381e65e175bSOded Gabbay return 0; 5382e65e175bSOded Gabbay } 5383e65e175bSOded Gabbay 5384e65e175bSOded Gabbay 5385e65e175bSOded Gabbay static struct hl_state_dump_specs_funcs goya_state_dump_funcs = { 5386e65e175bSOded Gabbay .monitor_valid = goya_monitor_valid, 5387e65e175bSOded Gabbay .print_single_monitor = goya_print_single_monitor, 5388e65e175bSOded Gabbay .gen_sync_to_engine_map = goya_gen_sync_to_engine_map, 5389e65e175bSOded Gabbay .print_fences_single_engine = goya_print_fences_single_engine, 5390e65e175bSOded Gabbay }; 5391e65e175bSOded Gabbay 5392e65e175bSOded Gabbay static void goya_state_dump_init(struct hl_device *hdev) 5393e65e175bSOded Gabbay { 5394e65e175bSOded Gabbay /* Not implemented */ 5395e65e175bSOded Gabbay hdev->state_dump_specs.props = goya_state_dump_specs_props; 5396e65e175bSOded Gabbay hdev->state_dump_specs.funcs = goya_state_dump_funcs; 5397e65e175bSOded Gabbay } 5398e65e175bSOded Gabbay 5399e65e175bSOded Gabbay static u32 goya_get_sob_addr(struct hl_device *hdev, u32 sob_id) 5400e65e175bSOded Gabbay { 5401e65e175bSOded Gabbay return 0; 5402e65e175bSOded Gabbay } 5403e65e175bSOded Gabbay 5404e65e175bSOded Gabbay static u32 *goya_get_stream_master_qid_arr(void) 5405e65e175bSOded Gabbay { 5406e65e175bSOded Gabbay return NULL; 5407e65e175bSOded Gabbay } 5408e65e175bSOded Gabbay 5409e65e175bSOded Gabbay static int goya_get_monitor_dump(struct hl_device *hdev, void *data) 5410e65e175bSOded Gabbay { 5411e65e175bSOded Gabbay return -EOPNOTSUPP; 5412e65e175bSOded Gabbay } 5413e65e175bSOded Gabbay 5414e65e175bSOded Gabbay static void goya_check_if_razwi_happened(struct hl_device *hdev) 5415e65e175bSOded Gabbay { 5416e65e175bSOded Gabbay } 5417e65e175bSOded Gabbay 5418e65e175bSOded Gabbay static int goya_scrub_device_dram(struct hl_device *hdev, u64 val) 5419e65e175bSOded Gabbay { 5420e65e175bSOded Gabbay return -EOPNOTSUPP; 5421e65e175bSOded Gabbay } 5422e65e175bSOded Gabbay 5423e65e175bSOded Gabbay static int goya_set_dram_properties(struct hl_device *hdev) 5424e65e175bSOded Gabbay { 5425e65e175bSOded Gabbay return 0; 5426e65e175bSOded Gabbay } 5427e65e175bSOded Gabbay 5428ab509d81SOhad Sharabi static int goya_set_binning_masks(struct hl_device *hdev) 5429ab509d81SOhad Sharabi { 5430ab509d81SOhad Sharabi return 0; 5431ab509d81SOhad Sharabi } 5432ab509d81SOhad Sharabi 5433e65e175bSOded Gabbay static int goya_send_device_activity(struct hl_device *hdev, bool open) 5434e65e175bSOded Gabbay { 5435e65e175bSOded Gabbay return 0; 5436e65e175bSOded Gabbay } 5437e65e175bSOded Gabbay 5438e65e175bSOded Gabbay static const struct hl_asic_funcs goya_funcs = { 5439e65e175bSOded Gabbay .early_init = goya_early_init, 5440e65e175bSOded Gabbay .early_fini = goya_early_fini, 5441e65e175bSOded Gabbay .late_init = goya_late_init, 5442e65e175bSOded Gabbay .late_fini = goya_late_fini, 5443e65e175bSOded Gabbay .sw_init = goya_sw_init, 5444e65e175bSOded Gabbay .sw_fini = goya_sw_fini, 5445e65e175bSOded Gabbay .hw_init = goya_hw_init, 5446e65e175bSOded Gabbay .hw_fini = goya_hw_fini, 5447e65e175bSOded Gabbay .halt_engines = goya_halt_engines, 5448e65e175bSOded Gabbay .suspend = goya_suspend, 5449e65e175bSOded Gabbay .resume = goya_resume, 5450e65e175bSOded Gabbay .mmap = goya_mmap, 5451e65e175bSOded Gabbay .ring_doorbell = goya_ring_doorbell, 5452e65e175bSOded Gabbay .pqe_write = goya_pqe_write, 5453e65e175bSOded Gabbay .asic_dma_alloc_coherent = goya_dma_alloc_coherent, 5454e65e175bSOded Gabbay .asic_dma_free_coherent = goya_dma_free_coherent, 5455e65e175bSOded Gabbay .scrub_device_mem = goya_scrub_device_mem, 5456e65e175bSOded Gabbay .scrub_device_dram = goya_scrub_device_dram, 5457e65e175bSOded Gabbay .get_int_queue_base = goya_get_int_queue_base, 5458e65e175bSOded Gabbay .test_queues = goya_test_queues, 5459e65e175bSOded Gabbay .asic_dma_pool_zalloc = goya_dma_pool_zalloc, 5460e65e175bSOded Gabbay .asic_dma_pool_free = goya_dma_pool_free, 5461e65e175bSOded Gabbay .cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc, 5462e65e175bSOded Gabbay .cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free, 5463e65e175bSOded Gabbay .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable, 5464e65e175bSOded Gabbay .cs_parser = goya_cs_parser, 5465e65e175bSOded Gabbay .asic_dma_map_sgtable = hl_dma_map_sgtable, 5466e65e175bSOded Gabbay .add_end_of_cb_packets = goya_add_end_of_cb_packets, 5467e65e175bSOded Gabbay .update_eq_ci = goya_update_eq_ci, 5468e65e175bSOded Gabbay .context_switch = goya_context_switch, 5469e65e175bSOded Gabbay .restore_phase_topology = goya_restore_phase_topology, 5470e65e175bSOded Gabbay .debugfs_read_dma = goya_debugfs_read_dma, 5471e65e175bSOded Gabbay .add_device_attr = goya_add_device_attr, 5472e65e175bSOded Gabbay .handle_eqe = goya_handle_eqe, 5473e65e175bSOded Gabbay .get_events_stat = goya_get_events_stat, 5474e65e175bSOded Gabbay .read_pte = goya_read_pte, 5475e65e175bSOded Gabbay .write_pte = goya_write_pte, 5476e65e175bSOded Gabbay .mmu_invalidate_cache = goya_mmu_invalidate_cache, 5477e65e175bSOded Gabbay .mmu_invalidate_cache_range = goya_mmu_invalidate_cache_range, 5478e65e175bSOded Gabbay .mmu_prefetch_cache_range = NULL, 5479e65e175bSOded Gabbay .send_heartbeat = goya_send_heartbeat, 5480e65e175bSOded Gabbay .debug_coresight = goya_debug_coresight, 5481e65e175bSOded Gabbay .is_device_idle = goya_is_device_idle, 5482e65e175bSOded Gabbay .compute_reset_late_init = goya_compute_reset_late_init, 5483e65e175bSOded Gabbay .hw_queues_lock = goya_hw_queues_lock, 5484e65e175bSOded Gabbay .hw_queues_unlock = goya_hw_queues_unlock, 5485e65e175bSOded Gabbay .get_pci_id = goya_get_pci_id, 5486e65e175bSOded Gabbay .get_eeprom_data = goya_get_eeprom_data, 5487e65e175bSOded Gabbay .get_monitor_dump = goya_get_monitor_dump, 5488e65e175bSOded Gabbay .send_cpu_message = goya_send_cpu_message, 5489e65e175bSOded Gabbay .pci_bars_map = goya_pci_bars_map, 5490e65e175bSOded Gabbay .init_iatu = goya_init_iatu, 5491e65e175bSOded Gabbay .rreg = hl_rreg, 5492e65e175bSOded Gabbay .wreg = hl_wreg, 5493e65e175bSOded Gabbay .halt_coresight = goya_halt_coresight, 5494e65e175bSOded Gabbay .ctx_init = goya_ctx_init, 5495e65e175bSOded Gabbay .ctx_fini = goya_ctx_fini, 5496e65e175bSOded Gabbay .pre_schedule_cs = goya_pre_schedule_cs, 5497e65e175bSOded Gabbay .get_queue_id_for_cq = goya_get_queue_id_for_cq, 5498e65e175bSOded Gabbay .load_firmware_to_device = goya_load_firmware_to_device, 5499e65e175bSOded Gabbay .load_boot_fit_to_device = goya_load_boot_fit_to_device, 5500e65e175bSOded Gabbay .get_signal_cb_size = goya_get_signal_cb_size, 5501e65e175bSOded Gabbay .get_wait_cb_size = goya_get_wait_cb_size, 5502e65e175bSOded Gabbay .gen_signal_cb = goya_gen_signal_cb, 5503e65e175bSOded Gabbay .gen_wait_cb = goya_gen_wait_cb, 5504e65e175bSOded Gabbay .reset_sob = goya_reset_sob, 5505e65e175bSOded Gabbay .reset_sob_group = goya_reset_sob_group, 5506e65e175bSOded Gabbay .get_device_time = goya_get_device_time, 5507e65e175bSOded Gabbay .pb_print_security_errors = NULL, 5508e65e175bSOded Gabbay .collective_wait_init_cs = goya_collective_wait_init_cs, 5509e65e175bSOded Gabbay .collective_wait_create_jobs = goya_collective_wait_create_jobs, 5510e65e175bSOded Gabbay .get_dec_base_addr = NULL, 5511e65e175bSOded Gabbay .scramble_addr = hl_mmu_scramble_addr, 5512e65e175bSOded Gabbay .descramble_addr = hl_mmu_descramble_addr, 5513e65e175bSOded Gabbay .ack_protection_bits_errors = goya_ack_protection_bits_errors, 5514e65e175bSOded Gabbay .get_hw_block_id = goya_get_hw_block_id, 5515e65e175bSOded Gabbay .hw_block_mmap = goya_block_mmap, 5516e65e175bSOded Gabbay .enable_events_from_fw = goya_enable_events_from_fw, 5517e65e175bSOded Gabbay .ack_mmu_errors = goya_ack_mmu_page_fault_or_access_error, 5518e65e175bSOded Gabbay .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx, 5519e65e175bSOded Gabbay .init_firmware_preload_params = goya_init_firmware_preload_params, 5520e65e175bSOded Gabbay .init_firmware_loader = goya_init_firmware_loader, 5521e65e175bSOded Gabbay .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram, 5522e65e175bSOded Gabbay .state_dump_init = goya_state_dump_init, 5523e65e175bSOded Gabbay .get_sob_addr = &goya_get_sob_addr, 5524e65e175bSOded Gabbay .set_pci_memory_regions = goya_set_pci_memory_regions, 5525e65e175bSOded Gabbay .get_stream_master_qid_arr = goya_get_stream_master_qid_arr, 5526e65e175bSOded Gabbay .check_if_razwi_happened = goya_check_if_razwi_happened, 5527e65e175bSOded Gabbay .mmu_get_real_page_size = hl_mmu_get_real_page_size, 5528e65e175bSOded Gabbay .access_dev_mem = hl_access_dev_mem, 5529e65e175bSOded Gabbay .set_dram_bar_base = goya_set_ddr_bar_base, 5530e65e175bSOded Gabbay .send_device_activity = goya_send_device_activity, 5531e65e175bSOded Gabbay .set_dram_properties = goya_set_dram_properties, 5532ab509d81SOhad Sharabi .set_binning_masks = goya_set_binning_masks, 5533e65e175bSOded Gabbay }; 5534e65e175bSOded Gabbay 5535e65e175bSOded Gabbay /* 5536e65e175bSOded Gabbay * goya_set_asic_funcs - set Goya function pointers 5537e65e175bSOded Gabbay * 5538e65e175bSOded Gabbay * @*hdev: pointer to hl_device structure 5539e65e175bSOded Gabbay * 5540e65e175bSOded Gabbay */ 5541e65e175bSOded Gabbay void goya_set_asic_funcs(struct hl_device *hdev) 5542e65e175bSOded Gabbay { 5543e65e175bSOded Gabbay hdev->asic_funcs = &goya_funcs; 5544e65e175bSOded Gabbay } 5545