1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <asm/page.h> 4 #include <linux/acpi.h> 5 #include <linux/bitmap.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/idr.h> 8 #include <linux/io.h> 9 #include <linux/irqreturn.h> 10 #include <linux/log2.h> 11 #include <linux/pm_runtime.h> 12 #include <linux/seq_file.h> 13 #include <linux/slab.h> 14 #include <linux/uacce.h> 15 #include <linux/uaccess.h> 16 #include <uapi/misc/uacce/hisi_qm.h> 17 #include <linux/hisi_acc_qm.h> 18 #include "qm_common.h" 19 20 /* eq/aeq irq enable */ 21 #define QM_VF_AEQ_INT_SOURCE 0x0 22 #define QM_VF_AEQ_INT_MASK 0x4 23 #define QM_VF_EQ_INT_SOURCE 0x8 24 #define QM_VF_EQ_INT_MASK 0xc 25 26 #define QM_IRQ_VECTOR_MASK GENMASK(15, 0) 27 #define QM_IRQ_TYPE_MASK GENMASK(15, 0) 28 #define QM_IRQ_TYPE_SHIFT 16 29 #define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0) 30 31 /* mailbox */ 32 #define QM_MB_PING_ALL_VFS 0xffff 33 #define QM_MB_STATUS_MASK GENMASK(12, 9) 34 #define QM_MB_BUSY_MASK BIT(13) 35 #define QM_MB_MAX_WAIT_TIMEOUT USEC_PER_SEC 36 #define QM_MB_MAX_STOP_TIMEOUT (5 * USEC_PER_SEC) 37 38 /* sqc shift */ 39 #define QM_SQ_HOP_NUM_SHIFT 0 40 #define QM_SQ_PAGE_SIZE_SHIFT 4 41 #define QM_SQ_BUF_SIZE_SHIFT 8 42 #define QM_SQ_SQE_SIZE_SHIFT 12 43 #define QM_SQ_PRIORITY_SHIFT 0 44 #define QM_SQ_ORDERS_SHIFT 4 45 #define QM_SQ_TYPE_SHIFT 8 46 #define QM_QC_PASID_ENABLE 0x1 47 #define QM_QC_PASID_ENABLE_SHIFT 7 48 49 #define QM_SQ_TYPE_MASK GENMASK(3, 0) 50 #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc).w11) >> 6) & 0x1) 51 #define QM_SQC_DISABLE_QP (1U << 6) 52 #define QM_XQC_RANDOM_DATA 0xaaaa 53 54 /* cqc shift */ 55 #define QM_CQ_HOP_NUM_SHIFT 0 56 #define QM_CQ_PAGE_SIZE_SHIFT 4 57 #define QM_CQ_BUF_SIZE_SHIFT 8 58 #define QM_CQ_CQE_SIZE_SHIFT 12 59 #define QM_CQ_PHASE_SHIFT 0 60 #define QM_CQ_FLAG_SHIFT 1 61 62 #define QM_CQE_PHASE(cqe) (le16_to_cpu((cqe)->w7) & 0x1) 63 #define QM_QC_CQE_SIZE 4 64 #define QM_CQ_TAIL_IDX(cqc) ((le16_to_cpu((cqc).w11) >> 6) & 0x1) 65 66 /* eqc shift */ 67 #define QM_EQE_AEQE_SIZE (2UL << 12) 68 #define QM_EQC_PHASE_SHIFT 16 69 70 #define QM_EQE_PHASE(dw0) (((dw0) >> 16) & 0x1) 71 #define QM_EQE_CQN_MASK GENMASK(15, 0) 72 73 #define QM_AEQE_PHASE(dw0) (((dw0) >> 16) & 0x1) 74 #define QM_AEQE_TYPE_SHIFT 17 75 #define QM_AEQE_TYPE_MASK 0xf 76 #define QM_AEQE_CQN_MASK GENMASK(15, 0) 77 #define QM_CQ_OVERFLOW 0 78 #define QM_EQ_OVERFLOW 1 79 #define QM_CQE_ERROR 2 80 81 #define QM_XQ_DEPTH_SHIFT 16 82 #define QM_XQ_DEPTH_MASK GENMASK(15, 0) 83 84 #define QM_DOORBELL_CMD_SQ 0 85 #define QM_DOORBELL_CMD_CQ 1 86 #define QM_DOORBELL_CMD_EQ 2 87 #define QM_DOORBELL_CMD_AEQ 3 88 89 #define QM_DOORBELL_BASE_V1 0x340 90 #define QM_DB_CMD_SHIFT_V1 16 91 #define QM_DB_INDEX_SHIFT_V1 32 92 #define QM_DB_PRIORITY_SHIFT_V1 48 93 #define QM_PAGE_SIZE 0x0034 94 #define QM_QP_DB_INTERVAL 0x10000 95 #define QM_DB_TIMEOUT_CFG 0x100074 96 #define QM_DB_TIMEOUT_SET 0x1fffff 97 98 #define QM_MEM_START_INIT 0x100040 99 #define QM_MEM_INIT_DONE 0x100044 100 #define QM_VFT_CFG_RDY 0x10006c 101 #define QM_VFT_CFG_OP_WR 0x100058 102 #define QM_VFT_CFG_TYPE 0x10005c 103 #define QM_VFT_CFG 0x100060 104 #define QM_VFT_CFG_OP_ENABLE 0x100054 105 #define QM_PM_CTRL 0x100148 106 #define QM_IDLE_DISABLE BIT(9) 107 108 #define QM_SUB_VERSION_ID 0x210 109 110 #define QM_VFT_CFG_DATA_L 0x100064 111 #define QM_VFT_CFG_DATA_H 0x100068 112 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8) 113 #define QM_SQC_VFT_SQC_SIZE (5ULL << 12) 114 #define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16) 115 #define QM_SQC_VFT_START_SQN_SHIFT 28 116 #define QM_SQC_VFT_VALID (1ULL << 44) 117 #define QM_SQC_VFT_SQN_SHIFT 45 118 #define QM_CQC_VFT_BUF_SIZE (7ULL << 8) 119 #define QM_CQC_VFT_SQC_SIZE (5ULL << 12) 120 #define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16) 121 #define QM_CQC_VFT_VALID (1ULL << 28) 122 123 #define QM_SQC_VFT_BASE_SHIFT_V2 28 124 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) 125 #define QM_SQC_VFT_NUM_SHIFT_V2 45 126 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) 127 #define QM_MAX_QC_TYPE 2 128 129 #define QM_ABNORMAL_INT_SOURCE 0x100000 130 #define QM_ABNORMAL_INT_MASK 0x100004 131 #define QM_ABNORMAL_INT_MASK_VALUE 0x7fff 132 #define QM_ABNORMAL_INT_STATUS 0x100008 133 #define QM_ABNORMAL_INT_SET 0x10000c 134 #define QM_ABNORMAL_INF00 0x100010 135 #define QM_FIFO_OVERFLOW_TYPE 0xc0 136 #define QM_FIFO_OVERFLOW_TYPE_SHIFT 6 137 #define QM_FIFO_OVERFLOW_VF 0x3f 138 #define QM_FIFO_OVERFLOW_QP_SHIFT 16 139 #define QM_ABNORMAL_INF01 0x100014 140 #define QM_DB_TIMEOUT_TYPE 0xc0 141 #define QM_DB_TIMEOUT_TYPE_SHIFT 6 142 #define QM_DB_TIMEOUT_VF 0x3f 143 #define QM_DB_TIMEOUT_QP_SHIFT 16 144 #define QM_ABNORMAL_INF02 0x100018 145 #define QM_AXI_POISON_ERR BIT(22) 146 #define QM_RAS_CE_ENABLE 0x1000ec 147 #define QM_RAS_FE_ENABLE 0x1000f0 148 #define QM_RAS_NFE_ENABLE 0x1000f4 149 #define QM_RAS_CE_THRESHOLD 0x1000f8 150 #define QM_RAS_CE_TIMES_PER_IRQ 1 151 #define QM_OOO_SHUTDOWN_SEL 0x1040f8 152 #define QM_AXI_RRESP_ERR BIT(0) 153 #define QM_DB_TIMEOUT BIT(10) 154 #define QM_OF_FIFO_OF BIT(11) 155 #define QM_RAS_AXI_ERROR (BIT(0) | BIT(1) | BIT(12)) 156 157 #define QM_RESET_WAIT_TIMEOUT 400 158 #define QM_PEH_VENDOR_ID 0x1000d8 159 #define ACC_VENDOR_ID_VALUE 0x5a5a 160 #define QM_PEH_DFX_INFO0 0x1000fc 161 #define QM_PEH_DFX_INFO1 0x100100 162 #define QM_PEH_DFX_MASK (BIT(0) | BIT(2)) 163 #define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16) 164 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3 165 #define ACC_PEH_MSI_DISABLE GENMASK(31, 0) 166 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1 167 #define ACC_MASTER_TRANS_RETURN_RW 3 168 #define ACC_MASTER_TRANS_RETURN 0x300150 169 #define ACC_MASTER_GLOBAL_CTRL 0x300000 170 #define ACC_AM_CFG_PORT_WR_EN 0x30001c 171 #define ACC_AM_ROB_ECC_INT_STS 0x300104 172 #define ACC_ROB_ECC_ERR_MULTPL BIT(1) 173 #define QM_MSI_CAP_ENABLE BIT(16) 174 175 /* interfunction communication */ 176 #define QM_IFC_READY_STATUS 0x100128 177 #define QM_IFC_INT_SET_P 0x100130 178 #define QM_IFC_INT_CFG 0x100134 179 #define QM_IFC_INT_SOURCE_P 0x100138 180 #define QM_IFC_INT_SOURCE_V 0x0020 181 #define QM_IFC_INT_MASK 0x0024 182 #define QM_IFC_INT_STATUS 0x0028 183 #define QM_IFC_INT_SET_V 0x002C 184 #define QM_PF2VF_PF_W 0x104700 185 #define QM_VF2PF_PF_R 0x104800 186 #define QM_VF2PF_VF_W 0x320 187 #define QM_PF2VF_VF_R 0x380 188 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) 189 #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) 190 #define QM_IFC_INT_SOURCE_MASK BIT(0) 191 #define QM_IFC_INT_DISABLE BIT(0) 192 #define QM_IFC_INT_STATUS_MASK BIT(0) 193 #define QM_IFC_INT_SET_MASK BIT(0) 194 #define QM_WAIT_DST_ACK 1000 195 #define QM_MAX_PF_WAIT_COUNT 20 196 #define QM_MAX_VF_WAIT_COUNT 40 197 #define QM_VF_RESET_WAIT_US 20000 198 #define QM_VF_RESET_WAIT_CNT 3000 199 #define QM_VF2PF_REG_SIZE 4 200 #define QM_IFC_CMD_MASK GENMASK(31, 0) 201 #define QM_IFC_DATA_SHIFT 32 202 #define QM_VF_RESET_WAIT_TIMEOUT_US \ 203 (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) 204 205 #define POLL_PERIOD 10 206 #define POLL_TIMEOUT 1000 207 #define WAIT_PERIOD_US_MAX 200 208 #define WAIT_PERIOD_US_MIN 100 209 #define MAX_WAIT_COUNTS 1000 210 #define QM_CACHE_WB_START 0x204 211 #define QM_CACHE_WB_DONE 0x208 212 #define QM_FUNC_CAPS_REG 0x3100 213 #define QM_CAPBILITY_VERSION GENMASK(7, 0) 214 215 #define PCI_BAR_2 2 216 #define PCI_BAR_4 4 217 #define QMC_ALIGN(sz) ALIGN(sz, 32) 218 219 #define QM_DBG_READ_LEN 256 220 #define QM_PCI_COMMAND_INVALID ~0 221 #define QM_RESET_STOP_TX_OFFSET 1 222 #define QM_RESET_STOP_RX_OFFSET 2 223 224 #define WAIT_PERIOD 20 225 #define REMOVE_WAIT_DELAY 10 226 227 #define QM_QOS_PARAM_NUM 2 228 #define QM_QOS_MAX_VAL 1000 229 #define QM_QOS_RATE 100 230 #define QM_QOS_EXPAND_RATE 1000 231 #define QM_SHAPER_CIR_B_MASK GENMASK(7, 0) 232 #define QM_SHAPER_CIR_U_MASK GENMASK(10, 8) 233 #define QM_SHAPER_CIR_S_MASK GENMASK(14, 11) 234 #define QM_SHAPER_FACTOR_CIR_U_SHIFT 8 235 #define QM_SHAPER_FACTOR_CIR_S_SHIFT 11 236 #define QM_SHAPER_FACTOR_CBS_B_SHIFT 15 237 #define QM_SHAPER_FACTOR_CBS_S_SHIFT 19 238 #define QM_SHAPER_CBS_B 1 239 #define QM_SHAPER_VFT_OFFSET 6 240 #define QM_QOS_MIN_ERROR_RATE 5 241 #define QM_SHAPER_MIN_CBS_S 8 242 #define QM_QOS_TICK 0x300U 243 #define QM_QOS_DIVISOR_CLK 0x1f40U 244 #define QM_QOS_MAX_CIR_B 200 245 #define QM_QOS_MIN_CIR_B 100 246 #define QM_QOS_MAX_CIR_U 6 247 #define QM_AUTOSUSPEND_DELAY 3000 248 249 /* abnormal status value for stopping queue */ 250 #define QM_STOP_QUEUE_FAIL 1 251 #define QM_DUMP_SQC_FAIL 3 252 #define QM_DUMP_CQC_FAIL 4 253 #define QM_FINISH_WAIT 5 254 255 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ 256 (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ 257 ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ 258 ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \ 259 ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) 260 261 #define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \ 262 ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) 263 264 #define QM_MK_SQC_W13(priority, orders, alg_type) \ 265 (((priority) << QM_SQ_PRIORITY_SHIFT) | \ 266 ((orders) << QM_SQ_ORDERS_SHIFT) | \ 267 (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT)) 268 269 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \ 270 (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \ 271 ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \ 272 ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \ 273 ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) 274 275 #define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \ 276 ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) 277 278 enum vft_type { 279 SQC_VFT = 0, 280 CQC_VFT, 281 SHAPER_VFT, 282 }; 283 284 enum qm_alg_type { 285 ALG_TYPE_0, 286 ALG_TYPE_1, 287 }; 288 289 enum qm_ifc_cmd { 290 QM_PF_FLR_PREPARE = 0x01, 291 QM_PF_SRST_PREPARE, 292 QM_PF_RESET_DONE, 293 QM_VF_PREPARE_DONE, 294 QM_VF_PREPARE_FAIL, 295 QM_VF_START_DONE, 296 QM_VF_START_FAIL, 297 QM_PF_SET_QOS, 298 QM_VF_GET_QOS, 299 }; 300 301 enum qm_basic_type { 302 QM_TOTAL_QP_NUM_CAP = 0x0, 303 QM_FUNC_MAX_QP_CAP, 304 QM_XEQ_DEPTH_CAP, 305 QM_QP_DEPTH_CAP, 306 QM_EQ_IRQ_TYPE_CAP, 307 QM_AEQ_IRQ_TYPE_CAP, 308 QM_ABN_IRQ_TYPE_CAP, 309 QM_PF2VF_IRQ_TYPE_CAP, 310 QM_PF_IRQ_NUM_CAP, 311 QM_VF_IRQ_NUM_CAP, 312 }; 313 314 enum qm_cap_table_type { 315 QM_CAP_VF = 0x0, 316 QM_AEQE_NUM, 317 QM_SCQE_NUM, 318 QM_EQ_IRQ, 319 QM_AEQ_IRQ, 320 QM_ABNORMAL_IRQ, 321 QM_MB_IRQ, 322 MAX_IRQ_NUM, 323 EXT_BAR_INDEX, 324 }; 325 326 static const struct hisi_qm_cap_query_info qm_cap_query_info[] = { 327 {QM_CAP_VF, "QM_CAP_VF ", 0x3100, 0x0, 0x0, 0x6F01}, 328 {QM_AEQE_NUM, "QM_AEQE_NUM ", 0x3104, 0x800, 0x4000800, 0x4000800}, 329 {QM_SCQE_NUM, "QM_SCQE_NUM ", 330 0x3108, 0x4000400, 0x4000400, 0x4000400}, 331 {QM_EQ_IRQ, "QM_EQ_IRQ ", 0x310c, 0x10000, 0x10000, 0x10000}, 332 {QM_AEQ_IRQ, "QM_AEQ_IRQ ", 0x3110, 0x0, 0x10001, 0x10001}, 333 {QM_ABNORMAL_IRQ, "QM_ABNORMAL_IRQ ", 0x3114, 0x0, 0x10003, 0x10003}, 334 {QM_MB_IRQ, "QM_MB_IRQ ", 0x3118, 0x0, 0x0, 0x10002}, 335 {MAX_IRQ_NUM, "MAX_IRQ_NUM ", 0x311c, 0x10001, 0x40002, 0x40003}, 336 {EXT_BAR_INDEX, "EXT_BAR_INDEX ", 0x3120, 0x0, 0x0, 0x14}, 337 }; 338 339 static const struct hisi_qm_cap_info qm_cap_info_comm[] = { 340 {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, 341 {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, 342 {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, 343 {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, 344 {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, 345 {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, 346 {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, 347 }; 348 349 static const struct hisi_qm_cap_info qm_cap_info_pf[] = { 350 {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1}, 351 }; 352 353 static const struct hisi_qm_cap_info qm_cap_info_vf[] = { 354 {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0}, 355 }; 356 357 static const struct hisi_qm_cap_info qm_basic_info[] = { 358 {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400}, 359 {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400}, 360 {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(31, 0), 0x800, 0x4000800, 0x4000800}, 361 {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400}, 362 {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000}, 363 {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001}, 364 {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003}, 365 {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002}, 366 {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4}, 367 {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, 368 }; 369 370 struct qm_mailbox { 371 __le16 w0; 372 __le16 queue_num; 373 __le32 base_l; 374 __le32 base_h; 375 __le32 rsvd; 376 }; 377 378 struct qm_doorbell { 379 __le16 queue_num; 380 __le16 cmd; 381 __le16 index; 382 __le16 priority; 383 }; 384 385 struct hisi_qm_resource { 386 struct hisi_qm *qm; 387 int distance; 388 struct list_head list; 389 }; 390 391 /** 392 * struct qm_hw_err - Structure describing the device errors 393 * @list: hardware error list 394 * @timestamp: timestamp when the error occurred 395 */ 396 struct qm_hw_err { 397 struct list_head list; 398 unsigned long long timestamp; 399 }; 400 401 struct hisi_qm_hw_ops { 402 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); 403 void (*qm_db)(struct hisi_qm *qm, u16 qn, 404 u8 cmd, u16 index, u8 priority); 405 int (*debug_init)(struct hisi_qm *qm); 406 void (*hw_error_init)(struct hisi_qm *qm); 407 void (*hw_error_uninit)(struct hisi_qm *qm); 408 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); 409 int (*set_msi)(struct hisi_qm *qm, bool set); 410 411 /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ 412 int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); 413 void (*set_ifc_end)(struct hisi_qm *qm); 414 int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); 415 }; 416 417 struct hisi_qm_hw_error { 418 u32 int_msk; 419 const char *msg; 420 }; 421 422 static const struct hisi_qm_hw_error qm_hw_error[] = { 423 { .int_msk = BIT(0), .msg = "qm_axi_rresp" }, 424 { .int_msk = BIT(1), .msg = "qm_axi_bresp" }, 425 { .int_msk = BIT(2), .msg = "qm_ecc_mbit" }, 426 { .int_msk = BIT(3), .msg = "qm_ecc_1bit" }, 427 { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" }, 428 { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" }, 429 { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" }, 430 { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" }, 431 { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" }, 432 { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" }, 433 { .int_msk = BIT(10), .msg = "qm_db_timeout" }, 434 { .int_msk = BIT(11), .msg = "qm_of_fifo_of" }, 435 { .int_msk = BIT(12), .msg = "qm_db_random_invalid" }, 436 { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" }, 437 { .int_msk = BIT(14), .msg = "qm_flr_timeout" }, 438 }; 439 440 static const char * const qm_db_timeout[] = { 441 "sq", "cq", "eq", "aeq", 442 }; 443 444 static const char * const qm_fifo_overflow[] = { 445 "cq", "eq", "aeq", 446 }; 447 448 struct qm_typical_qos_table { 449 u32 start; 450 u32 end; 451 u32 val; 452 }; 453 454 /* the qos step is 100 */ 455 static struct qm_typical_qos_table shaper_cir_s[] = { 456 {100, 100, 4}, 457 {200, 200, 3}, 458 {300, 500, 2}, 459 {600, 1000, 1}, 460 {1100, 100000, 0}, 461 }; 462 463 static struct qm_typical_qos_table shaper_cbs_s[] = { 464 {100, 200, 9}, 465 {300, 500, 11}, 466 {600, 1000, 12}, 467 {1100, 10000, 16}, 468 {10100, 25000, 17}, 469 {25100, 50000, 18}, 470 {50100, 100000, 19} 471 }; 472 473 static void qm_irqs_unregister(struct hisi_qm *qm); 474 static int qm_reset_device(struct hisi_qm *qm); 475 int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, 476 unsigned int device) 477 { 478 struct pci_dev *pdev; 479 u32 n, q_num; 480 int ret; 481 482 if (!val) 483 return -EINVAL; 484 485 pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL); 486 if (!pdev) { 487 q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); 488 pr_info("No device found currently, suppose queue number is %u\n", 489 q_num); 490 } else { 491 if (pdev->revision == QM_HW_V1) 492 q_num = QM_QNUM_V1; 493 else 494 q_num = QM_QNUM_V2; 495 496 pci_dev_put(pdev); 497 } 498 499 ret = kstrtou32(val, 10, &n); 500 if (ret || n < QM_MIN_QNUM || n > q_num) 501 return -EINVAL; 502 503 return param_set_int(val, kp); 504 } 505 EXPORT_SYMBOL_GPL(hisi_qm_q_num_set); 506 507 static u32 qm_get_hw_error_status(struct hisi_qm *qm) 508 { 509 return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); 510 } 511 512 static u32 qm_get_dev_err_status(struct hisi_qm *qm) 513 { 514 return qm->err_ini->get_dev_hw_err_status(qm); 515 } 516 517 /* Check if the error causes the master ooo block */ 518 static bool qm_check_dev_error(struct hisi_qm *qm) 519 { 520 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 521 u32 err_status; 522 523 if (pf_qm->fun_type == QM_HW_VF) 524 return false; 525 526 err_status = qm_get_hw_error_status(pf_qm); 527 if (err_status & pf_qm->err_info.qm_err.shutdown_mask) 528 return true; 529 530 if (pf_qm->err_ini->dev_is_abnormal) 531 return pf_qm->err_ini->dev_is_abnormal(pf_qm); 532 533 return false; 534 } 535 536 static int qm_wait_reset_finish(struct hisi_qm *qm) 537 { 538 int delay = 0; 539 540 /* All reset requests need to be queued for processing */ 541 while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { 542 msleep(++delay); 543 if (delay > QM_RESET_WAIT_TIMEOUT) 544 return -EBUSY; 545 } 546 547 return 0; 548 } 549 550 static int qm_reset_prepare_ready(struct hisi_qm *qm) 551 { 552 struct pci_dev *pdev = qm->pdev; 553 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 554 555 /* 556 * PF and VF on host doesnot support resetting at the 557 * same time on Kunpeng920. 558 */ 559 if (qm->ver < QM_HW_V3) 560 return qm_wait_reset_finish(pf_qm); 561 562 return qm_wait_reset_finish(qm); 563 } 564 565 static void qm_reset_bit_clear(struct hisi_qm *qm) 566 { 567 struct pci_dev *pdev = qm->pdev; 568 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 569 570 if (qm->ver < QM_HW_V3) 571 clear_bit(QM_RESETTING, &pf_qm->misc_ctl); 572 573 clear_bit(QM_RESETTING, &qm->misc_ctl); 574 } 575 576 static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, 577 u64 base, u16 queue, bool op) 578 { 579 mailbox->w0 = cpu_to_le16((cmd) | 580 ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) | 581 (0x1 << QM_MB_BUSY_SHIFT)); 582 mailbox->queue_num = cpu_to_le16(queue); 583 mailbox->base_l = cpu_to_le32(lower_32_bits(base)); 584 mailbox->base_h = cpu_to_le32(upper_32_bits(base)); 585 mailbox->rsvd = 0; 586 } 587 588 /* 589 * The mailbox is 128 bits and requires a single read/write operation. 590 * Since there is no general 128-bit IO memory access API in the current 591 * ARM64 architecture, this needs to be implemented in the driver. 592 */ 593 static struct qm_mailbox qm_mb_read(struct hisi_qm *qm) 594 { 595 struct qm_mailbox mailbox = {0}; 596 597 #if IS_ENABLED(CONFIG_ARM64) 598 const void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; 599 unsigned long tmp0, tmp1; 600 601 asm volatile("ldp %0, %1, %3\n" 602 "stp %0, %1, %2\n" 603 : "=&r" (tmp0), 604 "=&r" (tmp1), 605 "+Q" (mailbox) 606 : "Q" (*((char __iomem *)fun_base)) 607 : "memory"); 608 #endif 609 610 return mailbox; 611 } 612 613 /* 128 bit should be written to hardware at one time to trigger a mailbox */ 614 static void qm_mb_write(struct hisi_qm *qm, const void *src) 615 { 616 #if IS_ENABLED(CONFIG_ARM64) 617 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; 618 unsigned long tmp0, tmp1; 619 /* 620 * The dmb oshst instruction ensures that the data in the 621 * mailbox is written before it is sent to the hardware. 622 */ 623 asm volatile("ldp %0, %1, %3\n" 624 "dmb oshst\n" 625 "stp %0, %1, %2\n" 626 : "=&r" (tmp0), 627 "=&r" (tmp1), 628 "+Q" (*((char __iomem *)fun_base)) 629 : "Q" (*((char *)src)) 630 : "memory"); 631 #endif 632 } 633 634 int hisi_qm_wait_mb_ready(struct hisi_qm *qm) 635 { 636 struct qm_mailbox mailbox = {0}; 637 int ret; 638 639 ret = read_poll_timeout(qm_mb_read, mailbox, 640 !(le16_to_cpu(mailbox.w0) & QM_MB_BUSY_MASK), 641 POLL_PERIOD, POLL_TIMEOUT, 642 true, qm); 643 if (ret) 644 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); 645 646 return ret; 647 } 648 EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready); 649 650 static int qm_wait_mb_finish(struct hisi_qm *qm, struct qm_mailbox *mailbox, u32 wait_timeout) 651 { 652 struct device *dev = &qm->pdev->dev; 653 int ret; 654 655 ret = read_poll_timeout(qm_mb_read, *mailbox, 656 !(le16_to_cpu(mailbox->w0) & QM_MB_BUSY_MASK), 657 POLL_PERIOD, wait_timeout, 658 true, qm); 659 if (ret) { 660 dev_err(dev, "QM mailbox operation timeout!\n"); 661 return ret; 662 } 663 664 if (le16_to_cpu(mailbox->w0) & QM_MB_STATUS_MASK) { 665 dev_err(dev, "QM mailbox operation failed!\n"); 666 return -EIO; 667 } 668 669 return 0; 670 } 671 672 static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox, u32 wait_timeout) 673 { 674 int ret; 675 676 ret = hisi_qm_wait_mb_ready(qm); 677 if (ret) 678 goto mb_err_cnt_increase; 679 680 qm_mb_write(qm, mailbox); 681 682 ret = qm_wait_mb_finish(qm, mailbox, wait_timeout); 683 if (ret) 684 goto mb_err_cnt_increase; 685 686 return 0; 687 688 mb_err_cnt_increase: 689 atomic64_inc(&qm->debug.dfx.mb_err_cnt); 690 return ret; 691 } 692 693 int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, 694 bool op) 695 { 696 struct qm_mailbox mailbox; 697 u32 wait_timeout; 698 int ret; 699 700 if (cmd == QM_MB_CMD_STOP_QP || cmd == QM_MB_CMD_FLUSH_QM) 701 wait_timeout = QM_MB_MAX_STOP_TIMEOUT; 702 else 703 wait_timeout = QM_MB_MAX_WAIT_TIMEOUT; 704 705 /* No need to judge if master OOO is blocked. */ 706 if (qm_check_dev_error(qm)) { 707 dev_err(&qm->pdev->dev, "QM mailbox operation failed since qm is stop!\n"); 708 return -EIO; 709 } 710 711 qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); 712 713 mutex_lock(&qm->mailbox_lock); 714 ret = qm_mb_nolock(qm, &mailbox, wait_timeout); 715 mutex_unlock(&qm->mailbox_lock); 716 717 return ret; 718 } 719 EXPORT_SYMBOL_GPL(hisi_qm_mb); 720 721 int hisi_qm_mb_read(struct hisi_qm *qm, u64 *base, u8 cmd, u16 queue) 722 { 723 struct qm_mailbox mailbox; 724 int ret; 725 726 qm_mb_pre_init(&mailbox, cmd, 0, queue, 1); 727 mutex_lock(&qm->mailbox_lock); 728 ret = qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); 729 mutex_unlock(&qm->mailbox_lock); 730 if (ret) 731 return ret; 732 733 *base = le32_to_cpu(mailbox.base_l) | 734 ((u64)le32_to_cpu(mailbox.base_h) << 32); 735 736 return 0; 737 } 738 EXPORT_SYMBOL_GPL(hisi_qm_mb_read); 739 740 /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ 741 int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) 742 { 743 struct qm_mailbox mailbox; 744 dma_addr_t xqc_dma; 745 void *tmp_xqc; 746 size_t size; 747 int ret; 748 749 switch (cmd) { 750 case QM_MB_CMD_SQC: 751 size = sizeof(struct qm_sqc); 752 tmp_xqc = qm->xqc_buf.sqc; 753 xqc_dma = qm->xqc_buf.sqc_dma; 754 break; 755 case QM_MB_CMD_CQC: 756 size = sizeof(struct qm_cqc); 757 tmp_xqc = qm->xqc_buf.cqc; 758 xqc_dma = qm->xqc_buf.cqc_dma; 759 break; 760 case QM_MB_CMD_EQC: 761 size = sizeof(struct qm_eqc); 762 tmp_xqc = qm->xqc_buf.eqc; 763 xqc_dma = qm->xqc_buf.eqc_dma; 764 break; 765 case QM_MB_CMD_AEQC: 766 size = sizeof(struct qm_aeqc); 767 tmp_xqc = qm->xqc_buf.aeqc; 768 xqc_dma = qm->xqc_buf.aeqc_dma; 769 break; 770 default: 771 dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd); 772 return -EINVAL; 773 } 774 775 /* Setting xqc will fail if master OOO is blocked. */ 776 if (qm_check_dev_error(qm)) { 777 dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); 778 return -EIO; 779 } 780 781 mutex_lock(&qm->mailbox_lock); 782 if (!op) 783 memcpy(tmp_xqc, xqc, size); 784 785 qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op); 786 ret = qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); 787 if (!ret && op) 788 memcpy(xqc, tmp_xqc, size); 789 790 mutex_unlock(&qm->mailbox_lock); 791 792 return ret; 793 } 794 795 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 796 { 797 u64 doorbell; 798 799 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) | 800 ((u64)index << QM_DB_INDEX_SHIFT_V1) | 801 ((u64)priority << QM_DB_PRIORITY_SHIFT_V1); 802 803 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1); 804 } 805 806 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 807 { 808 void __iomem *io_base = qm->io_base; 809 u16 randata = 0; 810 u64 doorbell; 811 812 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) 813 io_base = qm->db_io_base + (u64)qn * qm->db_interval + 814 QM_DOORBELL_SQ_CQ_BASE_V2; 815 else 816 io_base += QM_DOORBELL_EQ_AEQ_BASE_V2; 817 818 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | 819 ((u64)randata << QM_DB_RAND_SHIFT_V2) | 820 ((u64)index << QM_DB_INDEX_SHIFT_V2) | 821 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); 822 823 writeq(doorbell, io_base); 824 } 825 826 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 827 { 828 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n", 829 qn, cmd, index); 830 831 qm->ops->qm_db(qm, qn, cmd, index, priority); 832 } 833 834 static void qm_disable_clock_gate(struct hisi_qm *qm) 835 { 836 u32 val; 837 838 /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */ 839 if (qm->ver < QM_HW_V3) 840 return; 841 842 val = readl(qm->io_base + QM_PM_CTRL); 843 val |= QM_IDLE_DISABLE; 844 writel(val, qm->io_base + QM_PM_CTRL); 845 } 846 847 static int qm_dev_mem_reset(struct hisi_qm *qm) 848 { 849 u32 val; 850 851 writel(0x1, qm->io_base + QM_MEM_START_INIT); 852 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, 853 val & BIT(0), POLL_PERIOD, 854 POLL_TIMEOUT); 855 } 856 857 /** 858 * hisi_qm_get_hw_info() - Get device information. 859 * @qm: The qm which want to get information. 860 * @info_table: Array for storing device information. 861 * @index: Index in info_table. 862 * @is_read: Whether read from reg, 0: not support read from reg. 863 * 864 * This function returns device information the caller needs. 865 */ 866 u32 hisi_qm_get_hw_info(struct hisi_qm *qm, 867 const struct hisi_qm_cap_info *info_table, 868 u32 index, bool is_read) 869 { 870 u32 val; 871 872 switch (qm->ver) { 873 case QM_HW_V1: 874 return info_table[index].v1_val; 875 case QM_HW_V2: 876 return info_table[index].v2_val; 877 default: 878 if (!is_read) 879 return info_table[index].v3_val; 880 881 val = readl(qm->io_base + info_table[index].offset); 882 return (val >> info_table[index].shift) & info_table[index].mask; 883 } 884 } 885 EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); 886 887 u32 hisi_qm_get_cap_value(struct hisi_qm *qm, 888 const struct hisi_qm_cap_query_info *info_table, 889 u32 index, bool is_read) 890 { 891 u32 val; 892 893 switch (qm->ver) { 894 case QM_HW_V1: 895 return info_table[index].v1_val; 896 case QM_HW_V2: 897 return info_table[index].v2_val; 898 default: 899 if (!is_read) 900 return info_table[index].v3_val; 901 902 val = readl(qm->io_base + info_table[index].offset); 903 return val; 904 } 905 } 906 EXPORT_SYMBOL_GPL(hisi_qm_get_cap_value); 907 908 static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, 909 u16 *high_bits, enum qm_basic_type type) 910 { 911 u32 depth; 912 913 depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); 914 *low_bits = depth & QM_XQ_DEPTH_MASK; 915 *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; 916 } 917 918 int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, 919 u32 dev_algs_size) 920 { 921 struct device *dev = &qm->pdev->dev; 922 char *algs, *ptr; 923 int i; 924 925 if (!qm->uacce) 926 return 0; 927 928 if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) { 929 dev_err(dev, "algs size %u is equal or larger than %d.\n", 930 dev_algs_size, QM_DEV_ALG_MAX_LEN); 931 return -EINVAL; 932 } 933 934 algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL); 935 if (!algs) 936 return -ENOMEM; 937 938 for (i = 0; i < dev_algs_size; i++) 939 if (alg_msk & dev_algs[i].alg_msk) 940 strcat(algs, dev_algs[i].alg); 941 942 ptr = strrchr(algs, '\n'); 943 if (ptr) 944 *ptr = '\0'; 945 946 qm->uacce->algs = algs; 947 948 return 0; 949 } 950 EXPORT_SYMBOL_GPL(hisi_qm_set_algs); 951 952 static u32 qm_get_irq_num(struct hisi_qm *qm) 953 { 954 if (qm->fun_type == QM_HW_PF) 955 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver); 956 957 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver); 958 } 959 960 static int qm_pm_get_sync(struct hisi_qm *qm) 961 { 962 struct device *dev = &qm->pdev->dev; 963 int ret; 964 965 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 966 return 0; 967 968 ret = pm_runtime_resume_and_get(dev); 969 if (ret < 0) { 970 dev_err(dev, "failed to get_sync(%d).\n", ret); 971 return ret; 972 } 973 974 return 0; 975 } 976 977 static void qm_pm_put_sync(struct hisi_qm *qm) 978 { 979 struct device *dev = &qm->pdev->dev; 980 981 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 982 return; 983 984 pm_runtime_put_autosuspend(dev); 985 } 986 987 static void qm_cq_head_update(struct hisi_qp *qp) 988 { 989 if (qp->qp_status.cq_head == qp->cq_depth - 1) { 990 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; 991 qp->qp_status.cq_head = 0; 992 } else { 993 qp->qp_status.cq_head++; 994 } 995 } 996 997 static void qm_poll_req_cb(struct hisi_qp *qp) 998 { 999 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; 1000 struct hisi_qm *qm = qp->qm; 1001 1002 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { 1003 dma_rmb(); 1004 qp->req_cb(qp, qp->sqe + qm->sqe_size * 1005 le16_to_cpu(cqe->sq_head)); 1006 qm_cq_head_update(qp); 1007 cqe = qp->cqe + qp->qp_status.cq_head; 1008 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, 1009 qp->qp_status.cq_head, 0); 1010 atomic_dec(&qp->qp_status.used); 1011 1012 cond_resched(); 1013 } 1014 1015 /* set c_flag */ 1016 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); 1017 } 1018 1019 static void qm_work_process(struct work_struct *work) 1020 { 1021 struct hisi_qm_poll_data *poll_data = 1022 container_of(work, struct hisi_qm_poll_data, work); 1023 struct hisi_qm *qm = poll_data->qm; 1024 u16 eqe_num = poll_data->eqe_num; 1025 struct hisi_qp *qp; 1026 int i; 1027 1028 for (i = eqe_num - 1; i >= 0; i--) { 1029 qp = &qm->qp_array[poll_data->qp_finish_id[i]]; 1030 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) 1031 continue; 1032 1033 if (qp->event_cb) { 1034 qp->event_cb(qp); 1035 continue; 1036 } 1037 1038 if (likely(qp->req_cb)) 1039 qm_poll_req_cb(qp); 1040 } 1041 } 1042 1043 static void qm_get_complete_eqe_num(struct hisi_qm *qm) 1044 { 1045 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; 1046 struct hisi_qm_poll_data *poll_data = NULL; 1047 u32 dw0 = le32_to_cpu(eqe->dw0); 1048 u16 eq_depth = qm->eq_depth; 1049 u16 cqn, eqe_num = 0; 1050 1051 if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { 1052 atomic64_inc(&qm->debug.dfx.err_irq_cnt); 1053 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 1054 return; 1055 } 1056 1057 cqn = dw0 & QM_EQE_CQN_MASK; 1058 if (unlikely(cqn >= qm->qp_num)) 1059 return; 1060 poll_data = &qm->poll_data[cqn]; 1061 1062 do { 1063 poll_data->qp_finish_id[eqe_num] = dw0 & QM_EQE_CQN_MASK; 1064 eqe_num++; 1065 1066 if (qm->status.eq_head == eq_depth - 1) { 1067 qm->status.eqc_phase = !qm->status.eqc_phase; 1068 eqe = qm->eqe; 1069 qm->status.eq_head = 0; 1070 } else { 1071 eqe++; 1072 qm->status.eq_head++; 1073 } 1074 1075 dw0 = le32_to_cpu(eqe->dw0); 1076 if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) 1077 break; 1078 } while (eqe_num < (eq_depth >> 1) - 1); 1079 1080 poll_data->eqe_num = eqe_num; 1081 queue_work(qm->wq, &poll_data->work); 1082 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 1083 } 1084 1085 static irqreturn_t qm_eq_irq(int irq, void *data) 1086 { 1087 struct hisi_qm *qm = data; 1088 1089 /* Get qp id of completed tasks and re-enable the interrupt */ 1090 qm_get_complete_eqe_num(qm); 1091 1092 return IRQ_HANDLED; 1093 } 1094 1095 static irqreturn_t qm_mb_cmd_irq(int irq, void *data) 1096 { 1097 struct hisi_qm *qm = data; 1098 u32 val; 1099 1100 val = readl(qm->io_base + QM_IFC_INT_STATUS); 1101 val &= QM_IFC_INT_STATUS_MASK; 1102 if (!val) 1103 return IRQ_NONE; 1104 1105 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) { 1106 dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n"); 1107 return IRQ_HANDLED; 1108 } 1109 1110 schedule_work(&qm->cmd_process); 1111 1112 return IRQ_HANDLED; 1113 } 1114 1115 static void qm_set_qp_disable(struct hisi_qp *qp, int offset) 1116 { 1117 u32 *addr; 1118 1119 if (qp->is_in_kernel) 1120 return; 1121 1122 addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset; 1123 *addr = 1; 1124 1125 /* make sure setup is completed */ 1126 smp_wmb(); 1127 } 1128 1129 static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) 1130 { 1131 struct hisi_qp *qp = &qm->qp_array[qp_id]; 1132 1133 qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET); 1134 hisi_qm_stop_qp(qp); 1135 qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET); 1136 } 1137 1138 static void qm_reset_function(struct hisi_qm *qm) 1139 { 1140 struct device *dev = &qm->pdev->dev; 1141 int ret; 1142 1143 if (qm_check_dev_error(qm)) 1144 return; 1145 1146 ret = qm_reset_prepare_ready(qm); 1147 if (ret) { 1148 dev_err(dev, "reset function not ready\n"); 1149 return; 1150 } 1151 1152 ret = hisi_qm_stop(qm, QM_DOWN); 1153 if (ret) { 1154 dev_err(dev, "failed to stop qm when reset function\n"); 1155 goto clear_bit; 1156 } 1157 1158 ret = hisi_qm_start(qm); 1159 if (ret) 1160 dev_err(dev, "failed to start qm when reset function\n"); 1161 1162 clear_bit: 1163 qm_reset_bit_clear(qm); 1164 } 1165 1166 static irqreturn_t qm_aeq_thread(int irq, void *data) 1167 { 1168 struct hisi_qm *qm = data; 1169 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; 1170 u32 dw0 = le32_to_cpu(aeqe->dw0); 1171 u16 aeq_depth = qm->aeq_depth; 1172 u32 type, qp_id; 1173 1174 atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); 1175 1176 while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { 1177 type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; 1178 qp_id = dw0 & QM_AEQE_CQN_MASK; 1179 1180 switch (type) { 1181 case QM_EQ_OVERFLOW: 1182 dev_err(&qm->pdev->dev, "eq overflow, reset function\n"); 1183 qm_reset_function(qm); 1184 return IRQ_HANDLED; 1185 case QM_CQ_OVERFLOW: 1186 dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n", 1187 qp_id); 1188 fallthrough; 1189 case QM_CQE_ERROR: 1190 qm_disable_qp(qm, qp_id); 1191 break; 1192 default: 1193 dev_err(&qm->pdev->dev, "unknown error type %u\n", 1194 type); 1195 break; 1196 } 1197 1198 if (qm->status.aeq_head == aeq_depth - 1) { 1199 qm->status.aeqc_phase = !qm->status.aeqc_phase; 1200 aeqe = qm->aeqe; 1201 qm->status.aeq_head = 0; 1202 } else { 1203 aeqe++; 1204 qm->status.aeq_head++; 1205 } 1206 dw0 = le32_to_cpu(aeqe->dw0); 1207 } 1208 1209 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); 1210 1211 return IRQ_HANDLED; 1212 } 1213 1214 static void qm_init_qp_status(struct hisi_qp *qp) 1215 { 1216 struct hisi_qp_status *qp_status = &qp->qp_status; 1217 1218 qp_status->sq_tail = 0; 1219 qp_status->cq_head = 0; 1220 qp_status->cqc_phase = true; 1221 atomic_set(&qp_status->used, 0); 1222 } 1223 1224 static void qm_init_prefetch(struct hisi_qm *qm) 1225 { 1226 struct device *dev = &qm->pdev->dev; 1227 u32 page_type = 0x0; 1228 1229 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) 1230 return; 1231 1232 switch (PAGE_SIZE) { 1233 case SZ_4K: 1234 page_type = 0x0; 1235 break; 1236 case SZ_16K: 1237 page_type = 0x1; 1238 break; 1239 case SZ_64K: 1240 page_type = 0x2; 1241 break; 1242 default: 1243 dev_err(dev, "system page size is not support: %lu, default set to 4KB", 1244 PAGE_SIZE); 1245 } 1246 1247 writel(page_type, qm->io_base + QM_PAGE_SIZE); 1248 } 1249 1250 /* 1251 * acc_shaper_para_calc() Get the IR value by the qos formula, the return value 1252 * is the expected qos calculated. 1253 * the formula: 1254 * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps 1255 * 1256 * IR_b * (2 ^ IR_u) * 8000 1257 * IR(Mbps) = ------------------------- 1258 * Tick * (2 ^ IR_s) 1259 */ 1260 static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) 1261 { 1262 return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) / 1263 (QM_QOS_TICK * (1 << cir_s)); 1264 } 1265 1266 static u32 acc_shaper_calc_cbs_s(u32 ir) 1267 { 1268 int table_size = ARRAY_SIZE(shaper_cbs_s); 1269 int i; 1270 1271 for (i = 0; i < table_size; i++) { 1272 if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end) 1273 return shaper_cbs_s[i].val; 1274 } 1275 1276 return QM_SHAPER_MIN_CBS_S; 1277 } 1278 1279 static u32 acc_shaper_calc_cir_s(u32 ir) 1280 { 1281 int table_size = ARRAY_SIZE(shaper_cir_s); 1282 int i; 1283 1284 for (i = 0; i < table_size; i++) { 1285 if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end) 1286 return shaper_cir_s[i].val; 1287 } 1288 1289 return 0; 1290 } 1291 1292 static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) 1293 { 1294 u32 cir_b, cir_u, cir_s, ir_calc; 1295 u32 error_rate; 1296 1297 factor->cbs_s = acc_shaper_calc_cbs_s(ir); 1298 cir_s = acc_shaper_calc_cir_s(ir); 1299 1300 for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) { 1301 for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) { 1302 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); 1303 1304 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; 1305 if (error_rate <= QM_QOS_MIN_ERROR_RATE) { 1306 factor->cir_b = cir_b; 1307 factor->cir_u = cir_u; 1308 factor->cir_s = cir_s; 1309 return 0; 1310 } 1311 } 1312 } 1313 1314 return -EINVAL; 1315 } 1316 1317 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, 1318 u32 number, struct qm_shaper_factor *factor) 1319 { 1320 u64 tmp = 0; 1321 1322 if (number > 0) { 1323 switch (type) { 1324 case SQC_VFT: 1325 if (qm->ver == QM_HW_V1) { 1326 tmp = QM_SQC_VFT_BUF_SIZE | 1327 QM_SQC_VFT_SQC_SIZE | 1328 QM_SQC_VFT_INDEX_NUMBER | 1329 QM_SQC_VFT_VALID | 1330 (u64)base << QM_SQC_VFT_START_SQN_SHIFT; 1331 } else { 1332 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT | 1333 QM_SQC_VFT_VALID | 1334 (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT; 1335 } 1336 break; 1337 case CQC_VFT: 1338 if (qm->ver == QM_HW_V1) { 1339 tmp = QM_CQC_VFT_BUF_SIZE | 1340 QM_CQC_VFT_SQC_SIZE | 1341 QM_CQC_VFT_INDEX_NUMBER | 1342 QM_CQC_VFT_VALID; 1343 } else { 1344 tmp = QM_CQC_VFT_VALID; 1345 } 1346 break; 1347 case SHAPER_VFT: 1348 if (factor) { 1349 tmp = factor->cir_b | 1350 (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | 1351 (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | 1352 (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) | 1353 (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); 1354 } 1355 break; 1356 /* 1357 * Note: The current logic only needs to handle the above three types 1358 * If new types are added, they need to be supplemented here, 1359 * otherwise undefined behavior may occur. 1360 */ 1361 default: 1362 break; 1363 } 1364 } 1365 1366 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L); 1367 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H); 1368 } 1369 1370 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, 1371 u32 fun_num, u32 base, u32 number) 1372 { 1373 struct qm_shaper_factor *factor = NULL; 1374 unsigned int val; 1375 int ret; 1376 1377 if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 1378 factor = &qm->factor[fun_num]; 1379 1380 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 1381 val & BIT(0), POLL_PERIOD, 1382 POLL_TIMEOUT); 1383 if (ret) 1384 return ret; 1385 1386 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); 1387 writel(type, qm->io_base + QM_VFT_CFG_TYPE); 1388 if (type == SHAPER_VFT) 1389 fun_num |= base << QM_SHAPER_VFT_OFFSET; 1390 1391 writel(fun_num, qm->io_base + QM_VFT_CFG); 1392 1393 qm_vft_data_cfg(qm, type, base, number, factor); 1394 1395 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); 1396 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); 1397 1398 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 1399 val & BIT(0), POLL_PERIOD, 1400 POLL_TIMEOUT); 1401 } 1402 1403 static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) 1404 { 1405 u32 qos = qm->factor[fun_num].func_qos; 1406 int ret, i; 1407 1408 ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]); 1409 if (ret) { 1410 dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); 1411 return ret; 1412 } 1413 writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG); 1414 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { 1415 /* The base number of queue reuse for different alg type */ 1416 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1); 1417 if (ret) 1418 return ret; 1419 } 1420 1421 return 0; 1422 } 1423 1424 /* The config should be conducted after qm_dev_mem_reset() */ 1425 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, 1426 u32 number) 1427 { 1428 int ret, i; 1429 1430 for (i = SQC_VFT; i <= CQC_VFT; i++) { 1431 ret = qm_set_vft_common(qm, i, fun_num, base, number); 1432 if (ret) 1433 return ret; 1434 } 1435 1436 /* init default shaper qos val */ 1437 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { 1438 ret = qm_shaper_init_vft(qm, fun_num); 1439 if (ret) 1440 goto back_sqc_cqc; 1441 } 1442 1443 return 0; 1444 back_sqc_cqc: 1445 for (i = SQC_VFT; i <= CQC_VFT; i++) 1446 qm_set_vft_common(qm, i, fun_num, 0, 0); 1447 1448 return ret; 1449 } 1450 1451 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) 1452 { 1453 u64 sqc_vft; 1454 int ret; 1455 1456 ret = hisi_qm_mb_read(qm, &sqc_vft, QM_MB_CMD_SQC_VFT_V2, 0); 1457 if (ret) 1458 return ret; 1459 1460 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); 1461 *number = (QM_SQC_VFT_NUM_MASK_V2 & 1462 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; 1463 1464 return 0; 1465 } 1466 1467 static void qm_hw_error_init_v1(struct hisi_qm *qm) 1468 { 1469 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); 1470 } 1471 1472 static void qm_hw_error_cfg(struct hisi_qm *qm) 1473 { 1474 struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; 1475 1476 qm->error_mask = qm_err->nfe | qm_err->ce | qm_err->fe; 1477 /* clear QM hw residual error source */ 1478 writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); 1479 1480 /* configure error type */ 1481 writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); 1482 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); 1483 writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); 1484 writel(qm_err->fe, qm->io_base + QM_RAS_FE_ENABLE); 1485 } 1486 1487 static void qm_hw_error_init_v2(struct hisi_qm *qm) 1488 { 1489 u32 irq_unmask; 1490 1491 qm_hw_error_cfg(qm); 1492 1493 irq_unmask = ~qm->error_mask; 1494 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1495 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); 1496 } 1497 1498 static void qm_hw_error_uninit_v2(struct hisi_qm *qm) 1499 { 1500 u32 irq_mask = qm->error_mask; 1501 1502 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1503 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); 1504 } 1505 1506 static void qm_hw_error_init_v3(struct hisi_qm *qm) 1507 { 1508 u32 irq_unmask; 1509 1510 qm_hw_error_cfg(qm); 1511 1512 /* enable close master ooo when hardware error happened */ 1513 writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); 1514 1515 irq_unmask = ~qm->error_mask; 1516 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1517 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); 1518 } 1519 1520 static void qm_hw_error_uninit_v3(struct hisi_qm *qm) 1521 { 1522 u32 irq_mask = qm->error_mask; 1523 1524 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1525 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); 1526 1527 /* disable close master ooo when hardware error happened */ 1528 writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); 1529 } 1530 1531 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) 1532 { 1533 const struct hisi_qm_hw_error *err; 1534 struct device *dev = &qm->pdev->dev; 1535 u32 reg_val, type, vf_num, qp_id; 1536 int i; 1537 1538 for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) { 1539 err = &qm_hw_error[i]; 1540 if (!(err->int_msk & error_status)) 1541 continue; 1542 1543 dev_err(dev, "%s [error status=0x%x] found\n", 1544 err->msg, err->int_msk); 1545 1546 if (err->int_msk & QM_DB_TIMEOUT) { 1547 reg_val = readl(qm->io_base + QM_ABNORMAL_INF01); 1548 type = (reg_val & QM_DB_TIMEOUT_TYPE) >> 1549 QM_DB_TIMEOUT_TYPE_SHIFT; 1550 vf_num = reg_val & QM_DB_TIMEOUT_VF; 1551 qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT; 1552 dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n", 1553 qm_db_timeout[type], vf_num, qp_id); 1554 } else if (err->int_msk & QM_OF_FIFO_OF) { 1555 reg_val = readl(qm->io_base + QM_ABNORMAL_INF00); 1556 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> 1557 QM_FIFO_OVERFLOW_TYPE_SHIFT; 1558 vf_num = reg_val & QM_FIFO_OVERFLOW_VF; 1559 qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT; 1560 if (type < ARRAY_SIZE(qm_fifo_overflow)) 1561 dev_err(dev, "qm %s fifo overflow in function %u qp %u\n", 1562 qm_fifo_overflow[type], vf_num, qp_id); 1563 else 1564 dev_err(dev, "unknown error type\n"); 1565 } else if (err->int_msk & QM_AXI_RRESP_ERR) { 1566 reg_val = readl(qm->io_base + QM_ABNORMAL_INF02); 1567 if (reg_val & QM_AXI_POISON_ERR) 1568 dev_err(dev, "qm axi poison error happened\n"); 1569 } 1570 } 1571 } 1572 1573 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) 1574 { 1575 struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; 1576 u32 error_status; 1577 1578 error_status = qm_get_hw_error_status(qm); 1579 if (error_status & qm->error_mask) { 1580 if (error_status & QM_ECC_MBIT) 1581 qm->err_status.is_qm_ecc_mbit = true; 1582 1583 qm_log_hw_error(qm, error_status); 1584 if (error_status & qm_err->reset_mask) { 1585 /* Disable the same error reporting until device is recovered. */ 1586 writel(qm_err->nfe & (~error_status), qm->io_base + QM_RAS_NFE_ENABLE); 1587 return ACC_ERR_NEED_RESET; 1588 } 1589 1590 /* Clear error source if not need reset. */ 1591 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); 1592 writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); 1593 writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); 1594 } 1595 1596 return ACC_ERR_RECOVERED; 1597 } 1598 1599 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) 1600 { 1601 u32 val; 1602 1603 if (qm->fun_type == QM_HW_PF) 1604 writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P); 1605 1606 val = readl(qm->io_base + QM_IFC_INT_SOURCE_V); 1607 val |= QM_IFC_INT_SOURCE_MASK; 1608 writel(val, qm->io_base + QM_IFC_INT_SOURCE_V); 1609 } 1610 1611 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) 1612 { 1613 struct device *dev = &qm->pdev->dev; 1614 enum qm_ifc_cmd cmd; 1615 int ret; 1616 1617 ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); 1618 if (ret) { 1619 dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); 1620 return; 1621 } 1622 1623 switch (cmd) { 1624 case QM_VF_PREPARE_FAIL: 1625 dev_err(dev, "failed to stop VF(%u)!\n", vf_id); 1626 break; 1627 case QM_VF_START_FAIL: 1628 dev_err(dev, "failed to start VF(%u)!\n", vf_id); 1629 break; 1630 case QM_VF_PREPARE_DONE: 1631 case QM_VF_START_DONE: 1632 break; 1633 default: 1634 dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); 1635 break; 1636 } 1637 } 1638 1639 static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) 1640 { 1641 struct device *dev = &qm->pdev->dev; 1642 u32 vfs_num = qm->vfs_num; 1643 int cnt = 0; 1644 int ret = 0; 1645 u64 val; 1646 u32 i; 1647 1648 if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 1649 return 0; 1650 1651 while (true) { 1652 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); 1653 /* All VFs send command to PF, break */ 1654 if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1)) 1655 break; 1656 1657 if (++cnt > QM_MAX_PF_WAIT_COUNT) { 1658 ret = -EBUSY; 1659 break; 1660 } 1661 1662 msleep(QM_WAIT_DST_ACK); 1663 } 1664 1665 /* PF check VFs msg */ 1666 for (i = 1; i <= vfs_num; i++) { 1667 if (val & BIT(i)) 1668 qm_handle_vf_msg(qm, i); 1669 else 1670 dev_err(dev, "VF(%u) not ping PF!\n", i); 1671 } 1672 1673 /* PF clear interrupt to ack VFs */ 1674 qm_clear_cmd_interrupt(qm, val); 1675 1676 return ret; 1677 } 1678 1679 static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) 1680 { 1681 u32 val; 1682 1683 val = readl(qm->io_base + QM_IFC_INT_CFG); 1684 val &= ~QM_IFC_SEND_ALL_VFS; 1685 val |= fun_num; 1686 writel(val, qm->io_base + QM_IFC_INT_CFG); 1687 1688 val = readl(qm->io_base + QM_IFC_INT_SET_P); 1689 val |= QM_IFC_INT_SET_MASK; 1690 writel(val, qm->io_base + QM_IFC_INT_SET_P); 1691 } 1692 1693 static void qm_trigger_pf_interrupt(struct hisi_qm *qm) 1694 { 1695 u32 val; 1696 1697 val = readl(qm->io_base + QM_IFC_INT_SET_V); 1698 val |= QM_IFC_INT_SET_MASK; 1699 writel(val, qm->io_base + QM_IFC_INT_SET_V); 1700 } 1701 1702 static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1703 { 1704 struct device *dev = &qm->pdev->dev; 1705 int cnt = 0; 1706 u64 val; 1707 int ret; 1708 1709 ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); 1710 if (ret) { 1711 dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); 1712 goto err_unlock; 1713 } 1714 1715 qm_trigger_vf_interrupt(qm, fun_num); 1716 while (true) { 1717 msleep(QM_WAIT_DST_ACK); 1718 val = readq(qm->io_base + QM_IFC_READY_STATUS); 1719 /* if VF respond, PF notifies VF successfully. */ 1720 if (!(val & BIT(fun_num))) 1721 goto err_unlock; 1722 1723 if (++cnt > QM_MAX_PF_WAIT_COUNT) { 1724 dev_err(dev, "failed to get response from VF(%u)!\n", fun_num); 1725 ret = -ETIMEDOUT; 1726 break; 1727 } 1728 } 1729 1730 err_unlock: 1731 qm->ops->set_ifc_end(qm); 1732 return ret; 1733 } 1734 1735 static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 1736 { 1737 struct device *dev = &qm->pdev->dev; 1738 u32 vfs_num = qm->vfs_num; 1739 u64 val = 0; 1740 int cnt = 0; 1741 int ret; 1742 u32 i; 1743 1744 ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); 1745 if (ret) { 1746 dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); 1747 qm->ops->set_ifc_end(qm); 1748 return ret; 1749 } 1750 1751 qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS); 1752 while (true) { 1753 msleep(QM_WAIT_DST_ACK); 1754 val = readq(qm->io_base + QM_IFC_READY_STATUS); 1755 /* If all VFs acked, PF notifies VFs successfully. */ 1756 if (!(val & GENMASK(vfs_num, 1))) { 1757 qm->ops->set_ifc_end(qm); 1758 return 0; 1759 } 1760 1761 if (++cnt > QM_MAX_PF_WAIT_COUNT) 1762 break; 1763 } 1764 1765 qm->ops->set_ifc_end(qm); 1766 1767 /* Check which vf respond timeout. */ 1768 for (i = 1; i <= vfs_num; i++) { 1769 if (val & BIT(i)) 1770 dev_err(dev, "failed to get response from VF(%u)!\n", i); 1771 } 1772 1773 return -ETIMEDOUT; 1774 } 1775 1776 static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 1777 { 1778 int cnt = 0; 1779 u32 val; 1780 int ret; 1781 1782 ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); 1783 if (ret) { 1784 dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); 1785 goto unlock; 1786 } 1787 1788 qm_trigger_pf_interrupt(qm); 1789 /* Waiting for PF response */ 1790 while (true) { 1791 msleep(QM_WAIT_DST_ACK); 1792 val = readl(qm->io_base + QM_IFC_INT_SET_V); 1793 if (!(val & QM_IFC_INT_STATUS_MASK)) 1794 break; 1795 1796 if (++cnt > QM_MAX_VF_WAIT_COUNT) { 1797 ret = -ETIMEDOUT; 1798 break; 1799 } 1800 } 1801 1802 unlock: 1803 qm->ops->set_ifc_end(qm); 1804 1805 return ret; 1806 } 1807 1808 static int qm_drain_qm(struct hisi_qm *qm) 1809 { 1810 return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0); 1811 } 1812 1813 static int qm_stop_qp(struct hisi_qp *qp) 1814 { 1815 return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); 1816 } 1817 1818 static int qm_set_msi(struct hisi_qm *qm, bool set) 1819 { 1820 struct pci_dev *pdev = qm->pdev; 1821 1822 if (set) { 1823 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, 1824 0); 1825 } else { 1826 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, 1827 ACC_PEH_MSI_DISABLE); 1828 if (qm->err_status.is_qm_ecc_mbit || 1829 qm->err_status.is_dev_ecc_mbit) 1830 return 0; 1831 1832 mdelay(1); 1833 if (readl(qm->io_base + QM_PEH_DFX_INFO0)) 1834 return -EFAULT; 1835 } 1836 1837 return 0; 1838 } 1839 1840 static void qm_wait_msi_finish(struct hisi_qm *qm) 1841 { 1842 struct pci_dev *pdev = qm->pdev; 1843 u32 cmd = ~0; 1844 int cnt = 0; 1845 u32 val; 1846 int ret; 1847 1848 while (true) { 1849 pci_read_config_dword(pdev, pdev->msi_cap + 1850 PCI_MSI_PENDING_64, &cmd); 1851 if (!cmd) 1852 break; 1853 1854 if (++cnt > MAX_WAIT_COUNTS) { 1855 pci_warn(pdev, "failed to empty MSI PENDING!\n"); 1856 break; 1857 } 1858 1859 udelay(1); 1860 } 1861 1862 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0, 1863 val, !(val & QM_PEH_DFX_MASK), 1864 POLL_PERIOD, POLL_TIMEOUT); 1865 if (ret) 1866 pci_warn(pdev, "failed to empty PEH MSI!\n"); 1867 1868 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1, 1869 val, !(val & QM_PEH_MSI_FINISH_MASK), 1870 POLL_PERIOD, POLL_TIMEOUT); 1871 if (ret) 1872 pci_warn(pdev, "failed to finish MSI operation!\n"); 1873 } 1874 1875 static int qm_set_msi_v3(struct hisi_qm *qm, bool set) 1876 { 1877 struct pci_dev *pdev = qm->pdev; 1878 int ret = -ETIMEDOUT; 1879 u32 cmd, i; 1880 1881 pci_read_config_dword(pdev, pdev->msi_cap, &cmd); 1882 if (set) 1883 cmd |= QM_MSI_CAP_ENABLE; 1884 else 1885 cmd &= ~QM_MSI_CAP_ENABLE; 1886 1887 pci_write_config_dword(pdev, pdev->msi_cap, cmd); 1888 if (set) { 1889 for (i = 0; i < MAX_WAIT_COUNTS; i++) { 1890 pci_read_config_dword(pdev, pdev->msi_cap, &cmd); 1891 if (cmd & QM_MSI_CAP_ENABLE) 1892 return 0; 1893 1894 udelay(1); 1895 } 1896 } else { 1897 udelay(WAIT_PERIOD_US_MIN); 1898 qm_wait_msi_finish(qm); 1899 ret = 0; 1900 } 1901 1902 return ret; 1903 } 1904 1905 static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1906 { 1907 struct qm_mailbox mailbox; 1908 u64 msg; 1909 1910 msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; 1911 1912 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); 1913 mutex_lock(&qm->mailbox_lock); 1914 return qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); 1915 } 1916 1917 static void qm_set_ifc_end_v3(struct hisi_qm *qm) 1918 { 1919 mutex_unlock(&qm->mailbox_lock); 1920 } 1921 1922 static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) 1923 { 1924 u64 msg; 1925 int ret; 1926 1927 ret = hisi_qm_mb_read(qm, &msg, QM_MB_CMD_DST, fun_num); 1928 if (ret) 1929 return ret; 1930 1931 *cmd = msg & QM_IFC_CMD_MASK; 1932 1933 if (data) 1934 *data = msg >> QM_IFC_DATA_SHIFT; 1935 1936 return 0; 1937 } 1938 1939 static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1940 { 1941 uintptr_t offset; 1942 u64 msg; 1943 1944 if (qm->fun_type == QM_HW_PF) 1945 offset = QM_PF2VF_PF_W; 1946 else 1947 offset = QM_VF2PF_VF_W; 1948 1949 msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; 1950 1951 mutex_lock(&qm->ifc_lock); 1952 writeq(msg, qm->io_base + offset); 1953 1954 return 0; 1955 } 1956 1957 static void qm_set_ifc_end_v4(struct hisi_qm *qm) 1958 { 1959 mutex_unlock(&qm->ifc_lock); 1960 } 1961 1962 static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) 1963 { 1964 uintptr_t offset; 1965 1966 offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; 1967 1968 return (u64)readl(qm->io_base + offset); 1969 } 1970 1971 static u64 qm_get_ifc_vf(struct hisi_qm *qm) 1972 { 1973 return readq(qm->io_base + QM_PF2VF_VF_R); 1974 } 1975 1976 static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) 1977 { 1978 u64 msg; 1979 1980 if (qm->fun_type == QM_HW_PF) 1981 msg = qm_get_ifc_pf(qm, fun_num); 1982 else 1983 msg = qm_get_ifc_vf(qm); 1984 1985 *cmd = msg & QM_IFC_CMD_MASK; 1986 1987 if (data) 1988 *data = msg >> QM_IFC_DATA_SHIFT; 1989 1990 return 0; 1991 } 1992 1993 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { 1994 .qm_db = qm_db_v1, 1995 .hw_error_init = qm_hw_error_init_v1, 1996 .set_msi = qm_set_msi, 1997 }; 1998 1999 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { 2000 .get_vft = qm_get_vft_v2, 2001 .qm_db = qm_db_v2, 2002 .hw_error_init = qm_hw_error_init_v2, 2003 .hw_error_uninit = qm_hw_error_uninit_v2, 2004 .hw_error_handle = qm_hw_error_handle_v2, 2005 .set_msi = qm_set_msi, 2006 }; 2007 2008 static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { 2009 .get_vft = qm_get_vft_v2, 2010 .qm_db = qm_db_v2, 2011 .hw_error_init = qm_hw_error_init_v3, 2012 .hw_error_uninit = qm_hw_error_uninit_v3, 2013 .hw_error_handle = qm_hw_error_handle_v2, 2014 .set_msi = qm_set_msi_v3, 2015 .set_ifc_begin = qm_set_ifc_begin_v3, 2016 .set_ifc_end = qm_set_ifc_end_v3, 2017 .get_ifc = qm_get_ifc_v3, 2018 }; 2019 2020 static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { 2021 .get_vft = qm_get_vft_v2, 2022 .qm_db = qm_db_v2, 2023 .hw_error_init = qm_hw_error_init_v3, 2024 .hw_error_uninit = qm_hw_error_uninit_v3, 2025 .hw_error_handle = qm_hw_error_handle_v2, 2026 .set_msi = qm_set_msi_v3, 2027 .set_ifc_begin = qm_set_ifc_begin_v4, 2028 .set_ifc_end = qm_set_ifc_end_v4, 2029 .get_ifc = qm_get_ifc_v4, 2030 }; 2031 2032 static void *qm_get_avail_sqe(struct hisi_qp *qp) 2033 { 2034 struct hisi_qp_status *qp_status = &qp->qp_status; 2035 u16 sq_tail = qp_status->sq_tail; 2036 2037 if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1)) 2038 return NULL; 2039 2040 return qp->sqe + sq_tail * qp->qm->sqe_size; 2041 } 2042 2043 static void hisi_qm_unset_hw_reset(struct hisi_qp *qp) 2044 { 2045 u64 *addr; 2046 2047 /* Use last 64 bits of DUS to reset status. */ 2048 addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET; 2049 *addr = 0; 2050 } 2051 2052 static struct hisi_qp *find_shareable_qp(struct hisi_qm *qm, u8 alg_type, bool is_in_kernel) 2053 { 2054 struct device *dev = &qm->pdev->dev; 2055 struct hisi_qp *share_qp = NULL; 2056 struct hisi_qp *qp; 2057 u32 ref_count = ~0; 2058 int i; 2059 2060 if (!is_in_kernel) 2061 goto queues_busy; 2062 2063 for (i = 0; i < qm->qp_num; i++) { 2064 qp = &qm->qp_array[i]; 2065 if (qp->is_in_kernel && qp->alg_type == alg_type && qp->ref_count < ref_count) { 2066 ref_count = qp->ref_count; 2067 share_qp = qp; 2068 } 2069 } 2070 2071 if (share_qp) { 2072 share_qp->ref_count++; 2073 return share_qp; 2074 } 2075 2076 queues_busy: 2077 dev_info_ratelimited(dev, "All %u queues of QM are busy and no shareable queue\n", 2078 qm->qp_num); 2079 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); 2080 return ERR_PTR(-EBUSY); 2081 } 2082 2083 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type, bool is_in_kernel) 2084 { 2085 struct device *dev = &qm->pdev->dev; 2086 struct hisi_qp *qp; 2087 int qp_id; 2088 2089 if (atomic_read(&qm->status.flags) == QM_STOP) { 2090 dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n"); 2091 return ERR_PTR(-EPERM); 2092 } 2093 2094 /* Try to find a shareable queue when all queues are busy */ 2095 if (qm->qp_in_used == qm->qp_num) 2096 return find_shareable_qp(qm, alg_type, is_in_kernel); 2097 2098 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC); 2099 if (qp_id < 0) { 2100 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", 2101 qm->qp_num); 2102 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); 2103 return ERR_PTR(-EBUSY); 2104 } 2105 2106 qp = &qm->qp_array[qp_id]; 2107 hisi_qm_unset_hw_reset(qp); 2108 memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth); 2109 2110 qp->event_cb = NULL; 2111 qp->req_cb = NULL; 2112 qp->alg_type = alg_type; 2113 qp->is_in_kernel = is_in_kernel; 2114 qm->qp_in_used++; 2115 qp->ref_count = 1; 2116 2117 return qp; 2118 } 2119 2120 /** 2121 * hisi_qm_create_qp() - Create a queue pair from qm. 2122 * @qm: The qm we create a qp from. 2123 * @alg_type: Accelerator specific algorithm type in sqc. 2124 * 2125 * Return created qp, negative error code if failed. 2126 */ 2127 static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) 2128 { 2129 struct hisi_qp *qp; 2130 int ret; 2131 2132 ret = qm_pm_get_sync(qm); 2133 if (ret) 2134 return ERR_PTR(ret); 2135 2136 down_write(&qm->qps_lock); 2137 qp = qm_create_qp_nolock(qm, alg_type, false); 2138 up_write(&qm->qps_lock); 2139 2140 if (IS_ERR(qp)) 2141 qm_pm_put_sync(qm); 2142 2143 return qp; 2144 } 2145 2146 /** 2147 * hisi_qm_release_qp() - Release a qp back to its qm. 2148 * @qp: The qp we want to release. 2149 * 2150 * This function releases the resource of a qp. 2151 */ 2152 static void hisi_qm_release_qp(struct hisi_qp *qp) 2153 { 2154 struct hisi_qm *qm = qp->qm; 2155 2156 down_write(&qm->qps_lock); 2157 2158 qm->qp_in_used--; 2159 idr_remove(&qm->qp_idr, qp->qp_id); 2160 2161 up_write(&qm->qps_lock); 2162 2163 qm_pm_put_sync(qm); 2164 } 2165 2166 static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) 2167 { 2168 struct hisi_qm *qm = qp->qm; 2169 enum qm_hw_ver ver = qm->ver; 2170 struct qm_sqc sqc = {0}; 2171 2172 if (ver == QM_HW_V1) { 2173 sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size)); 2174 sqc.w8 = cpu_to_le16(qp->sq_depth - 1); 2175 } else { 2176 sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); 2177 sqc.w8 = 0; /* rand_qc */ 2178 } 2179 sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type)); 2180 sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma)); 2181 sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma)); 2182 sqc.cq_num = cpu_to_le16(qp_id); 2183 sqc.pasid = cpu_to_le16(pasid); 2184 2185 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) 2186 sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE << 2187 QM_QC_PASID_ENABLE_SHIFT); 2188 2189 return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0); 2190 } 2191 2192 static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) 2193 { 2194 struct hisi_qm *qm = qp->qm; 2195 enum qm_hw_ver ver = qm->ver; 2196 struct qm_cqc cqc = {0}; 2197 2198 if (ver == QM_HW_V1) { 2199 cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE)); 2200 cqc.w8 = cpu_to_le16(qp->cq_depth - 1); 2201 } else { 2202 cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); 2203 cqc.w8 = 0; /* rand_qc */ 2204 } 2205 /* 2206 * Enable request finishing interrupts defaultly. 2207 * So, there will be some interrupts until disabling 2208 * this. 2209 */ 2210 cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); 2211 cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma)); 2212 cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma)); 2213 cqc.pasid = cpu_to_le16(pasid); 2214 2215 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) 2216 cqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE); 2217 2218 return qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 0); 2219 } 2220 2221 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) 2222 { 2223 int ret; 2224 2225 qm_init_qp_status(qp); 2226 2227 ret = qm_sq_ctx_cfg(qp, qp_id, pasid); 2228 if (ret) 2229 return ret; 2230 2231 return qm_cq_ctx_cfg(qp, qp_id, pasid); 2232 } 2233 2234 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) 2235 { 2236 struct hisi_qm *qm = qp->qm; 2237 struct device *dev = &qm->pdev->dev; 2238 int qp_id = qp->qp_id; 2239 u32 pasid = arg; 2240 int ret; 2241 2242 if (atomic_read(&qm->status.flags) == QM_STOP) { 2243 dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n"); 2244 return -EPERM; 2245 } 2246 2247 ret = qm_qp_ctx_cfg(qp, qp_id, pasid); 2248 if (ret) 2249 return ret; 2250 2251 atomic_set(&qp->qp_status.flags, QP_START); 2252 dev_dbg(dev, "queue %d started\n", qp_id); 2253 2254 return 0; 2255 } 2256 2257 /** 2258 * hisi_qm_start_qp() - Start a qp into running. 2259 * @qp: The qp we want to start to run. 2260 * @arg: Accelerator specific argument. 2261 * 2262 * After this function, qp can receive request from user. Return 0 if 2263 * successful, negative error code if failed. 2264 */ 2265 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) 2266 { 2267 struct hisi_qm *qm = qp->qm; 2268 int ret; 2269 2270 down_write(&qm->qps_lock); 2271 ret = qm_start_qp_nolock(qp, arg); 2272 up_write(&qm->qps_lock); 2273 2274 return ret; 2275 } 2276 EXPORT_SYMBOL_GPL(hisi_qm_start_qp); 2277 2278 /** 2279 * qp_stop_fail_cb() - call request cb. 2280 * @qp: stopped failed qp. 2281 * 2282 * Callback function should be called whether task completed or not. 2283 */ 2284 static void qp_stop_fail_cb(struct hisi_qp *qp) 2285 { 2286 int qp_used = atomic_read(&qp->qp_status.used); 2287 u16 cur_tail = qp->qp_status.sq_tail; 2288 u16 sq_depth = qp->sq_depth; 2289 u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth; 2290 struct hisi_qm *qm = qp->qm; 2291 u16 pos; 2292 int i; 2293 2294 for (i = 0; i < qp_used; i++) { 2295 pos = (i + cur_head) % sq_depth; 2296 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); 2297 qm_cq_head_update(qp); 2298 atomic_dec(&qp->qp_status.used); 2299 } 2300 } 2301 2302 static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) 2303 { 2304 struct device *dev = &qm->pdev->dev; 2305 struct qm_sqc sqc; 2306 struct qm_cqc cqc; 2307 int ret, i = 0; 2308 2309 while (++i) { 2310 ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1); 2311 if (ret) { 2312 dev_err_ratelimited(dev, "Failed to dump sqc!\n"); 2313 *state = QM_DUMP_SQC_FAIL; 2314 return ret; 2315 } 2316 2317 ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1); 2318 if (ret) { 2319 dev_err_ratelimited(dev, "Failed to dump cqc!\n"); 2320 *state = QM_DUMP_CQC_FAIL; 2321 return ret; 2322 } 2323 2324 if ((sqc.tail == cqc.tail) && 2325 (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc))) 2326 break; 2327 2328 if (i == MAX_WAIT_COUNTS) { 2329 dev_err(dev, "Fail to empty queue %u!\n", qp_id); 2330 *state = QM_STOP_QUEUE_FAIL; 2331 return -ETIMEDOUT; 2332 } 2333 2334 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX); 2335 } 2336 2337 return 0; 2338 } 2339 2340 /** 2341 * qm_drain_qp() - Drain a qp. 2342 * @qp: The qp we want to drain. 2343 * 2344 * If the device does not support stopping queue by sending mailbox, 2345 * determine whether the queue is cleared by judging the tail pointers of 2346 * sq and cq. 2347 */ 2348 static int qm_drain_qp(struct hisi_qp *qp) 2349 { 2350 struct hisi_qm *qm = qp->qm; 2351 u32 state = 0; 2352 int ret; 2353 2354 /* No need to judge if master OOO is blocked. */ 2355 if (qm_check_dev_error(qm)) 2356 return 0; 2357 2358 /* HW V3 supports drain qp by device */ 2359 if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { 2360 ret = qm_stop_qp(qp); 2361 if (ret) { 2362 dev_err(&qm->pdev->dev, "Failed to stop qp!\n"); 2363 state = QM_STOP_QUEUE_FAIL; 2364 goto set_dev_state; 2365 } 2366 return ret; 2367 } 2368 2369 ret = qm_wait_qp_empty(qm, &state, qp->qp_id); 2370 if (ret) 2371 goto set_dev_state; 2372 2373 return 0; 2374 2375 set_dev_state: 2376 if (qm->debug.dev_dfx.dev_timeout) 2377 qm->debug.dev_dfx.dev_state = state; 2378 2379 return ret; 2380 } 2381 2382 static void qm_stop_qp_nolock(struct hisi_qp *qp) 2383 { 2384 struct hisi_qm *qm = qp->qm; 2385 struct device *dev = &qm->pdev->dev; 2386 int ret; 2387 2388 /* 2389 * It is allowed to stop and release qp when reset, If the qp is 2390 * stopped when reset but still want to be released then, the 2391 * is_resetting flag should be set negative so that this qp will not 2392 * be restarted after reset. 2393 */ 2394 if (atomic_read(&qp->qp_status.flags) != QP_START) { 2395 qp->is_resetting = false; 2396 return; 2397 } 2398 2399 atomic_set(&qp->qp_status.flags, QP_STOP); 2400 2401 /* V3 supports direct stop function when FLR prepare */ 2402 if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) { 2403 ret = qm_drain_qp(qp); 2404 if (ret) 2405 dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id); 2406 } 2407 2408 flush_workqueue(qm->wq); 2409 if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) 2410 qp_stop_fail_cb(qp); 2411 2412 dev_dbg(dev, "stop queue %u!", qp->qp_id); 2413 } 2414 2415 /** 2416 * hisi_qm_stop_qp() - Stop a qp in qm. 2417 * @qp: The qp we want to stop. 2418 * 2419 * This function is reverse of hisi_qm_start_qp. 2420 */ 2421 void hisi_qm_stop_qp(struct hisi_qp *qp) 2422 { 2423 down_write(&qp->qm->qps_lock); 2424 qm_stop_qp_nolock(qp); 2425 up_write(&qp->qm->qps_lock); 2426 } 2427 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); 2428 2429 /** 2430 * hisi_qp_send() - Queue up a task in the hardware queue. 2431 * @qp: The qp in which to put the message. 2432 * @msg: The message. 2433 * 2434 * This function will return -EBUSY if qp is currently full, and -EAGAIN 2435 * if qp related qm is resetting. 2436 * 2437 * Note: This function may run with qm_irq_thread and ACC reset at same time. 2438 * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC 2439 * reset may happen, we have no lock here considering performance. This 2440 * causes current qm_db sending fail or can not receive sended sqe. QM 2441 * sync/async receive function should handle the error sqe. ACC reset 2442 * done function should clear used sqe to 0. 2443 */ 2444 int hisi_qp_send(struct hisi_qp *qp, const void *msg) 2445 { 2446 struct hisi_qp_status *qp_status = &qp->qp_status; 2447 u16 sq_tail, sq_tail_next; 2448 void *sqe; 2449 2450 spin_lock_bh(&qp->qp_lock); 2451 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || 2452 atomic_read(&qp->qm->status.flags) == QM_STOP || 2453 qp->is_resetting)) { 2454 spin_unlock_bh(&qp->qp_lock); 2455 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); 2456 return -EAGAIN; 2457 } 2458 2459 sqe = qm_get_avail_sqe(qp); 2460 if (!sqe) { 2461 spin_unlock_bh(&qp->qp_lock); 2462 return -EBUSY; 2463 } 2464 2465 sq_tail = qp_status->sq_tail; 2466 sq_tail_next = (sq_tail + 1) % qp->sq_depth; 2467 memcpy(sqe, msg, qp->qm->sqe_size); 2468 qp->msg[sq_tail] = msg; 2469 2470 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); 2471 atomic_inc(&qp->qp_status.used); 2472 qp_status->sq_tail = sq_tail_next; 2473 spin_unlock_bh(&qp->qp_lock); 2474 2475 return 0; 2476 } 2477 EXPORT_SYMBOL_GPL(hisi_qp_send); 2478 2479 static void hisi_qm_cache_wb(struct hisi_qm *qm) 2480 { 2481 unsigned int val; 2482 2483 if (qm->ver == QM_HW_V1) 2484 return; 2485 2486 writel(0x1, qm->io_base + QM_CACHE_WB_START); 2487 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, 2488 val, val & BIT(0), POLL_PERIOD, 2489 POLL_TIMEOUT)) 2490 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); 2491 } 2492 2493 static void qm_qp_event_notifier(struct hisi_qp *qp) 2494 { 2495 wake_up_interruptible(&qp->uacce_q->wait); 2496 } 2497 2498 /* This function returns free number of qp in qm. */ 2499 static int hisi_qm_get_available_instances(struct uacce_device *uacce) 2500 { 2501 struct hisi_qm *qm = uacce->priv; 2502 int ret; 2503 2504 down_read(&qm->qps_lock); 2505 ret = qm->qp_num - qm->qp_in_used; 2506 up_read(&qm->qps_lock); 2507 2508 return ret; 2509 } 2510 2511 static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) 2512 { 2513 int i; 2514 2515 for (i = 0; i < qm->qp_num; i++) 2516 qm_set_qp_disable(&qm->qp_array[i], offset); 2517 } 2518 2519 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, 2520 unsigned long arg, 2521 struct uacce_queue *q) 2522 { 2523 struct hisi_qm *qm = uacce->priv; 2524 struct hisi_qp *qp; 2525 u8 alg_type = 0; 2526 2527 qp = hisi_qm_create_qp(qm, alg_type); 2528 if (IS_ERR(qp)) 2529 return PTR_ERR(qp); 2530 2531 q->priv = qp; 2532 q->uacce = uacce; 2533 qp->uacce_q = q; 2534 qp->event_cb = qm_qp_event_notifier; 2535 qp->pasid = arg; 2536 2537 return 0; 2538 } 2539 2540 static void hisi_qm_uacce_put_queue(struct uacce_queue *q) 2541 { 2542 struct hisi_qp *qp = q->priv; 2543 2544 hisi_qm_release_qp(qp); 2545 } 2546 2547 /* map sq/cq/doorbell to user space */ 2548 static int hisi_qm_uacce_mmap(struct uacce_queue *q, 2549 struct vm_area_struct *vma, 2550 struct uacce_qfile_region *qfr) 2551 { 2552 struct hisi_qp *qp = q->priv; 2553 struct hisi_qm *qm = qp->qm; 2554 resource_size_t phys_base = qm->db_phys_base + 2555 qp->qp_id * qm->db_interval; 2556 size_t sz = vma->vm_end - vma->vm_start; 2557 struct pci_dev *pdev = qm->pdev; 2558 struct device *dev = &pdev->dev; 2559 unsigned long vm_pgoff; 2560 int ret; 2561 2562 switch (qfr->type) { 2563 case UACCE_QFRT_MMIO: 2564 if (qm->ver == QM_HW_V1) { 2565 if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) 2566 return -EINVAL; 2567 } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { 2568 if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + 2569 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) 2570 return -EINVAL; 2571 } else { 2572 if (sz > qm->db_interval) 2573 return -EINVAL; 2574 } 2575 2576 vm_flags_set(vma, VM_IO); 2577 2578 return remap_pfn_range(vma, vma->vm_start, 2579 phys_base >> PAGE_SHIFT, 2580 sz, pgprot_noncached(vma->vm_page_prot)); 2581 case UACCE_QFRT_DUS: 2582 if (sz != qp->qdma.size) 2583 return -EINVAL; 2584 2585 /* 2586 * dma_mmap_coherent() requires vm_pgoff as 0 2587 * restore vm_pfoff to initial value for mmap() 2588 */ 2589 vm_pgoff = vma->vm_pgoff; 2590 vma->vm_pgoff = 0; 2591 ret = dma_mmap_coherent(dev, vma, qp->qdma.va, 2592 qp->qdma.dma, sz); 2593 vma->vm_pgoff = vm_pgoff; 2594 return ret; 2595 2596 default: 2597 return -EINVAL; 2598 } 2599 } 2600 2601 static int hisi_qm_uacce_start_queue(struct uacce_queue *q) 2602 { 2603 struct hisi_qp *qp = q->priv; 2604 2605 return hisi_qm_start_qp(qp, qp->pasid); 2606 } 2607 2608 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) 2609 { 2610 struct hisi_qp *qp = q->priv; 2611 struct hisi_qm *qm = qp->qm; 2612 struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx; 2613 u32 i = 0; 2614 2615 hisi_qm_stop_qp(qp); 2616 2617 if (!dev_dfx->dev_timeout || !dev_dfx->dev_state) 2618 return; 2619 2620 /* 2621 * After the queue fails to be stopped, 2622 * wait for a period of time before releasing the queue. 2623 */ 2624 while (++i) { 2625 msleep(WAIT_PERIOD); 2626 2627 /* Since dev_timeout maybe modified, check i >= dev_timeout */ 2628 if (i >= dev_dfx->dev_timeout) { 2629 dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n", 2630 qp->qp_id, dev_dfx->dev_state); 2631 dev_dfx->dev_state = QM_FINISH_WAIT; 2632 break; 2633 } 2634 } 2635 } 2636 2637 static int hisi_qm_is_q_updated(struct uacce_queue *q) 2638 { 2639 struct hisi_qp *qp = q->priv; 2640 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; 2641 int updated = 0; 2642 2643 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { 2644 /* make sure to read data from memory */ 2645 dma_rmb(); 2646 qm_cq_head_update(qp); 2647 cqe = qp->cqe + qp->qp_status.cq_head; 2648 updated = 1; 2649 } 2650 2651 return updated; 2652 } 2653 2654 static void qm_set_sqctype(struct uacce_queue *q, u16 type) 2655 { 2656 struct hisi_qm *qm = q->uacce->priv; 2657 struct hisi_qp *qp = q->priv; 2658 2659 down_write(&qm->qps_lock); 2660 qp->alg_type = type; 2661 up_write(&qm->qps_lock); 2662 } 2663 2664 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, 2665 unsigned long arg) 2666 { 2667 struct hisi_qp *qp = q->priv; 2668 struct hisi_qp_info qp_info; 2669 struct hisi_qp_ctx qp_ctx; 2670 2671 if (cmd == UACCE_CMD_QM_SET_QP_CTX) { 2672 if (copy_from_user(&qp_ctx, (void __user *)arg, 2673 sizeof(struct hisi_qp_ctx))) 2674 return -EFAULT; 2675 2676 if (qp_ctx.qc_type > QM_MAX_QC_TYPE) 2677 return -EINVAL; 2678 2679 qm_set_sqctype(q, qp_ctx.qc_type); 2680 qp_ctx.id = qp->qp_id; 2681 2682 if (copy_to_user((void __user *)arg, &qp_ctx, 2683 sizeof(struct hisi_qp_ctx))) 2684 return -EFAULT; 2685 2686 return 0; 2687 } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) { 2688 if (copy_from_user(&qp_info, (void __user *)arg, 2689 sizeof(struct hisi_qp_info))) 2690 return -EFAULT; 2691 2692 qp_info.sqe_size = qp->qm->sqe_size; 2693 qp_info.sq_depth = qp->sq_depth; 2694 qp_info.cq_depth = qp->cq_depth; 2695 2696 if (copy_to_user((void __user *)arg, &qp_info, 2697 sizeof(struct hisi_qp_info))) 2698 return -EFAULT; 2699 2700 return 0; 2701 } 2702 2703 return -EINVAL; 2704 } 2705 2706 /** 2707 * qm_hw_err_isolate() - Try to set the isolation status of the uacce device 2708 * according to user's configuration of error threshold. 2709 * @qm: the uacce device 2710 */ 2711 static int qm_hw_err_isolate(struct hisi_qm *qm) 2712 { 2713 struct qm_hw_err *err, *tmp, *hw_err; 2714 struct qm_err_isolate *isolate; 2715 u32 count = 0; 2716 2717 isolate = &qm->isolate_data; 2718 2719 #define SECONDS_PER_HOUR 3600 2720 2721 /* All the hw errs are processed by PF driver */ 2722 if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold) 2723 return 0; 2724 2725 hw_err = kzalloc(sizeof(*hw_err), GFP_KERNEL); 2726 if (!hw_err) 2727 return -ENOMEM; 2728 2729 /* 2730 * Time-stamp every slot AER error. Then check the AER error log when the 2731 * next device AER error occurred. if the device slot AER error count exceeds 2732 * the setting error threshold in one hour, the isolated state will be set 2733 * to true. And the AER error logs that exceed one hour will be cleared. 2734 */ 2735 mutex_lock(&isolate->isolate_lock); 2736 hw_err->timestamp = jiffies; 2737 list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) { 2738 if ((hw_err->timestamp - err->timestamp) / HZ > 2739 SECONDS_PER_HOUR) { 2740 list_del(&err->list); 2741 kfree(err); 2742 } else { 2743 count++; 2744 } 2745 } 2746 list_add(&hw_err->list, &isolate->qm_hw_errs); 2747 2748 if (count >= isolate->err_threshold) 2749 isolate->is_isolate = true; 2750 mutex_unlock(&isolate->isolate_lock); 2751 2752 return 0; 2753 } 2754 2755 static void qm_hw_err_destroy(struct hisi_qm *qm) 2756 { 2757 struct qm_hw_err *err, *tmp; 2758 2759 list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { 2760 list_del(&err->list); 2761 kfree(err); 2762 } 2763 } 2764 2765 static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) 2766 { 2767 struct hisi_qm *qm = uacce->priv; 2768 struct hisi_qm *pf_qm; 2769 2770 if (uacce->is_vf) 2771 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 2772 else 2773 pf_qm = qm; 2774 2775 return pf_qm->isolate_data.is_isolate ? 2776 UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL; 2777 } 2778 2779 static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) 2780 { 2781 struct hisi_qm *qm = uacce->priv; 2782 2783 /* Must be set by PF */ 2784 if (uacce->is_vf) 2785 return -EPERM; 2786 2787 if (qm->isolate_data.is_isolate) 2788 return -EPERM; 2789 2790 mutex_lock(&qm->isolate_data.isolate_lock); 2791 qm->isolate_data.err_threshold = num; 2792 2793 /* After the policy is updated, need to reset the hardware err list */ 2794 qm_hw_err_destroy(qm); 2795 mutex_unlock(&qm->isolate_data.isolate_lock); 2796 2797 return 0; 2798 } 2799 2800 static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce) 2801 { 2802 struct hisi_qm *qm = uacce->priv; 2803 struct hisi_qm *pf_qm; 2804 2805 if (uacce->is_vf) { 2806 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 2807 return pf_qm->isolate_data.err_threshold; 2808 } 2809 2810 return qm->isolate_data.err_threshold; 2811 } 2812 2813 static const struct uacce_ops uacce_qm_ops = { 2814 .get_available_instances = hisi_qm_get_available_instances, 2815 .get_queue = hisi_qm_uacce_get_queue, 2816 .put_queue = hisi_qm_uacce_put_queue, 2817 .start_queue = hisi_qm_uacce_start_queue, 2818 .stop_queue = hisi_qm_uacce_stop_queue, 2819 .mmap = hisi_qm_uacce_mmap, 2820 .ioctl = hisi_qm_uacce_ioctl, 2821 .is_q_updated = hisi_qm_is_q_updated, 2822 .get_isolate_state = hisi_qm_get_isolate_state, 2823 .isolate_err_threshold_write = hisi_qm_isolate_threshold_write, 2824 .isolate_err_threshold_read = hisi_qm_isolate_threshold_read, 2825 }; 2826 2827 static void qm_remove_uacce(struct hisi_qm *qm) 2828 { 2829 struct uacce_device *uacce = qm->uacce; 2830 2831 if (qm->use_sva) { 2832 mutex_lock(&qm->isolate_data.isolate_lock); 2833 qm_hw_err_destroy(qm); 2834 mutex_unlock(&qm->isolate_data.isolate_lock); 2835 2836 uacce_remove(uacce); 2837 qm->uacce = NULL; 2838 } 2839 } 2840 2841 static void qm_uacce_api_ver_init(struct hisi_qm *qm) 2842 { 2843 struct uacce_device *uacce = qm->uacce; 2844 2845 switch (qm->ver) { 2846 case QM_HW_V1: 2847 uacce->api_ver = HISI_QM_API_VER_BASE; 2848 break; 2849 case QM_HW_V2: 2850 uacce->api_ver = HISI_QM_API_VER2_BASE; 2851 break; 2852 case QM_HW_V3: 2853 case QM_HW_V4: 2854 uacce->api_ver = HISI_QM_API_VER3_BASE; 2855 break; 2856 default: 2857 uacce->api_ver = HISI_QM_API_VER5_BASE; 2858 break; 2859 } 2860 } 2861 2862 static int qm_alloc_uacce(struct hisi_qm *qm) 2863 { 2864 struct pci_dev *pdev = qm->pdev; 2865 struct uacce_device *uacce; 2866 unsigned long mmio_page_nr; 2867 unsigned long dus_page_nr; 2868 u16 sq_depth, cq_depth; 2869 struct uacce_interface interface = { 2870 .flags = UACCE_DEV_SVA, 2871 .ops = &uacce_qm_ops, 2872 }; 2873 int ret; 2874 2875 ret = strscpy(interface.name, dev_driver_string(&pdev->dev), 2876 sizeof(interface.name)); 2877 if (ret < 0) 2878 return -ENAMETOOLONG; 2879 2880 uacce = uacce_alloc(&pdev->dev, &interface); 2881 if (IS_ERR(uacce)) 2882 return PTR_ERR(uacce); 2883 2884 if (uacce->flags & UACCE_DEV_SVA) { 2885 qm->use_sva = true; 2886 } else { 2887 /* only consider sva case */ 2888 qm_remove_uacce(qm); 2889 return -EINVAL; 2890 } 2891 2892 uacce->is_vf = pdev->is_virtfn; 2893 uacce->priv = qm; 2894 2895 if (qm->ver == QM_HW_V1) 2896 mmio_page_nr = QM_DOORBELL_PAGE_NR; 2897 else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) 2898 mmio_page_nr = QM_DOORBELL_PAGE_NR + 2899 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; 2900 else 2901 mmio_page_nr = qm->db_interval / PAGE_SIZE; 2902 2903 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); 2904 2905 /* Add one more page for device or qp status */ 2906 dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + 2907 sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> 2908 PAGE_SHIFT; 2909 2910 uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; 2911 uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; 2912 2913 qm->uacce = uacce; 2914 qm_uacce_api_ver_init(qm); 2915 INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); 2916 mutex_init(&qm->isolate_data.isolate_lock); 2917 2918 return 0; 2919 } 2920 2921 /** 2922 * qm_frozen() - Try to froze QM to cut continuous queue request. If 2923 * there is user on the QM, return failure without doing anything. 2924 * @qm: The qm needed to be fronzen. 2925 * 2926 * This function frozes QM, then we can do SRIOV disabling. 2927 */ 2928 static int qm_frozen(struct hisi_qm *qm) 2929 { 2930 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) 2931 return 0; 2932 2933 down_write(&qm->qps_lock); 2934 2935 if (!qm->qp_in_used) { 2936 qm->qp_in_used = qm->qp_num; 2937 up_write(&qm->qps_lock); 2938 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl); 2939 return 0; 2940 } 2941 2942 up_write(&qm->qps_lock); 2943 2944 return -EBUSY; 2945 } 2946 2947 static int qm_try_frozen_vfs(struct pci_dev *pdev, 2948 struct hisi_qm_list *qm_list) 2949 { 2950 struct hisi_qm *qm, *vf_qm; 2951 struct pci_dev *dev; 2952 int ret = 0; 2953 2954 if (!qm_list || !pdev) 2955 return -EINVAL; 2956 2957 /* Try to frozen all the VFs as disable SRIOV */ 2958 mutex_lock(&qm_list->lock); 2959 list_for_each_entry(qm, &qm_list->list, list) { 2960 dev = qm->pdev; 2961 if (dev == pdev) 2962 continue; 2963 if (pci_physfn(dev) == pdev) { 2964 vf_qm = pci_get_drvdata(dev); 2965 ret = qm_frozen(vf_qm); 2966 if (ret) 2967 goto frozen_fail; 2968 } 2969 } 2970 2971 frozen_fail: 2972 mutex_unlock(&qm_list->lock); 2973 2974 return ret; 2975 } 2976 2977 /** 2978 * hisi_qm_wait_task_finish() - Wait until the task is finished 2979 * when removing the driver. 2980 * @qm: The qm needed to wait for the task to finish. 2981 * @qm_list: The list of all available devices. 2982 */ 2983 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) 2984 { 2985 while (qm_frozen(qm) || 2986 ((qm->fun_type == QM_HW_PF) && 2987 qm_try_frozen_vfs(qm->pdev, qm_list))) { 2988 msleep(WAIT_PERIOD); 2989 } 2990 2991 while (test_bit(QM_RST_SCHED, &qm->misc_ctl) || 2992 test_bit(QM_RESETTING, &qm->misc_ctl)) 2993 msleep(WAIT_PERIOD); 2994 2995 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 2996 flush_work(&qm->cmd_process); 2997 2998 udelay(REMOVE_WAIT_DELAY); 2999 } 3000 EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); 3001 3002 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) 3003 { 3004 struct device *dev = &qm->pdev->dev; 3005 struct hisi_qp *qp; 3006 int i; 3007 3008 for (i = num - 1; i >= 0; i--) { 3009 qp = &qm->qp_array[i]; 3010 dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); 3011 kfree(qp->msg); 3012 kfree(qm->poll_data[i].qp_finish_id); 3013 } 3014 3015 kfree(qm->poll_data); 3016 kfree(qm->qp_array); 3017 } 3018 3019 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, 3020 u16 sq_depth, u16 cq_depth) 3021 { 3022 struct device *dev = &qm->pdev->dev; 3023 size_t off = qm->sqe_size * sq_depth; 3024 struct hisi_qp *qp; 3025 int ret = -ENOMEM; 3026 3027 qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), 3028 GFP_KERNEL); 3029 if (!qm->poll_data[id].qp_finish_id) 3030 return -ENOMEM; 3031 3032 qp = &qm->qp_array[id]; 3033 qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); 3034 if (!qp->msg) 3035 goto err_free_qp_finish_id; 3036 3037 qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, 3038 GFP_KERNEL); 3039 if (!qp->qdma.va) 3040 goto err_free_qp_msg; 3041 3042 qp->sqe = qp->qdma.va; 3043 qp->sqe_dma = qp->qdma.dma; 3044 qp->cqe = qp->qdma.va + off; 3045 qp->cqe_dma = qp->qdma.dma + off; 3046 qp->qdma.size = dma_size; 3047 qp->sq_depth = sq_depth; 3048 qp->cq_depth = cq_depth; 3049 qp->qm = qm; 3050 qp->qp_id = id; 3051 3052 spin_lock_init(&qp->qp_lock); 3053 spin_lock_init(&qp->backlog.lock); 3054 INIT_LIST_HEAD(&qp->backlog.list); 3055 3056 return 0; 3057 3058 err_free_qp_msg: 3059 kfree(qp->msg); 3060 err_free_qp_finish_id: 3061 kfree(qm->poll_data[id].qp_finish_id); 3062 return ret; 3063 } 3064 3065 static void hisi_qm_pre_init(struct hisi_qm *qm) 3066 { 3067 struct pci_dev *pdev = qm->pdev; 3068 3069 if (qm->ver == QM_HW_V1) 3070 qm->ops = &qm_hw_ops_v1; 3071 else if (qm->ver == QM_HW_V2) 3072 qm->ops = &qm_hw_ops_v2; 3073 else if (qm->ver == QM_HW_V3) 3074 qm->ops = &qm_hw_ops_v3; 3075 else 3076 qm->ops = &qm_hw_ops_v4; 3077 3078 pci_set_drvdata(pdev, qm); 3079 mutex_init(&qm->mailbox_lock); 3080 mutex_init(&qm->ifc_lock); 3081 init_rwsem(&qm->qps_lock); 3082 qm->qp_in_used = 0; 3083 if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { 3084 if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev))) 3085 dev_info(&pdev->dev, "_PS0 and _PR0 are not defined"); 3086 } 3087 } 3088 3089 static void qm_cmd_uninit(struct hisi_qm *qm) 3090 { 3091 u32 val; 3092 3093 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 3094 return; 3095 3096 val = readl(qm->io_base + QM_IFC_INT_MASK); 3097 val |= QM_IFC_INT_DISABLE; 3098 writel(val, qm->io_base + QM_IFC_INT_MASK); 3099 } 3100 3101 static void qm_cmd_init(struct hisi_qm *qm) 3102 { 3103 u32 val; 3104 3105 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 3106 return; 3107 3108 /* Clear communication interrupt source */ 3109 qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR); 3110 3111 /* Enable pf to vf communication reg. */ 3112 val = readl(qm->io_base + QM_IFC_INT_MASK); 3113 val &= ~QM_IFC_INT_DISABLE; 3114 writel(val, qm->io_base + QM_IFC_INT_MASK); 3115 } 3116 3117 static void qm_put_pci_res(struct hisi_qm *qm) 3118 { 3119 struct pci_dev *pdev = qm->pdev; 3120 3121 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) 3122 iounmap(qm->db_io_base); 3123 3124 iounmap(qm->io_base); 3125 pci_release_mem_regions(pdev); 3126 } 3127 3128 static void hisi_mig_region_clear(struct hisi_qm *qm) 3129 { 3130 u32 val; 3131 3132 /* Clear migration region set of PF */ 3133 if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { 3134 val = readl(qm->io_base + QM_MIG_REGION_SEL); 3135 val &= ~QM_MIG_REGION_EN; 3136 writel(val, qm->io_base + QM_MIG_REGION_SEL); 3137 } 3138 } 3139 3140 static void hisi_mig_region_enable(struct hisi_qm *qm) 3141 { 3142 u32 val; 3143 3144 /* Select migration region of PF */ 3145 if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { 3146 val = readl(qm->io_base + QM_MIG_REGION_SEL); 3147 val |= QM_MIG_REGION_EN; 3148 writel(val, qm->io_base + QM_MIG_REGION_SEL); 3149 } 3150 } 3151 3152 static void hisi_qm_pci_uninit(struct hisi_qm *qm) 3153 { 3154 struct pci_dev *pdev = qm->pdev; 3155 3156 pci_free_irq_vectors(pdev); 3157 hisi_mig_region_clear(qm); 3158 qm_put_pci_res(qm); 3159 pci_disable_device(pdev); 3160 } 3161 3162 static void hisi_qm_set_state(struct hisi_qm *qm, u8 state) 3163 { 3164 if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF) 3165 writel(state, qm->io_base + QM_VF_STATE); 3166 } 3167 3168 static void hisi_qm_unint_work(struct hisi_qm *qm) 3169 { 3170 destroy_workqueue(qm->wq); 3171 } 3172 3173 static void hisi_qm_free_rsv_buf(struct hisi_qm *qm) 3174 { 3175 struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma; 3176 struct device *dev = &qm->pdev->dev; 3177 3178 dma_free_coherent(dev, xqc_dma->size, xqc_dma->va, xqc_dma->dma); 3179 } 3180 3181 static void hisi_qm_memory_uninit(struct hisi_qm *qm) 3182 { 3183 struct device *dev = &qm->pdev->dev; 3184 3185 hisi_qp_memory_uninit(qm, qm->qp_num); 3186 hisi_qm_free_rsv_buf(qm); 3187 if (qm->qdma.va) { 3188 hisi_qm_cache_wb(qm); 3189 dma_free_coherent(dev, qm->qdma.size, 3190 qm->qdma.va, qm->qdma.dma); 3191 } 3192 3193 idr_destroy(&qm->qp_idr); 3194 3195 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 3196 kfree(qm->factor); 3197 } 3198 3199 /** 3200 * hisi_qm_uninit() - Uninitialize qm. 3201 * @qm: The qm needed uninit. 3202 * 3203 * This function uninits qm related device resources. 3204 */ 3205 void hisi_qm_uninit(struct hisi_qm *qm) 3206 { 3207 qm_cmd_uninit(qm); 3208 hisi_qm_unint_work(qm); 3209 3210 down_write(&qm->qps_lock); 3211 hisi_qm_memory_uninit(qm); 3212 hisi_qm_set_state(qm, QM_NOT_READY); 3213 up_write(&qm->qps_lock); 3214 3215 qm_remove_uacce(qm); 3216 qm_irqs_unregister(qm); 3217 hisi_qm_pci_uninit(qm); 3218 } 3219 EXPORT_SYMBOL_GPL(hisi_qm_uninit); 3220 3221 /** 3222 * hisi_qm_get_vft() - Get vft from a qm. 3223 * @qm: The qm we want to get its vft. 3224 * @base: The base number of queue in vft. 3225 * @number: The number of queues in vft. 3226 * 3227 * We can allocate multiple queues to a qm by configuring virtual function 3228 * table. We get related configures by this function. Normally, we call this 3229 * function in VF driver to get the queue information. 3230 * 3231 * qm hw v1 does not support this interface. 3232 */ 3233 static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) 3234 { 3235 if (!base || !number) 3236 return -EINVAL; 3237 3238 if (!qm->ops->get_vft) { 3239 dev_err(&qm->pdev->dev, "Don't support vft read!\n"); 3240 return -EINVAL; 3241 } 3242 3243 return qm->ops->get_vft(qm, base, number); 3244 } 3245 3246 /** 3247 * hisi_qm_set_vft() - Set vft to a qm. 3248 * @qm: The qm we want to set its vft. 3249 * @fun_num: The function number. 3250 * @base: The base number of queue in vft. 3251 * @number: The number of queues in vft. 3252 * 3253 * This function is alway called in PF driver, it is used to assign queues 3254 * among PF and VFs. 3255 * 3256 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1) 3257 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1) 3258 * (VF function number 0x2) 3259 */ 3260 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, 3261 u32 number) 3262 { 3263 u32 max_q_num = qm->ctrl_qp_num; 3264 3265 if (base >= max_q_num || number > max_q_num || 3266 (base + number) > max_q_num) 3267 return -EINVAL; 3268 3269 return qm_set_sqc_cqc_vft(qm, fun_num, base, number); 3270 } 3271 3272 static void qm_init_eq_aeq_status(struct hisi_qm *qm) 3273 { 3274 struct hisi_qm_status *status = &qm->status; 3275 3276 status->eq_head = 0; 3277 status->aeq_head = 0; 3278 status->eqc_phase = true; 3279 status->aeqc_phase = true; 3280 } 3281 3282 static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm) 3283 { 3284 /* Clear eq/aeq interrupt source */ 3285 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); 3286 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 3287 3288 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); 3289 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); 3290 } 3291 3292 static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm) 3293 { 3294 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); 3295 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); 3296 } 3297 3298 static int qm_eq_ctx_cfg(struct hisi_qm *qm) 3299 { 3300 struct qm_eqc eqc = {0}; 3301 3302 eqc.base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma)); 3303 eqc.base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); 3304 if (qm->ver == QM_HW_V1) 3305 eqc.dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); 3306 eqc.dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); 3307 3308 return qm_set_and_get_xqc(qm, QM_MB_CMD_EQC, &eqc, 0, 0); 3309 } 3310 3311 static int qm_aeq_ctx_cfg(struct hisi_qm *qm) 3312 { 3313 struct qm_aeqc aeqc = {0}; 3314 3315 aeqc.base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); 3316 aeqc.base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); 3317 aeqc.dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); 3318 3319 return qm_set_and_get_xqc(qm, QM_MB_CMD_AEQC, &aeqc, 0, 0); 3320 } 3321 3322 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) 3323 { 3324 struct device *dev = &qm->pdev->dev; 3325 int ret; 3326 3327 qm_init_eq_aeq_status(qm); 3328 3329 /* Before starting the dev, clear the memory and then configure to device using. */ 3330 memset(qm->qdma.va, 0, qm->qdma.size); 3331 3332 ret = qm_eq_ctx_cfg(qm); 3333 if (ret) { 3334 dev_err(dev, "Set eqc failed!\n"); 3335 return ret; 3336 } 3337 3338 return qm_aeq_ctx_cfg(qm); 3339 } 3340 3341 static int __hisi_qm_start(struct hisi_qm *qm) 3342 { 3343 struct device *dev = &qm->pdev->dev; 3344 int ret; 3345 3346 if (!qm->qdma.va) { 3347 dev_err(dev, "qm qdma is NULL!\n"); 3348 return -EINVAL; 3349 } 3350 3351 if (qm->fun_type == QM_HW_PF) { 3352 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); 3353 if (ret) 3354 return ret; 3355 } 3356 3357 ret = qm_eq_aeq_ctx_cfg(qm); 3358 if (ret) 3359 return ret; 3360 3361 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); 3362 if (ret) 3363 return ret; 3364 3365 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); 3366 if (ret) 3367 return ret; 3368 3369 qm_init_prefetch(qm); 3370 qm_enable_eq_aeq_interrupts(qm); 3371 3372 return 0; 3373 } 3374 3375 /** 3376 * hisi_qm_start() - start qm 3377 * @qm: The qm to be started. 3378 * 3379 * This function starts a qm, then we can allocate qp from this qm. 3380 */ 3381 int hisi_qm_start(struct hisi_qm *qm) 3382 { 3383 struct device *dev = &qm->pdev->dev; 3384 int ret = 0; 3385 3386 down_write(&qm->qps_lock); 3387 3388 dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); 3389 3390 if (!qm->qp_num) { 3391 dev_err(dev, "qp_num should not be 0\n"); 3392 ret = -EINVAL; 3393 goto err_unlock; 3394 } 3395 3396 ret = __hisi_qm_start(qm); 3397 if (ret) 3398 goto err_unlock; 3399 3400 atomic_set(&qm->status.flags, QM_WORK); 3401 hisi_qm_set_state(qm, QM_READY); 3402 3403 err_unlock: 3404 up_write(&qm->qps_lock); 3405 return ret; 3406 } 3407 EXPORT_SYMBOL_GPL(hisi_qm_start); 3408 3409 static int qm_restart(struct hisi_qm *qm) 3410 { 3411 struct device *dev = &qm->pdev->dev; 3412 struct hisi_qp *qp; 3413 int ret, i; 3414 3415 ret = hisi_qm_start(qm); 3416 if (ret < 0) 3417 return ret; 3418 3419 down_write(&qm->qps_lock); 3420 for (i = 0; i < qm->qp_num; i++) { 3421 qp = &qm->qp_array[i]; 3422 if (atomic_read(&qp->qp_status.flags) == QP_STOP && 3423 qp->is_resetting == true && qp->is_in_kernel == true) { 3424 ret = qm_start_qp_nolock(qp, 0); 3425 if (ret < 0) { 3426 dev_err(dev, "Failed to start qp%d!\n", i); 3427 3428 up_write(&qm->qps_lock); 3429 return ret; 3430 } 3431 qp->is_resetting = false; 3432 } 3433 } 3434 up_write(&qm->qps_lock); 3435 3436 return 0; 3437 } 3438 3439 /* Stop started qps in reset flow */ 3440 static void qm_stop_started_qp(struct hisi_qm *qm) 3441 { 3442 struct hisi_qp *qp; 3443 int i; 3444 3445 for (i = 0; i < qm->qp_num; i++) { 3446 qp = &qm->qp_array[i]; 3447 if (atomic_read(&qp->qp_status.flags) == QP_START) { 3448 qp->is_resetting = true; 3449 qm_stop_qp_nolock(qp); 3450 } 3451 } 3452 } 3453 3454 /** 3455 * qm_invalid_queues() - invalid all queues in use. 3456 * @qm: The qm in which the queues will be invalidated. 3457 * 3458 * This function invalid all queues in use. If the doorbell command is sent 3459 * to device in user space after the device is reset, the device discards 3460 * the doorbell command. 3461 */ 3462 static void qm_invalid_queues(struct hisi_qm *qm) 3463 { 3464 struct hisi_qp *qp; 3465 struct qm_sqc *sqc; 3466 struct qm_cqc *cqc; 3467 int i; 3468 3469 /* 3470 * Normal stop queues is no longer used and does not need to be 3471 * invalid queues. 3472 */ 3473 if (qm->status.stop_reason == QM_NORMAL) 3474 return; 3475 3476 if (qm->status.stop_reason == QM_DOWN) 3477 hisi_qm_cache_wb(qm); 3478 3479 for (i = 0; i < qm->qp_num; i++) { 3480 qp = &qm->qp_array[i]; 3481 if (!qp->is_resetting) 3482 continue; 3483 3484 /* Modify random data and set sqc close bit to invalid queue. */ 3485 sqc = qm->sqc + i; 3486 cqc = qm->cqc + i; 3487 sqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); 3488 sqc->w13 = cpu_to_le16(QM_SQC_DISABLE_QP); 3489 cqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); 3490 if (qp->is_in_kernel) 3491 memset(qp->qdma.va, 0, qp->qdma.size); 3492 } 3493 } 3494 3495 /** 3496 * hisi_qm_stop() - Stop a qm. 3497 * @qm: The qm which will be stopped. 3498 * @r: The reason to stop qm. 3499 * 3500 * This function stops qm and its qps, then qm can not accept request. 3501 * Related resources are not released at this state, we can use hisi_qm_start 3502 * to let qm start again. 3503 */ 3504 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) 3505 { 3506 struct device *dev = &qm->pdev->dev; 3507 int ret = 0; 3508 3509 down_write(&qm->qps_lock); 3510 3511 if (atomic_read(&qm->status.flags) == QM_STOP) 3512 goto err_unlock; 3513 3514 /* Stop all the request sending at first. */ 3515 atomic_set(&qm->status.flags, QM_STOP); 3516 qm->status.stop_reason = r; 3517 3518 if (qm->status.stop_reason != QM_NORMAL) { 3519 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 3520 /* 3521 * When performing soft reset, the hardware will no longer 3522 * do tasks, and the tasks in the device will be flushed 3523 * out directly since the master ooo is closed. 3524 */ 3525 if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) && 3526 r != QM_SOFT_RESET) { 3527 ret = qm_drain_qm(qm); 3528 if (ret) { 3529 dev_err(dev, "failed to drain qm!\n"); 3530 goto err_unlock; 3531 } 3532 } 3533 3534 qm_stop_started_qp(qm); 3535 3536 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 3537 } 3538 3539 qm_disable_eq_aeq_interrupts(qm); 3540 if (qm->fun_type == QM_HW_PF) { 3541 ret = hisi_qm_set_vft(qm, 0, 0, 0); 3542 if (ret < 0) { 3543 dev_err(dev, "Failed to set vft!\n"); 3544 ret = -EBUSY; 3545 goto err_unlock; 3546 } 3547 } 3548 3549 qm_invalid_queues(qm); 3550 qm->status.stop_reason = QM_NORMAL; 3551 3552 err_unlock: 3553 up_write(&qm->qps_lock); 3554 return ret; 3555 } 3556 EXPORT_SYMBOL_GPL(hisi_qm_stop); 3557 3558 static void qm_hw_error_init(struct hisi_qm *qm) 3559 { 3560 if (!qm->ops->hw_error_init) { 3561 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); 3562 return; 3563 } 3564 3565 qm->ops->hw_error_init(qm); 3566 } 3567 3568 static void qm_hw_error_uninit(struct hisi_qm *qm) 3569 { 3570 if (!qm->ops->hw_error_uninit) { 3571 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n"); 3572 return; 3573 } 3574 3575 qm->ops->hw_error_uninit(qm); 3576 } 3577 3578 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm) 3579 { 3580 if (!qm->ops->hw_error_handle) { 3581 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n"); 3582 return ACC_ERR_NONE; 3583 } 3584 3585 return qm->ops->hw_error_handle(qm); 3586 } 3587 3588 /** 3589 * hisi_qm_dev_err_init() - Initialize device error configuration. 3590 * @qm: The qm for which we want to do error initialization. 3591 * 3592 * Initialize QM and device error related configuration. 3593 */ 3594 void hisi_qm_dev_err_init(struct hisi_qm *qm) 3595 { 3596 if (qm->fun_type == QM_HW_VF) 3597 return; 3598 3599 qm_hw_error_init(qm); 3600 3601 if (!qm->err_ini->hw_err_enable) { 3602 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n"); 3603 return; 3604 } 3605 qm->err_ini->hw_err_enable(qm); 3606 } 3607 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init); 3608 3609 /** 3610 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration. 3611 * @qm: The qm for which we want to do error uninitialization. 3612 * 3613 * Uninitialize QM and device error related configuration. 3614 */ 3615 void hisi_qm_dev_err_uninit(struct hisi_qm *qm) 3616 { 3617 if (qm->fun_type == QM_HW_VF) 3618 return; 3619 3620 qm_hw_error_uninit(qm); 3621 3622 if (!qm->err_ini->hw_err_disable) { 3623 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n"); 3624 return; 3625 } 3626 qm->err_ini->hw_err_disable(qm); 3627 } 3628 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); 3629 3630 static void qm_release_qp_nolock(struct hisi_qp *qp) 3631 { 3632 struct hisi_qm *qm = qp->qm; 3633 3634 if (--qp->ref_count) 3635 return; 3636 3637 qm->qp_in_used--; 3638 idr_remove(&qm->qp_idr, qp->qp_id); 3639 } 3640 3641 /** 3642 * hisi_qm_free_qps() - free multiple queue pairs. 3643 * @qps: The queue pairs need to be freed. 3644 * @qp_num: The num of queue pairs. 3645 */ 3646 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) 3647 { 3648 int i; 3649 3650 if (!qps || qp_num <= 0) 3651 return; 3652 3653 down_write(&qps[0]->qm->qps_lock); 3654 3655 for (i = qp_num - 1; i >= 0; i--) { 3656 if (qps[i]->ref_count == 1) 3657 qm_stop_qp_nolock(qps[i]); 3658 3659 qm_release_qp_nolock(qps[i]); 3660 } 3661 3662 up_write(&qps[0]->qm->qps_lock); 3663 qm_pm_put_sync(qps[0]->qm); 3664 } 3665 EXPORT_SYMBOL_GPL(hisi_qm_free_qps); 3666 3667 static void qm_insert_sorted(struct list_head *head, struct hisi_qm_resource *res) 3668 { 3669 struct hisi_qm_resource *tmp; 3670 struct list_head *n = head; 3671 3672 list_for_each_entry(tmp, head, list) { 3673 if (res->distance < tmp->distance) { 3674 n = &tmp->list; 3675 break; 3676 } 3677 } 3678 list_add_tail(&res->list, n); 3679 } 3680 3681 static void free_list(struct list_head *head) 3682 { 3683 struct hisi_qm_resource *res, *tmp; 3684 3685 list_for_each_entry_safe(res, tmp, head, list) { 3686 list_del(&res->list); 3687 kfree(res); 3688 } 3689 } 3690 3691 static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) 3692 { 3693 int i, ret; 3694 3695 ret = qm_pm_get_sync(qm); 3696 if (ret) 3697 return ret; 3698 3699 down_write(&qm->qps_lock); 3700 for (i = 0; i < qp_num; i++) { 3701 qps[i] = qm_create_qp_nolock(qm, alg_type[i], true); 3702 if (IS_ERR(qps[i])) { 3703 ret = -ENODEV; 3704 goto stop_and_free; 3705 } 3706 3707 if (qps[i]->ref_count != 1) 3708 continue; 3709 3710 ret = qm_start_qp_nolock(qps[i], 0); 3711 if (ret) { 3712 qm_release_qp_nolock(qps[i]); 3713 goto stop_and_free; 3714 } 3715 } 3716 up_write(&qm->qps_lock); 3717 3718 return 0; 3719 3720 stop_and_free: 3721 for (i--; i >= 0; i--) { 3722 if (qps[i]->ref_count == 1) 3723 qm_stop_qp_nolock(qps[i]); 3724 3725 qm_release_qp_nolock(qps[i]); 3726 } 3727 up_write(&qm->qps_lock); 3728 qm_pm_put_sync(qm); 3729 3730 return ret; 3731 } 3732 3733 static int hisi_qm_sort_devices(int node, struct list_head *head, 3734 struct hisi_qm_list *qm_list) 3735 { 3736 struct hisi_qm_resource *res; 3737 struct hisi_qm *qm; 3738 struct device *dev; 3739 int dev_node; 3740 LIST_HEAD(non_full_list); 3741 LIST_HEAD(full_list); 3742 3743 list_for_each_entry(qm, &qm_list->list, list) { 3744 dev = &qm->pdev->dev; 3745 3746 dev_node = dev_to_node(dev); 3747 if (dev_node < 0) 3748 dev_node = 0; 3749 3750 res = kzalloc(sizeof(*res), GFP_KERNEL); 3751 if (!res) 3752 return -ENOMEM; 3753 3754 res->qm = qm; 3755 res->distance = node_distance(dev_node, node); 3756 3757 if (qm->qp_in_used == qm->qp_num) 3758 qm_insert_sorted(&full_list, res); 3759 else 3760 qm_insert_sorted(&non_full_list, res); 3761 } 3762 3763 list_splice_tail(&non_full_list, head); 3764 list_splice_tail(&full_list, head); 3765 3766 return 0; 3767 } 3768 3769 /** 3770 * hisi_qm_alloc_qps_node() - Create multiple queue pairs. 3771 * @qm_list: The list of all available devices. 3772 * @qp_num: The number of queue pairs need created. 3773 * @alg_type: The algorithm type. 3774 * @node: The numa node. 3775 * @qps: The queue pairs need created. 3776 * 3777 * This function will sort all available device according to numa distance. 3778 * Then try to create all queue pairs from one device, if all devices do 3779 * not meet the requirements will return error. 3780 */ 3781 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, 3782 u8 *alg_type, int node, struct hisi_qp **qps) 3783 { 3784 struct hisi_qm_resource *tmp; 3785 int ret = -ENODEV; 3786 LIST_HEAD(head); 3787 3788 if (!qps || !qm_list || qp_num <= 0) 3789 return -EINVAL; 3790 3791 mutex_lock(&qm_list->lock); 3792 if (hisi_qm_sort_devices(node, &head, qm_list)) { 3793 mutex_unlock(&qm_list->lock); 3794 goto err; 3795 } 3796 3797 list_for_each_entry(tmp, &head, list) { 3798 ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); 3799 if (!ret) 3800 break; 3801 } 3802 3803 mutex_unlock(&qm_list->lock); 3804 if (ret) 3805 pr_info("Failed to create qps, node[%d], qp[%d]!\n", 3806 node, qp_num); 3807 3808 err: 3809 free_list(&head); 3810 return ret; 3811 } 3812 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node); 3813 3814 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) 3815 { 3816 u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j; 3817 u32 max_qp_num = qm->max_qp_num; 3818 u32 q_base = qm->qp_num; 3819 int ret; 3820 3821 if (!num_vfs) 3822 return -EINVAL; 3823 3824 vfs_q_num = qm->ctrl_qp_num - qm->qp_num; 3825 3826 /* If vfs_q_num is less than num_vfs, return error. */ 3827 if (vfs_q_num < num_vfs) 3828 return -EINVAL; 3829 3830 q_num = vfs_q_num / num_vfs; 3831 remain_q_num = vfs_q_num % num_vfs; 3832 3833 for (i = num_vfs; i > 0; i--) { 3834 /* 3835 * if q_num + remain_q_num > max_qp_num in last vf, divide the 3836 * remaining queues equally. 3837 */ 3838 if (i == num_vfs && q_num + remain_q_num <= max_qp_num) { 3839 act_q_num = q_num + remain_q_num; 3840 remain_q_num = 0; 3841 } else if (remain_q_num > 0) { 3842 act_q_num = q_num + 1; 3843 remain_q_num--; 3844 } else { 3845 act_q_num = q_num; 3846 } 3847 3848 act_q_num = min(act_q_num, max_qp_num); 3849 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num); 3850 if (ret) { 3851 for (j = num_vfs; j > i; j--) 3852 hisi_qm_set_vft(qm, j, 0, 0); 3853 return ret; 3854 } 3855 q_base += act_q_num; 3856 } 3857 3858 return 0; 3859 } 3860 3861 static void qm_clear_vft_config(struct hisi_qm *qm) 3862 { 3863 u32 i; 3864 3865 /* 3866 * When disabling SR-IOV, clear the configuration of each VF in the hardware 3867 * sequentially. Failure to clear a single VF should not affect the clearing 3868 * operation of other VFs. 3869 */ 3870 for (i = 1; i <= qm->vfs_num; i++) 3871 (void)hisi_qm_set_vft(qm, i, 0, 0); 3872 3873 qm->vfs_num = 0; 3874 } 3875 3876 static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) 3877 { 3878 struct device *dev = &qm->pdev->dev; 3879 struct qm_shaper_factor t_factor; 3880 u32 ir = qos * QM_QOS_RATE; 3881 int ret, total_vfs, i; 3882 3883 total_vfs = pci_sriov_get_totalvfs(qm->pdev); 3884 if (fun_index > total_vfs) 3885 return -EINVAL; 3886 3887 memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); 3888 qm->factor[fun_index].func_qos = qos; 3889 3890 ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); 3891 if (ret) { 3892 dev_err(dev, "failed to calculate shaper parameter!\n"); 3893 return -EINVAL; 3894 } 3895 3896 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { 3897 /* The base number of queue reuse for different alg type */ 3898 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); 3899 if (ret) { 3900 dev_err(dev, "type: %d, failed to set shaper vft!\n", i); 3901 goto back_func_qos; 3902 } 3903 } 3904 3905 return 0; 3906 3907 back_func_qos: 3908 memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); 3909 for (i--; i >= ALG_TYPE_0; i--) { 3910 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); 3911 if (ret) 3912 dev_err(dev, "failed to restore shaper vft during rollback!\n"); 3913 } 3914 3915 return -EINVAL; 3916 } 3917 3918 static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) 3919 { 3920 u64 cir_u = 0, cir_b = 0, cir_s = 0; 3921 u64 shaper_vft, ir_calc, ir; 3922 unsigned int val; 3923 u32 error_rate; 3924 int ret; 3925 3926 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 3927 val & BIT(0), POLL_PERIOD, 3928 POLL_TIMEOUT); 3929 if (ret) 3930 return 0; 3931 3932 writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); 3933 writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE); 3934 writel(fun_index, qm->io_base + QM_VFT_CFG); 3935 3936 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); 3937 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); 3938 3939 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 3940 val & BIT(0), POLL_PERIOD, 3941 POLL_TIMEOUT); 3942 if (ret) 3943 return 0; 3944 3945 shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | 3946 ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32); 3947 3948 cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK; 3949 cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK; 3950 cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT; 3951 3952 cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK; 3953 cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT; 3954 3955 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); 3956 3957 ir = qm->factor[fun_index].func_qos * QM_QOS_RATE; 3958 3959 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; 3960 if (error_rate > QM_QOS_MIN_ERROR_RATE) { 3961 pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate); 3962 return 0; 3963 } 3964 3965 return ir; 3966 } 3967 3968 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) 3969 { 3970 struct device *dev = &qm->pdev->dev; 3971 u32 qos; 3972 int ret; 3973 3974 qos = qm_get_shaper_vft_qos(qm, fun_num); 3975 if (!qos) { 3976 dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num); 3977 return; 3978 } 3979 3980 ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); 3981 if (ret) 3982 dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); 3983 } 3984 3985 static int qm_vf_read_qos(struct hisi_qm *qm) 3986 { 3987 int cnt = 0; 3988 int ret = -EINVAL; 3989 3990 /* reset mailbox qos val */ 3991 qm->mb_qos = 0; 3992 3993 /* vf ping pf to get function qos */ 3994 ret = qm_ping_pf(qm, QM_VF_GET_QOS); 3995 if (ret) { 3996 pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); 3997 return ret; 3998 } 3999 4000 while (true) { 4001 msleep(QM_WAIT_DST_ACK); 4002 if (qm->mb_qos) 4003 break; 4004 4005 if (++cnt > QM_MAX_VF_WAIT_COUNT) { 4006 pci_err(qm->pdev, "PF ping VF timeout!\n"); 4007 return -ETIMEDOUT; 4008 } 4009 } 4010 4011 return ret; 4012 } 4013 4014 static ssize_t qm_algqos_read(struct file *filp, char __user *buf, 4015 size_t count, loff_t *pos) 4016 { 4017 struct hisi_qm *qm = filp->private_data; 4018 char tbuf[QM_DBG_READ_LEN]; 4019 u32 qos_val, ir; 4020 int ret; 4021 4022 ret = hisi_qm_get_dfx_access(qm); 4023 if (ret) 4024 return ret; 4025 4026 /* Mailbox and reset cannot be operated at the same time */ 4027 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { 4028 pci_err(qm->pdev, "dev resetting, read alg qos failed!\n"); 4029 ret = -EAGAIN; 4030 goto err_put_dfx_access; 4031 } 4032 4033 if (qm->fun_type == QM_HW_PF) { 4034 ir = qm_get_shaper_vft_qos(qm, 0); 4035 } else { 4036 ret = qm_vf_read_qos(qm); 4037 if (ret) 4038 goto err_get_status; 4039 ir = qm->mb_qos; 4040 } 4041 4042 qos_val = ir / QM_QOS_RATE; 4043 ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val); 4044 4045 ret = simple_read_from_buffer(buf, count, pos, tbuf, ret); 4046 4047 err_get_status: 4048 clear_bit(QM_RESETTING, &qm->misc_ctl); 4049 err_put_dfx_access: 4050 hisi_qm_put_dfx_access(qm); 4051 return ret; 4052 } 4053 4054 static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, 4055 unsigned long *val, 4056 unsigned int *fun_index) 4057 { 4058 const struct bus_type *bus_type = qm->pdev->dev.bus; 4059 char tbuf_bdf[QM_DBG_READ_LEN] = {0}; 4060 char val_buf[QM_DBG_READ_LEN] = {0}; 4061 struct pci_dev *pdev; 4062 struct device *dev; 4063 int ret; 4064 4065 ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf); 4066 if (ret != QM_QOS_PARAM_NUM) 4067 return -EINVAL; 4068 4069 ret = kstrtoul(val_buf, 10, val); 4070 if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) { 4071 pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); 4072 return -EINVAL; 4073 } 4074 4075 dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf); 4076 if (!dev) { 4077 pci_err(qm->pdev, "input pci bdf number is error!\n"); 4078 return -ENODEV; 4079 } 4080 4081 pdev = container_of(dev, struct pci_dev, dev); 4082 if (pci_physfn(pdev) != qm->pdev) { 4083 pci_err(qm->pdev, "the pdev input does not match the pf!\n"); 4084 put_device(dev); 4085 return -EINVAL; 4086 } 4087 4088 *fun_index = pdev->devfn; 4089 put_device(dev); 4090 4091 return 0; 4092 } 4093 4094 static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, 4095 size_t count, loff_t *pos) 4096 { 4097 struct hisi_qm *qm = filp->private_data; 4098 char tbuf[QM_DBG_READ_LEN]; 4099 unsigned int fun_index; 4100 unsigned long val; 4101 int len, ret; 4102 4103 if (*pos != 0) 4104 return 0; 4105 4106 if (count >= QM_DBG_READ_LEN) 4107 return -ENOSPC; 4108 4109 len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); 4110 if (len < 0) 4111 return len; 4112 4113 tbuf[len] = '\0'; 4114 ret = qm_get_qos_value(qm, tbuf, &val, &fun_index); 4115 if (ret) 4116 return ret; 4117 4118 /* Mailbox and reset cannot be operated at the same time */ 4119 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { 4120 pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); 4121 return -EAGAIN; 4122 } 4123 4124 ret = qm_pm_get_sync(qm); 4125 if (ret) { 4126 ret = -EINVAL; 4127 goto err_get_status; 4128 } 4129 4130 ret = qm_func_shaper_enable(qm, fun_index, val); 4131 if (ret) { 4132 pci_err(qm->pdev, "failed to enable function shaper!\n"); 4133 ret = -EINVAL; 4134 goto err_put_sync; 4135 } 4136 4137 pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n", 4138 fun_index, val); 4139 ret = count; 4140 4141 err_put_sync: 4142 qm_pm_put_sync(qm); 4143 err_get_status: 4144 clear_bit(QM_RESETTING, &qm->misc_ctl); 4145 return ret; 4146 } 4147 4148 static const struct file_operations qm_algqos_fops = { 4149 .owner = THIS_MODULE, 4150 .open = simple_open, 4151 .read = qm_algqos_read, 4152 .write = qm_algqos_write, 4153 }; 4154 4155 /** 4156 * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. 4157 * @qm: The qm for which we want to add debugfs files. 4158 * 4159 * Create function qos debugfs files, VF ping PF to get function qos. 4160 */ 4161 void hisi_qm_set_algqos_init(struct hisi_qm *qm) 4162 { 4163 if (qm->fun_type == QM_HW_PF) 4164 debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, 4165 qm, &qm_algqos_fops); 4166 else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 4167 debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, 4168 qm, &qm_algqos_fops); 4169 } 4170 4171 static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) 4172 { 4173 int i; 4174 4175 for (i = 1; i <= total_func; i++) 4176 qm->factor[i].func_qos = QM_QOS_MAX_VAL; 4177 } 4178 4179 /** 4180 * hisi_qm_sriov_enable() - enable virtual functions 4181 * @pdev: the PCIe device 4182 * @max_vfs: the number of virtual functions to enable 4183 * 4184 * Returns the number of enabled VFs. If there are VFs enabled already or 4185 * max_vfs is more than the total number of device can be enabled, returns 4186 * failure. 4187 */ 4188 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) 4189 { 4190 struct hisi_qm *qm = pci_get_drvdata(pdev); 4191 int pre_existing_vfs, num_vfs, total_vfs, ret; 4192 4193 ret = qm_pm_get_sync(qm); 4194 if (ret) 4195 return ret; 4196 4197 total_vfs = pci_sriov_get_totalvfs(pdev); 4198 pre_existing_vfs = pci_num_vf(pdev); 4199 if (pre_existing_vfs) { 4200 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n", 4201 pre_existing_vfs); 4202 goto err_put_sync; 4203 } 4204 4205 if (max_vfs > total_vfs) { 4206 pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs); 4207 ret = -ERANGE; 4208 goto err_put_sync; 4209 } 4210 4211 num_vfs = max_vfs; 4212 4213 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 4214 hisi_qm_init_vf_qos(qm, num_vfs); 4215 4216 ret = qm_vf_q_assign(qm, num_vfs); 4217 if (ret) { 4218 pci_err(pdev, "Can't assign queues for VF!\n"); 4219 goto err_put_sync; 4220 } 4221 4222 qm->vfs_num = num_vfs; 4223 ret = pci_enable_sriov(pdev, num_vfs); 4224 if (ret) { 4225 pci_err(pdev, "Can't enable VF!\n"); 4226 qm_clear_vft_config(qm); 4227 goto err_put_sync; 4228 } 4229 4230 pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs); 4231 4232 return num_vfs; 4233 4234 err_put_sync: 4235 qm_pm_put_sync(qm); 4236 return ret; 4237 } 4238 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); 4239 4240 /** 4241 * hisi_qm_sriov_disable - disable virtual functions 4242 * @pdev: the PCI device. 4243 * @is_frozen: true when all the VFs are frozen. 4244 * 4245 * Return failure if there are VFs assigned already or VF is in used. 4246 */ 4247 int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) 4248 { 4249 struct hisi_qm *qm = pci_get_drvdata(pdev); 4250 4251 if (pci_vfs_assigned(pdev)) { 4252 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n"); 4253 return -EPERM; 4254 } 4255 4256 /* While VF is in used, SRIOV cannot be disabled. */ 4257 if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { 4258 pci_err(pdev, "Task is using its VF!\n"); 4259 return -EBUSY; 4260 } 4261 4262 pci_disable_sriov(pdev); 4263 qm_clear_vft_config(qm); 4264 qm_pm_put_sync(qm); 4265 4266 return 0; 4267 } 4268 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); 4269 4270 /** 4271 * hisi_qm_sriov_configure - configure the number of VFs 4272 * @pdev: The PCI device 4273 * @num_vfs: The number of VFs need enabled 4274 * 4275 * Enable SR-IOV according to num_vfs, 0 means disable. 4276 */ 4277 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) 4278 { 4279 if (num_vfs == 0) 4280 return hisi_qm_sriov_disable(pdev, false); 4281 else 4282 return hisi_qm_sriov_enable(pdev, num_vfs); 4283 } 4284 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure); 4285 4286 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) 4287 { 4288 if (!qm->err_ini->get_err_result) { 4289 dev_err(&qm->pdev->dev, "Device doesn't support reset!\n"); 4290 return ACC_ERR_NONE; 4291 } 4292 4293 return qm->err_ini->get_err_result(qm); 4294 } 4295 4296 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) 4297 { 4298 enum acc_err_result qm_ret, dev_ret; 4299 4300 /* log qm error */ 4301 qm_ret = qm_hw_error_handle(qm); 4302 4303 /* log device error */ 4304 dev_ret = qm_dev_err_handle(qm); 4305 4306 return (qm_ret == ACC_ERR_NEED_RESET || 4307 dev_ret == ACC_ERR_NEED_RESET) ? 4308 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; 4309 } 4310 4311 /** 4312 * hisi_qm_dev_err_detected() - Get device and qm error status then log it. 4313 * @pdev: The PCI device which need report error. 4314 * @state: The connectivity between CPU and device. 4315 * 4316 * We register this function into PCIe AER handlers, It will report device or 4317 * qm hardware error status when error occur. 4318 */ 4319 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, 4320 pci_channel_state_t state) 4321 { 4322 struct hisi_qm *qm = pci_get_drvdata(pdev); 4323 enum acc_err_result ret; 4324 4325 if (pdev->is_virtfn) 4326 return PCI_ERS_RESULT_NONE; 4327 4328 pci_info(pdev, "PCI error detected, state(=%u)!!\n", state); 4329 if (state == pci_channel_io_perm_failure) 4330 return PCI_ERS_RESULT_DISCONNECT; 4331 4332 ret = qm_process_dev_error(qm); 4333 if (ret == ACC_ERR_NEED_RESET) 4334 return PCI_ERS_RESULT_NEED_RESET; 4335 4336 return PCI_ERS_RESULT_RECOVERED; 4337 } 4338 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); 4339 4340 static int qm_check_req_recv(struct hisi_qm *qm) 4341 { 4342 struct pci_dev *pdev = qm->pdev; 4343 int ret; 4344 u32 val; 4345 4346 if (qm->ver >= QM_HW_V3) 4347 return 0; 4348 4349 writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID); 4350 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, 4351 (val == ACC_VENDOR_ID_VALUE), 4352 POLL_PERIOD, POLL_TIMEOUT); 4353 if (ret) { 4354 dev_err(&pdev->dev, "Fails to read QM reg!\n"); 4355 return ret; 4356 } 4357 4358 writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID); 4359 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, 4360 (val == PCI_VENDOR_ID_HUAWEI), 4361 POLL_PERIOD, POLL_TIMEOUT); 4362 if (ret) 4363 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n"); 4364 4365 return ret; 4366 } 4367 4368 static int qm_set_pf_mse(struct hisi_qm *qm, bool set) 4369 { 4370 struct pci_dev *pdev = qm->pdev; 4371 u16 cmd; 4372 int i; 4373 4374 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 4375 if (set) 4376 cmd |= PCI_COMMAND_MEMORY; 4377 else 4378 cmd &= ~PCI_COMMAND_MEMORY; 4379 4380 pci_write_config_word(pdev, PCI_COMMAND, cmd); 4381 for (i = 0; i < MAX_WAIT_COUNTS; i++) { 4382 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 4383 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1)) 4384 return 0; 4385 4386 udelay(1); 4387 } 4388 4389 return -ETIMEDOUT; 4390 } 4391 4392 static int qm_set_vf_mse(struct hisi_qm *qm, bool set) 4393 { 4394 struct pci_dev *pdev = qm->pdev; 4395 u16 sriov_ctrl; 4396 int pos; 4397 int i; 4398 4399 /* 4400 * Since function qm_set_vf_mse is called only after SRIOV is enabled, 4401 * pci_find_ext_capability cannot return 0, pos does not need to be 4402 * checked. 4403 */ 4404 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 4405 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); 4406 if (set) 4407 sriov_ctrl |= PCI_SRIOV_CTRL_MSE; 4408 else 4409 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE; 4410 pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl); 4411 4412 for (i = 0; i < MAX_WAIT_COUNTS; i++) { 4413 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); 4414 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >> 4415 ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT) 4416 return 0; 4417 4418 udelay(1); 4419 } 4420 4421 return -ETIMEDOUT; 4422 } 4423 4424 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) 4425 { 4426 u32 nfe_enb = 0; 4427 4428 /* Kunpeng930 hardware automatically close master ooo when NFE occurs */ 4429 if (qm->ver >= QM_HW_V3) 4430 return; 4431 4432 if (!qm->err_status.is_dev_ecc_mbit && 4433 qm->err_status.is_qm_ecc_mbit && 4434 qm->err_ini->close_axi_master_ooo) { 4435 qm->err_ini->close_axi_master_ooo(qm); 4436 } else if (qm->err_status.is_dev_ecc_mbit && 4437 !qm->err_status.is_qm_ecc_mbit && 4438 !qm->err_ini->close_axi_master_ooo) { 4439 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE); 4440 writel(nfe_enb & ~qm->err_info.qm_err.ecc_2bits_mask, 4441 qm->io_base + QM_RAS_NFE_ENABLE); 4442 writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SET); 4443 } 4444 } 4445 4446 static int qm_vf_reset_prepare(struct hisi_qm *qm, 4447 enum qm_stop_reason stop_reason) 4448 { 4449 struct hisi_qm_list *qm_list = qm->qm_list; 4450 struct pci_dev *pdev = qm->pdev; 4451 struct pci_dev *virtfn; 4452 struct hisi_qm *vf_qm; 4453 int ret = 0; 4454 4455 mutex_lock(&qm_list->lock); 4456 list_for_each_entry(vf_qm, &qm_list->list, list) { 4457 virtfn = vf_qm->pdev; 4458 if (virtfn == pdev) 4459 continue; 4460 4461 if (pci_physfn(virtfn) == pdev) { 4462 /* save VFs PCIE BAR configuration */ 4463 pci_save_state(virtfn); 4464 4465 ret = hisi_qm_stop(vf_qm, stop_reason); 4466 if (ret) 4467 goto stop_fail; 4468 } 4469 } 4470 4471 stop_fail: 4472 mutex_unlock(&qm_list->lock); 4473 return ret; 4474 } 4475 4476 static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, 4477 enum qm_stop_reason stop_reason) 4478 { 4479 struct pci_dev *pdev = qm->pdev; 4480 int ret; 4481 4482 if (!qm->vfs_num) 4483 return 0; 4484 4485 /* Kunpeng930 supports to notify VFs to stop before PF reset */ 4486 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { 4487 ret = qm_ping_all_vfs(qm, cmd); 4488 if (ret) 4489 pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); 4490 } else { 4491 ret = qm_vf_reset_prepare(qm, stop_reason); 4492 if (ret) 4493 pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret); 4494 } 4495 4496 return ret; 4497 } 4498 4499 static int qm_controller_reset_prepare(struct hisi_qm *qm) 4500 { 4501 struct pci_dev *pdev = qm->pdev; 4502 int ret; 4503 4504 if (qm->err_ini->set_priv_status) { 4505 ret = qm->err_ini->set_priv_status(qm); 4506 if (ret) 4507 return ret; 4508 } 4509 4510 ret = qm_reset_prepare_ready(qm); 4511 if (ret) { 4512 pci_err(pdev, "Controller reset not ready!\n"); 4513 return ret; 4514 } 4515 4516 qm_dev_ecc_mbit_handle(qm); 4517 4518 /* PF obtains the information of VF by querying the register. */ 4519 qm_cmd_uninit(qm); 4520 4521 /* Whether VFs stop successfully, soft reset will continue. */ 4522 ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET); 4523 if (ret) 4524 pci_err(pdev, "failed to stop vfs by pf in soft reset.\n"); 4525 4526 ret = hisi_qm_stop(qm, QM_SOFT_RESET); 4527 if (ret) { 4528 pci_err(pdev, "Fails to stop QM!\n"); 4529 qm_reset_bit_clear(qm); 4530 return ret; 4531 } 4532 4533 if (qm->use_sva) { 4534 ret = qm_hw_err_isolate(qm); 4535 if (ret) 4536 pci_err(pdev, "failed to isolate hw err!\n"); 4537 } 4538 4539 ret = qm_wait_vf_prepare_finish(qm); 4540 if (ret) 4541 pci_err(pdev, "failed to stop by vfs in soft reset!\n"); 4542 4543 clear_bit(QM_RST_SCHED, &qm->misc_ctl); 4544 4545 return 0; 4546 } 4547 4548 static int qm_master_ooo_check(struct hisi_qm *qm) 4549 { 4550 u32 val; 4551 int ret; 4552 4553 /* Check the ooo register of the device before resetting the device. */ 4554 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 4555 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN, 4556 val, (val == ACC_MASTER_TRANS_RETURN_RW), 4557 POLL_PERIOD, POLL_TIMEOUT); 4558 if (ret) 4559 pci_warn(qm->pdev, "Bus lock! Please reset system.\n"); 4560 4561 return ret; 4562 } 4563 4564 static int qm_soft_reset_prepare(struct hisi_qm *qm) 4565 { 4566 struct pci_dev *pdev = qm->pdev; 4567 int ret; 4568 4569 /* Ensure all doorbells and mailboxes received by QM */ 4570 ret = qm_check_req_recv(qm); 4571 if (ret) 4572 return ret; 4573 4574 if (qm->vfs_num) { 4575 ret = qm_set_vf_mse(qm, false); 4576 if (ret) { 4577 pci_err(pdev, "Fails to disable vf MSE bit.\n"); 4578 return ret; 4579 } 4580 } 4581 4582 ret = qm->ops->set_msi(qm, false); 4583 if (ret) { 4584 pci_err(pdev, "Fails to disable PEH MSI bit.\n"); 4585 return ret; 4586 } 4587 4588 ret = qm_master_ooo_check(qm); 4589 if (ret) 4590 return ret; 4591 4592 if (qm->err_ini->close_sva_prefetch) 4593 qm->err_ini->close_sva_prefetch(qm); 4594 4595 ret = qm_set_pf_mse(qm, false); 4596 if (ret) 4597 pci_err(pdev, "Fails to disable pf MSE bit.\n"); 4598 4599 return ret; 4600 } 4601 4602 static int qm_reset_device(struct hisi_qm *qm) 4603 { 4604 struct pci_dev *pdev = qm->pdev; 4605 4606 /* The reset related sub-control registers are not in PCI BAR */ 4607 if (ACPI_HANDLE(&pdev->dev)) { 4608 unsigned long long value = 0; 4609 acpi_status s; 4610 4611 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev), 4612 qm->err_info.acpi_rst, 4613 NULL, &value); 4614 if (ACPI_FAILURE(s)) { 4615 pci_err(pdev, "NO controller reset method!\n"); 4616 return -EIO; 4617 } 4618 4619 if (value) { 4620 pci_err(pdev, "Reset step %llu failed!\n", value); 4621 return -EIO; 4622 } 4623 4624 return 0; 4625 } 4626 4627 pci_err(pdev, "No reset method!\n"); 4628 return -EINVAL; 4629 } 4630 4631 static int qm_soft_reset(struct hisi_qm *qm) 4632 { 4633 int ret; 4634 4635 ret = qm_soft_reset_prepare(qm); 4636 if (ret) 4637 return ret; 4638 4639 return qm_reset_device(qm); 4640 } 4641 4642 static int qm_vf_reset_done(struct hisi_qm *qm) 4643 { 4644 struct hisi_qm_list *qm_list = qm->qm_list; 4645 struct pci_dev *pdev = qm->pdev; 4646 struct pci_dev *virtfn; 4647 struct hisi_qm *vf_qm; 4648 int ret = 0; 4649 4650 mutex_lock(&qm_list->lock); 4651 list_for_each_entry(vf_qm, &qm_list->list, list) { 4652 virtfn = vf_qm->pdev; 4653 if (virtfn == pdev) 4654 continue; 4655 4656 if (pci_physfn(virtfn) == pdev) { 4657 /* enable VFs PCIE BAR configuration */ 4658 pci_restore_state(virtfn); 4659 4660 ret = qm_restart(vf_qm); 4661 if (ret) 4662 goto restart_fail; 4663 } 4664 } 4665 4666 restart_fail: 4667 mutex_unlock(&qm_list->lock); 4668 return ret; 4669 } 4670 4671 static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 4672 { 4673 struct pci_dev *pdev = qm->pdev; 4674 int ret; 4675 4676 if (!qm->vfs_num) 4677 return 0; 4678 4679 ret = qm_vf_q_assign(qm, qm->vfs_num); 4680 if (ret) { 4681 pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret); 4682 return ret; 4683 } 4684 4685 /* Kunpeng930 supports to notify VFs to start after PF reset. */ 4686 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { 4687 ret = qm_ping_all_vfs(qm, cmd); 4688 if (ret) 4689 pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); 4690 } else { 4691 ret = qm_vf_reset_done(qm); 4692 if (ret) 4693 pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret); 4694 } 4695 4696 return ret; 4697 } 4698 4699 static int qm_dev_hw_init(struct hisi_qm *qm) 4700 { 4701 return qm->err_ini->hw_init(qm); 4702 } 4703 4704 static void qm_restart_prepare(struct hisi_qm *qm) 4705 { 4706 u32 value; 4707 4708 if (qm->ver >= QM_HW_V3) 4709 return; 4710 4711 if (!qm->err_status.is_qm_ecc_mbit && 4712 !qm->err_status.is_dev_ecc_mbit) 4713 return; 4714 4715 /* temporarily close the OOO port used for PEH to write out MSI */ 4716 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4717 writel(value & ~qm->err_info.msi_wr_port, 4718 qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4719 4720 /* clear dev ecc 2bit error source if having */ 4721 value = qm_get_dev_err_status(qm) & qm->err_info.dev_err.ecc_2bits_mask; 4722 if (value && qm->err_ini->clear_dev_hw_err_status) 4723 qm->err_ini->clear_dev_hw_err_status(qm, value); 4724 4725 /* clear QM ecc mbit error source */ 4726 writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); 4727 4728 /* clear AM Reorder Buffer ecc mbit source */ 4729 writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); 4730 } 4731 4732 static void qm_restart_done(struct hisi_qm *qm) 4733 { 4734 u32 value; 4735 4736 if (qm->ver >= QM_HW_V3) 4737 goto clear_flags; 4738 4739 if (!qm->err_status.is_qm_ecc_mbit && 4740 !qm->err_status.is_dev_ecc_mbit) 4741 return; 4742 4743 /* open the OOO port for PEH to write out MSI */ 4744 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4745 value |= qm->err_info.msi_wr_port; 4746 writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4747 4748 clear_flags: 4749 qm->err_status.is_qm_ecc_mbit = false; 4750 qm->err_status.is_dev_ecc_mbit = false; 4751 } 4752 4753 static void qm_disable_axi_error(struct hisi_qm *qm) 4754 { 4755 struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; 4756 u32 val; 4757 4758 val = ~(qm->error_mask & (~QM_RAS_AXI_ERROR)); 4759 writel(val, qm->io_base + QM_ABNORMAL_INT_MASK); 4760 if (qm->ver > QM_HW_V2) 4761 writel(qm_err->shutdown_mask & (~QM_RAS_AXI_ERROR), 4762 qm->io_base + QM_OOO_SHUTDOWN_SEL); 4763 4764 if (qm->err_ini->disable_axi_error) 4765 qm->err_ini->disable_axi_error(qm); 4766 } 4767 4768 static void qm_enable_axi_error(struct hisi_qm *qm) 4769 { 4770 /* clear axi error source */ 4771 writel(QM_RAS_AXI_ERROR, qm->io_base + QM_ABNORMAL_INT_SOURCE); 4772 4773 writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); 4774 if (qm->ver > QM_HW_V2) 4775 writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); 4776 4777 if (qm->err_ini->enable_axi_error) 4778 qm->err_ini->enable_axi_error(qm); 4779 } 4780 4781 static int qm_controller_reset_done(struct hisi_qm *qm) 4782 { 4783 struct pci_dev *pdev = qm->pdev; 4784 int ret; 4785 4786 ret = qm->ops->set_msi(qm, true); 4787 if (ret) { 4788 pci_err(pdev, "Fails to enable PEH MSI bit!\n"); 4789 return ret; 4790 } 4791 4792 ret = qm_set_pf_mse(qm, true); 4793 if (ret) { 4794 pci_err(pdev, "Fails to enable pf MSE bit!\n"); 4795 return ret; 4796 } 4797 4798 if (qm->vfs_num) { 4799 ret = qm_set_vf_mse(qm, true); 4800 if (ret) { 4801 pci_err(pdev, "Fails to enable vf MSE bit!\n"); 4802 return ret; 4803 } 4804 } 4805 4806 ret = qm_dev_hw_init(qm); 4807 if (ret) { 4808 pci_err(pdev, "Failed to init device\n"); 4809 return ret; 4810 } 4811 4812 qm_restart_prepare(qm); 4813 hisi_qm_dev_err_init(qm); 4814 qm_disable_axi_error(qm); 4815 if (qm->err_ini->open_axi_master_ooo) 4816 qm->err_ini->open_axi_master_ooo(qm); 4817 4818 ret = qm_dev_mem_reset(qm); 4819 if (ret) { 4820 pci_err(pdev, "failed to reset device memory\n"); 4821 return ret; 4822 } 4823 4824 ret = qm_restart(qm); 4825 if (ret) { 4826 pci_err(pdev, "Failed to start QM!\n"); 4827 return ret; 4828 } 4829 4830 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); 4831 if (ret) 4832 pci_err(pdev, "failed to start vfs by pf in soft reset.\n"); 4833 4834 ret = qm_wait_vf_prepare_finish(qm); 4835 if (ret) 4836 pci_err(pdev, "failed to start by vfs in soft reset!\n"); 4837 qm_enable_axi_error(qm); 4838 qm_cmd_init(qm); 4839 qm_restart_done(qm); 4840 4841 qm_reset_bit_clear(qm); 4842 4843 return 0; 4844 } 4845 4846 static int qm_controller_reset(struct hisi_qm *qm) 4847 { 4848 struct pci_dev *pdev = qm->pdev; 4849 int ret; 4850 4851 pci_info(pdev, "Controller resetting...\n"); 4852 4853 ret = qm_controller_reset_prepare(qm); 4854 if (ret) { 4855 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 4856 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 4857 clear_bit(QM_RST_SCHED, &qm->misc_ctl); 4858 return ret; 4859 } 4860 4861 hisi_qm_show_last_dfx_regs(qm); 4862 if (qm->err_ini->show_last_dfx_regs) 4863 qm->err_ini->show_last_dfx_regs(qm); 4864 4865 ret = qm_soft_reset(qm); 4866 if (ret) 4867 goto err_reset; 4868 4869 ret = qm_controller_reset_done(qm); 4870 if (ret) 4871 goto err_reset; 4872 4873 pci_info(pdev, "Controller reset complete\n"); 4874 4875 return 0; 4876 4877 err_reset: 4878 pci_err(pdev, "Controller reset failed (%d)\n", ret); 4879 qm_reset_bit_clear(qm); 4880 4881 /* if resetting fails, isolate the device */ 4882 if (qm->use_sva) 4883 qm->isolate_data.is_isolate = true; 4884 return ret; 4885 } 4886 4887 /** 4888 * hisi_qm_dev_slot_reset() - slot reset 4889 * @pdev: the PCIe device 4890 * 4891 * This function offers QM relate PCIe device reset interface. Drivers which 4892 * use QM can use this function as slot_reset in its struct pci_error_handlers. 4893 */ 4894 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) 4895 { 4896 struct hisi_qm *qm = pci_get_drvdata(pdev); 4897 int ret; 4898 4899 if (pdev->is_virtfn) 4900 return PCI_ERS_RESULT_RECOVERED; 4901 4902 /* reset pcie device controller */ 4903 ret = qm_controller_reset(qm); 4904 if (ret) { 4905 pci_err(pdev, "Controller reset failed (%d)\n", ret); 4906 return PCI_ERS_RESULT_DISCONNECT; 4907 } 4908 4909 return PCI_ERS_RESULT_RECOVERED; 4910 } 4911 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset); 4912 4913 void hisi_qm_reset_prepare(struct pci_dev *pdev) 4914 { 4915 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 4916 struct hisi_qm *qm = pci_get_drvdata(pdev); 4917 u32 delay = 0; 4918 int ret; 4919 4920 hisi_qm_dev_err_uninit(pf_qm); 4921 4922 /* 4923 * Check whether there is an ECC mbit error, If it occurs, need to 4924 * wait for soft reset to fix it. 4925 */ 4926 while (qm_check_dev_error(qm)) { 4927 msleep(++delay); 4928 if (delay > QM_RESET_WAIT_TIMEOUT) 4929 return; 4930 } 4931 4932 ret = qm_reset_prepare_ready(qm); 4933 if (ret) { 4934 pci_err(pdev, "FLR not ready!\n"); 4935 return; 4936 } 4937 4938 /* PF obtains the information of VF by querying the register. */ 4939 if (qm->fun_type == QM_HW_PF) 4940 qm_cmd_uninit(qm); 4941 4942 ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN); 4943 if (ret) 4944 pci_err(pdev, "failed to stop vfs by pf in FLR.\n"); 4945 4946 ret = hisi_qm_stop(qm, QM_DOWN); 4947 if (ret) { 4948 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); 4949 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 4950 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 4951 return; 4952 } 4953 4954 ret = qm_wait_vf_prepare_finish(qm); 4955 if (ret) 4956 pci_err(pdev, "failed to stop by vfs in FLR!\n"); 4957 4958 pci_info(pdev, "FLR resetting...\n"); 4959 } 4960 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare); 4961 4962 static bool qm_flr_reset_complete(struct pci_dev *pdev) 4963 { 4964 struct pci_dev *pf_pdev = pci_physfn(pdev); 4965 struct hisi_qm *qm = pci_get_drvdata(pf_pdev); 4966 u32 id; 4967 4968 pci_read_config_dword(qm->pdev, PCI_COMMAND, &id); 4969 if (id == QM_PCI_COMMAND_INVALID) { 4970 pci_err(pdev, "Device can not be used!\n"); 4971 return false; 4972 } 4973 4974 return true; 4975 } 4976 4977 void hisi_qm_reset_done(struct pci_dev *pdev) 4978 { 4979 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 4980 struct hisi_qm *qm = pci_get_drvdata(pdev); 4981 int ret; 4982 4983 if (qm->fun_type == QM_HW_PF) { 4984 ret = qm_dev_hw_init(qm); 4985 if (ret) { 4986 pci_err(pdev, "Failed to init PF, ret = %d.\n", ret); 4987 goto flr_done; 4988 } 4989 } 4990 4991 hisi_qm_dev_err_init(pf_qm); 4992 4993 ret = qm_restart(qm); 4994 if (ret) { 4995 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret); 4996 goto flr_done; 4997 } 4998 4999 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); 5000 if (ret) 5001 pci_err(pdev, "failed to start vfs by pf in FLR.\n"); 5002 5003 ret = qm_wait_vf_prepare_finish(qm); 5004 if (ret) 5005 pci_err(pdev, "failed to start by vfs in FLR!\n"); 5006 5007 flr_done: 5008 if (qm->fun_type == QM_HW_PF) 5009 qm_cmd_init(qm); 5010 5011 if (qm_flr_reset_complete(pdev)) 5012 pci_info(pdev, "FLR reset complete\n"); 5013 5014 qm_reset_bit_clear(qm); 5015 } 5016 EXPORT_SYMBOL_GPL(hisi_qm_reset_done); 5017 5018 static irqreturn_t qm_rsvd_irq(int irq, void *data) 5019 { 5020 struct hisi_qm *qm = data; 5021 5022 dev_info(&qm->pdev->dev, "Reserved interrupt, ignore!\n"); 5023 5024 return IRQ_HANDLED; 5025 } 5026 5027 static irqreturn_t qm_abnormal_irq(int irq, void *data) 5028 { 5029 struct hisi_qm *qm = data; 5030 enum acc_err_result ret; 5031 5032 atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); 5033 ret = qm_process_dev_error(qm); 5034 if (ret == ACC_ERR_NEED_RESET && 5035 !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) && 5036 !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl)) 5037 schedule_work(&qm->rst_work); 5038 5039 return IRQ_HANDLED; 5040 } 5041 5042 /** 5043 * hisi_qm_dev_shutdown() - Shutdown device. 5044 * @pdev: The device will be shutdown. 5045 * 5046 * This function will stop qm when OS shutdown or rebooting. 5047 */ 5048 void hisi_qm_dev_shutdown(struct pci_dev *pdev) 5049 { 5050 struct hisi_qm *qm = pci_get_drvdata(pdev); 5051 int ret; 5052 5053 ret = hisi_qm_stop(qm, QM_DOWN); 5054 if (ret) 5055 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); 5056 } 5057 EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); 5058 5059 static void hisi_qm_controller_reset(struct work_struct *rst_work) 5060 { 5061 struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); 5062 int ret; 5063 5064 ret = qm_pm_get_sync(qm); 5065 if (ret) { 5066 clear_bit(QM_RST_SCHED, &qm->misc_ctl); 5067 return; 5068 } 5069 5070 /* reset pcie device controller */ 5071 ret = qm_controller_reset(qm); 5072 if (ret) 5073 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret); 5074 5075 qm_pm_put_sync(qm); 5076 } 5077 5078 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, 5079 enum qm_stop_reason stop_reason) 5080 { 5081 enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; 5082 struct pci_dev *pdev = qm->pdev; 5083 int ret; 5084 5085 ret = qm_reset_prepare_ready(qm); 5086 if (ret) { 5087 dev_err(&pdev->dev, "reset prepare not ready!\n"); 5088 atomic_set(&qm->status.flags, QM_STOP); 5089 cmd = QM_VF_PREPARE_FAIL; 5090 goto err_prepare; 5091 } 5092 5093 ret = hisi_qm_stop(qm, stop_reason); 5094 if (ret) { 5095 dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret); 5096 atomic_set(&qm->status.flags, QM_STOP); 5097 cmd = QM_VF_PREPARE_FAIL; 5098 goto err_prepare; 5099 } else { 5100 goto out; 5101 } 5102 5103 err_prepare: 5104 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 5105 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 5106 out: 5107 pci_save_state(pdev); 5108 ret = qm_ping_pf(qm, cmd); 5109 if (ret) 5110 dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); 5111 } 5112 5113 static void qm_pf_reset_vf_done(struct hisi_qm *qm) 5114 { 5115 enum qm_ifc_cmd cmd = QM_VF_START_DONE; 5116 struct pci_dev *pdev = qm->pdev; 5117 int ret; 5118 5119 pci_restore_state(pdev); 5120 ret = hisi_qm_start(qm); 5121 if (ret) { 5122 dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); 5123 cmd = QM_VF_START_FAIL; 5124 } 5125 5126 qm_cmd_init(qm); 5127 ret = qm_ping_pf(qm, cmd); 5128 if (ret) 5129 dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); 5130 5131 qm_reset_bit_clear(qm); 5132 } 5133 5134 static int qm_wait_pf_reset_finish(struct hisi_qm *qm) 5135 { 5136 struct device *dev = &qm->pdev->dev; 5137 u32 val, cmd; 5138 int ret; 5139 5140 /* Wait for reset to finish */ 5141 ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val, 5142 val == BIT(0), QM_VF_RESET_WAIT_US, 5143 QM_VF_RESET_WAIT_TIMEOUT_US); 5144 /* hardware completion status should be available by this time */ 5145 if (ret) { 5146 dev_err(dev, "couldn't get reset done status from PF, timeout!\n"); 5147 return -ETIMEDOUT; 5148 } 5149 5150 /* 5151 * Whether message is got successfully, 5152 * VF needs to ack PF by clearing the interrupt. 5153 */ 5154 ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); 5155 qm_clear_cmd_interrupt(qm, 0); 5156 if (ret) { 5157 dev_err(dev, "failed to get command from PF in reset done!\n"); 5158 return ret; 5159 } 5160 5161 if (cmd != QM_PF_RESET_DONE) { 5162 dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); 5163 ret = -EINVAL; 5164 } 5165 5166 return ret; 5167 } 5168 5169 static void qm_pf_reset_vf_process(struct hisi_qm *qm, 5170 enum qm_stop_reason stop_reason) 5171 { 5172 struct device *dev = &qm->pdev->dev; 5173 int ret; 5174 5175 dev_info(dev, "device reset start...\n"); 5176 5177 /* The message is obtained by querying the register during resetting */ 5178 qm_cmd_uninit(qm); 5179 qm_pf_reset_vf_prepare(qm, stop_reason); 5180 5181 ret = qm_wait_pf_reset_finish(qm); 5182 if (ret) 5183 goto err_get_status; 5184 5185 qm_pf_reset_vf_done(qm); 5186 5187 dev_info(dev, "device reset done.\n"); 5188 5189 return; 5190 5191 err_get_status: 5192 qm_cmd_init(qm); 5193 qm_reset_bit_clear(qm); 5194 } 5195 5196 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) 5197 { 5198 struct device *dev = &qm->pdev->dev; 5199 enum qm_ifc_cmd cmd; 5200 u32 data; 5201 int ret; 5202 5203 /* 5204 * Get the msg from source by sending mailbox. Whether message is got 5205 * successfully, destination needs to ack source by clearing the interrupt. 5206 */ 5207 ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); 5208 qm_clear_cmd_interrupt(qm, BIT(fun_num)); 5209 if (ret) { 5210 dev_err(dev, "failed to get command from source!\n"); 5211 return; 5212 } 5213 5214 switch (cmd) { 5215 case QM_PF_FLR_PREPARE: 5216 qm_pf_reset_vf_process(qm, QM_DOWN); 5217 break; 5218 case QM_PF_SRST_PREPARE: 5219 qm_pf_reset_vf_process(qm, QM_SOFT_RESET); 5220 break; 5221 case QM_VF_GET_QOS: 5222 qm_vf_get_qos(qm, fun_num); 5223 break; 5224 case QM_PF_SET_QOS: 5225 qm->mb_qos = data; 5226 break; 5227 default: 5228 dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); 5229 break; 5230 } 5231 } 5232 5233 static void qm_cmd_process(struct work_struct *cmd_process) 5234 { 5235 struct hisi_qm *qm = container_of(cmd_process, 5236 struct hisi_qm, cmd_process); 5237 u32 vfs_num = qm->vfs_num; 5238 u64 val; 5239 u32 i; 5240 5241 if (qm->fun_type == QM_HW_PF) { 5242 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); 5243 if (!val) 5244 return; 5245 5246 for (i = 1; i <= vfs_num; i++) { 5247 if (val & BIT(i)) 5248 qm_handle_cmd_msg(qm, i); 5249 } 5250 5251 return; 5252 } 5253 5254 qm_handle_cmd_msg(qm, 0); 5255 } 5256 5257 /** 5258 * hisi_qm_alg_register() - Register alg to crypto. 5259 * @qm: The qm needs add. 5260 * @qm_list: The qm list. 5261 * @guard: Guard of qp_num. 5262 * 5263 * Register algorithm to crypto when the function is satisfy guard. 5264 */ 5265 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) 5266 { 5267 struct device *dev = &qm->pdev->dev; 5268 5269 if (qm->ver <= QM_HW_V2 && qm->use_sva) { 5270 dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n"); 5271 return 0; 5272 } 5273 5274 if (qm->qp_num < guard) { 5275 dev_info(dev, "qp_num is less than task need.\n"); 5276 return 0; 5277 } 5278 5279 return qm_list->register_to_crypto(qm); 5280 } 5281 EXPORT_SYMBOL_GPL(hisi_qm_alg_register); 5282 5283 /** 5284 * hisi_qm_alg_unregister() - Unregister alg from crypto. 5285 * @qm: The qm needs delete. 5286 * @qm_list: The qm list. 5287 * @guard: Guard of qp_num. 5288 * 5289 * Unregister algorithm from crypto when the last function is satisfy guard. 5290 */ 5291 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) 5292 { 5293 if (qm->ver <= QM_HW_V2 && qm->use_sva) 5294 return; 5295 5296 if (qm->qp_num < guard) 5297 return; 5298 5299 qm_list->unregister_from_crypto(qm); 5300 } 5301 EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); 5302 5303 static void qm_unregister_abnormal_irq(struct hisi_qm *qm) 5304 { 5305 struct pci_dev *pdev = qm->pdev; 5306 u32 irq_vector, val; 5307 5308 if (qm->fun_type == QM_HW_VF && qm->ver < QM_HW_V3) 5309 return; 5310 5311 val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; 5312 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) 5313 return; 5314 5315 irq_vector = val & QM_IRQ_VECTOR_MASK; 5316 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5317 } 5318 5319 static int qm_register_abnormal_irq(struct hisi_qm *qm) 5320 { 5321 struct pci_dev *pdev = qm->pdev; 5322 u32 irq_vector, val; 5323 int ret; 5324 5325 val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; 5326 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) 5327 return 0; 5328 irq_vector = val & QM_IRQ_VECTOR_MASK; 5329 5330 /* For VF, this is a reserved interrupt in V3 version. */ 5331 if (qm->fun_type == QM_HW_VF) { 5332 if (qm->ver < QM_HW_V3) 5333 return 0; 5334 5335 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_rsvd_irq, 5336 IRQF_NO_AUTOEN, qm->dev_name, qm); 5337 if (ret) { 5338 dev_err(&pdev->dev, "failed to request reserved irq, ret = %d!\n", ret); 5339 return ret; 5340 } 5341 return 0; 5342 } 5343 5344 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); 5345 if (ret) 5346 dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d!\n", ret); 5347 5348 return ret; 5349 } 5350 5351 static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) 5352 { 5353 struct pci_dev *pdev = qm->pdev; 5354 u32 irq_vector, val; 5355 5356 val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; 5357 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5358 return; 5359 5360 irq_vector = val & QM_IRQ_VECTOR_MASK; 5361 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5362 } 5363 5364 static int qm_register_mb_cmd_irq(struct hisi_qm *qm) 5365 { 5366 struct pci_dev *pdev = qm->pdev; 5367 u32 irq_vector, val; 5368 int ret; 5369 5370 val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; 5371 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5372 return 0; 5373 5374 irq_vector = val & QM_IRQ_VECTOR_MASK; 5375 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); 5376 if (ret) 5377 dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret); 5378 5379 return ret; 5380 } 5381 5382 static void qm_unregister_aeq_irq(struct hisi_qm *qm) 5383 { 5384 struct pci_dev *pdev = qm->pdev; 5385 u32 irq_vector, val; 5386 5387 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; 5388 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5389 return; 5390 5391 irq_vector = val & QM_IRQ_VECTOR_MASK; 5392 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5393 } 5394 5395 static int qm_register_aeq_irq(struct hisi_qm *qm) 5396 { 5397 struct pci_dev *pdev = qm->pdev; 5398 u32 irq_vector, val; 5399 int ret; 5400 5401 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; 5402 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5403 return 0; 5404 5405 irq_vector = val & QM_IRQ_VECTOR_MASK; 5406 ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL, 5407 qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm); 5408 if (ret) 5409 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); 5410 5411 return ret; 5412 } 5413 5414 static void qm_unregister_eq_irq(struct hisi_qm *qm) 5415 { 5416 struct pci_dev *pdev = qm->pdev; 5417 u32 irq_vector, val; 5418 5419 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; 5420 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5421 return; 5422 5423 irq_vector = val & QM_IRQ_VECTOR_MASK; 5424 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5425 } 5426 5427 static int qm_register_eq_irq(struct hisi_qm *qm) 5428 { 5429 struct pci_dev *pdev = qm->pdev; 5430 u32 irq_vector, val; 5431 int ret; 5432 5433 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; 5434 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5435 return 0; 5436 5437 irq_vector = val & QM_IRQ_VECTOR_MASK; 5438 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm); 5439 if (ret) 5440 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); 5441 5442 return ret; 5443 } 5444 5445 static void qm_irqs_unregister(struct hisi_qm *qm) 5446 { 5447 qm_unregister_mb_cmd_irq(qm); 5448 qm_unregister_abnormal_irq(qm); 5449 qm_unregister_aeq_irq(qm); 5450 qm_unregister_eq_irq(qm); 5451 } 5452 5453 static int qm_irqs_register(struct hisi_qm *qm) 5454 { 5455 int ret; 5456 5457 ret = qm_register_eq_irq(qm); 5458 if (ret) 5459 return ret; 5460 5461 ret = qm_register_aeq_irq(qm); 5462 if (ret) 5463 goto free_eq_irq; 5464 5465 ret = qm_register_abnormal_irq(qm); 5466 if (ret) 5467 goto free_aeq_irq; 5468 5469 ret = qm_register_mb_cmd_irq(qm); 5470 if (ret) 5471 goto free_abnormal_irq; 5472 5473 return 0; 5474 5475 free_abnormal_irq: 5476 qm_unregister_abnormal_irq(qm); 5477 free_aeq_irq: 5478 qm_unregister_aeq_irq(qm); 5479 free_eq_irq: 5480 qm_unregister_eq_irq(qm); 5481 return ret; 5482 } 5483 5484 static int qm_get_qp_num(struct hisi_qm *qm) 5485 { 5486 struct device *dev = &qm->pdev->dev; 5487 bool is_db_isolation; 5488 5489 /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */ 5490 if (qm->fun_type == QM_HW_VF) { 5491 if (qm->ver != QM_HW_V1) 5492 /* v2 starts to support get vft by mailbox */ 5493 return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); 5494 5495 return 0; 5496 } 5497 5498 is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); 5499 qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true); 5500 qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, 5501 QM_FUNC_MAX_QP_CAP, is_db_isolation); 5502 5503 if (qm->qp_num <= qm->max_qp_num) 5504 return 0; 5505 5506 if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) { 5507 /* Check whether the set qp number is valid */ 5508 dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n", 5509 qm->qp_num, qm->max_qp_num); 5510 return -EINVAL; 5511 } 5512 5513 dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n", 5514 qm->qp_num, qm->max_qp_num); 5515 qm->qp_num = qm->max_qp_num; 5516 qm->debug.curr_qm_qp_num = qm->qp_num; 5517 5518 return 0; 5519 } 5520 5521 static int qm_pre_store_caps(struct hisi_qm *qm) 5522 { 5523 struct hisi_qm_cap_record *qm_cap; 5524 struct pci_dev *pdev = qm->pdev; 5525 size_t i, size; 5526 5527 size = ARRAY_SIZE(qm_cap_query_info); 5528 qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL); 5529 if (!qm_cap) 5530 return -ENOMEM; 5531 5532 for (i = 0; i < size; i++) { 5533 qm_cap[i].type = qm_cap_query_info[i].type; 5534 qm_cap[i].name = qm_cap_query_info[i].name; 5535 qm_cap[i].cap_val = hisi_qm_get_cap_value(qm, qm_cap_query_info, 5536 i, qm->cap_ver); 5537 } 5538 5539 qm->cap_tables.qm_cap_table = qm_cap; 5540 qm->cap_tables.qm_cap_size = size; 5541 5542 return 0; 5543 } 5544 5545 static int qm_get_hw_caps(struct hisi_qm *qm) 5546 { 5547 const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? 5548 qm_cap_info_pf : qm_cap_info_vf; 5549 u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) : 5550 ARRAY_SIZE(qm_cap_info_vf); 5551 u32 val, i; 5552 5553 /* Doorbell isolate register is a independent register. */ 5554 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true); 5555 if (val) 5556 set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); 5557 5558 if (qm->ver >= QM_HW_V3) { 5559 val = readl(qm->io_base + QM_FUNC_CAPS_REG); 5560 qm->cap_ver = val & QM_CAPBILITY_VERSION; 5561 } 5562 5563 /* Get PF/VF common capbility */ 5564 for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) { 5565 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver); 5566 if (val) 5567 set_bit(qm_cap_info_comm[i].type, &qm->caps); 5568 } 5569 5570 /* Get PF/VF different capbility */ 5571 for (i = 0; i < size; i++) { 5572 val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver); 5573 if (val) 5574 set_bit(cap_info[i].type, &qm->caps); 5575 } 5576 5577 /* Fetch and save the value of qm capability registers */ 5578 return qm_pre_store_caps(qm); 5579 } 5580 5581 static void qm_get_version(struct hisi_qm *qm) 5582 { 5583 struct pci_dev *pdev = qm->pdev; 5584 u32 sub_version_id; 5585 5586 qm->ver = pdev->revision; 5587 5588 if (pdev->revision == QM_HW_V3) { 5589 sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); 5590 if (sub_version_id) 5591 qm->ver = sub_version_id; 5592 } 5593 } 5594 5595 static int qm_get_pci_res(struct hisi_qm *qm) 5596 { 5597 struct pci_dev *pdev = qm->pdev; 5598 struct device *dev = &pdev->dev; 5599 int ret; 5600 5601 ret = pci_request_mem_regions(pdev, qm->dev_name); 5602 if (ret < 0) { 5603 dev_err(dev, "Failed to request mem regions!\n"); 5604 return ret; 5605 } 5606 5607 qm->phys_base = pci_resource_start(pdev, PCI_BAR_2); 5608 qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2)); 5609 if (!qm->io_base) { 5610 ret = -EIO; 5611 goto err_request_mem_regions; 5612 } 5613 5614 qm_get_version(qm); 5615 5616 ret = qm_get_hw_caps(qm); 5617 if (ret) 5618 goto err_ioremap; 5619 5620 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { 5621 qm->db_interval = QM_QP_DB_INTERVAL; 5622 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); 5623 qm->db_io_base = ioremap(qm->db_phys_base, 5624 pci_resource_len(pdev, PCI_BAR_4)); 5625 if (!qm->db_io_base) { 5626 ret = -EIO; 5627 goto err_ioremap; 5628 } 5629 } else { 5630 qm->db_phys_base = qm->phys_base; 5631 qm->db_io_base = qm->io_base; 5632 qm->db_interval = 0; 5633 } 5634 5635 hisi_qm_pre_init(qm); 5636 ret = qm_get_qp_num(qm); 5637 if (ret) 5638 goto err_db_ioremap; 5639 5640 return 0; 5641 5642 err_db_ioremap: 5643 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) 5644 iounmap(qm->db_io_base); 5645 err_ioremap: 5646 iounmap(qm->io_base); 5647 err_request_mem_regions: 5648 pci_release_mem_regions(pdev); 5649 return ret; 5650 } 5651 5652 static int qm_clear_device(struct hisi_qm *qm) 5653 { 5654 acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev); 5655 int ret; 5656 5657 if (qm->fun_type == QM_HW_VF) 5658 return 0; 5659 5660 /* Device does not support reset, return */ 5661 if (!qm->err_ini->err_info_init) 5662 return 0; 5663 qm->err_ini->err_info_init(qm); 5664 5665 if (!handle) 5666 return 0; 5667 5668 /* No reset method, return */ 5669 if (!acpi_has_method(handle, qm->err_info.acpi_rst)) 5670 return 0; 5671 5672 ret = qm_master_ooo_check(qm); 5673 if (ret) { 5674 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 5675 return ret; 5676 } 5677 5678 if (qm->err_ini->set_priv_status) { 5679 ret = qm->err_ini->set_priv_status(qm); 5680 if (ret) { 5681 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 5682 return ret; 5683 } 5684 } 5685 5686 return qm_reset_device(qm); 5687 } 5688 5689 static int hisi_qm_pci_init(struct hisi_qm *qm) 5690 { 5691 struct pci_dev *pdev = qm->pdev; 5692 struct device *dev = &pdev->dev; 5693 unsigned int num_vec; 5694 int ret; 5695 5696 ret = pci_enable_device_mem(pdev); 5697 if (ret < 0) { 5698 dev_err(dev, "Failed to enable device mem!\n"); 5699 return ret; 5700 } 5701 5702 ret = qm_get_pci_res(qm); 5703 if (ret) 5704 goto err_disable_pcidev; 5705 5706 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 5707 if (ret < 0) 5708 goto err_get_pci_res; 5709 pci_set_master(pdev); 5710 5711 num_vec = qm_get_irq_num(qm); 5712 if (!num_vec) { 5713 dev_err(dev, "Device irq num is zero!\n"); 5714 ret = -EINVAL; 5715 goto err_get_pci_res; 5716 } 5717 num_vec = roundup_pow_of_two(num_vec); 5718 ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); 5719 if (ret < 0) { 5720 dev_err(dev, "Failed to enable MSI vectors!\n"); 5721 goto err_get_pci_res; 5722 } 5723 5724 ret = qm_clear_device(qm); 5725 if (ret) 5726 goto err_free_vectors; 5727 5728 return 0; 5729 5730 err_free_vectors: 5731 pci_free_irq_vectors(pdev); 5732 err_get_pci_res: 5733 qm_put_pci_res(qm); 5734 err_disable_pcidev: 5735 pci_disable_device(pdev); 5736 return ret; 5737 } 5738 5739 static int hisi_qm_init_work(struct hisi_qm *qm) 5740 { 5741 int i; 5742 5743 for (i = 0; i < qm->qp_num; i++) 5744 INIT_WORK(&qm->poll_data[i].work, qm_work_process); 5745 5746 if (qm->fun_type == QM_HW_PF) 5747 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); 5748 5749 if (qm->ver > QM_HW_V2) 5750 INIT_WORK(&qm->cmd_process, qm_cmd_process); 5751 5752 qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | 5753 WQ_UNBOUND, num_online_cpus(), 5754 pci_name(qm->pdev)); 5755 if (!qm->wq) { 5756 pci_err(qm->pdev, "failed to alloc workqueue!\n"); 5757 return -ENOMEM; 5758 } 5759 5760 return 0; 5761 } 5762 5763 static int hisi_qp_alloc_memory(struct hisi_qm *qm) 5764 { 5765 struct device *dev = &qm->pdev->dev; 5766 u16 sq_depth, cq_depth; 5767 size_t qp_dma_size; 5768 int i, ret; 5769 5770 qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); 5771 if (!qm->qp_array) 5772 return -ENOMEM; 5773 5774 qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); 5775 if (!qm->poll_data) { 5776 kfree(qm->qp_array); 5777 return -ENOMEM; 5778 } 5779 5780 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); 5781 5782 /* one more page for device or qp statuses */ 5783 qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; 5784 qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; 5785 for (i = 0; i < qm->qp_num; i++) { 5786 qm->poll_data[i].qm = qm; 5787 ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); 5788 if (ret) 5789 goto err_init_qp_mem; 5790 5791 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); 5792 } 5793 5794 return 0; 5795 err_init_qp_mem: 5796 hisi_qp_memory_uninit(qm, i); 5797 5798 return ret; 5799 } 5800 5801 static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm) 5802 { 5803 struct qm_rsv_buf *xqc_buf = &qm->xqc_buf; 5804 struct qm_dma *xqc_dma = &xqc_buf->qcdma; 5805 struct device *dev = &qm->pdev->dev; 5806 size_t off = 0; 5807 5808 #define QM_XQC_BUF_INIT(xqc_buf, type) do { \ 5809 (xqc_buf)->type = ((xqc_buf)->qcdma.va + (off)); \ 5810 (xqc_buf)->type##_dma = (xqc_buf)->qcdma.dma + (off); \ 5811 off += QMC_ALIGN(sizeof(struct qm_##type)); \ 5812 } while (0) 5813 5814 xqc_dma->size = QMC_ALIGN(sizeof(struct qm_eqc)) + 5815 QMC_ALIGN(sizeof(struct qm_aeqc)) + 5816 QMC_ALIGN(sizeof(struct qm_sqc)) + 5817 QMC_ALIGN(sizeof(struct qm_cqc)); 5818 xqc_dma->va = dma_alloc_coherent(dev, xqc_dma->size, 5819 &xqc_dma->dma, GFP_KERNEL); 5820 if (!xqc_dma->va) 5821 return -ENOMEM; 5822 5823 QM_XQC_BUF_INIT(xqc_buf, eqc); 5824 QM_XQC_BUF_INIT(xqc_buf, aeqc); 5825 QM_XQC_BUF_INIT(xqc_buf, sqc); 5826 QM_XQC_BUF_INIT(xqc_buf, cqc); 5827 5828 return 0; 5829 } 5830 5831 static int hisi_qm_memory_init(struct hisi_qm *qm) 5832 { 5833 struct device *dev = &qm->pdev->dev; 5834 int ret, total_func; 5835 size_t off = 0; 5836 5837 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { 5838 total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; 5839 qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); 5840 if (!qm->factor) 5841 return -ENOMEM; 5842 5843 /* Only the PF value needs to be initialized */ 5844 qm->factor[0].func_qos = QM_QOS_MAX_VAL; 5845 } 5846 5847 #define QM_INIT_BUF(qm, type, num) do { \ 5848 (qm)->type = ((qm)->qdma.va + (off)); \ 5849 (qm)->type##_dma = (qm)->qdma.dma + (off); \ 5850 off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ 5851 } while (0) 5852 5853 idr_init(&qm->qp_idr); 5854 qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); 5855 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + 5856 QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + 5857 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + 5858 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); 5859 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, 5860 GFP_ATOMIC); 5861 dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); 5862 if (!qm->qdma.va) { 5863 ret = -ENOMEM; 5864 goto err_destroy_idr; 5865 } 5866 5867 QM_INIT_BUF(qm, eqe, qm->eq_depth); 5868 QM_INIT_BUF(qm, aeqe, qm->aeq_depth); 5869 QM_INIT_BUF(qm, sqc, qm->qp_num); 5870 QM_INIT_BUF(qm, cqc, qm->qp_num); 5871 5872 ret = hisi_qm_alloc_rsv_buf(qm); 5873 if (ret) 5874 goto err_free_qdma; 5875 5876 ret = hisi_qp_alloc_memory(qm); 5877 if (ret) 5878 goto err_free_reserve_buf; 5879 5880 return 0; 5881 5882 err_free_reserve_buf: 5883 hisi_qm_free_rsv_buf(qm); 5884 err_free_qdma: 5885 dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); 5886 err_destroy_idr: 5887 idr_destroy(&qm->qp_idr); 5888 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 5889 kfree(qm->factor); 5890 5891 return ret; 5892 } 5893 5894 /** 5895 * hisi_qm_init() - Initialize configures about qm. 5896 * @qm: The qm needing init. 5897 * 5898 * This function init qm, then we can call hisi_qm_start to put qm into work. 5899 */ 5900 int hisi_qm_init(struct hisi_qm *qm) 5901 { 5902 struct pci_dev *pdev = qm->pdev; 5903 struct device *dev = &pdev->dev; 5904 int ret; 5905 5906 ret = hisi_qm_pci_init(qm); 5907 if (ret) 5908 return ret; 5909 5910 ret = qm_irqs_register(qm); 5911 if (ret) 5912 goto err_pci_init; 5913 5914 if (qm->fun_type == QM_HW_PF) { 5915 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ 5916 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); 5917 qm_disable_clock_gate(qm); 5918 ret = qm_dev_mem_reset(qm); 5919 if (ret) { 5920 dev_err(dev, "failed to reset device memory\n"); 5921 goto err_irq_register; 5922 } 5923 } 5924 5925 if (qm->mode == UACCE_MODE_SVA) { 5926 ret = qm_alloc_uacce(qm); 5927 if (ret < 0) 5928 dev_warn(dev, "fail to alloc uacce (%d)\n", ret); 5929 } 5930 5931 ret = hisi_qm_memory_init(qm); 5932 if (ret) 5933 goto err_alloc_uacce; 5934 5935 ret = hisi_qm_init_work(qm); 5936 if (ret) 5937 goto err_free_qm_memory; 5938 5939 qm_cmd_init(qm); 5940 hisi_mig_region_enable(qm); 5941 5942 return 0; 5943 5944 err_free_qm_memory: 5945 hisi_qm_memory_uninit(qm); 5946 err_alloc_uacce: 5947 qm_remove_uacce(qm); 5948 err_irq_register: 5949 qm_irqs_unregister(qm); 5950 err_pci_init: 5951 hisi_qm_pci_uninit(qm); 5952 return ret; 5953 } 5954 EXPORT_SYMBOL_GPL(hisi_qm_init); 5955 5956 /** 5957 * hisi_qm_get_dfx_access() - Try to get dfx access. 5958 * @qm: pointer to accelerator device. 5959 * 5960 * Try to get dfx access, then user can get message. 5961 * 5962 * If device is in suspended, return failure, otherwise 5963 * bump up the runtime PM usage counter. 5964 */ 5965 int hisi_qm_get_dfx_access(struct hisi_qm *qm) 5966 { 5967 struct device *dev = &qm->pdev->dev; 5968 5969 if (pm_runtime_suspended(dev)) { 5970 dev_info(dev, "can not read/write - device in suspended.\n"); 5971 return -EAGAIN; 5972 } 5973 5974 return qm_pm_get_sync(qm); 5975 } 5976 EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access); 5977 5978 /** 5979 * hisi_qm_put_dfx_access() - Put dfx access. 5980 * @qm: pointer to accelerator device. 5981 * 5982 * Put dfx access, drop runtime PM usage counter. 5983 */ 5984 void hisi_qm_put_dfx_access(struct hisi_qm *qm) 5985 { 5986 qm_pm_put_sync(qm); 5987 } 5988 EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access); 5989 5990 /** 5991 * hisi_qm_pm_init() - Initialize qm runtime PM. 5992 * @qm: pointer to accelerator device. 5993 * 5994 * Function that initialize qm runtime PM. 5995 */ 5996 void hisi_qm_pm_init(struct hisi_qm *qm) 5997 { 5998 struct device *dev = &qm->pdev->dev; 5999 6000 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 6001 return; 6002 6003 pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY); 6004 pm_runtime_use_autosuspend(dev); 6005 pm_runtime_put_noidle(dev); 6006 } 6007 EXPORT_SYMBOL_GPL(hisi_qm_pm_init); 6008 6009 /** 6010 * hisi_qm_pm_uninit() - Uninitialize qm runtime PM. 6011 * @qm: pointer to accelerator device. 6012 * 6013 * Function that uninitialize qm runtime PM. 6014 */ 6015 void hisi_qm_pm_uninit(struct hisi_qm *qm) 6016 { 6017 struct device *dev = &qm->pdev->dev; 6018 6019 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 6020 return; 6021 6022 pm_runtime_get_noresume(dev); 6023 pm_runtime_dont_use_autosuspend(dev); 6024 } 6025 EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit); 6026 6027 static int qm_prepare_for_suspend(struct hisi_qm *qm) 6028 { 6029 struct pci_dev *pdev = qm->pdev; 6030 int ret; 6031 6032 ret = qm->ops->set_msi(qm, false); 6033 if (ret) { 6034 pci_err(pdev, "failed to disable MSI before suspending!\n"); 6035 return ret; 6036 } 6037 6038 ret = qm_master_ooo_check(qm); 6039 if (ret) 6040 return ret; 6041 6042 if (qm->err_ini->set_priv_status) { 6043 ret = qm->err_ini->set_priv_status(qm); 6044 if (ret) 6045 return ret; 6046 } 6047 6048 ret = qm_set_pf_mse(qm, false); 6049 if (ret) 6050 pci_err(pdev, "failed to disable MSE before suspending!\n"); 6051 6052 return ret; 6053 } 6054 6055 static int qm_rebuild_for_resume(struct hisi_qm *qm) 6056 { 6057 struct pci_dev *pdev = qm->pdev; 6058 int ret; 6059 6060 ret = qm_set_pf_mse(qm, true); 6061 if (ret) { 6062 pci_err(pdev, "failed to enable MSE after resuming!\n"); 6063 return ret; 6064 } 6065 6066 ret = qm->ops->set_msi(qm, true); 6067 if (ret) { 6068 pci_err(pdev, "failed to enable MSI after resuming!\n"); 6069 return ret; 6070 } 6071 6072 ret = qm_dev_hw_init(qm); 6073 if (ret) { 6074 pci_err(pdev, "failed to init device after resuming\n"); 6075 return ret; 6076 } 6077 6078 qm_cmd_init(qm); 6079 hisi_mig_region_enable(qm); 6080 hisi_qm_dev_err_init(qm); 6081 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ 6082 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); 6083 qm_disable_clock_gate(qm); 6084 ret = qm_dev_mem_reset(qm); 6085 if (ret) 6086 pci_err(pdev, "failed to reset device memory\n"); 6087 6088 return ret; 6089 } 6090 6091 /** 6092 * hisi_qm_suspend() - Runtime suspend of given device. 6093 * @dev: device to suspend. 6094 * 6095 * Function that suspend the device. 6096 */ 6097 int hisi_qm_suspend(struct device *dev) 6098 { 6099 struct pci_dev *pdev = to_pci_dev(dev); 6100 struct hisi_qm *qm = pci_get_drvdata(pdev); 6101 int ret; 6102 6103 pci_info(pdev, "entering suspended state\n"); 6104 6105 ret = hisi_qm_stop(qm, QM_NORMAL); 6106 if (ret) { 6107 pci_err(pdev, "failed to stop qm(%d)\n", ret); 6108 return ret; 6109 } 6110 6111 ret = qm_prepare_for_suspend(qm); 6112 if (ret) 6113 pci_err(pdev, "failed to prepare suspended(%d)\n", ret); 6114 6115 return ret; 6116 } 6117 EXPORT_SYMBOL_GPL(hisi_qm_suspend); 6118 6119 /** 6120 * hisi_qm_resume() - Runtime resume of given device. 6121 * @dev: device to resume. 6122 * 6123 * Function that resume the device. 6124 */ 6125 int hisi_qm_resume(struct device *dev) 6126 { 6127 struct pci_dev *pdev = to_pci_dev(dev); 6128 struct hisi_qm *qm = pci_get_drvdata(pdev); 6129 int ret; 6130 6131 pci_info(pdev, "resuming from suspend state\n"); 6132 6133 ret = qm_rebuild_for_resume(qm); 6134 if (ret) { 6135 pci_err(pdev, "failed to rebuild resume(%d)\n", ret); 6136 return ret; 6137 } 6138 6139 ret = hisi_qm_start(qm); 6140 if (ret) { 6141 if (qm_check_dev_error(qm)) { 6142 pci_info(pdev, "failed to start qm due to device error, device will be reset!\n"); 6143 return 0; 6144 } 6145 6146 pci_err(pdev, "failed to start qm(%d)!\n", ret); 6147 } 6148 6149 return ret; 6150 } 6151 EXPORT_SYMBOL_GPL(hisi_qm_resume); 6152 6153 MODULE_LICENSE("GPL v2"); 6154 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 6155 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); 6156