1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 4 #include <linux/acpi.h> 5 #include <linux/bitops.h> 6 #include <linux/debugfs.h> 7 #include <linux/init.h> 8 #include <linux/io.h> 9 #include <linux/iommu.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/pm_runtime.h> 14 #include <linux/seq_file.h> 15 #include <linux/topology.h> 16 #include <linux/uacce.h> 17 #include "sec.h" 18 19 #define CAP_FILE_PERMISSION 0444 20 #define SEC_VF_NUM 63 21 #define SEC_QUEUE_NUM_V1 4096 22 #define PCI_DEVICE_ID_HUAWEI_SEC_PF 0xa255 23 24 #define SEC_BD_ERR_CHK_EN0 0xEFFFFFFF 25 #define SEC_BD_ERR_CHK_EN1 0x7ffff7fd 26 #define SEC_BD_ERR_CHK_EN3 0xffffbfff 27 28 #define SEC_SQE_SIZE 128 29 #define SEC_PF_DEF_Q_NUM 256 30 #define SEC_PF_DEF_Q_BASE 0 31 #define SEC_CTX_Q_NUM_DEF 2 32 #define SEC_CTX_Q_NUM_MAX 32 33 34 #define SEC_CTRL_CNT_CLR_CE 0x301120 35 #define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) 36 #define SEC_CORE_INT_SOURCE 0x301010 37 #define SEC_CORE_INT_MASK 0x301000 38 #define SEC_CORE_INT_STATUS 0x301008 39 #define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14 40 #define SEC_ECC_NUM 16 41 #define SEC_ECC_MASH 0xFF 42 #define SEC_CORE_INT_DISABLE 0x0 43 44 #define SEC_RAS_CE_REG 0x301050 45 #define SEC_RAS_FE_REG 0x301054 46 #define SEC_RAS_NFE_REG 0x301058 47 #define SEC_RAS_FE_ENB_MSK 0x0 48 #define SEC_OOO_SHUTDOWN_SEL 0x301014 49 #define SEC_RAS_DISABLE 0x0 50 #define SEC_AXI_ERROR_MASK (BIT(0) | BIT(1)) 51 #define SEC_RAS_CLEAR_ALL GENMASK(31, 0) 52 53 #define SEC_MEM_START_INIT_REG 0x301100 54 #define SEC_MEM_INIT_DONE_REG 0x301104 55 56 /* clock gating */ 57 #define SEC_CONTROL_REG 0x301200 58 #define SEC_DYNAMIC_GATE_REG 0x30121c 59 #define SEC_CORE_AUTO_GATE 0x30212c 60 #define SEC_DYNAMIC_GATE_EN 0x7fff 61 #define SEC_CORE_AUTO_GATE_EN GENMASK(3, 0) 62 #define SEC_CLK_GATE_ENABLE BIT(3) 63 #define SEC_CLK_GATE_DISABLE (~BIT(3)) 64 65 #define SEC_TRNG_EN_SHIFT 8 66 #define SEC_AXI_SHUTDOWN_ENABLE BIT(12) 67 #define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF 68 69 #define SEC_INTERFACE_USER_CTRL0_REG 0x301220 70 #define SEC_INTERFACE_USER_CTRL1_REG 0x301224 71 #define SEC_SAA_EN_REG 0x301270 72 #define SEC_BD_ERR_CHK_EN_REG0 0x301380 73 #define SEC_BD_ERR_CHK_EN_REG1 0x301384 74 #define SEC_BD_ERR_CHK_EN_REG3 0x30138c 75 76 #define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) 77 #define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) 78 #define SEC_USER1_ENABLE_CONTEXT_SSV BIT(24) 79 #define SEC_USER1_ENABLE_DATA_SSV BIT(16) 80 #define SEC_USER1_WB_CONTEXT_SSV BIT(8) 81 #define SEC_USER1_WB_DATA_SSV BIT(0) 82 #define SEC_USER1_SVA_SET (SEC_USER1_ENABLE_CONTEXT_SSV | \ 83 SEC_USER1_ENABLE_DATA_SSV | \ 84 SEC_USER1_WB_CONTEXT_SSV | \ 85 SEC_USER1_WB_DATA_SSV) 86 #define SEC_USER1_SMMU_SVA (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET) 87 #define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) 88 #define SEC_INTERFACE_USER_CTRL0_REG_V3 0x302220 89 #define SEC_INTERFACE_USER_CTRL1_REG_V3 0x302224 90 #define SEC_USER1_SMMU_NORMAL_V3 (BIT(23) | BIT(17) | BIT(11) | BIT(5)) 91 #define SEC_USER1_SMMU_MASK_V3 0xFF79E79E 92 #define SEC_CORE_INT_STATUS_M_ECC BIT(2) 93 94 #define SEC_PREFETCH_CFG 0x301130 95 #define SEC_SVA_TRANS 0x301EC4 96 #define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) 97 #define SEC_PREFETCH_DISABLE BIT(1) 98 #define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) 99 #define SEC_SVA_PREFETCH_INFO 0x301ED4 100 #define SEC_SVA_STALL_NUM GENMASK(23, 8) 101 #define SEC_SVA_PREFETCH_NUM GENMASK(2, 0) 102 #define SEC_WAIT_SVA_READY 500000 103 #define SEC_READ_SVA_STATUS_TIMES 3 104 #define SEC_WAIT_US_MIN 10 105 #define SEC_WAIT_US_MAX 20 106 #define SEC_WAIT_QP_US_MIN 1000 107 #define SEC_WAIT_QP_US_MAX 2000 108 #define SEC_MAX_WAIT_TIMES 2000 109 110 #define SEC_DELAY_10_US 10 111 #define SEC_POLL_TIMEOUT_US 1000 112 #define SEC_DBGFS_VAL_MAX_LEN 20 113 #define SEC_SINGLE_PORT_MAX_TRANS 0x2060 114 115 #define SEC_SQE_MASK_OFFSET 16 116 #define SEC_SQE_MASK_LEN 108 117 #define SEC_SHAPER_TYPE_RATE 400 118 119 #define SEC_DFX_BASE 0x301000 120 #define SEC_DFX_CORE 0x302100 121 #define SEC_DFX_COMMON1 0x301600 122 #define SEC_DFX_COMMON2 0x301C00 123 #define SEC_DFX_BASE_LEN 0x9D 124 #define SEC_DFX_CORE_LEN 0x32B 125 #define SEC_DFX_COMMON1_LEN 0x45 126 #define SEC_DFX_COMMON2_LEN 0xBA 127 128 #define SEC_ALG_BITMAP_SHIFT 32 129 130 #define SEC_CIPHER_BITMAP (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \ 131 GENMASK(24, 21)) 132 #define SEC_DIGEST_BITMAP (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \ 133 GENMASK_ULL(42, 25)) 134 #define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ 135 GENMASK_ULL(45, 43)) 136 137 #define SEC_MAX_CHANNEL_NUM 1 138 139 struct sec_hw_error { 140 u32 int_msk; 141 const char *msg; 142 }; 143 144 struct sec_dfx_item { 145 const char *name; 146 u32 offset; 147 }; 148 149 static const char sec_name[] = "hisi_sec2"; 150 static struct dentry *sec_debugfs_root; 151 152 static struct hisi_qm_list sec_devices = { 153 .register_to_crypto = sec_register_to_crypto, 154 .unregister_from_crypto = sec_unregister_from_crypto, 155 }; 156 157 static const struct hisi_qm_cap_info sec_basic_info[] = { 158 {SEC_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77}, 159 {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77}, 160 {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, 161 {SEC_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, 162 {SEC_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177}, 163 {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177}, 164 {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177}, 165 {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088}, 166 {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1}, 167 {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1}, 168 {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, 169 {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, 170 {SEC_CORE_ENABLE_BITMAP, 0x3140, 0, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, 171 {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF}, 172 {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, 173 {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 174 {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 175 {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 176 {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 177 {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 178 {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 179 {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 180 {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 181 {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 182 {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 183 }; 184 185 static const struct hisi_qm_cap_query_info sec_cap_query_info[] = { 186 {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C77, 0x7C77}, 187 {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77}, 188 {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, 189 {SEC_RAS_NFE_TYPE, "SEC_RAS_NFE_TYPE ", 0x3130, 0x0, 0x177, 0x60177}, 190 {SEC_RAS_NFE_RESET, "SEC_RAS_NFE_RESET ", 0x3134, 0x0, 0x177, 0x177}, 191 {SEC_RAS_CE_TYPE, "SEC_RAS_CE_TYPE ", 0x3138, 0x0, 0x88, 0xC088}, 192 {SEC_CORE_INFO, "SEC_CORE_INFO ", 0x313c, 0x110404, 0x110404, 0x110404}, 193 {SEC_CORE_EN, "SEC_CORE_EN ", 0x3140, 0x17F, 0x17F, 0xF}, 194 {SEC_DRV_ALG_BITMAP_LOW_TB, "SEC_DRV_ALG_BITMAP_LOW ", 195 0x3144, 0x18050CB, 0x18050CB, 0x18670CF}, 196 {SEC_DRV_ALG_BITMAP_HIGH_TB, "SEC_DRV_ALG_BITMAP_HIGH ", 197 0x3148, 0x395C, 0x395C, 0x395C}, 198 {SEC_ALG_BITMAP_LOW, "SEC_ALG_BITMAP_LOW ", 199 0x314c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 200 {SEC_ALG_BITMAP_HIGH, "SEC_ALG_BITMAP_HIGH ", 0x3150, 0x3FFF, 0x3FFF, 0x3FFF}, 201 {SEC_CORE1_BITMAP_LOW, "SEC_CORE1_BITMAP_LOW ", 202 0x3154, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 203 {SEC_CORE1_BITMAP_HIGH, "SEC_CORE1_BITMAP_HIGH ", 0x3158, 0x3FFF, 0x3FFF, 0x3FFF}, 204 {SEC_CORE2_BITMAP_LOW, "SEC_CORE2_BITMAP_LOW ", 205 0x315c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 206 {SEC_CORE2_BITMAP_HIGH, "SEC_CORE2_BITMAP_HIGH ", 0x3160, 0x3FFF, 0x3FFF, 0x3FFF}, 207 {SEC_CORE3_BITMAP_LOW, "SEC_CORE3_BITMAP_LOW ", 208 0x3164, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 209 {SEC_CORE3_BITMAP_HIGH, "SEC_CORE3_BITMAP_HIGH ", 0x3168, 0x3FFF, 0x3FFF, 0x3FFF}, 210 {SEC_CORE4_BITMAP_LOW, "SEC_CORE4_BITMAP_LOW ", 211 0x316c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 212 {SEC_CORE4_BITMAP_HIGH, "SEC_CORE4_BITMAP_HIGH ", 0x3170, 0x3FFF, 0x3FFF, 0x3FFF}, 213 }; 214 215 static const struct qm_dev_alg sec_dev_algs[] = { { 216 .alg_msk = SEC_CIPHER_BITMAP, 217 .alg = "cipher\n", 218 }, { 219 .alg_msk = SEC_DIGEST_BITMAP, 220 .alg = "digest\n", 221 }, { 222 .alg_msk = SEC_AEAD_BITMAP, 223 .alg = "aead\n", 224 }, 225 }; 226 227 static const struct sec_hw_error sec_hw_errors[] = { 228 { 229 .int_msk = BIT(0), 230 .msg = "sec_axi_rresp_err_rint" 231 }, 232 { 233 .int_msk = BIT(1), 234 .msg = "sec_axi_bresp_err_rint" 235 }, 236 { 237 .int_msk = BIT(2), 238 .msg = "sec_ecc_2bit_err_rint" 239 }, 240 { 241 .int_msk = BIT(3), 242 .msg = "sec_ecc_1bit_err_rint" 243 }, 244 { 245 .int_msk = BIT(4), 246 .msg = "sec_req_trng_timeout_rint" 247 }, 248 { 249 .int_msk = BIT(5), 250 .msg = "sec_fsm_hbeat_rint" 251 }, 252 { 253 .int_msk = BIT(6), 254 .msg = "sec_channel_req_rng_timeout_rint" 255 }, 256 { 257 .int_msk = BIT(7), 258 .msg = "sec_bd_err_rint" 259 }, 260 { 261 .int_msk = BIT(8), 262 .msg = "sec_chain_buff_err_rint" 263 }, 264 { 265 .int_msk = BIT(14), 266 .msg = "sec_no_secure_access" 267 }, 268 { 269 .int_msk = BIT(15), 270 .msg = "sec_wrapping_key_auth_err" 271 }, 272 { 273 .int_msk = BIT(16), 274 .msg = "sec_km_key_crc_fail" 275 }, 276 { 277 .int_msk = BIT(17), 278 .msg = "sec_axi_poison_err" 279 }, 280 { 281 .int_msk = BIT(18), 282 .msg = "sec_sva_err" 283 }, 284 {} 285 }; 286 287 static const char * const sec_dbg_file_name[] = { 288 [SEC_CLEAR_ENABLE] = "clear_enable", 289 }; 290 291 static struct sec_dfx_item sec_dfx_labels[] = { 292 {"send_cnt", offsetof(struct sec_dfx, send_cnt)}, 293 {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)}, 294 {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)}, 295 {"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)}, 296 {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)}, 297 {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)}, 298 {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)}, 299 }; 300 301 static const struct debugfs_reg32 sec_dfx_regs[] = { 302 {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010}, 303 {"SEC_SAA_EN ", 0x301270}, 304 {"SEC_BD_LATENCY_MIN ", 0x301600}, 305 {"SEC_BD_LATENCY_MAX ", 0x301608}, 306 {"SEC_BD_LATENCY_AVG ", 0x30160C}, 307 {"SEC_BD_NUM_IN_SAA0 ", 0x301670}, 308 {"SEC_BD_NUM_IN_SAA1 ", 0x301674}, 309 {"SEC_BD_NUM_IN_SEC ", 0x301680}, 310 {"SEC_ECC_1BIT_CNT ", 0x301C00}, 311 {"SEC_ECC_1BIT_INFO ", 0x301C04}, 312 {"SEC_ECC_2BIT_CNT ", 0x301C10}, 313 {"SEC_ECC_2BIT_INFO ", 0x301C14}, 314 {"SEC_BD_SAA0 ", 0x301C20}, 315 {"SEC_BD_SAA1 ", 0x301C24}, 316 {"SEC_BD_SAA2 ", 0x301C28}, 317 {"SEC_BD_SAA3 ", 0x301C2C}, 318 {"SEC_BD_SAA4 ", 0x301C30}, 319 {"SEC_BD_SAA5 ", 0x301C34}, 320 {"SEC_BD_SAA6 ", 0x301C38}, 321 {"SEC_BD_SAA7 ", 0x301C3C}, 322 {"SEC_BD_SAA8 ", 0x301C40}, 323 {"SEC_RAS_CE_ENABLE ", 0x301050}, 324 {"SEC_RAS_FE_ENABLE ", 0x301054}, 325 {"SEC_RAS_NFE_ENABLE ", 0x301058}, 326 {"SEC_REQ_TRNG_TIME_TH ", 0x30112C}, 327 {"SEC_CHANNEL_RNG_REQ_THLD ", 0x302110}, 328 }; 329 330 /* define the SEC's dfx regs region and region length */ 331 static struct dfx_diff_registers sec_diff_regs[] = { 332 { 333 .reg_offset = SEC_DFX_BASE, 334 .reg_len = SEC_DFX_BASE_LEN, 335 }, { 336 .reg_offset = SEC_DFX_COMMON1, 337 .reg_len = SEC_DFX_COMMON1_LEN, 338 }, { 339 .reg_offset = SEC_DFX_COMMON2, 340 .reg_len = SEC_DFX_COMMON2_LEN, 341 }, { 342 .reg_offset = SEC_DFX_CORE, 343 .reg_len = SEC_DFX_CORE_LEN, 344 }, 345 }; 346 347 static int sec_diff_regs_show(struct seq_file *s, void *unused) 348 { 349 struct hisi_qm *qm = s->private; 350 351 hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs, 352 ARRAY_SIZE(sec_diff_regs)); 353 354 return 0; 355 } 356 DEFINE_SHOW_ATTRIBUTE(sec_diff_regs); 357 358 static bool pf_q_num_flag; 359 static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) 360 { 361 pf_q_num_flag = true; 362 363 return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF); 364 } 365 366 static const struct kernel_param_ops sec_pf_q_num_ops = { 367 .set = sec_pf_q_num_set, 368 .get = param_get_int, 369 }; 370 371 static u32 pf_q_num = SEC_PF_DEF_Q_NUM; 372 module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); 373 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); 374 375 static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) 376 { 377 u32 ctx_q_num; 378 int ret; 379 380 if (!val) 381 return -EINVAL; 382 383 ret = kstrtou32(val, 10, &ctx_q_num); 384 if (ret) 385 return -EINVAL; 386 387 if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) { 388 pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num); 389 return -EINVAL; 390 } 391 392 return param_set_int(val, kp); 393 } 394 395 static const struct kernel_param_ops sec_ctx_q_num_ops = { 396 .set = sec_ctx_q_num_set, 397 .get = param_get_int, 398 }; 399 static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; 400 module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); 401 MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)"); 402 403 static const struct kernel_param_ops vfs_num_ops = { 404 .set = vfs_num_set, 405 .get = param_get_int, 406 }; 407 408 static u32 vfs_num; 409 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); 410 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); 411 412 void sec_destroy_qps(struct hisi_qp **qps, int qp_num) 413 { 414 hisi_qm_free_qps(qps, qp_num); 415 kfree(qps); 416 } 417 418 struct hisi_qp **sec_create_qps(void) 419 { 420 int node = cpu_to_node(raw_smp_processor_id()); 421 u32 ctx_num = ctx_q_num; 422 struct hisi_qp **qps; 423 u8 *type; 424 int ret; 425 426 qps = kzalloc_objs(struct hisi_qp *, ctx_num); 427 if (!qps) 428 return NULL; 429 430 /* The type of SEC is all 0, so just allocated by kcalloc */ 431 type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); 432 if (!type) { 433 kfree(qps); 434 return NULL; 435 } 436 437 ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); 438 if (ret) { 439 kfree(type); 440 kfree(qps); 441 return NULL; 442 } 443 444 kfree(type); 445 return qps; 446 } 447 448 u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) 449 { 450 u32 cap_val_h, cap_val_l; 451 452 cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val; 453 cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val; 454 455 return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; 456 } 457 458 static const struct kernel_param_ops sec_uacce_mode_ops = { 459 .set = uacce_mode_set, 460 .get = param_get_int, 461 }; 462 463 /* 464 * uacce_mode = 0 means sec only register to crypto, 465 * uacce_mode = 1 means sec both register to crypto and uacce. 466 */ 467 static u32 uacce_mode = UACCE_MODE_NOUACCE; 468 module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444); 469 MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); 470 471 static const struct pci_device_id sec_dev_ids[] = { 472 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_PF) }, 473 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) }, 474 { 0, } 475 }; 476 MODULE_DEVICE_TABLE(pci, sec_dev_ids); 477 478 static void sec_set_endian(struct hisi_qm *qm) 479 { 480 u32 reg; 481 482 reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 483 reg &= ~(BIT(1) | BIT(0)); 484 if (!IS_ENABLED(CONFIG_64BIT)) 485 reg |= BIT(1); 486 487 if (!IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) 488 reg |= BIT(0); 489 490 writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); 491 } 492 493 static int sec_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) 494 { 495 u32 val, try_times = 0; 496 u8 count = 0; 497 498 /* 499 * Read the register value every 10-20us. If the value is 0 for three 500 * consecutive times, the SVA module is ready. 501 */ 502 do { 503 val = readl(qm->io_base + offset); 504 if (val & mask) 505 count = 0; 506 else if (++count == SEC_READ_SVA_STATUS_TIMES) 507 break; 508 509 usleep_range(SEC_WAIT_US_MIN, SEC_WAIT_US_MAX); 510 } while (++try_times < SEC_WAIT_SVA_READY); 511 512 if (try_times == SEC_WAIT_SVA_READY) { 513 pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); 514 return -ETIMEDOUT; 515 } 516 517 return 0; 518 } 519 520 static void sec_close_sva_prefetch(struct hisi_qm *qm) 521 { 522 u32 val; 523 int ret; 524 525 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) 526 return; 527 528 val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); 529 val |= SEC_PREFETCH_DISABLE; 530 writel(val, qm->io_base + SEC_PREFETCH_CFG); 531 532 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, 533 val, !(val & SEC_SVA_DISABLE_READY), 534 SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); 535 if (ret) 536 pci_err(qm->pdev, "failed to close sva prefetch\n"); 537 538 (void)sec_wait_sva_ready(qm, SEC_SVA_PREFETCH_INFO, SEC_SVA_STALL_NUM); 539 } 540 541 static void sec_open_sva_prefetch(struct hisi_qm *qm) 542 { 543 u32 val; 544 int ret; 545 546 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) 547 return; 548 549 /* Enable prefetch */ 550 val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); 551 val &= SEC_PREFETCH_ENABLE; 552 writel(val, qm->io_base + SEC_PREFETCH_CFG); 553 554 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, 555 val, !(val & SEC_PREFETCH_DISABLE), 556 SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); 557 if (ret) { 558 pci_err(qm->pdev, "failed to open sva prefetch\n"); 559 sec_close_sva_prefetch(qm); 560 return; 561 } 562 563 ret = sec_wait_sva_ready(qm, SEC_SVA_TRANS, SEC_SVA_PREFETCH_NUM); 564 if (ret) 565 sec_close_sva_prefetch(qm); 566 } 567 568 static void sec_engine_sva_config(struct hisi_qm *qm) 569 { 570 u32 reg; 571 572 if (qm->ver > QM_HW_V2) { 573 reg = readl_relaxed(qm->io_base + 574 SEC_INTERFACE_USER_CTRL0_REG_V3); 575 reg |= SEC_USER0_SMMU_NORMAL; 576 writel_relaxed(reg, qm->io_base + 577 SEC_INTERFACE_USER_CTRL0_REG_V3); 578 579 reg = readl_relaxed(qm->io_base + 580 SEC_INTERFACE_USER_CTRL1_REG_V3); 581 reg &= SEC_USER1_SMMU_MASK_V3; 582 reg |= SEC_USER1_SMMU_NORMAL_V3; 583 writel_relaxed(reg, qm->io_base + 584 SEC_INTERFACE_USER_CTRL1_REG_V3); 585 } else { 586 reg = readl_relaxed(qm->io_base + 587 SEC_INTERFACE_USER_CTRL0_REG); 588 reg |= SEC_USER0_SMMU_NORMAL; 589 writel_relaxed(reg, qm->io_base + 590 SEC_INTERFACE_USER_CTRL0_REG); 591 reg = readl_relaxed(qm->io_base + 592 SEC_INTERFACE_USER_CTRL1_REG); 593 reg &= SEC_USER1_SMMU_MASK; 594 if (qm->use_sva) 595 reg |= SEC_USER1_SMMU_SVA; 596 else 597 reg |= SEC_USER1_SMMU_NORMAL; 598 writel_relaxed(reg, qm->io_base + 599 SEC_INTERFACE_USER_CTRL1_REG); 600 } 601 sec_open_sva_prefetch(qm); 602 } 603 604 static void sec_enable_clock_gate(struct hisi_qm *qm) 605 { 606 u32 val; 607 608 if (qm->ver < QM_HW_V3) 609 return; 610 611 val = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 612 val |= SEC_CLK_GATE_ENABLE; 613 writel_relaxed(val, qm->io_base + SEC_CONTROL_REG); 614 615 val = readl(qm->io_base + SEC_DYNAMIC_GATE_REG); 616 val |= SEC_DYNAMIC_GATE_EN; 617 writel(val, qm->io_base + SEC_DYNAMIC_GATE_REG); 618 619 val = readl(qm->io_base + SEC_CORE_AUTO_GATE); 620 val |= SEC_CORE_AUTO_GATE_EN; 621 writel(val, qm->io_base + SEC_CORE_AUTO_GATE); 622 } 623 624 static void sec_disable_clock_gate(struct hisi_qm *qm) 625 { 626 u32 val; 627 628 /* Kunpeng920 needs to close clock gating */ 629 val = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 630 val &= SEC_CLK_GATE_DISABLE; 631 writel_relaxed(val, qm->io_base + SEC_CONTROL_REG); 632 } 633 634 static int sec_engine_init(struct hisi_qm *qm) 635 { 636 int ret; 637 u32 reg; 638 639 /* disable clock gate control before mem init */ 640 sec_disable_clock_gate(qm); 641 642 writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG); 643 644 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG, 645 reg, reg & 0x1, SEC_DELAY_10_US, 646 SEC_POLL_TIMEOUT_US); 647 if (ret) { 648 pci_err(qm->pdev, "fail to init sec mem\n"); 649 return ret; 650 } 651 652 reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 653 reg |= (0x1 << SEC_TRNG_EN_SHIFT); 654 writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); 655 656 sec_engine_sva_config(qm); 657 658 writel(SEC_SINGLE_PORT_MAX_TRANS, 659 qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); 660 661 reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver); 662 writel(reg, qm->io_base + SEC_SAA_EN_REG); 663 664 if (qm->ver < QM_HW_V3) { 665 /* HW V2 enable sm4 extra mode, as ctr/ecb */ 666 writel_relaxed(SEC_BD_ERR_CHK_EN0, 667 qm->io_base + SEC_BD_ERR_CHK_EN_REG0); 668 669 /* HW V2 enable sm4 xts mode multiple iv */ 670 writel_relaxed(SEC_BD_ERR_CHK_EN1, 671 qm->io_base + SEC_BD_ERR_CHK_EN_REG1); 672 writel_relaxed(SEC_BD_ERR_CHK_EN3, 673 qm->io_base + SEC_BD_ERR_CHK_EN_REG3); 674 } 675 676 /* config endian */ 677 sec_set_endian(qm); 678 679 sec_enable_clock_gate(qm); 680 681 return 0; 682 } 683 684 static int sec_set_user_domain_and_cache(struct hisi_qm *qm) 685 { 686 /* qm user domain */ 687 writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1); 688 writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE); 689 writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1); 690 writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE); 691 writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE); 692 693 /* qm cache */ 694 writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG); 695 writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE); 696 697 /* disable FLR triggered by BME(bus master enable) */ 698 writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG); 699 writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); 700 701 /* enable sqc,cqc writeback */ 702 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | 703 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | 704 FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL); 705 706 return sec_engine_init(qm); 707 } 708 709 /* sec_debug_regs_clear() - clear the sec debug regs */ 710 static void sec_debug_regs_clear(struct hisi_qm *qm) 711 { 712 int i; 713 714 /* clear sec dfx regs */ 715 writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE); 716 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) 717 readl(qm->io_base + sec_dfx_regs[i].offset); 718 719 /* clear rdclr_en */ 720 writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE); 721 722 hisi_qm_debug_regs_clear(qm); 723 } 724 725 static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) 726 { 727 u32 val1, val2; 728 729 val1 = readl(qm->io_base + SEC_CONTROL_REG); 730 if (enable) { 731 val1 |= SEC_AXI_SHUTDOWN_ENABLE; 732 val2 = qm->err_info.dev_err.shutdown_mask; 733 } else { 734 val1 &= SEC_AXI_SHUTDOWN_DISABLE; 735 val2 = 0x0; 736 } 737 738 if (qm->ver > QM_HW_V2) 739 writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL); 740 741 writel(val1, qm->io_base + SEC_CONTROL_REG); 742 } 743 744 static void sec_hw_error_enable(struct hisi_qm *qm) 745 { 746 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; 747 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; 748 749 if (qm->ver == QM_HW_V1) { 750 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); 751 pci_info(qm->pdev, "V1 not support hw error handle\n"); 752 return; 753 } 754 755 /* clear SEC hw error source if having */ 756 writel(SEC_RAS_CLEAR_ALL, qm->io_base + SEC_CORE_INT_SOURCE); 757 758 /* enable RAS int */ 759 writel(dev_err->ce, qm->io_base + SEC_RAS_CE_REG); 760 writel(dev_err->fe, qm->io_base + SEC_RAS_FE_REG); 761 writel(dev_err->nfe, qm->io_base + SEC_RAS_NFE_REG); 762 763 /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ 764 sec_master_ooo_ctrl(qm, true); 765 766 /* enable SEC hw error interrupts */ 767 writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); 768 } 769 770 static void sec_hw_error_disable(struct hisi_qm *qm) 771 { 772 /* disable SEC hw error interrupts */ 773 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); 774 775 /* disable SEC block master OOO when nfe occurs on Kunpeng930 */ 776 sec_master_ooo_ctrl(qm, false); 777 778 /* disable RAS int */ 779 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); 780 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); 781 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); 782 } 783 784 static u32 sec_clear_enable_read(struct hisi_qm *qm) 785 { 786 return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & 787 SEC_CTRL_CNT_CLR_CE_BIT; 788 } 789 790 static int sec_clear_enable_write(struct hisi_qm *qm, u32 val) 791 { 792 u32 tmp; 793 794 if (val != 1 && val) 795 return -EINVAL; 796 797 tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & 798 ~SEC_CTRL_CNT_CLR_CE_BIT) | val; 799 writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE); 800 801 return 0; 802 } 803 804 static ssize_t sec_debug_read(struct file *filp, char __user *buf, 805 size_t count, loff_t *pos) 806 { 807 struct sec_debug_file *file = filp->private_data; 808 char tbuf[SEC_DBGFS_VAL_MAX_LEN]; 809 struct hisi_qm *qm = file->qm; 810 u32 val; 811 int ret; 812 813 ret = hisi_qm_get_dfx_access(qm); 814 if (ret) 815 return ret; 816 817 spin_lock_irq(&file->lock); 818 819 switch (file->index) { 820 case SEC_CLEAR_ENABLE: 821 val = sec_clear_enable_read(qm); 822 break; 823 default: 824 goto err_input; 825 } 826 827 spin_unlock_irq(&file->lock); 828 829 hisi_qm_put_dfx_access(qm); 830 ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val); 831 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 832 833 err_input: 834 spin_unlock_irq(&file->lock); 835 hisi_qm_put_dfx_access(qm); 836 return -EINVAL; 837 } 838 839 static ssize_t sec_debug_write(struct file *filp, const char __user *buf, 840 size_t count, loff_t *pos) 841 { 842 struct sec_debug_file *file = filp->private_data; 843 char tbuf[SEC_DBGFS_VAL_MAX_LEN]; 844 struct hisi_qm *qm = file->qm; 845 unsigned long val; 846 int len, ret; 847 848 if (*pos != 0) 849 return 0; 850 851 if (count >= SEC_DBGFS_VAL_MAX_LEN) 852 return -ENOSPC; 853 854 len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1, 855 pos, buf, count); 856 if (len < 0) 857 return len; 858 859 tbuf[len] = '\0'; 860 if (kstrtoul(tbuf, 0, &val)) 861 return -EFAULT; 862 863 ret = hisi_qm_get_dfx_access(qm); 864 if (ret) 865 return ret; 866 867 spin_lock_irq(&file->lock); 868 869 switch (file->index) { 870 case SEC_CLEAR_ENABLE: 871 ret = sec_clear_enable_write(qm, val); 872 if (ret) 873 goto err_input; 874 break; 875 default: 876 ret = -EINVAL; 877 goto err_input; 878 } 879 880 ret = count; 881 882 err_input: 883 spin_unlock_irq(&file->lock); 884 hisi_qm_put_dfx_access(qm); 885 return ret; 886 } 887 888 static const struct file_operations sec_dbg_fops = { 889 .owner = THIS_MODULE, 890 .open = simple_open, 891 .read = sec_debug_read, 892 .write = sec_debug_write, 893 }; 894 895 static int sec_debugfs_atomic64_get(void *data, u64 *val) 896 { 897 *val = atomic64_read((atomic64_t *)data); 898 899 return 0; 900 } 901 902 static int sec_debugfs_atomic64_set(void *data, u64 val) 903 { 904 if (val) 905 return -EINVAL; 906 907 atomic64_set((atomic64_t *)data, 0); 908 909 return 0; 910 } 911 912 DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get, 913 sec_debugfs_atomic64_set, "%llu\n"); 914 915 static int sec_regs_show(struct seq_file *s, void *unused) 916 { 917 hisi_qm_regs_dump(s, s->private); 918 919 return 0; 920 } 921 922 DEFINE_SHOW_ATTRIBUTE(sec_regs); 923 924 static int sec_cap_regs_show(struct seq_file *s, void *unused) 925 { 926 struct hisi_qm *qm = s->private; 927 u32 i, size; 928 929 size = qm->cap_tables.qm_cap_size; 930 for (i = 0; i < size; i++) 931 seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, 932 qm->cap_tables.qm_cap_table[i].cap_val); 933 934 size = qm->cap_tables.dev_cap_size; 935 for (i = 0; i < size; i++) 936 seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, 937 qm->cap_tables.dev_cap_table[i].cap_val); 938 939 return 0; 940 } 941 942 DEFINE_SHOW_ATTRIBUTE(sec_cap_regs); 943 944 static int sec_core_debug_init(struct hisi_qm *qm) 945 { 946 struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs; 947 struct sec_dev *sec = container_of(qm, struct sec_dev, qm); 948 struct device *dev = &qm->pdev->dev; 949 struct sec_dfx *dfx = &sec->debug.dfx; 950 struct debugfs_regset32 *regset; 951 struct dentry *tmp_d; 952 int i; 953 954 tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root); 955 956 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 957 if (!regset) 958 return -ENOMEM; 959 960 regset->regs = sec_dfx_regs; 961 regset->nregs = ARRAY_SIZE(sec_dfx_regs); 962 regset->base = qm->io_base; 963 regset->dev = dev; 964 965 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) 966 debugfs_create_file("regs", 0444, tmp_d, regset, &sec_regs_fops); 967 if (qm->fun_type == QM_HW_PF && sec_regs) 968 debugfs_create_file("diff_regs", 0444, tmp_d, 969 qm, &sec_diff_regs_fops); 970 971 for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) { 972 atomic64_t *data = (atomic64_t *)((uintptr_t)dfx + 973 sec_dfx_labels[i].offset); 974 debugfs_create_file(sec_dfx_labels[i].name, 0644, 975 tmp_d, data, &sec_atomic64_ops); 976 } 977 978 debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, 979 qm->debug.debug_root, qm, &sec_cap_regs_fops); 980 981 return 0; 982 } 983 984 static int sec_debug_init(struct hisi_qm *qm) 985 { 986 struct sec_dev *sec = container_of(qm, struct sec_dev, qm); 987 int i; 988 989 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) { 990 for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) { 991 spin_lock_init(&sec->debug.files[i].lock); 992 sec->debug.files[i].index = i; 993 sec->debug.files[i].qm = qm; 994 995 debugfs_create_file(sec_dbg_file_name[i], 0600, 996 qm->debug.debug_root, 997 sec->debug.files + i, 998 &sec_dbg_fops); 999 } 1000 } 1001 1002 return sec_core_debug_init(qm); 1003 } 1004 1005 static int sec_debugfs_init(struct hisi_qm *qm) 1006 { 1007 struct device *dev = &qm->pdev->dev; 1008 int ret; 1009 1010 ret = hisi_qm_regs_debugfs_init(qm, sec_diff_regs, ARRAY_SIZE(sec_diff_regs)); 1011 if (ret) { 1012 dev_warn(dev, "Failed to init SEC diff regs!\n"); 1013 return ret; 1014 } 1015 1016 qm->debug.debug_root = debugfs_create_dir(dev_name(dev), 1017 sec_debugfs_root); 1018 qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; 1019 qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; 1020 1021 hisi_qm_debug_init(qm); 1022 1023 ret = sec_debug_init(qm); 1024 if (ret) 1025 goto debugfs_remove; 1026 1027 return 0; 1028 1029 debugfs_remove: 1030 debugfs_remove_recursive(qm->debug.debug_root); 1031 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); 1032 return ret; 1033 } 1034 1035 static void sec_debugfs_exit(struct hisi_qm *qm) 1036 { 1037 debugfs_remove_recursive(qm->debug.debug_root); 1038 1039 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); 1040 } 1041 1042 static int sec_show_last_regs_init(struct hisi_qm *qm) 1043 { 1044 struct qm_debug *debug = &qm->debug; 1045 int i; 1046 1047 debug->last_words = kcalloc(ARRAY_SIZE(sec_dfx_regs), 1048 sizeof(unsigned int), GFP_KERNEL); 1049 if (!debug->last_words) 1050 return -ENOMEM; 1051 1052 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) 1053 debug->last_words[i] = readl_relaxed(qm->io_base + 1054 sec_dfx_regs[i].offset); 1055 1056 return 0; 1057 } 1058 1059 static void sec_show_last_regs_uninit(struct hisi_qm *qm) 1060 { 1061 struct qm_debug *debug = &qm->debug; 1062 1063 if (qm->fun_type == QM_HW_VF || !debug->last_words) 1064 return; 1065 1066 kfree(debug->last_words); 1067 debug->last_words = NULL; 1068 } 1069 1070 static void sec_show_last_dfx_regs(struct hisi_qm *qm) 1071 { 1072 struct qm_debug *debug = &qm->debug; 1073 struct pci_dev *pdev = qm->pdev; 1074 u32 val; 1075 int i; 1076 1077 if (qm->fun_type == QM_HW_VF || !debug->last_words) 1078 return; 1079 1080 /* dumps last word of the debugging registers during controller reset */ 1081 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) { 1082 val = readl_relaxed(qm->io_base + sec_dfx_regs[i].offset); 1083 if (val != debug->last_words[i]) 1084 pci_info(pdev, "%s \t= 0x%08x => 0x%08x\n", 1085 sec_dfx_regs[i].name, debug->last_words[i], val); 1086 } 1087 } 1088 1089 static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) 1090 { 1091 const struct sec_hw_error *errs = sec_hw_errors; 1092 struct device *dev = &qm->pdev->dev; 1093 u32 err_val; 1094 1095 while (errs->msg) { 1096 if (errs->int_msk & err_sts) { 1097 dev_err(dev, "%s [error status=0x%x] found\n", 1098 errs->msg, errs->int_msk); 1099 1100 if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { 1101 err_val = readl(qm->io_base + 1102 SEC_CORE_SRAM_ECC_ERR_INFO); 1103 dev_err(dev, "multi ecc sram num=0x%x\n", 1104 ((err_val) >> SEC_ECC_NUM) & 1105 SEC_ECC_MASH); 1106 } 1107 } 1108 errs++; 1109 } 1110 } 1111 1112 static u32 sec_get_hw_err_status(struct hisi_qm *qm) 1113 { 1114 return readl(qm->io_base + SEC_CORE_INT_STATUS); 1115 } 1116 1117 static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) 1118 { 1119 writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); 1120 } 1121 1122 static void sec_disable_error_report(struct hisi_qm *qm, u32 err_type) 1123 { 1124 u32 nfe_mask = qm->err_info.dev_err.nfe; 1125 1126 writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG); 1127 } 1128 1129 static void sec_enable_error_report(struct hisi_qm *qm) 1130 { 1131 u32 nfe_mask = qm->err_info.dev_err.nfe; 1132 u32 ce_mask = qm->err_info.dev_err.ce; 1133 1134 writel(nfe_mask, qm->io_base + SEC_RAS_NFE_REG); 1135 writel(ce_mask, qm->io_base + SEC_RAS_CE_REG); 1136 } 1137 1138 static void sec_open_axi_master_ooo(struct hisi_qm *qm) 1139 { 1140 u32 val; 1141 1142 val = readl(qm->io_base + SEC_CONTROL_REG); 1143 writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG); 1144 writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG); 1145 } 1146 1147 static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) 1148 { 1149 u32 err_status; 1150 1151 err_status = sec_get_hw_err_status(qm); 1152 if (err_status) { 1153 if (err_status & qm->err_info.dev_err.ecc_2bits_mask) 1154 qm->err_status.is_dev_ecc_mbit = true; 1155 sec_log_hw_error(qm, err_status); 1156 1157 if (err_status & qm->err_info.dev_err.reset_mask) { 1158 /* Disable the same error reporting until device is recovered. */ 1159 sec_disable_error_report(qm, err_status); 1160 return ACC_ERR_NEED_RESET; 1161 } 1162 sec_clear_hw_err_status(qm, err_status); 1163 /* Avoid firmware disable error report, re-enable. */ 1164 sec_enable_error_report(qm); 1165 } 1166 1167 return ACC_ERR_RECOVERED; 1168 } 1169 1170 static bool sec_dev_is_abnormal(struct hisi_qm *qm) 1171 { 1172 u32 err_status; 1173 1174 err_status = sec_get_hw_err_status(qm); 1175 if (err_status & qm->err_info.dev_err.shutdown_mask) 1176 return true; 1177 1178 return false; 1179 } 1180 1181 static void sec_disable_axi_error(struct hisi_qm *qm) 1182 { 1183 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; 1184 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; 1185 1186 writel(err_mask & ~SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_MASK); 1187 1188 if (qm->ver > QM_HW_V2) 1189 writel(dev_err->shutdown_mask & (~SEC_AXI_ERROR_MASK), 1190 qm->io_base + SEC_OOO_SHUTDOWN_SEL); 1191 } 1192 1193 static void sec_enable_axi_error(struct hisi_qm *qm) 1194 { 1195 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; 1196 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; 1197 1198 /* clear axi error source */ 1199 writel(SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_SOURCE); 1200 1201 writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); 1202 1203 if (qm->ver > QM_HW_V2) 1204 writel(dev_err->shutdown_mask, qm->io_base + SEC_OOO_SHUTDOWN_SEL); 1205 } 1206 1207 static void sec_err_info_init(struct hisi_qm *qm) 1208 { 1209 struct hisi_qm_err_info *err_info = &qm->err_info; 1210 struct hisi_qm_err_mask *qm_err = &err_info->qm_err; 1211 struct hisi_qm_err_mask *dev_err = &err_info->dev_err; 1212 1213 qm_err->fe = SEC_RAS_FE_ENB_MSK; 1214 qm_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); 1215 qm_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); 1216 qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1217 SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); 1218 qm_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1219 SEC_QM_RESET_MASK_CAP, qm->cap_ver); 1220 qm_err->ecc_2bits_mask = QM_ECC_MBIT; 1221 1222 dev_err->fe = SEC_RAS_FE_ENB_MSK; 1223 dev_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); 1224 dev_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); 1225 dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1226 SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); 1227 dev_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1228 SEC_RESET_MASK_CAP, qm->cap_ver); 1229 dev_err->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; 1230 1231 err_info->msi_wr_port = BIT(0); 1232 err_info->acpi_rst = "SRST"; 1233 } 1234 1235 static const struct hisi_qm_err_ini sec_err_ini = { 1236 .hw_init = sec_set_user_domain_and_cache, 1237 .hw_err_enable = sec_hw_error_enable, 1238 .hw_err_disable = sec_hw_error_disable, 1239 .get_dev_hw_err_status = sec_get_hw_err_status, 1240 .clear_dev_hw_err_status = sec_clear_hw_err_status, 1241 .open_axi_master_ooo = sec_open_axi_master_ooo, 1242 .open_sva_prefetch = sec_open_sva_prefetch, 1243 .close_sva_prefetch = sec_close_sva_prefetch, 1244 .show_last_dfx_regs = sec_show_last_dfx_regs, 1245 .err_info_init = sec_err_info_init, 1246 .get_err_result = sec_get_err_result, 1247 .dev_is_abnormal = sec_dev_is_abnormal, 1248 .disable_axi_error = sec_disable_axi_error, 1249 .enable_axi_error = sec_enable_axi_error, 1250 }; 1251 1252 static int sec_pf_probe_init(struct sec_dev *sec) 1253 { 1254 struct hisi_qm *qm = &sec->qm; 1255 int ret; 1256 1257 ret = sec_set_user_domain_and_cache(qm); 1258 if (ret) 1259 return ret; 1260 1261 hisi_qm_dev_err_init(qm); 1262 sec_debug_regs_clear(qm); 1263 ret = sec_show_last_regs_init(qm); 1264 if (ret) 1265 pci_err(qm->pdev, "Failed to init last word regs!\n"); 1266 1267 return ret; 1268 } 1269 1270 static int sec_pre_store_cap_reg(struct hisi_qm *qm) 1271 { 1272 struct hisi_qm_cap_record *sec_cap; 1273 struct pci_dev *pdev = qm->pdev; 1274 size_t i, size; 1275 1276 size = ARRAY_SIZE(sec_cap_query_info); 1277 sec_cap = devm_kcalloc(&pdev->dev, size, sizeof(*sec_cap), GFP_KERNEL); 1278 if (!sec_cap) 1279 return -ENOMEM; 1280 1281 for (i = 0; i < size; i++) { 1282 sec_cap[i].type = sec_cap_query_info[i].type; 1283 sec_cap[i].name = sec_cap_query_info[i].name; 1284 sec_cap[i].cap_val = hisi_qm_get_cap_value(qm, sec_cap_query_info, 1285 i, qm->cap_ver); 1286 } 1287 1288 qm->cap_tables.dev_cap_table = sec_cap; 1289 qm->cap_tables.dev_cap_size = size; 1290 1291 return 0; 1292 } 1293 1294 static void sec_set_channels(struct hisi_qm *qm) 1295 { 1296 struct qm_channel *channel_data = &qm->channel_data; 1297 1298 channel_data->channel_num = SEC_MAX_CHANNEL_NUM; 1299 channel_data->channel_name[0] = "SEC"; 1300 } 1301 1302 static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) 1303 { 1304 u64 alg_msk; 1305 int ret; 1306 1307 qm->pdev = pdev; 1308 qm->mode = uacce_mode; 1309 qm->sqe_size = SEC_SQE_SIZE; 1310 qm->dev_name = sec_name; 1311 1312 qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) ? 1313 QM_HW_PF : QM_HW_VF; 1314 if (qm->fun_type == QM_HW_PF) { 1315 qm->qp_base = SEC_PF_DEF_Q_BASE; 1316 qm->qp_num = pf_q_num; 1317 qm->debug.curr_qm_qp_num = pf_q_num; 1318 qm->qm_list = &sec_devices; 1319 qm->err_ini = &sec_err_ini; 1320 if (pf_q_num_flag) 1321 set_bit(QM_MODULE_PARAM, &qm->misc_ctl); 1322 } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { 1323 /* 1324 * have no way to get qm configure in VM in v1 hardware, 1325 * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force 1326 * to trigger only one VF in v1 hardware. 1327 * v2 hardware has no such problem. 1328 */ 1329 qm->qp_base = SEC_PF_DEF_Q_NUM; 1330 qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; 1331 } 1332 1333 ret = hisi_qm_init(qm); 1334 if (ret) { 1335 pci_err(qm->pdev, "Failed to init sec qm configures!\n"); 1336 return ret; 1337 } 1338 1339 sec_set_channels(qm); 1340 /* Fetch and save the value of capability registers */ 1341 ret = sec_pre_store_cap_reg(qm); 1342 if (ret) { 1343 pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); 1344 hisi_qm_uninit(qm); 1345 return ret; 1346 } 1347 alg_msk = sec_get_alg_bitmap(qm, SEC_ALG_BITMAP_HIGH, SEC_ALG_BITMAP_LOW); 1348 ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); 1349 if (ret) { 1350 pci_err(qm->pdev, "Failed to set sec algs!\n"); 1351 hisi_qm_uninit(qm); 1352 } 1353 1354 return ret; 1355 } 1356 1357 static void sec_qm_uninit(struct hisi_qm *qm) 1358 { 1359 hisi_qm_uninit(qm); 1360 } 1361 1362 static int sec_probe_init(struct sec_dev *sec) 1363 { 1364 u32 type_rate = SEC_SHAPER_TYPE_RATE; 1365 struct hisi_qm *qm = &sec->qm; 1366 int ret; 1367 1368 if (qm->fun_type == QM_HW_PF) { 1369 ret = sec_pf_probe_init(sec); 1370 if (ret) 1371 return ret; 1372 /* enable shaper type 0 */ 1373 if (qm->ver >= QM_HW_V3) { 1374 type_rate |= QM_SHAPER_ENABLE; 1375 qm->type_rate = type_rate; 1376 } 1377 } 1378 1379 return 0; 1380 } 1381 1382 static void sec_probe_uninit(struct hisi_qm *qm) 1383 { 1384 if (qm->fun_type == QM_HW_VF) 1385 return; 1386 1387 sec_debug_regs_clear(qm); 1388 sec_show_last_regs_uninit(qm); 1389 sec_close_sva_prefetch(qm); 1390 hisi_qm_dev_err_uninit(qm); 1391 } 1392 1393 static void sec_iommu_used_check(struct sec_dev *sec) 1394 { 1395 struct iommu_domain *domain; 1396 struct device *dev = &sec->qm.pdev->dev; 1397 1398 domain = iommu_get_domain_for_dev(dev); 1399 1400 /* Check if iommu is used */ 1401 sec->iommu_used = false; 1402 if (domain) { 1403 if (domain->type & __IOMMU_DOMAIN_PAGING) 1404 sec->iommu_used = true; 1405 dev_info(dev, "SMMU Opened, the iommu type = %u\n", 1406 domain->type); 1407 } 1408 } 1409 1410 static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1411 { 1412 struct sec_dev *sec; 1413 struct hisi_qm *qm; 1414 int ret; 1415 1416 sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL); 1417 if (!sec) 1418 return -ENOMEM; 1419 1420 qm = &sec->qm; 1421 ret = sec_qm_init(qm, pdev); 1422 if (ret) { 1423 pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret); 1424 return ret; 1425 } 1426 1427 sec->ctx_q_num = ctx_q_num; 1428 sec_iommu_used_check(sec); 1429 1430 ret = sec_probe_init(sec); 1431 if (ret) { 1432 pci_err(pdev, "Failed to probe!\n"); 1433 goto err_qm_uninit; 1434 } 1435 1436 ret = hisi_qm_start(qm); 1437 if (ret) { 1438 pci_err(pdev, "Failed to start sec qm!\n"); 1439 goto err_probe_uninit; 1440 } 1441 1442 ret = sec_debugfs_init(qm); 1443 if (ret) 1444 pci_warn(pdev, "Failed to init debugfs!\n"); 1445 1446 hisi_qm_add_list(qm, &sec_devices); 1447 ret = hisi_qm_alg_register(qm, &sec_devices, ctx_q_num); 1448 if (ret < 0) { 1449 pr_err("Failed to register driver to crypto.\n"); 1450 goto err_qm_del_list; 1451 } 1452 1453 ret = hisi_qm_register_uacce(qm); 1454 if (ret) { 1455 pci_err(pdev, "failed to register uacce (%d)!\n", ret); 1456 goto err_alg_unregister; 1457 } 1458 1459 if (qm->fun_type == QM_HW_PF && vfs_num) { 1460 ret = hisi_qm_sriov_enable(pdev, vfs_num); 1461 if (ret < 0) 1462 goto err_alg_unregister; 1463 } 1464 1465 hisi_qm_pm_init(qm); 1466 1467 return 0; 1468 1469 err_alg_unregister: 1470 hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); 1471 err_qm_del_list: 1472 hisi_qm_del_list(qm, &sec_devices); 1473 sec_debugfs_exit(qm); 1474 hisi_qm_stop(qm, QM_NORMAL); 1475 err_probe_uninit: 1476 sec_probe_uninit(qm); 1477 err_qm_uninit: 1478 sec_qm_uninit(qm); 1479 return ret; 1480 } 1481 1482 static void sec_remove(struct pci_dev *pdev) 1483 { 1484 struct hisi_qm *qm = pci_get_drvdata(pdev); 1485 1486 hisi_qm_pm_uninit(qm); 1487 hisi_qm_wait_task_finish(qm, &sec_devices); 1488 hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); 1489 hisi_qm_del_list(qm, &sec_devices); 1490 1491 if (qm->fun_type == QM_HW_PF && qm->vfs_num) 1492 hisi_qm_sriov_disable(pdev, true); 1493 1494 sec_debugfs_exit(qm); 1495 1496 (void)hisi_qm_stop(qm, QM_NORMAL); 1497 sec_probe_uninit(qm); 1498 1499 sec_qm_uninit(qm); 1500 } 1501 1502 static const struct dev_pm_ops sec_pm_ops = { 1503 SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL) 1504 }; 1505 1506 static const struct pci_error_handlers sec_err_handler = { 1507 .error_detected = hisi_qm_dev_err_detected, 1508 .slot_reset = hisi_qm_dev_slot_reset, 1509 .reset_prepare = hisi_qm_reset_prepare, 1510 .reset_done = hisi_qm_reset_done, 1511 }; 1512 1513 static struct pci_driver sec_pci_driver = { 1514 .name = "hisi_sec2", 1515 .id_table = sec_dev_ids, 1516 .probe = sec_probe, 1517 .remove = sec_remove, 1518 .err_handler = &sec_err_handler, 1519 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? 1520 hisi_qm_sriov_configure : NULL, 1521 .shutdown = hisi_qm_dev_shutdown, 1522 .driver.pm = &sec_pm_ops, 1523 }; 1524 1525 struct pci_driver *hisi_sec_get_pf_driver(void) 1526 { 1527 return &sec_pci_driver; 1528 } 1529 EXPORT_SYMBOL_GPL(hisi_sec_get_pf_driver); 1530 1531 static void sec_register_debugfs(void) 1532 { 1533 if (!debugfs_initialized()) 1534 return; 1535 1536 sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL); 1537 } 1538 1539 static void sec_unregister_debugfs(void) 1540 { 1541 debugfs_remove_recursive(sec_debugfs_root); 1542 } 1543 1544 static int __init sec_init(void) 1545 { 1546 int ret; 1547 1548 hisi_qm_init_list(&sec_devices); 1549 sec_register_debugfs(); 1550 1551 ret = pci_register_driver(&sec_pci_driver); 1552 if (ret < 0) { 1553 sec_unregister_debugfs(); 1554 pr_err("Failed to register pci driver.\n"); 1555 return ret; 1556 } 1557 1558 return 0; 1559 } 1560 1561 static void __exit sec_exit(void) 1562 { 1563 pci_unregister_driver(&sec_pci_driver); 1564 sec_unregister_debugfs(); 1565 } 1566 1567 module_init(sec_init); 1568 module_exit(sec_exit); 1569 1570 MODULE_LICENSE("GPL v2"); 1571 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>"); 1572 MODULE_AUTHOR("Longfang Liu <liulongfang@huawei.com>"); 1573 MODULE_AUTHOR("Kai Ye <yekai13@huawei.com>"); 1574 MODULE_AUTHOR("Wei Zhang <zhangwei375@huawei.com>"); 1575 MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator"); 1576