1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2025 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_transport_access_macros.h" 12 #include "adf_transport_internal.h" 13 #include <linux/delay.h> 14 #include "adf_accel_devices.h" 15 #include "adf_common_drv.h" 16 #include "icp_qat_hal.h" 17 #include "icp_qat_uclo.h" 18 19 #define BAD_REGADDR 0xffff 20 #define MAX_RETRY_TIMES 1000000 21 #define INIT_CTX_ARB_VALUE 0x0 22 #define INIT_CTX_ENABLE_VALUE 0x0 23 #define INIT_PC_VALUE 0x0 24 #define INIT_WAKEUP_EVENTS_VALUE 0x1 25 #define INIT_SIG_EVENTS_VALUE 0x1 26 #define INIT_CCENABLE_VALUE 0x2000 27 #define RST_CSR_QAT_LSB 20 28 #define RST_CSR_AE_LSB 0 29 #define MC_TIMESTAMP_ENABLE (0x1 << 7) 30 31 #define IGNORE_W1C_MASK \ 32 ((~(1 << CE_BREAKPOINT_BITPOS)) & \ 33 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ 34 (~(1 << CE_REG_PAR_ERR_BITPOS))) 35 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \ 36 (inst = ((inst & 0xFFFF00C03FFull) | \ 37 ((((const_val) << 12) & 0x0FF00000ull) | \ 38 (((const_val) << 10) & 0x0003FC00ull)))) 39 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \ 40 (inst = ((inst & 0xFFFF00FFF00ull) | \ 41 ((((const_val) << 12) & 0x0FF00000ull) | \ 42 (((const_val) << 0) & 0x000000FFull)))) 43 44 #define AE(handle, ae) ((handle)->hal_handle->aes[ae]) 45 46 static const uint64_t inst_4b[] = { 0x0F0400C0000ull, 0x0F4400C0000ull, 47 0x0F040000300ull, 0x0F440000300ull, 48 0x0FC066C0000ull, 0x0F0000C0300ull, 49 0x0F0000C0300ull, 0x0F0000C0300ull, 50 0x0A021000000ull }; 51 52 static const uint64_t inst[] = { 53 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, 54 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 55 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, 56 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 57 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, 58 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 59 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 60 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 61 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 62 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 63 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 64 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 65 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 66 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 67 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 68 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 69 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 70 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 71 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 72 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 73 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 74 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, 75 }; 76 77 static const uint64_t inst_CPM2X[] = { 78 0x0F0000C0000ull, 0x0D802C00011ull, 0x0F0000C0001ull, 0x0FC066C0001ull, 79 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F000500300ull, 80 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0A0580C0000ull, 81 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 0x0A0584C0000ull, 82 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 0x0A0588C0000ull, 83 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 0x0A058CC0000ull, 84 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 0x0A05C0C0000ull, 85 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 0x0A05C4C0000ull, 86 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 0x0A05C8C0000ull, 87 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 0x0A05CCC0000ull, 88 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 0x0A0400C0000ull, 89 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 0x0A0402C0000ull, 90 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 0x0A0404C0000ull, 91 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 0x0A0406C0000ull, 92 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 0x0A0408C0000ull, 93 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 0x0A040AC0000ull, 94 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 0x0A040CC0000ull, 95 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 0x0A040EC0000ull, 96 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 0x0D81341C010ull, 97 0x0E000000001ull, 0x0E000010000ull, 98 }; 99 100 void 101 qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, 102 unsigned char ae, 103 unsigned int ctx_mask) 104 { 105 AE(handle, ae).live_ctx_mask = ctx_mask; 106 } 107 108 #define CSR_RETRY_TIMES 500 109 static int 110 qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, 111 unsigned char ae, 112 unsigned int csr, 113 unsigned int *value) 114 { 115 unsigned int iterations = CSR_RETRY_TIMES; 116 117 do { 118 *value = GET_AE_CSR(handle, ae, csr); 119 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) 120 return 0; 121 } while (iterations--); 122 123 pr_err("QAT: Read CSR timeout\n"); 124 return EFAULT; 125 } 126 127 static int 128 qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, 129 unsigned char ae, 130 unsigned int csr, 131 unsigned int value) 132 { 133 unsigned int iterations = CSR_RETRY_TIMES; 134 135 do { 136 SET_AE_CSR(handle, ae, csr, value); 137 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) 138 return 0; 139 } while (iterations--); 140 141 pr_err("QAT: Write CSR Timeout\n"); 142 return EFAULT; 143 } 144 145 static void 146 qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, 147 unsigned char ae, 148 unsigned char ctx, 149 unsigned int *events) 150 { 151 unsigned int cur_ctx; 152 153 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 154 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 155 qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events); 156 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 157 } 158 159 static int 160 qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, 161 unsigned char ae, 162 unsigned int cycles, 163 int chk_inactive) 164 { 165 unsigned int base_cnt = 0, cur_cnt = 0; 166 unsigned int csr = (1 << ACS_ABO_BITPOS); 167 int times = MAX_RETRY_TIMES; 168 int elapsed_cycles = 0; 169 170 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt); 171 base_cnt &= 0xffff; 172 while ((int)cycles > elapsed_cycles && times--) { 173 if (chk_inactive) 174 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr); 175 176 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt); 177 cur_cnt &= 0xffff; 178 elapsed_cycles = cur_cnt - base_cnt; 179 180 if (elapsed_cycles < 0) 181 elapsed_cycles += 0x10000; 182 183 /* ensure at least 8 time cycles elapsed in wait_cycles */ 184 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) 185 return 0; 186 } 187 if (times < 0) { 188 pr_err("QAT: wait_num_cycles time out\n"); 189 return EFAULT; 190 } 191 return 0; 192 } 193 194 void 195 qat_hal_get_scs_neigh_ae(unsigned char ae, unsigned char *ae_neigh) 196 { 197 *ae_neigh = (ae & 0x1) ? (ae - 1) : (ae + 1); 198 } 199 200 #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) 201 #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) 202 203 int 204 qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, 205 unsigned char ae, 206 unsigned char mode) 207 { 208 unsigned int csr, new_csr; 209 210 if (mode != 4 && mode != 8) { 211 pr_err("QAT: bad ctx mode=%d\n", mode); 212 return EINVAL; 213 } 214 215 /* Sets the accelaration engine context mode to either four or eight */ 216 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 217 csr = IGNORE_W1C_MASK & csr; 218 new_csr = (mode == 4) ? SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : 219 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); 220 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 221 return 0; 222 } 223 224 int 225 qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, 226 unsigned char ae, 227 unsigned char mode) 228 { 229 unsigned int csr, new_csr; 230 231 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 232 pr_err("QAT: No next neigh for CPM2X\n"); 233 return EINVAL; 234 } 235 236 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 237 csr &= IGNORE_W1C_MASK; 238 239 new_csr = (mode) ? SET_BIT(csr, CE_NN_MODE_BITPOS) : 240 CLR_BIT(csr, CE_NN_MODE_BITPOS); 241 242 if (new_csr != csr) 243 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 244 245 return 0; 246 } 247 248 int 249 qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, 250 unsigned char ae, 251 enum icp_qat_uof_regtype lm_type, 252 unsigned char mode) 253 { 254 unsigned int csr, new_csr; 255 256 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 257 csr &= IGNORE_W1C_MASK; 258 switch (lm_type) { 259 case ICP_LMEM0: 260 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : 261 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); 262 break; 263 case ICP_LMEM1: 264 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : 265 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); 266 break; 267 case ICP_LMEM2: 268 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : 269 CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); 270 break; 271 case ICP_LMEM3: 272 new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : 273 CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); 274 break; 275 default: 276 pr_err("QAT: lmType = 0x%x\n", lm_type); 277 return EINVAL; 278 } 279 280 if (new_csr != csr) 281 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 282 return 0; 283 } 284 285 void 286 qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, 287 unsigned char ae, 288 unsigned char mode) 289 { 290 unsigned int csr, new_csr; 291 292 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 293 csr &= IGNORE_W1C_MASK; 294 new_csr = (mode) ? SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : 295 CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); 296 if (new_csr != csr) 297 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 298 } 299 300 void 301 qat_hal_set_ae_scs_mode(struct icp_qat_fw_loader_handle *handle, 302 unsigned char ae, 303 unsigned char mode) 304 { 305 unsigned int csr, new_csr; 306 307 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr); 308 new_csr = (mode) ? SET_BIT(csr, MMC_SHARE_CS_BITPOS) : 309 CLR_BIT(csr, MMC_SHARE_CS_BITPOS); 310 if (new_csr != csr) 311 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, new_csr); 312 } 313 314 static unsigned short 315 qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num) 316 { 317 unsigned short reg_addr; 318 319 switch (type) { 320 case ICP_GPA_ABS: 321 case ICP_GPB_ABS: 322 reg_addr = 0x80 | (reg_num & 0x7f); 323 break; 324 case ICP_GPA_REL: 325 case ICP_GPB_REL: 326 reg_addr = reg_num & 0x1f; 327 break; 328 case ICP_SR_RD_REL: 329 case ICP_SR_WR_REL: 330 case ICP_SR_REL: 331 reg_addr = 0x180 | (reg_num & 0x1f); 332 break; 333 case ICP_SR_ABS: 334 reg_addr = 0x140 | ((reg_num & 0x3) << 1); 335 break; 336 case ICP_DR_RD_REL: 337 case ICP_DR_WR_REL: 338 case ICP_DR_REL: 339 reg_addr = 0x1c0 | (reg_num & 0x1f); 340 break; 341 case ICP_DR_ABS: 342 reg_addr = 0x100 | ((reg_num & 0x3) << 1); 343 break; 344 case ICP_NEIGH_REL: 345 reg_addr = 0x280 | (reg_num & 0x1f); 346 break; 347 case ICP_LMEM0: 348 reg_addr = 0x200; 349 break; 350 case ICP_LMEM1: 351 reg_addr = 0x220; 352 break; 353 case ICP_LMEM2: 354 reg_addr = 0x2c0; 355 break; 356 case ICP_LMEM3: 357 reg_addr = 0x2e0; 358 break; 359 case ICP_NO_DEST: 360 reg_addr = 0x300 | (reg_num & 0xff); 361 break; 362 default: 363 reg_addr = BAD_REGADDR; 364 break; 365 } 366 return reg_addr; 367 } 368 369 static u32 370 qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle *handle) 371 { 372 u32 tg = 0, ae; 373 u32 valid_ae_mask = 0; 374 375 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { 376 if (handle->hal_handle->ae_mask & (1 << ae)) { 377 tg = ae / 4; 378 valid_ae_mask |= (1 << (tg * 2)); 379 } 380 } 381 return valid_ae_mask; 382 } 383 384 void 385 qat_hal_reset(struct icp_qat_fw_loader_handle *handle) 386 { 387 unsigned int ae_reset_csr[MAX_CPP_NUM]; 388 unsigned int ae_reset_val[MAX_CPP_NUM]; 389 unsigned int valid_ae_mask, valid_slice_mask; 390 unsigned int cpp_num = 1; 391 unsigned int i; 392 393 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 394 ae_reset_csr[0] = ICP_RESET_CPP0; 395 ae_reset_csr[1] = ICP_RESET_CPP1; 396 if (handle->hal_handle->ae_mask > 0xffff) 397 ++cpp_num; 398 } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 399 ae_reset_csr[0] = ICP_RESET_CPP0; 400 } else { 401 ae_reset_csr[0] = ICP_RESET; 402 } 403 404 for (i = 0; i < cpp_num; i++) { 405 if (i == 0) { 406 if (IS_QAT_GEN4( 407 pci_get_device(GET_DEV(handle->accel_dev)))) { 408 valid_ae_mask = 409 qat_hal_get_ae_mask_gen4(handle); 410 valid_slice_mask = 411 handle->hal_handle->slice_mask; 412 } else { 413 valid_ae_mask = 414 handle->hal_handle->ae_mask & 0xFFFF; 415 valid_slice_mask = 416 handle->hal_handle->slice_mask & 0x3F; 417 } 418 } else { 419 valid_ae_mask = 420 (handle->hal_handle->ae_mask >> AES_PER_CPP) & 421 0xFFFF; 422 valid_slice_mask = 423 (handle->hal_handle->slice_mask >> SLICES_PER_CPP) & 424 0x3F; 425 } 426 427 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); 428 ae_reset_val[i] |= valid_ae_mask << RST_CSR_AE_LSB; 429 ae_reset_val[i] |= valid_slice_mask << RST_CSR_QAT_LSB; 430 SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]); 431 } 432 } 433 434 static void 435 qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, 436 unsigned char ae, 437 unsigned int ctx_mask, 438 unsigned int ae_csr, 439 unsigned int csr_val) 440 { 441 unsigned int ctx, cur_ctx; 442 443 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 444 445 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 446 if (!(ctx_mask & (1 << ctx))) 447 continue; 448 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 449 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); 450 } 451 452 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 453 } 454 455 static void 456 qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, 457 unsigned char ae, 458 unsigned char ctx, 459 unsigned int ae_csr, 460 unsigned int *csr_val) 461 { 462 unsigned int cur_ctx; 463 464 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 465 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 466 qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val); 467 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 468 } 469 470 static void 471 qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, 472 unsigned char ae, 473 unsigned int ctx_mask, 474 unsigned int events) 475 { 476 unsigned int ctx, cur_ctx; 477 478 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 479 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 480 if (!(ctx_mask & (1 << ctx))) 481 continue; 482 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 483 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); 484 } 485 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 486 } 487 488 static void 489 qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, 490 unsigned char ae, 491 unsigned int ctx_mask, 492 unsigned int events) 493 { 494 unsigned int ctx, cur_ctx; 495 496 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 497 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 498 if (!(ctx_mask & (1 << ctx))) 499 continue; 500 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 501 qat_hal_wr_ae_csr(handle, 502 ae, 503 CTX_WAKEUP_EVENTS_INDIRECT, 504 events); 505 } 506 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 507 } 508 509 static int 510 qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) 511 { 512 unsigned int base_cnt, cur_cnt; 513 unsigned char ae; 514 unsigned long ae_mask = handle->hal_handle->ae_mask; 515 int times = MAX_RETRY_TIMES; 516 517 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 518 { 519 qat_hal_rd_ae_csr(handle, 520 ae, 521 PROFILE_COUNT, 522 (unsigned int *)&base_cnt); 523 base_cnt &= 0xffff; 524 525 do { 526 qat_hal_rd_ae_csr(handle, 527 ae, 528 PROFILE_COUNT, 529 (unsigned int *)&cur_cnt); 530 cur_cnt &= 0xffff; 531 } while (times-- && (cur_cnt == base_cnt)); 532 533 if (times < 0) { 534 pr_err("QAT: AE%d is inactive!!\n", ae); 535 return EFAULT; 536 } 537 } 538 539 return 0; 540 } 541 542 int 543 qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, 544 unsigned int ae) 545 { 546 unsigned int enable = 0, active = 0; 547 548 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable); 549 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active); 550 if ((enable & (0xff << CE_ENABLE_BITPOS)) || 551 (active & (1 << ACS_ABO_BITPOS))) 552 return 1; 553 else 554 return 0; 555 } 556 557 static void 558 qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) 559 { 560 unsigned int misc_ctl_csr, misc_ctl; 561 unsigned char ae; 562 unsigned long ae_mask = handle->hal_handle->ae_mask; 563 564 misc_ctl_csr = 565 (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) ? 566 MISC_CONTROL_C4XXX : 567 MISC_CONTROL; 568 /* stop the timestamp timers */ 569 misc_ctl = GET_GLB_CSR(handle, misc_ctl_csr); 570 if (misc_ctl & MC_TIMESTAMP_ENABLE) 571 SET_GLB_CSR(handle, 572 misc_ctl_csr, 573 misc_ctl & (~MC_TIMESTAMP_ENABLE)); 574 575 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 576 { 577 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); 578 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); 579 } 580 /* start timestamp timers */ 581 SET_GLB_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); 582 } 583 584 #define ESRAM_AUTO_TINIT BIT(2) 585 #define ESRAM_AUTO_TINIT_DONE BIT(3) 586 #define ESRAM_AUTO_INIT_USED_CYCLES (1640) 587 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C 588 589 static int 590 qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) 591 { 592 uintptr_t csr_addr = 593 ((uintptr_t)handle->hal_ep_csr_addr_v + ESRAM_AUTO_INIT_CSR_OFFSET); 594 unsigned int csr_val; 595 int times = 30; 596 597 if (pci_get_device(GET_DEV(handle->accel_dev)) != 598 ADF_DH895XCC_PCI_DEVICE_ID) 599 return 0; 600 601 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); 602 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) 603 return 0; 604 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); 605 csr_val |= ESRAM_AUTO_TINIT; 606 607 ADF_CSR_WR(handle->hal_misc_addr_v, csr_addr, csr_val); 608 do { 609 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); 610 csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); 611 612 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); 613 if (times < 0) { 614 pr_err("QAT: Fail to init eSram!\n"); 615 return EFAULT; 616 } 617 return 0; 618 } 619 620 #define SHRAM_INIT_CYCLES 2060 621 int 622 qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) 623 { 624 unsigned int ae_reset_csr[MAX_CPP_NUM]; 625 unsigned int ae_reset_val[MAX_CPP_NUM]; 626 unsigned int cpp_num = 1; 627 unsigned int valid_ae_mask, valid_slice_mask; 628 unsigned char ae; 629 unsigned int i; 630 unsigned int clk_csr[MAX_CPP_NUM]; 631 unsigned int clk_val[MAX_CPP_NUM]; 632 unsigned int times = 100; 633 unsigned long ae_mask = handle->hal_handle->ae_mask; 634 635 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 636 ae_reset_csr[0] = ICP_RESET_CPP0; 637 ae_reset_csr[1] = ICP_RESET_CPP1; 638 clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0; 639 clk_csr[1] = ICP_GLOBAL_CLK_ENABLE_CPP1; 640 if (handle->hal_handle->ae_mask > 0xffff) 641 ++cpp_num; 642 } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 643 ae_reset_csr[0] = ICP_RESET_CPP0; 644 clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0; 645 } else { 646 ae_reset_csr[0] = ICP_RESET; 647 clk_csr[0] = ICP_GLOBAL_CLK_ENABLE; 648 } 649 650 for (i = 0; i < cpp_num; i++) { 651 if (i == 0) { 652 if (IS_QAT_GEN4( 653 pci_get_device(GET_DEV(handle->accel_dev)))) { 654 valid_ae_mask = 655 qat_hal_get_ae_mask_gen4(handle); 656 valid_slice_mask = 657 handle->hal_handle->slice_mask; 658 } else { 659 valid_ae_mask = 660 handle->hal_handle->ae_mask & 0xFFFF; 661 valid_slice_mask = 662 handle->hal_handle->slice_mask & 0x3F; 663 } 664 } else { 665 valid_ae_mask = 666 (handle->hal_handle->ae_mask >> AES_PER_CPP) & 667 0xFFFF; 668 valid_slice_mask = 669 (handle->hal_handle->slice_mask >> SLICES_PER_CPP) & 670 0x3F; 671 } 672 /* write to the reset csr */ 673 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); 674 ae_reset_val[i] &= ~(valid_ae_mask << RST_CSR_AE_LSB); 675 ae_reset_val[i] &= ~(valid_slice_mask << RST_CSR_QAT_LSB); 676 do { 677 SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]); 678 if (!(times--)) 679 goto out_err; 680 ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); 681 } while ( 682 (valid_ae_mask | (valid_slice_mask << RST_CSR_QAT_LSB)) & 683 ae_reset_val[i]); 684 /* enable clock */ 685 clk_val[i] = GET_GLB_CSR(handle, clk_csr[i]); 686 clk_val[i] |= valid_ae_mask << 0; 687 clk_val[i] |= valid_slice_mask << 20; 688 SET_GLB_CSR(handle, clk_csr[i], clk_val[i]); 689 } 690 if (qat_hal_check_ae_alive(handle)) 691 goto out_err; 692 693 /* Set undefined power-up/reset states to reasonable default values */ 694 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 695 { 696 qat_hal_wr_ae_csr(handle, 697 ae, 698 CTX_ENABLES, 699 INIT_CTX_ENABLE_VALUE); 700 qat_hal_wr_indr_csr(handle, 701 ae, 702 ICP_QAT_UCLO_AE_ALL_CTX, 703 CTX_STS_INDIRECT, 704 handle->hal_handle->upc_mask & 705 INIT_PC_VALUE); 706 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); 707 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); 708 qat_hal_put_wakeup_event(handle, 709 ae, 710 ICP_QAT_UCLO_AE_ALL_CTX, 711 INIT_WAKEUP_EVENTS_VALUE); 712 qat_hal_put_sig_event(handle, 713 ae, 714 ICP_QAT_UCLO_AE_ALL_CTX, 715 INIT_SIG_EVENTS_VALUE); 716 } 717 if (qat_hal_init_esram(handle)) 718 goto out_err; 719 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) 720 goto out_err; 721 qat_hal_reset_timestamp(handle); 722 723 return 0; 724 out_err: 725 pr_err("QAT: failed to get device out of reset\n"); 726 return EFAULT; 727 } 728 729 static void 730 qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, 731 unsigned char ae, 732 unsigned int ctx_mask) 733 { 734 unsigned int ctx; 735 736 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); 737 ctx &= IGNORE_W1C_MASK & 738 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); 739 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); 740 } 741 742 static uint64_t 743 qat_hal_parity_64bit(uint64_t word) 744 { 745 word ^= word >> 1; 746 word ^= word >> 2; 747 word ^= word >> 4; 748 word ^= word >> 8; 749 word ^= word >> 16; 750 word ^= word >> 32; 751 return word & 1; 752 } 753 754 static uint64_t 755 qat_hal_set_uword_ecc(uint64_t uword) 756 { 757 uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, 758 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, 759 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, 760 bit6_mask = 0xdaf69a46910ULL; 761 762 /* clear the ecc bits */ 763 uword &= ~(0x7fULL << 0x2C); 764 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; 765 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; 766 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; 767 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; 768 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; 769 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; 770 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; 771 return uword; 772 } 773 774 void 775 qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, 776 unsigned char ae, 777 unsigned int uaddr, 778 unsigned int words_num, 779 const uint64_t *uword) 780 { 781 unsigned int ustore_addr; 782 unsigned int i, ae_in_group; 783 784 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 785 ae_in_group = ae / 4 * 4; 786 787 for (i = 0; i < AE_TG_NUM_CPM2X; i++) { 788 if (ae_in_group + i == ae) 789 continue; 790 if (ae_in_group + i >= handle->hal_handle->ae_max_num) 791 break; 792 if (qat_hal_check_ae_active(handle, ae_in_group + i)) { 793 pr_err( 794 "ae%d in T_group is active, cannot write to ustore!\n", 795 ae_in_group + i); 796 return; 797 } 798 } 799 } 800 801 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 802 uaddr |= UA_ECS; 803 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 804 for (i = 0; i < words_num; i++) { 805 unsigned int uwrd_lo, uwrd_hi; 806 uint64_t tmp; 807 808 tmp = qat_hal_set_uword_ecc(uword[i]); 809 uwrd_lo = (unsigned int)(tmp & 0xffffffff); 810 uwrd_hi = (unsigned int)(tmp >> 0x20); 811 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 812 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 813 } 814 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 815 } 816 817 void 818 qat_hal_wr_coalesce_uwords(struct icp_qat_fw_loader_handle *handle, 819 unsigned char ae, 820 unsigned int uaddr, 821 unsigned int words_num, 822 u64 *uword) 823 { 824 u64 *even_uwrods, *odd_uwords; 825 unsigned char neigh_ae, odd_ae, even_ae; 826 int i, even_cpy_cnt = 0, odd_cpy_cnt = 0; 827 828 even_uwrods = 829 malloc(16 * 1024 * sizeof(*uword), M_QAT, M_WAITOK | M_ZERO); 830 odd_uwords = 831 malloc(16 * 1024 * sizeof(*uword), M_QAT, M_WAITOK | M_ZERO); 832 qat_hal_get_scs_neigh_ae(ae, &neigh_ae); 833 if (ae & 1) { 834 odd_ae = ae; 835 even_ae = neigh_ae; 836 } else { 837 odd_ae = neigh_ae; 838 even_ae = ae; 839 } 840 for (i = 0; i < words_num; i++) { 841 if ((uaddr + i) & 1) 842 odd_uwords[odd_cpy_cnt++] = uword[i]; 843 else 844 even_uwrods[even_cpy_cnt++] = uword[i]; 845 } 846 if (even_cpy_cnt) 847 qat_hal_wr_uwords(handle, 848 even_ae, 849 (uaddr + 1) / 2, 850 even_cpy_cnt, 851 even_uwrods); 852 if (odd_cpy_cnt) 853 qat_hal_wr_uwords( 854 handle, odd_ae, uaddr / 2, odd_cpy_cnt, odd_uwords); 855 free(even_uwrods, M_QAT); 856 free(odd_uwords, M_QAT); 857 } 858 859 static void 860 qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, 861 unsigned char ae, 862 unsigned int ctx_mask) 863 { 864 unsigned int ctx; 865 866 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); 867 ctx &= IGNORE_W1C_MASK; 868 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; 869 ctx |= (ctx_mask << CE_ENABLE_BITPOS); 870 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); 871 } 872 873 static void 874 qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) 875 { 876 unsigned char ae; 877 unsigned short reg; 878 unsigned long ae_mask = handle->hal_handle->ae_mask; 879 880 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 881 { 882 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { 883 qat_hal_init_rd_xfer( 884 handle, ae, 0, ICP_SR_RD_ABS, reg, 0); 885 qat_hal_init_rd_xfer( 886 handle, ae, 0, ICP_DR_RD_ABS, reg, 0); 887 } 888 } 889 } 890 891 static int 892 qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) 893 { 894 unsigned char ae; 895 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; 896 int times = MAX_RETRY_TIMES; 897 unsigned int csr_val = 0; 898 unsigned int savctx = 0; 899 unsigned int scs_flag = 0; 900 unsigned long ae_mask = handle->hal_handle->ae_mask; 901 int ret = 0; 902 903 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 904 { 905 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 906 scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS); 907 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); 908 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); 909 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val); 910 csr_val &= IGNORE_W1C_MASK; 911 if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 912 csr_val |= CE_NN_MODE; 913 } 914 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); 915 916 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 917 if (ae % 4 == 0) 918 qat_hal_wr_uwords(handle, 919 ae, 920 0, 921 ARRAY_SIZE(inst_CPM2X), 922 (const uint64_t *)inst_CPM2X); 923 } else { 924 qat_hal_wr_uwords(handle, 925 ae, 926 0, 927 ARRAY_SIZE(inst), 928 (const uint64_t *)inst); 929 } 930 qat_hal_wr_indr_csr(handle, 931 ae, 932 ctx_mask, 933 CTX_STS_INDIRECT, 934 handle->hal_handle->upc_mask & 935 INIT_PC_VALUE); 936 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); 937 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); 938 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); 939 qat_hal_wr_indr_csr( 940 handle, ae, ctx_mask, CTX_SIG_EVENTS_INDIRECT, 0); 941 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); 942 qat_hal_enable_ctx(handle, ae, ctx_mask); 943 } 944 945 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 946 { 947 /* wait for AE to finish */ 948 do { 949 ret = qat_hal_wait_cycles(handle, ae, 20, 1); 950 } while (ret && times--); 951 952 if (times < 0) { 953 pr_err("QAT: clear GPR of AE %d failed", ae); 954 return EINVAL; 955 } 956 qat_hal_disable_ctx(handle, ae, ctx_mask); 957 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 958 if (scs_flag) 959 csr_val |= (1 << MMC_SHARE_CS_BITPOS); 960 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); 961 qat_hal_wr_ae_csr(handle, 962 ae, 963 ACTIVE_CTX_STATUS, 964 savctx & ACS_ACNO); 965 qat_hal_wr_ae_csr(handle, 966 ae, 967 CTX_ENABLES, 968 INIT_CTX_ENABLE_VALUE); 969 qat_hal_wr_indr_csr(handle, 970 ae, 971 ctx_mask, 972 CTX_STS_INDIRECT, 973 handle->hal_handle->upc_mask & 974 INIT_PC_VALUE); 975 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); 976 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); 977 qat_hal_put_wakeup_event(handle, 978 ae, 979 ctx_mask, 980 INIT_WAKEUP_EVENTS_VALUE); 981 qat_hal_put_sig_event(handle, 982 ae, 983 ctx_mask, 984 INIT_SIG_EVENTS_VALUE); 985 } 986 return 0; 987 } 988 989 static int 990 qat_hal_check_imr(struct icp_qat_fw_loader_handle *handle) 991 { 992 device_t dev = accel_to_pci_dev(handle->accel_dev); 993 u8 reg_val = 0; 994 995 if (pci_get_device(GET_DEV(handle->accel_dev)) != 996 ADF_C3XXX_PCI_DEVICE_ID && 997 pci_get_device(GET_DEV(handle->accel_dev)) != 998 ADF_200XX_PCI_DEVICE_ID) 999 return 0; 1000 1001 reg_val = pci_read_config(dev, 0x04, 1); 1002 /* 1003 * PCI command register memory bit and rambaseaddr_lo address 1004 * are checked to confirm IMR2 is enabled in BIOS settings 1005 */ 1006 if ((reg_val & 0x2) && GET_FCU_CSR(handle, FCU_RAMBASE_ADDR_LO)) 1007 return 0; 1008 1009 return EINVAL; 1010 } 1011 1012 int 1013 qat_hal_init(struct adf_accel_dev *accel_dev) 1014 { 1015 unsigned char ae; 1016 unsigned int cap_offset, ae_offset, ep_offset; 1017 unsigned int sram_offset = 0; 1018 unsigned int max_en_ae_id = 0; 1019 int ret = 0; 1020 unsigned long ae_mask; 1021 struct icp_qat_fw_loader_handle *handle; 1022 if (!accel_dev) { 1023 return EFAULT; 1024 } 1025 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; 1026 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1027 struct adf_bar *misc_bar = 1028 &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; 1029 struct adf_bar *sram_bar; 1030 1031 handle = malloc(sizeof(*handle), M_QAT, M_WAITOK | M_ZERO); 1032 1033 handle->hal_misc_addr_v = misc_bar->virt_addr; 1034 handle->accel_dev = accel_dev; 1035 if (pci_get_device(GET_DEV(handle->accel_dev)) == 1036 ADF_DH895XCC_PCI_DEVICE_ID || 1037 IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 1038 sram_bar = 1039 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; 1040 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) 1041 sram_offset = 1042 0x400000 + accel_dev->aram_info->mmp_region_offset; 1043 handle->hal_sram_addr_v = sram_bar->virt_addr; 1044 handle->hal_sram_offset = sram_offset; 1045 handle->hal_sram_size = sram_bar->size; 1046 } 1047 GET_CSR_OFFSET(pci_get_device(GET_DEV(handle->accel_dev)), 1048 cap_offset, 1049 ae_offset, 1050 ep_offset); 1051 handle->hal_cap_g_ctl_csr_addr_v = cap_offset; 1052 handle->hal_cap_ae_xfer_csr_addr_v = ae_offset; 1053 handle->hal_ep_csr_addr_v = ep_offset; 1054 handle->hal_cap_ae_local_csr_addr_v = 1055 ((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); 1056 handle->fw_auth = (pci_get_device(GET_DEV(handle->accel_dev)) == 1057 ADF_DH895XCC_PCI_DEVICE_ID) ? 1058 false : 1059 true; 1060 if (handle->fw_auth && qat_hal_check_imr(handle)) { 1061 device_printf(GET_DEV(accel_dev), "IMR2 not enabled in BIOS\n"); 1062 ret = EINVAL; 1063 goto out_hal_handle; 1064 } 1065 1066 handle->hal_handle = 1067 malloc(sizeof(*handle->hal_handle), M_QAT, M_WAITOK | M_ZERO); 1068 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; 1069 handle->hal_handle->ae_mask = hw_data->ae_mask; 1070 handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; 1071 handle->hal_handle->slice_mask = hw_data->accel_mask; 1072 handle->cfg_ae_mask = 0xFFFFFFFF; 1073 /* create AE objects */ 1074 if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 1075 handle->hal_handle->upc_mask = 0xffff; 1076 handle->hal_handle->max_ustore = 0x2000; 1077 } else { 1078 handle->hal_handle->upc_mask = 0x1ffff; 1079 handle->hal_handle->max_ustore = 0x4000; 1080 } 1081 1082 ae_mask = hw_data->ae_mask; 1083 1084 for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) 1085 { 1086 handle->hal_handle->aes[ae].free_addr = 0; 1087 handle->hal_handle->aes[ae].free_size = 1088 handle->hal_handle->max_ustore; 1089 handle->hal_handle->aes[ae].ustore_size = 1090 handle->hal_handle->max_ustore; 1091 handle->hal_handle->aes[ae].live_ctx_mask = 1092 ICP_QAT_UCLO_AE_ALL_CTX; 1093 max_en_ae_id = ae; 1094 } 1095 handle->hal_handle->ae_max_num = max_en_ae_id + 1; 1096 /* take all AEs out of reset */ 1097 if (qat_hal_clr_reset(handle)) { 1098 device_printf(GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); 1099 ret = EIO; 1100 goto out_err; 1101 } 1102 qat_hal_clear_xfer(handle); 1103 if (!handle->fw_auth) { 1104 if (qat_hal_clear_gpr(handle)) { 1105 ret = EIO; 1106 goto out_err; 1107 } 1108 } 1109 1110 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ 1111 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1112 { 1113 unsigned int csr_val = 0; 1114 1115 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val); 1116 csr_val |= 0x1; 1117 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); 1118 } 1119 accel_dev->fw_loader->fw_loader = handle; 1120 return 0; 1121 1122 out_err: 1123 free(handle->hal_handle, M_QAT); 1124 out_hal_handle: 1125 free(handle, M_QAT); 1126 return ret; 1127 } 1128 1129 void 1130 qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) 1131 { 1132 if (!handle) 1133 return; 1134 free(handle->hal_handle, M_QAT); 1135 free(handle, M_QAT); 1136 } 1137 1138 int 1139 qat_hal_start(struct icp_qat_fw_loader_handle *handle) 1140 { 1141 unsigned char ae = 0; 1142 int retry = 0; 1143 unsigned int fcu_sts = 0; 1144 unsigned int fcu_ctl_csr, fcu_sts_csr; 1145 unsigned long ae_mask = handle->hal_handle->ae_mask; 1146 u32 ae_ctr = 0; 1147 1148 if (handle->fw_auth) { 1149 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1150 { 1151 ae_ctr++; 1152 } 1153 if (IS_QAT_GEN3_OR_GEN4( 1154 pci_get_device(GET_DEV(handle->accel_dev)))) { 1155 fcu_ctl_csr = FCU_CONTROL_C4XXX; 1156 fcu_sts_csr = FCU_STATUS_C4XXX; 1157 1158 } else { 1159 fcu_ctl_csr = FCU_CONTROL; 1160 fcu_sts_csr = FCU_STATUS; 1161 } 1162 SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); 1163 do { 1164 pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); 1165 fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); 1166 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) 1167 return ae_ctr; 1168 } while (retry++ < FW_AUTH_MAX_RETRY); 1169 pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", 1170 ae, 1171 fcu_sts); 1172 return 0; 1173 } else { 1174 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1175 { 1176 qat_hal_put_wakeup_event(handle, 1177 ae, 1178 0, 1179 IS_QAT_GEN4( 1180 pci_get_device(GET_DEV( 1181 handle->accel_dev))) ? 1182 0x80000000 : 1183 0x10000); 1184 qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); 1185 ae_ctr++; 1186 } 1187 return ae_ctr; 1188 } 1189 } 1190 1191 void 1192 qat_hal_stop(struct icp_qat_fw_loader_handle *handle, 1193 unsigned char ae, 1194 unsigned int ctx_mask) 1195 { 1196 if (!handle->fw_auth) 1197 qat_hal_disable_ctx(handle, ae, ctx_mask); 1198 } 1199 1200 void 1201 qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, 1202 unsigned char ae, 1203 unsigned int ctx_mask, 1204 unsigned int upc) 1205 { 1206 qat_hal_wr_indr_csr(handle, 1207 ae, 1208 ctx_mask, 1209 CTX_STS_INDIRECT, 1210 handle->hal_handle->upc_mask & upc); 1211 } 1212 1213 static void 1214 qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, 1215 unsigned char ae, 1216 unsigned int uaddr, 1217 unsigned int words_num, 1218 uint64_t *uword) 1219 { 1220 unsigned int i, uwrd_lo, uwrd_hi; 1221 unsigned int ustore_addr, misc_control; 1222 unsigned int scs_flag = 0; 1223 1224 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control); 1225 scs_flag = misc_control & (0x1 << MMC_SHARE_CS_BITPOS); 1226 /*disable scs*/ 1227 qat_hal_wr_ae_csr(handle, 1228 ae, 1229 AE_MISC_CONTROL, 1230 misc_control & 0xfffffffb); 1231 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 1232 uaddr |= UA_ECS; 1233 for (i = 0; i < words_num; i++) { 1234 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 1235 uaddr++; 1236 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo); 1237 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi); 1238 uword[i] = uwrd_hi; 1239 uword[i] = (uword[i] << 0x20) | uwrd_lo; 1240 } 1241 if (scs_flag) 1242 misc_control |= (0x1 << MMC_SHARE_CS_BITPOS); 1243 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); 1244 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 1245 } 1246 1247 void 1248 qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, 1249 unsigned char ae, 1250 unsigned int uaddr, 1251 unsigned int words_num, 1252 unsigned int *data) 1253 { 1254 unsigned int i, ustore_addr; 1255 1256 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 1257 uaddr |= UA_ECS; 1258 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 1259 for (i = 0; i < words_num; i++) { 1260 unsigned int uwrd_lo, uwrd_hi, tmp; 1261 1262 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | 1263 ((data[i] & 0xff00) << 2) | (0x3 << 8) | (data[i] & 0xff); 1264 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); 1265 uwrd_hi |= (bitcount32(data[i] & 0xffff) & 0x1) << 8; 1266 tmp = ((data[i] >> 0x10) & 0xffff); 1267 uwrd_hi |= (bitcount32(tmp) & 0x1) << 9; 1268 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 1269 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 1270 } 1271 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 1272 } 1273 1274 #define MAX_EXEC_INST 100 1275 static int 1276 qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, 1277 unsigned char ae, 1278 unsigned char ctx, 1279 uint64_t *micro_inst, 1280 unsigned int inst_num, 1281 int code_off, 1282 unsigned int max_cycle, 1283 unsigned int *endpc) 1284 { 1285 u64 *savuwords = NULL; 1286 unsigned int ind_lm_addr0, ind_lm_addr1; 1287 unsigned int ind_lm_addr2, ind_lm_addr3; 1288 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; 1289 unsigned int ind_lm_addr_byte2, ind_lm_addr_byte3; 1290 unsigned int ind_t_index, ind_t_index_byte; 1291 unsigned int ind_cnt_sig; 1292 unsigned int ind_sig, act_sig; 1293 unsigned int csr_val = 0, newcsr_val; 1294 unsigned int savctx, scs_flag; 1295 unsigned int savcc, wakeup_events, savpc; 1296 unsigned int ctxarb_ctl, ctx_enables; 1297 1298 if (inst_num > handle->hal_handle->max_ustore || !micro_inst) { 1299 pr_err("QAT: invalid instruction num %d\n", inst_num); 1300 return EINVAL; 1301 } 1302 1303 savuwords = kzalloc(sizeof(u64) * MAX_EXEC_INST, GFP_KERNEL); 1304 if (!savuwords) 1305 return ENOMEM; 1306 1307 /* save current context */ 1308 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0); 1309 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1); 1310 qat_hal_rd_indr_csr( 1311 handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &ind_lm_addr_byte0); 1312 qat_hal_rd_indr_csr( 1313 handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &ind_lm_addr_byte1); 1314 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1315 qat_hal_rd_indr_csr( 1316 handle, ae, ctx, LM_ADDR_2_INDIRECT, &ind_lm_addr2); 1317 qat_hal_rd_indr_csr( 1318 handle, ae, ctx, LM_ADDR_3_INDIRECT, &ind_lm_addr3); 1319 qat_hal_rd_indr_csr(handle, 1320 ae, 1321 ctx, 1322 INDIRECT_LM_ADDR_2_BYTE_INDEX, 1323 &ind_lm_addr_byte2); 1324 qat_hal_rd_indr_csr(handle, 1325 ae, 1326 ctx, 1327 INDIRECT_LM_ADDR_3_BYTE_INDEX, 1328 &ind_lm_addr_byte3); 1329 qat_hal_rd_indr_csr( 1330 handle, ae, ctx, INDIRECT_T_INDEX, &ind_t_index); 1331 qat_hal_rd_indr_csr(handle, 1332 ae, 1333 ctx, 1334 INDIRECT_T_INDEX_BYTE_INDEX, 1335 &ind_t_index_byte); 1336 } 1337 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 1338 scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS); 1339 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 1340 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 1341 if (inst_num <= MAX_EXEC_INST) 1342 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); 1343 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); 1344 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc); 1345 savpc = (savpc & handle->hal_handle->upc_mask) >> 0; 1346 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 1347 ctx_enables &= IGNORE_W1C_MASK; 1348 qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc); 1349 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); 1350 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl); 1351 qat_hal_rd_indr_csr( 1352 handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, &ind_cnt_sig); 1353 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig); 1354 qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig); 1355 /* execute micro codes */ 1356 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1357 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); 1358 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); 1359 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); 1360 if (code_off) 1361 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); 1362 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); 1363 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); 1364 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1365 qat_hal_enable_ctx(handle, ae, (1 << ctx)); 1366 /* wait for micro codes to finish */ 1367 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) { 1368 kfree(savuwords); 1369 return EFAULT; 1370 } 1371 if (endpc) { 1372 unsigned int ctx_status; 1373 1374 qat_hal_rd_indr_csr( 1375 handle, ae, ctx, CTX_STS_INDIRECT, &ctx_status); 1376 *endpc = ctx_status & handle->hal_handle->upc_mask; 1377 } 1378 /* retore to saved context */ 1379 qat_hal_disable_ctx(handle, ae, (1 << ctx)); 1380 if (inst_num <= MAX_EXEC_INST) 1381 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); 1382 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); 1383 qat_hal_wr_indr_csr(handle, 1384 ae, 1385 (1 << ctx), 1386 CTX_STS_INDIRECT, 1387 handle->hal_handle->upc_mask & savpc); 1388 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 1389 newcsr_val = scs_flag ? SET_BIT(csr_val, MMC_SHARE_CS_BITPOS) : 1390 CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 1391 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 1392 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); 1393 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); 1394 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); 1395 qat_hal_wr_indr_csr( 1396 handle, ae, (1 << ctx), LM_ADDR_0_INDIRECT, ind_lm_addr0); 1397 qat_hal_wr_indr_csr( 1398 handle, ae, (1 << ctx), LM_ADDR_1_INDIRECT, ind_lm_addr1); 1399 qat_hal_wr_indr_csr(handle, 1400 ae, 1401 (1 << ctx), 1402 INDIRECT_LM_ADDR_0_BYTE_INDEX, 1403 ind_lm_addr_byte0); 1404 qat_hal_wr_indr_csr(handle, 1405 ae, 1406 (1 << ctx), 1407 INDIRECT_LM_ADDR_1_BYTE_INDEX, 1408 ind_lm_addr_byte1); 1409 if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1410 qat_hal_wr_indr_csr( 1411 handle, ae, (1 << ctx), LM_ADDR_2_INDIRECT, ind_lm_addr2); 1412 qat_hal_wr_indr_csr( 1413 handle, ae, (1 << ctx), LM_ADDR_3_INDIRECT, ind_lm_addr3); 1414 qat_hal_wr_indr_csr(handle, 1415 ae, 1416 (1 << ctx), 1417 INDIRECT_LM_ADDR_2_BYTE_INDEX, 1418 ind_lm_addr_byte2); 1419 qat_hal_wr_indr_csr(handle, 1420 ae, 1421 (1 << ctx), 1422 INDIRECT_LM_ADDR_3_BYTE_INDEX, 1423 ind_lm_addr_byte3); 1424 qat_hal_wr_indr_csr( 1425 handle, ae, (1 << ctx), INDIRECT_T_INDEX, ind_t_index); 1426 qat_hal_wr_indr_csr(handle, 1427 ae, 1428 (1 << ctx), 1429 INDIRECT_T_INDEX_BYTE_INDEX, 1430 ind_t_index_byte); 1431 } 1432 qat_hal_wr_indr_csr( 1433 handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); 1434 qat_hal_wr_indr_csr( 1435 handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, ind_sig); 1436 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); 1437 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1438 kfree(savuwords); 1439 1440 return 0; 1441 } 1442 1443 static int 1444 qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, 1445 unsigned char ae, 1446 unsigned char ctx, 1447 enum icp_qat_uof_regtype reg_type, 1448 unsigned short reg_num, 1449 unsigned int *data) 1450 { 1451 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; 1452 unsigned int ctxarb_cntl, ustore_addr, ctx_enables; 1453 unsigned short reg_addr; 1454 int status = 0; 1455 unsigned int scs_flag = 0; 1456 unsigned int csr_val = 0, newcsr_val = 0; 1457 u64 insts, savuword; 1458 1459 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); 1460 if (reg_addr == BAD_REGADDR) { 1461 pr_err("QAT: bad regaddr=0x%x\n", reg_addr); 1462 return EINVAL; 1463 } 1464 switch (reg_type) { 1465 case ICP_GPA_REL: 1466 insts = 0xA070000000ull | (reg_addr & 0x3ff); 1467 break; 1468 default: 1469 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10); 1470 break; 1471 } 1472 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 1473 scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS); 1474 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 1475 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 1476 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); 1477 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl); 1478 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 1479 ctx_enables &= IGNORE_W1C_MASK; 1480 if (ctx != (savctx & ACS_ACNO)) 1481 qat_hal_wr_ae_csr(handle, 1482 ae, 1483 ACTIVE_CTX_STATUS, 1484 ctx & ACS_ACNO); 1485 qat_hal_get_uwords(handle, ae, 0, 1, &savuword); 1486 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1487 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 1488 uaddr = UA_ECS; 1489 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 1490 insts = qat_hal_set_uword_ecc(insts); 1491 uwrd_lo = (unsigned int)(insts & 0xffffffff); 1492 uwrd_hi = (unsigned int)(insts >> 0x20); 1493 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 1494 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 1495 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 1496 /* delay for at least 8 cycles */ 1497 qat_hal_wait_cycles(handle, ae, 0x8, 0); 1498 /* 1499 * read ALU output 1500 * the instruction should have been executed 1501 * prior to clearing the ECS in putUwords 1502 */ 1503 qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data); 1504 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 1505 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); 1506 if (ctx != (savctx & ACS_ACNO)) 1507 qat_hal_wr_ae_csr(handle, 1508 ae, 1509 ACTIVE_CTX_STATUS, 1510 savctx & ACS_ACNO); 1511 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); 1512 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 1513 newcsr_val = scs_flag ? SET_BIT(csr_val, MMC_SHARE_CS_BITPOS) : 1514 CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 1515 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 1516 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1517 1518 return status; 1519 } 1520 1521 static int 1522 qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, 1523 unsigned char ae, 1524 unsigned char ctx, 1525 enum icp_qat_uof_regtype reg_type, 1526 unsigned short reg_num, 1527 unsigned int data) 1528 { 1529 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; 1530 uint64_t insts[] = { 0x0F440000000ull, 1531 0x0F040000000ull, 1532 0x0F0000C0300ull, 1533 0x0E000010000ull }; 1534 const int num_inst = ARRAY_SIZE(insts), code_off = 1; 1535 const int imm_w1 = 0, imm_w0 = 1; 1536 1537 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); 1538 if (dest_addr == BAD_REGADDR) { 1539 pr_err("QAT: bad destAddr=0x%x\n", dest_addr); 1540 return EINVAL; 1541 } 1542 1543 data16lo = 0xffff & data; 1544 data16hi = 0xffff & (data >> 0x10); 1545 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 1546 (unsigned short)(0xff & data16hi)); 1547 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 1548 (unsigned short)(0xff & data16lo)); 1549 switch (reg_type) { 1550 case ICP_GPA_REL: 1551 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | 1552 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); 1553 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | 1554 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); 1555 break; 1556 default: 1557 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | 1558 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); 1559 1560 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | 1561 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); 1562 break; 1563 } 1564 1565 return qat_hal_exec_micro_inst( 1566 handle, ae, ctx, insts, num_inst, code_off, num_inst * 0x5, NULL); 1567 } 1568 1569 int 1570 qat_hal_get_ins_num(void) 1571 { 1572 return ARRAY_SIZE(inst_4b); 1573 } 1574 1575 static int 1576 qat_hal_concat_micro_code(uint64_t *micro_inst, 1577 unsigned int inst_num, 1578 unsigned int size, 1579 unsigned int addr, 1580 unsigned int *value) 1581 { 1582 int i; 1583 unsigned int cur_value; 1584 const uint64_t *inst_arr; 1585 unsigned int fixup_offset; 1586 int usize = 0; 1587 unsigned int orig_num; 1588 unsigned int delta; 1589 1590 orig_num = inst_num; 1591 fixup_offset = inst_num; 1592 cur_value = value[0]; 1593 inst_arr = inst_4b; 1594 usize = ARRAY_SIZE(inst_4b); 1595 for (i = 0; i < usize; i++) 1596 micro_inst[inst_num++] = inst_arr[i]; 1597 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); 1598 fixup_offset++; 1599 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); 1600 fixup_offset++; 1601 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); 1602 fixup_offset++; 1603 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); 1604 1605 delta = inst_num - orig_num; 1606 1607 return (int)delta; 1608 } 1609 1610 static int 1611 qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, 1612 unsigned char ae, 1613 unsigned char ctx, 1614 int *pfirst_exec, 1615 uint64_t *micro_inst, 1616 unsigned int inst_num) 1617 { 1618 int stat = 0; 1619 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; 1620 unsigned int gprb0 = 0, gprb1 = 0; 1621 1622 if (*pfirst_exec) { 1623 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); 1624 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); 1625 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); 1626 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); 1627 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); 1628 *pfirst_exec = 0; 1629 } 1630 stat = qat_hal_exec_micro_inst( 1631 handle, ae, ctx, micro_inst, inst_num, 1, inst_num * 0x5, NULL); 1632 if (stat != 0) 1633 return EFAULT; 1634 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); 1635 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); 1636 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); 1637 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); 1638 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); 1639 1640 return 0; 1641 } 1642 1643 int 1644 qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, 1645 unsigned char ae, 1646 struct icp_qat_uof_batch_init *lm_init_header) 1647 { 1648 struct icp_qat_uof_batch_init *plm_init; 1649 uint64_t *micro_inst_arry; 1650 int micro_inst_num; 1651 int alloc_inst_size; 1652 int first_exec = 1; 1653 int stat = 0; 1654 1655 if (!lm_init_header) 1656 return 0; 1657 plm_init = lm_init_header->next; 1658 alloc_inst_size = lm_init_header->size; 1659 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) 1660 alloc_inst_size = handle->hal_handle->max_ustore; 1661 micro_inst_arry = malloc(alloc_inst_size * sizeof(uint64_t), 1662 M_QAT, 1663 M_WAITOK | M_ZERO); 1664 micro_inst_num = 0; 1665 while (plm_init) { 1666 unsigned int addr, *value, size; 1667 1668 ae = plm_init->ae; 1669 addr = plm_init->addr; 1670 value = plm_init->value; 1671 size = plm_init->size; 1672 micro_inst_num += qat_hal_concat_micro_code( 1673 micro_inst_arry, micro_inst_num, size, addr, value); 1674 plm_init = plm_init->next; 1675 } 1676 /* exec micro codes */ 1677 if (micro_inst_arry && micro_inst_num > 0) { 1678 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; 1679 stat = qat_hal_exec_micro_init_lm(handle, 1680 ae, 1681 0, 1682 &first_exec, 1683 micro_inst_arry, 1684 micro_inst_num); 1685 } 1686 free(micro_inst_arry, M_QAT); 1687 return stat; 1688 } 1689 1690 static int 1691 qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, 1692 unsigned char ae, 1693 unsigned char ctx, 1694 enum icp_qat_uof_regtype reg_type, 1695 unsigned short reg_num, 1696 unsigned int val) 1697 { 1698 int status = 0; 1699 unsigned int reg_addr; 1700 unsigned int ctx_enables; 1701 unsigned short mask; 1702 unsigned short dr_offset = 0x10; 1703 1704 status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 1705 if (CE_INUSE_CONTEXTS & ctx_enables) { 1706 if (ctx & 0x1) { 1707 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); 1708 return EINVAL; 1709 } 1710 mask = 0x1f; 1711 dr_offset = 0x20; 1712 } else { 1713 mask = 0x0f; 1714 } 1715 if (reg_num & ~mask) 1716 return EINVAL; 1717 reg_addr = reg_num + (ctx << 0x5); 1718 switch (reg_type) { 1719 case ICP_SR_RD_REL: 1720 case ICP_SR_REL: 1721 SET_AE_XFER(handle, ae, reg_addr, val); 1722 break; 1723 case ICP_DR_RD_REL: 1724 case ICP_DR_REL: 1725 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); 1726 break; 1727 default: 1728 status = EINVAL; 1729 break; 1730 } 1731 return status; 1732 } 1733 1734 static int 1735 qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, 1736 unsigned char ae, 1737 unsigned char ctx, 1738 enum icp_qat_uof_regtype reg_type, 1739 unsigned short reg_num, 1740 unsigned int data) 1741 { 1742 unsigned int gprval, ctx_enables; 1743 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, 1744 data16low; 1745 unsigned short reg_mask; 1746 int status = 0; 1747 uint64_t micro_inst[] = { 0x0F440000000ull, 1748 0x0F040000000ull, 1749 0x0A000000000ull, 1750 0x0F0000C0300ull, 1751 0x0E000010000ull }; 1752 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; 1753 const unsigned short gprnum = 0, dly = num_inst * 0x5; 1754 1755 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 1756 if (CE_INUSE_CONTEXTS & ctx_enables) { 1757 if (ctx & 0x1) { 1758 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); 1759 return EINVAL; 1760 } 1761 reg_mask = (unsigned short)~0x1f; 1762 } else { 1763 reg_mask = (unsigned short)~0xf; 1764 } 1765 if (reg_num & reg_mask) 1766 return EINVAL; 1767 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); 1768 if (xfr_addr == BAD_REGADDR) { 1769 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); 1770 return EINVAL; 1771 } 1772 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); 1773 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); 1774 data16low = 0xffff & data; 1775 data16hi = 0xffff & (data >> 0x10); 1776 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 1777 (unsigned short)(0xff & data16hi)); 1778 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 1779 (unsigned short)(0xff & data16low)); 1780 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | 1781 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); 1782 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | 1783 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); 1784 micro_inst[0x2] = micro_inst[0x2] | ((xfr_addr & 0x3ff) << 20) | 1785 ((gpr_addr & 0x3ff) << 10); 1786 status = qat_hal_exec_micro_inst( 1787 handle, ae, ctx, micro_inst, num_inst, code_off, dly, NULL); 1788 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); 1789 return status; 1790 } 1791 1792 static int 1793 qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, 1794 unsigned char ae, 1795 unsigned char ctx, 1796 unsigned short nn, 1797 unsigned int val) 1798 { 1799 unsigned int ctx_enables; 1800 int stat = 0; 1801 1802 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 1803 ctx_enables &= IGNORE_W1C_MASK; 1804 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); 1805 1806 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); 1807 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 1808 return stat; 1809 } 1810 1811 static int 1812 qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle *handle, 1813 unsigned char ae, 1814 unsigned short absreg_num, 1815 unsigned short *relreg, 1816 unsigned char *ctx) 1817 { 1818 unsigned int ctx_enables; 1819 1820 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 1821 if (ctx_enables & CE_INUSE_CONTEXTS) { 1822 /* 4-ctx mode */ 1823 *relreg = absreg_num & 0x1F; 1824 *ctx = (absreg_num >> 0x4) & 0x6; 1825 } else { 1826 /* 8-ctx mode */ 1827 *relreg = absreg_num & 0x0F; 1828 *ctx = (absreg_num >> 0x4) & 0x7; 1829 } 1830 return 0; 1831 } 1832 1833 int 1834 qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, 1835 unsigned char ae, 1836 unsigned long ctx_mask, 1837 enum icp_qat_uof_regtype reg_type, 1838 unsigned short reg_num, 1839 unsigned int regdata) 1840 { 1841 int stat = 0; 1842 unsigned short reg; 1843 unsigned char ctx = 0; 1844 enum icp_qat_uof_regtype type; 1845 1846 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) 1847 return EINVAL; 1848 1849 do { 1850 if (ctx_mask == 0) { 1851 qat_hal_convert_abs_to_rel( 1852 handle, ae, reg_num, ®, &ctx); 1853 type = reg_type - 1; 1854 } else { 1855 reg = reg_num; 1856 type = reg_type; 1857 if (!test_bit(ctx, &ctx_mask)) 1858 continue; 1859 } 1860 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); 1861 if (stat) { 1862 pr_err("QAT: write gpr fail\n"); 1863 return EINVAL; 1864 } 1865 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 1866 1867 return 0; 1868 } 1869 1870 int 1871 qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, 1872 unsigned char ae, 1873 unsigned long ctx_mask, 1874 enum icp_qat_uof_regtype reg_type, 1875 unsigned short reg_num, 1876 unsigned int regdata) 1877 { 1878 int stat = 0; 1879 unsigned short reg; 1880 unsigned char ctx = 0; 1881 enum icp_qat_uof_regtype type; 1882 1883 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) 1884 return EINVAL; 1885 1886 do { 1887 if (ctx_mask == 0) { 1888 qat_hal_convert_abs_to_rel( 1889 handle, ae, reg_num, ®, &ctx); 1890 type = reg_type - 3; 1891 } else { 1892 reg = reg_num; 1893 type = reg_type; 1894 if (!test_bit(ctx, &ctx_mask)) 1895 continue; 1896 } 1897 stat = qat_hal_put_rel_wr_xfer( 1898 handle, ae, ctx, type, reg, regdata); 1899 if (stat) { 1900 pr_err("QAT: write wr xfer fail\n"); 1901 return EINVAL; 1902 } 1903 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 1904 1905 return 0; 1906 } 1907 1908 int 1909 qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, 1910 unsigned char ae, 1911 unsigned long ctx_mask, 1912 enum icp_qat_uof_regtype reg_type, 1913 unsigned short reg_num, 1914 unsigned int regdata) 1915 { 1916 int stat = 0; 1917 unsigned short reg; 1918 unsigned char ctx = 0; 1919 enum icp_qat_uof_regtype type; 1920 1921 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) 1922 return EINVAL; 1923 1924 do { 1925 if (ctx_mask == 0) { 1926 qat_hal_convert_abs_to_rel( 1927 handle, ae, reg_num, ®, &ctx); 1928 type = reg_type - 3; 1929 } else { 1930 reg = reg_num; 1931 type = reg_type; 1932 if (!test_bit(ctx, &ctx_mask)) 1933 continue; 1934 } 1935 stat = qat_hal_put_rel_rd_xfer( 1936 handle, ae, ctx, type, reg, regdata); 1937 if (stat) { 1938 pr_err("QAT: write rd xfer fail\n"); 1939 return EINVAL; 1940 } 1941 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 1942 1943 return 0; 1944 } 1945 1946 int 1947 qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, 1948 unsigned char ae, 1949 unsigned long ctx_mask, 1950 unsigned short reg_num, 1951 unsigned int regdata) 1952 { 1953 int stat = 0; 1954 unsigned char ctx; 1955 1956 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1957 pr_err("QAT: No next neigh for CPM2X\n"); 1958 return EINVAL; 1959 } 1960 1961 if (ctx_mask == 0) 1962 return EINVAL; 1963 1964 for_each_set_bit(ctx, &ctx_mask, ICP_QAT_UCLO_MAX_CTX) 1965 { 1966 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); 1967 if (stat) { 1968 pr_err("QAT: write neigh error\n"); 1969 return EINVAL; 1970 } 1971 } 1972 1973 return 0; 1974 } 1975