1 /* SPDX-License-Identifier: GPL-2.0-only 2 * Copyright (C) 2020 Marvell. 3 */ 4 #ifndef __OTX2_CPTLF_H 5 #define __OTX2_CPTLF_H 6 7 #include <linux/soc/marvell/octeontx2/asm.h> 8 #include <linux/bitfield.h> 9 #include <mbox.h> 10 #include <rvu.h> 11 #include "otx2_cpt_common.h" 12 #include "otx2_cpt_reqmgr.h" 13 14 /* 15 * CPT instruction and pending queues user requested length in CPT_INST_S msgs 16 */ 17 #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 18 19 /* 20 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S 21 * messages. 22 */ 23 #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) 24 25 /* 26 * CPT instruction and pending queues length in CPT_INST_S messages 27 */ 28 #define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) 29 30 /* 31 * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction 32 * queue has less than 320 free entries. So, increase HW instruction queue 33 * size by 320 and give 320 entries less for SW/NIX RX as a workaround. 34 */ 35 #define OTX2_CPT_INST_QLEN_EXTRA_BYTES (320 * OTX2_CPT_INST_SIZE) 36 #define OTX2_CPT_EXTRA_SIZE_DIV40 (320/40) 37 38 /* CPT instruction queue length in bytes */ 39 #define OTX2_CPT_INST_QLEN_BYTES \ 40 ((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) + \ 41 OTX2_CPT_INST_QLEN_EXTRA_BYTES) 42 43 /* CPT instruction group queue length in bytes */ 44 #define OTX2_CPT_INST_GRP_QLEN_BYTES \ 45 ((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16) 46 47 /* CPT FC length in bytes */ 48 #define OTX2_CPT_Q_FC_LEN 128 49 50 /* CPT instruction queue alignment */ 51 #define OTX2_CPT_INST_Q_ALIGNMENT 128 52 53 /* Mask which selects all engine groups */ 54 #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF 55 56 /* Maximum LFs supported in OcteonTX2 for CPT */ 57 #define OTX2_CPT_MAX_LFS_NUM 64 58 59 /* Queue priority */ 60 #define OTX2_CPT_QUEUE_HI_PRIO 0x1 61 #define OTX2_CPT_QUEUE_LOW_PRIO 0x0 62 63 enum otx2_cptlf_state { 64 OTX2_CPTLF_IN_RESET, 65 OTX2_CPTLF_STARTED, 66 }; 67 68 struct otx2_cpt_inst_queue { 69 u8 *vaddr; 70 u8 *real_vaddr; 71 dma_addr_t dma_addr; 72 dma_addr_t real_dma_addr; 73 u32 size; 74 }; 75 76 struct otx2_cptlfs_info; 77 struct otx2_cptlf_wqe { 78 struct tasklet_struct work; 79 struct otx2_cptlfs_info *lfs; 80 u8 lf_num; 81 }; 82 83 struct otx2_cptlf_info { 84 struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ 85 void __iomem *lmtline; /* Address of LMTLINE */ 86 void __iomem *ioreg; /* LMTLINE send register */ 87 int msix_offset; /* MSI-X interrupts offset */ 88 cpumask_var_t affinity_mask; /* IRQs affinity mask */ 89 u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ 90 u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ 91 u8 slot; /* Slot number of this LF */ 92 93 struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ 94 struct otx2_cpt_pending_queue pqueue; /* Pending queue */ 95 struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ 96 }; 97 98 struct cpt_hw_ops { 99 void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num, 100 struct otx2_cptlf_info *lf); 101 u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result); 102 u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result); 103 struct otx2_cpt_inst_info * 104 (*cpt_sg_info_create)(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 105 gfp_t gfp); 106 }; 107 108 #define LMTLINE_SIZE 128 109 #define LMTLINE_ALIGN 128 110 struct otx2_lmt_info { 111 void *base; 112 dma_addr_t iova; 113 u32 size; 114 u8 align; 115 }; 116 117 struct otx2_cptlfs_info { 118 /* Registers start address of VF/PF LFs are attached to */ 119 void __iomem *reg_base; 120 struct otx2_lmt_info lmt_info; 121 struct pci_dev *pdev; /* Device LFs are attached to */ 122 struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; 123 struct otx2_mbox *mbox; 124 struct cpt_hw_ops *ops; 125 u8 are_lfs_attached; /* Whether CPT LFs are attached */ 126 u8 lfs_num; /* Number of CPT LFs */ 127 u8 kcrypto_se_eng_grp_num; /* Crypto symmetric engine group number */ 128 u8 kcrypto_ae_eng_grp_num; /* Crypto asymmetric engine group number */ 129 u8 kvf_limits; /* Kernel crypto limits */ 130 atomic_t state; /* LF's state. started/reset */ 131 int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */ 132 int global_slot; /* Global slot across the blocks */ 133 u8 ctx_ilen; 134 u8 ctx_ilen_ovrd; 135 }; 136 137 static inline void otx2_cpt_free_instruction_queues( 138 struct otx2_cptlfs_info *lfs) 139 { 140 struct otx2_cpt_inst_queue *iq; 141 int i; 142 143 for (i = 0; i < lfs->lfs_num; i++) { 144 iq = &lfs->lf[i].iqueue; 145 if (iq->real_vaddr) 146 dma_free_coherent(&lfs->pdev->dev, 147 iq->size, 148 iq->real_vaddr, 149 iq->real_dma_addr); 150 iq->real_vaddr = NULL; 151 iq->vaddr = NULL; 152 } 153 } 154 155 static inline int otx2_cpt_alloc_instruction_queues( 156 struct otx2_cptlfs_info *lfs) 157 { 158 struct otx2_cpt_inst_queue *iq; 159 int ret = 0, i; 160 161 if (!lfs->lfs_num) 162 return -EINVAL; 163 164 for (i = 0; i < lfs->lfs_num; i++) { 165 iq = &lfs->lf[i].iqueue; 166 iq->size = OTX2_CPT_INST_QLEN_BYTES + 167 OTX2_CPT_Q_FC_LEN + 168 OTX2_CPT_INST_GRP_QLEN_BYTES + 169 OTX2_CPT_INST_Q_ALIGNMENT; 170 iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, 171 &iq->real_dma_addr, GFP_KERNEL); 172 if (!iq->real_vaddr) { 173 ret = -ENOMEM; 174 goto error; 175 } 176 iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; 177 iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; 178 179 /* Align pointers */ 180 iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); 181 iq->dma_addr = PTR_ALIGN(iq->dma_addr, 182 OTX2_CPT_INST_Q_ALIGNMENT); 183 } 184 return 0; 185 186 error: 187 otx2_cpt_free_instruction_queues(lfs); 188 return ret; 189 } 190 191 static inline void otx2_cptlf_set_iqueues_base_addr( 192 struct otx2_cptlfs_info *lfs) 193 { 194 union otx2_cptx_lf_q_base lf_q_base; 195 int slot; 196 197 for (slot = 0; slot < lfs->lfs_num; slot++) { 198 lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; 199 otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, 200 OTX2_CPT_LF_Q_BASE, lf_q_base.u); 201 } 202 } 203 204 static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) 205 { 206 union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; 207 208 lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 + 209 OTX2_CPT_EXTRA_SIZE_DIV40; 210 otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, 211 OTX2_CPT_LF_Q_SIZE, lf_q_size.u); 212 } 213 214 static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) 215 { 216 int slot; 217 218 for (slot = 0; slot < lfs->lfs_num; slot++) 219 otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); 220 } 221 222 #define INFLIGHT GENMASK_ULL(8, 0) 223 #define GRB_CNT GENMASK_ULL(39, 32) 224 #define GWB_CNT GENMASK_ULL(47, 40) 225 #define XQ_XOR GENMASK_ULL(63, 63) 226 #define DQPTR GENMASK_ULL(19, 0) 227 #define NQPTR GENMASK_ULL(51, 32) 228 229 static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) 230 { 231 void __iomem *reg_base = lf->lfs->reg_base; 232 struct pci_dev *pdev = lf->lfs->pdev; 233 u8 blkaddr = lf->lfs->blkaddr; 234 int timeout = 1000000; 235 u64 inprog, inst_ptr; 236 u64 slot = lf->slot; 237 u64 qsize, pending; 238 int i = 0; 239 240 /* Disable instructions enqueuing */ 241 otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_CTL, 0x0); 242 243 inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG); 244 inprog |= BIT_ULL(16); 245 otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG, inprog); 246 247 qsize = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_SIZE) & 0x7FFF; 248 do { 249 inst_ptr = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_INST_PTR); 250 pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) + 251 FIELD_GET(NQPTR, inst_ptr) - FIELD_GET(DQPTR, inst_ptr); 252 udelay(1); 253 timeout--; 254 } while ((pending != 0) && (timeout != 0)); 255 256 if (timeout == 0) 257 dev_warn(&pdev->dev, "TIMEOUT: CPT poll on pending instructions\n"); 258 259 timeout = 1000000; 260 /* Wait for CPT queue to become execution-quiescent */ 261 do { 262 inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG); 263 264 if ((FIELD_GET(INFLIGHT, inprog) == 0) && 265 (FIELD_GET(GRB_CNT, inprog) == 0)) { 266 i++; 267 } else { 268 i = 0; 269 timeout--; 270 } 271 } while ((timeout != 0) && (i < 10)); 272 273 if (timeout == 0) 274 dev_warn(&pdev->dev, "TIMEOUT: CPT poll on inflight count\n"); 275 /* Wait for 2 us to flush all queue writes to memory */ 276 udelay(2); 277 } 278 279 static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) 280 { 281 int slot; 282 283 for (slot = 0; slot < lfs->lfs_num; slot++) { 284 otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); 285 otx2_cpt_lf_reset_msg(lfs, lfs->global_slot + slot); 286 } 287 } 288 289 static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, 290 bool enable) 291 { 292 u8 blkaddr = lf->lfs->blkaddr; 293 union otx2_cptx_lf_ctl lf_ctl; 294 295 lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, 296 OTX2_CPT_LF_CTL); 297 298 /* Set iqueue's enqueuing */ 299 lf_ctl.s.ena = enable ? 0x1 : 0x0; 300 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 301 OTX2_CPT_LF_CTL, lf_ctl.u); 302 } 303 304 static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) 305 { 306 otx2_cptlf_set_iqueue_enq(lf, true); 307 } 308 309 static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, 310 bool enable) 311 { 312 union otx2_cptx_lf_inprog lf_inprog; 313 u8 blkaddr = lf->lfs->blkaddr; 314 315 lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, 316 OTX2_CPT_LF_INPROG); 317 318 /* Set iqueue's execution */ 319 lf_inprog.s.eena = enable ? 0x1 : 0x0; 320 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 321 OTX2_CPT_LF_INPROG, lf_inprog.u); 322 } 323 324 static inline void otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info *lf) 325 { 326 u8 blkaddr = lf->lfs->blkaddr; 327 u64 val; 328 329 val = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, 330 OTX2_CPT_LF_CTX_CTL); 331 val |= BIT_ULL(0); 332 333 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 334 OTX2_CPT_LF_CTX_CTL, val); 335 } 336 337 static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) 338 { 339 otx2_cptlf_set_iqueue_exec(lf, true); 340 } 341 342 static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) 343 { 344 otx2_cptlf_set_iqueue_exec(lf, false); 345 } 346 347 static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) 348 { 349 int slot; 350 351 for (slot = 0; slot < lfs->lfs_num; slot++) { 352 /* Enable flush on FLR for Errata */ 353 if (is_dev_cn10kb(lfs->pdev)) 354 otx2_cptlf_set_ctx_flr_flush(&lfs->lf[slot]); 355 356 otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); 357 otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); 358 } 359 } 360 361 static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, 362 struct otx2_cpt_iq_command *iq_cmd, 363 u64 comp_baddr) 364 { 365 cptinst->u[0] = 0x0; 366 cptinst->s.doneint = true; 367 cptinst->s.res_addr = comp_baddr; 368 cptinst->u[2] = 0x0; 369 cptinst->u[3] = 0x0; 370 cptinst->s.ei0 = iq_cmd->cmd.u; 371 cptinst->s.ei1 = iq_cmd->dptr; 372 cptinst->s.ei2 = iq_cmd->rptr; 373 cptinst->s.ei3 = iq_cmd->cptr.u; 374 } 375 376 /* 377 * On OcteonTX2 platform the parameter insts_num is used as a count of 378 * instructions to be enqueued. The valid values for insts_num are: 379 * 1 - 1 CPT instruction will be enqueued during LMTST operation 380 * 2 - 2 CPT instructions will be enqueued during LMTST operation 381 */ 382 static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, 383 u32 insts_num, struct otx2_cptlf_info *lf) 384 { 385 void __iomem *lmtline = lf->lmtline; 386 long ret; 387 388 /* 389 * Make sure memory areas pointed in CPT_INST_S 390 * are flushed before the instruction is sent to CPT 391 */ 392 dma_wmb(); 393 394 do { 395 /* Copy CPT command to LMTLINE */ 396 memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); 397 398 /* 399 * LDEOR initiates atomic transfer to I/O device 400 * The following will cause the LMTST to fail (the LDEOR 401 * returns zero): 402 * - No stores have been performed to the LMTLINE since it was 403 * last invalidated. 404 * - The bytes which have been stored to LMTLINE since it was 405 * last invalidated form a pattern that is non-contiguous, does 406 * not start at byte 0, or does not end on a 8-byte boundary. 407 * (i.e.comprises a formation of other than 1–16 8-byte 408 * words.) 409 * 410 * These rules are designed such that an operating system 411 * context switch or hypervisor guest switch need have no 412 * knowledge of the LMTST operations; the switch code does not 413 * need to store to LMTCANCEL. Also note as LMTLINE data cannot 414 * be read, there is no information leakage between processes. 415 */ 416 ret = otx2_lmt_flush(lf->ioreg); 417 418 } while (!ret); 419 } 420 421 static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) 422 { 423 return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; 424 } 425 426 static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs, 427 struct pci_dev *pdev, 428 void __iomem *reg_base, 429 struct otx2_mbox *mbox, 430 int blkaddr) 431 { 432 lfs->pdev = pdev; 433 lfs->reg_base = reg_base; 434 lfs->mbox = mbox; 435 lfs->blkaddr = blkaddr; 436 } 437 438 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, 439 int lfs_num); 440 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); 441 int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs); 442 int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs); 443 void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs); 444 void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs); 445 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); 446 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); 447 448 #endif /* __OTX2_CPTLF_H */ 449