1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Marvell. */ 3 4 #include "otx2_cptvf.h" 5 #include "otx2_cpt_common.h" 6 7 /* SG list header size in bytes */ 8 #define SG_LIST_HDR_SIZE 8 9 10 /* Default timeout when waiting for free pending entry in us */ 11 #define CPT_PENTRY_TIMEOUT 1000 12 #define CPT_PENTRY_STEP 50 13 14 /* Default threshold for stopping and resuming sender requests */ 15 #define CPT_IQ_STOP_MARGIN 128 16 #define CPT_IQ_RESUME_MARGIN 512 17 18 /* Default command timeout in seconds */ 19 #define CPT_COMMAND_TIMEOUT 4 20 #define CPT_TIME_IN_RESET_COUNT 5 21 22 static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, 23 struct otx2_cpt_req_info *req) 24 { 25 int i; 26 27 pr_debug("Gather list size %d\n", req->in_cnt); 28 for (i = 0; i < req->in_cnt; i++) { 29 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, 30 req->in[i].size, req->in[i].vptr, 31 (void *) req->in[i].dma_addr); 32 pr_debug("Buffer hexdump (%d bytes)\n", 33 req->in[i].size); 34 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, 35 req->in[i].vptr, req->in[i].size, false); 36 } 37 pr_debug("Scatter list size %d\n", req->out_cnt); 38 for (i = 0; i < req->out_cnt; i++) { 39 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, 40 req->out[i].size, req->out[i].vptr, 41 (void *) req->out[i].dma_addr); 42 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); 43 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, 44 req->out[i].vptr, req->out[i].size, false); 45 } 46 } 47 48 static inline struct otx2_cpt_pending_entry *get_free_pending_entry( 49 struct otx2_cpt_pending_queue *q, 50 int qlen) 51 { 52 struct otx2_cpt_pending_entry *ent = NULL; 53 54 ent = &q->head[q->rear]; 55 if (unlikely(ent->busy)) 56 return NULL; 57 58 q->rear++; 59 if (unlikely(q->rear == qlen)) 60 q->rear = 0; 61 62 return ent; 63 } 64 65 static inline u32 modulo_inc(u32 index, u32 length, u32 inc) 66 { 67 if (WARN_ON(inc > length)) 68 inc = length; 69 70 index += inc; 71 if (unlikely(index >= length)) 72 index -= length; 73 74 return index; 75 } 76 77 static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) 78 { 79 pentry->completion_addr = NULL; 80 pentry->info = NULL; 81 pentry->callback = NULL; 82 pentry->areq = NULL; 83 pentry->resume_sender = false; 84 pentry->busy = false; 85 } 86 87 static inline int setup_sgio_components(struct pci_dev *pdev, 88 struct otx2_cpt_buf_ptr *list, 89 int buf_count, u8 *buffer) 90 { 91 struct otx2_cpt_sglist_component *sg_ptr = NULL; 92 int ret = 0, i, j; 93 int components; 94 95 if (unlikely(!list)) { 96 dev_err(&pdev->dev, "Input list pointer is NULL\n"); 97 return -EFAULT; 98 } 99 100 for (i = 0; i < buf_count; i++) { 101 if (unlikely(!list[i].vptr)) 102 continue; 103 list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, 104 list[i].size, 105 DMA_BIDIRECTIONAL); 106 if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { 107 dev_err(&pdev->dev, "Dma mapping failed\n"); 108 ret = -EIO; 109 goto sg_cleanup; 110 } 111 } 112 components = buf_count / 4; 113 sg_ptr = (struct otx2_cpt_sglist_component *)buffer; 114 for (i = 0; i < components; i++) { 115 sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); 116 sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); 117 sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); 118 sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size); 119 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 120 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 121 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 122 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); 123 sg_ptr++; 124 } 125 components = buf_count % 4; 126 127 switch (components) { 128 case 3: 129 sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); 130 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 131 fallthrough; 132 case 2: 133 sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); 134 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 135 fallthrough; 136 case 1: 137 sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); 138 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 139 break; 140 default: 141 break; 142 } 143 return ret; 144 145 sg_cleanup: 146 for (j = 0; j < i; j++) { 147 if (list[j].dma_addr) { 148 dma_unmap_single(&pdev->dev, list[j].dma_addr, 149 list[j].size, DMA_BIDIRECTIONAL); 150 } 151 152 list[j].dma_addr = 0; 153 } 154 return ret; 155 } 156 157 static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev, 158 struct otx2_cpt_req_info *req, 159 gfp_t gfp) 160 { 161 int align = OTX2_CPT_DMA_MINALIGN; 162 struct otx2_cpt_inst_info *info; 163 u32 dlen, align_dlen, info_len; 164 u16 g_sz_bytes, s_sz_bytes; 165 u32 total_mem_len; 166 167 if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || 168 req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { 169 dev_err(&pdev->dev, "Error too many sg components\n"); 170 return NULL; 171 } 172 173 g_sz_bytes = ((req->in_cnt + 3) / 4) * 174 sizeof(struct otx2_cpt_sglist_component); 175 s_sz_bytes = ((req->out_cnt + 3) / 4) * 176 sizeof(struct otx2_cpt_sglist_component); 177 178 dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; 179 align_dlen = ALIGN(dlen, align); 180 info_len = ALIGN(sizeof(*info), align); 181 total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); 182 183 info = kzalloc(total_mem_len, gfp); 184 if (unlikely(!info)) 185 return NULL; 186 187 info->dlen = dlen; 188 info->in_buffer = (u8 *)info + info_len; 189 190 ((u16 *)info->in_buffer)[0] = req->out_cnt; 191 ((u16 *)info->in_buffer)[1] = req->in_cnt; 192 ((u16 *)info->in_buffer)[2] = 0; 193 ((u16 *)info->in_buffer)[3] = 0; 194 cpu_to_be64s((u64 *)info->in_buffer); 195 196 /* Setup gather (input) components */ 197 if (setup_sgio_components(pdev, req->in, req->in_cnt, 198 &info->in_buffer[8])) { 199 dev_err(&pdev->dev, "Failed to setup gather list\n"); 200 goto destroy_info; 201 } 202 203 if (setup_sgio_components(pdev, req->out, req->out_cnt, 204 &info->in_buffer[8 + g_sz_bytes])) { 205 dev_err(&pdev->dev, "Failed to setup scatter list\n"); 206 goto destroy_info; 207 } 208 209 info->dma_len = total_mem_len - info_len; 210 info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, 211 info->dma_len, DMA_BIDIRECTIONAL); 212 if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { 213 dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); 214 goto destroy_info; 215 } 216 /* 217 * Get buffer for union otx2_cpt_res_s response 218 * structure and its physical address 219 */ 220 info->completion_addr = info->in_buffer + align_dlen; 221 info->comp_baddr = info->dptr_baddr + align_dlen; 222 223 return info; 224 225 destroy_info: 226 otx2_cpt_info_destroy(pdev, info); 227 return NULL; 228 } 229 230 static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 231 struct otx2_cpt_pending_queue *pqueue, 232 struct otx2_cptlf_info *lf) 233 { 234 struct otx2_cptvf_request *cpt_req = &req->req; 235 struct otx2_cpt_pending_entry *pentry = NULL; 236 union otx2_cpt_ctrl_info *ctrl = &req->ctrl; 237 struct otx2_cpt_inst_info *info = NULL; 238 union otx2_cpt_res_s *result = NULL; 239 struct otx2_cpt_iq_command iq_cmd; 240 union otx2_cpt_inst_s cptinst; 241 int retry, ret = 0; 242 u8 resume_sender; 243 gfp_t gfp; 244 245 gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : 246 GFP_ATOMIC; 247 if (unlikely(!otx2_cptlf_started(lf->lfs))) 248 return -ENODEV; 249 250 info = info_create(pdev, req, gfp); 251 if (unlikely(!info)) { 252 dev_err(&pdev->dev, "Setting up cpt inst info failed"); 253 return -ENOMEM; 254 } 255 cpt_req->dlen = info->dlen; 256 257 result = info->completion_addr; 258 result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; 259 260 spin_lock_bh(&pqueue->lock); 261 pentry = get_free_pending_entry(pqueue, pqueue->qlen); 262 retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; 263 while (unlikely(!pentry) && retry--) { 264 spin_unlock_bh(&pqueue->lock); 265 udelay(CPT_PENTRY_STEP); 266 spin_lock_bh(&pqueue->lock); 267 pentry = get_free_pending_entry(pqueue, pqueue->qlen); 268 } 269 270 if (unlikely(!pentry)) { 271 ret = -ENOSPC; 272 goto destroy_info; 273 } 274 275 /* 276 * Check if we are close to filling in entire pending queue, 277 * if so then tell the sender to stop/sleep by returning -EBUSY 278 * We do it only for context which can sleep (GFP_KERNEL) 279 */ 280 if (gfp == GFP_KERNEL && 281 pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { 282 pentry->resume_sender = true; 283 } else 284 pentry->resume_sender = false; 285 resume_sender = pentry->resume_sender; 286 pqueue->pending_count++; 287 288 pentry->completion_addr = info->completion_addr; 289 pentry->info = info; 290 pentry->callback = req->callback; 291 pentry->areq = req->areq; 292 pentry->busy = true; 293 info->pentry = pentry; 294 info->time_in = jiffies; 295 info->req = req; 296 297 /* Fill in the command */ 298 iq_cmd.cmd.u = 0; 299 iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); 300 iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); 301 iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); 302 iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); 303 304 /* 64-bit swap for microcode data reads, not needed for addresses*/ 305 cpu_to_be64s(&iq_cmd.cmd.u); 306 iq_cmd.dptr = info->dptr_baddr; 307 iq_cmd.rptr = 0; 308 iq_cmd.cptr.u = 0; 309 iq_cmd.cptr.s.grp = ctrl->s.grp; 310 311 /* Fill in the CPT_INST_S type command for HW interpretation */ 312 otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr); 313 314 /* Print debug info if enabled */ 315 otx2_cpt_dump_sg_list(pdev, req); 316 pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE); 317 print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); 318 pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); 319 print_hex_dump_debug("", 0, 16, 1, info->in_buffer, 320 cpt_req->dlen, false); 321 322 /* Send CPT command */ 323 lf->lfs->ops->send_cmd(&cptinst, 1, lf); 324 325 /* 326 * We allocate and prepare pending queue entry in critical section 327 * together with submitting CPT instruction to CPT instruction queue 328 * to make sure that order of CPT requests is the same in both 329 * pending and instruction queues 330 */ 331 spin_unlock_bh(&pqueue->lock); 332 333 ret = resume_sender ? -EBUSY : -EINPROGRESS; 334 return ret; 335 336 destroy_info: 337 spin_unlock_bh(&pqueue->lock); 338 otx2_cpt_info_destroy(pdev, info); 339 return ret; 340 } 341 342 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 343 int cpu_num) 344 { 345 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); 346 struct otx2_cptlfs_info *lfs = &cptvf->lfs; 347 348 return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue, 349 &lfs->lf[cpu_num]); 350 } 351 352 static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, 353 union otx2_cpt_res_s *cpt_status, 354 struct otx2_cpt_inst_info *info, 355 u32 *res_code) 356 { 357 u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status); 358 u8 ccode = lfs->ops->cpt_get_compcode(cpt_status); 359 struct pci_dev *pdev = lfs->pdev; 360 361 switch (ccode) { 362 case OTX2_CPT_COMP_E_FAULT: 363 dev_err(&pdev->dev, 364 "Request failed with DMA fault\n"); 365 otx2_cpt_dump_sg_list(pdev, info->req); 366 break; 367 368 case OTX2_CPT_COMP_E_HWERR: 369 dev_err(&pdev->dev, 370 "Request failed with hardware error\n"); 371 otx2_cpt_dump_sg_list(pdev, info->req); 372 break; 373 374 case OTX2_CPT_COMP_E_INSTERR: 375 dev_err(&pdev->dev, 376 "Request failed with instruction error\n"); 377 otx2_cpt_dump_sg_list(pdev, info->req); 378 break; 379 380 case OTX2_CPT_COMP_E_NOTDONE: 381 /* check for timeout */ 382 if (time_after_eq(jiffies, info->time_in + 383 CPT_COMMAND_TIMEOUT * HZ)) 384 dev_warn(&pdev->dev, 385 "Request timed out 0x%p", info->req); 386 else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { 387 info->time_in = jiffies; 388 info->extra_time++; 389 } 390 return 1; 391 392 case OTX2_CPT_COMP_E_GOOD: 393 case OTX2_CPT_COMP_E_WARN: 394 /* 395 * Check microcode completion code, it is only valid 396 * when completion code is CPT_COMP_E::GOOD 397 */ 398 if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { 399 /* 400 * If requested hmac is truncated and ucode returns 401 * s/g write length error then we report success 402 * because ucode writes as many bytes of calculated 403 * hmac as available in gather buffer and reports 404 * s/g write length error if number of bytes in gather 405 * buffer is less than full hmac size. 406 */ 407 if (info->req->is_trunc_hmac && 408 uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { 409 *res_code = 0; 410 break; 411 } 412 413 dev_err(&pdev->dev, 414 "Request failed with software error code 0x%x\n", 415 cpt_status->s.uc_compcode); 416 otx2_cpt_dump_sg_list(pdev, info->req); 417 break; 418 } 419 /* Request has been processed with success */ 420 *res_code = 0; 421 break; 422 423 default: 424 dev_err(&pdev->dev, 425 "Request returned invalid status %d\n", ccode); 426 break; 427 } 428 return 0; 429 } 430 431 static inline void process_pending_queue(struct otx2_cptlfs_info *lfs, 432 struct otx2_cpt_pending_queue *pqueue) 433 { 434 struct otx2_cpt_pending_entry *resume_pentry = NULL; 435 void (*callback)(int status, void *arg, void *req); 436 struct otx2_cpt_pending_entry *pentry = NULL; 437 union otx2_cpt_res_s *cpt_status = NULL; 438 struct otx2_cpt_inst_info *info = NULL; 439 struct otx2_cpt_req_info *req = NULL; 440 struct crypto_async_request *areq; 441 struct pci_dev *pdev = lfs->pdev; 442 u32 res_code, resume_index; 443 444 while (1) { 445 spin_lock_bh(&pqueue->lock); 446 pentry = &pqueue->head[pqueue->front]; 447 448 if (WARN_ON(!pentry)) { 449 spin_unlock_bh(&pqueue->lock); 450 break; 451 } 452 453 res_code = -EINVAL; 454 if (unlikely(!pentry->busy)) { 455 spin_unlock_bh(&pqueue->lock); 456 break; 457 } 458 459 if (unlikely(!pentry->callback)) { 460 dev_err(&pdev->dev, "Callback NULL\n"); 461 goto process_pentry; 462 } 463 464 info = pentry->info; 465 if (unlikely(!info)) { 466 dev_err(&pdev->dev, "Pending entry post arg NULL\n"); 467 goto process_pentry; 468 } 469 470 req = info->req; 471 if (unlikely(!req)) { 472 dev_err(&pdev->dev, "Request NULL\n"); 473 goto process_pentry; 474 } 475 476 cpt_status = pentry->completion_addr; 477 if (unlikely(!cpt_status)) { 478 dev_err(&pdev->dev, "Completion address NULL\n"); 479 goto process_pentry; 480 } 481 482 if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) { 483 spin_unlock_bh(&pqueue->lock); 484 return; 485 } 486 info->pdev = pdev; 487 488 process_pentry: 489 /* 490 * Check if we should inform sending side to resume 491 * We do it CPT_IQ_RESUME_MARGIN elements in advance before 492 * pending queue becomes empty 493 */ 494 resume_index = modulo_inc(pqueue->front, pqueue->qlen, 495 CPT_IQ_RESUME_MARGIN); 496 resume_pentry = &pqueue->head[resume_index]; 497 if (resume_pentry && 498 resume_pentry->resume_sender) { 499 resume_pentry->resume_sender = false; 500 callback = resume_pentry->callback; 501 areq = resume_pentry->areq; 502 503 if (callback) { 504 spin_unlock_bh(&pqueue->lock); 505 506 /* 507 * EINPROGRESS is an indication for sending 508 * side that it can resume sending requests 509 */ 510 callback(-EINPROGRESS, areq, info); 511 spin_lock_bh(&pqueue->lock); 512 } 513 } 514 515 callback = pentry->callback; 516 areq = pentry->areq; 517 free_pentry(pentry); 518 519 pqueue->pending_count--; 520 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); 521 spin_unlock_bh(&pqueue->lock); 522 523 /* 524 * Call callback after current pending entry has been 525 * processed, we don't do it if the callback pointer is 526 * invalid. 527 */ 528 if (callback) 529 callback(res_code, areq, info); 530 } 531 } 532 533 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) 534 { 535 process_pending_queue(wqe->lfs, 536 &wqe->lfs->lf[wqe->lf_num].pqueue); 537 } 538 539 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) 540 { 541 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); 542 543 return cptvf->lfs.kcrypto_eng_grp_num; 544 } 545