1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/gfp.h> 3 #include <linux/workqueue.h> 4 #include <crypto/internal/skcipher.h> 5 6 #include "nitrox_common.h" 7 #include "nitrox_dev.h" 8 #include "nitrox_req.h" 9 #include "nitrox_csr.h" 10 11 /* SLC_STORE_INFO */ 12 #define MIN_UDD_LEN 16 13 /* PKT_IN_HDR + SLC_STORE_INFO */ 14 #define FDATA_SIZE 32 15 /* Base destination port for the solicited requests */ 16 #define SOLICIT_BASE_DPORT 256 17 18 #define REQ_NOT_POSTED 1 19 #define REQ_BACKLOG 2 20 #define REQ_POSTED 3 21 22 /* 23 * Response codes from SE microcode 24 * 0x00 - Success 25 * Completion with no error 26 * 0x43 - ERR_GC_DATA_LEN_INVALID 27 * Invalid Data length if Encryption Data length is 28 * less than 16 bytes for AES-XTS and AES-CTS. 29 * 0x45 - ERR_GC_CTX_LEN_INVALID 30 * Invalid context length: CTXL != 23 words. 31 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID 32 * DOCSIS support is enabled with other than 33 * AES/DES-CBC mode encryption. 34 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID 35 * Authentication offset is other than 0 with 36 * Encryption IV source = 0. 37 * Authentication offset is other than 8 (DES)/16 (AES) 38 * with Encryption IV source = 1 39 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION 40 * CRC32 is enabled for other than DOCSIS encryption. 41 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID 42 * Invalid flag options in AES-CCM IV. 43 */ 44 45 static inline int incr_index(int index, int count, int max) 46 { 47 if ((index + count) >= max) 48 index = index + count - max; 49 else 50 index += count; 51 52 return index; 53 } 54 55 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) 56 { 57 struct nitrox_device *ndev = sr->ndev; 58 struct device *dev = DEV(ndev); 59 60 61 dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg), 62 DMA_BIDIRECTIONAL); 63 dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, 64 DMA_TO_DEVICE); 65 kfree(sr->in.sgcomp); 66 sr->in.sg = NULL; 67 sr->in.sgmap_cnt = 0; 68 69 dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg), 70 DMA_BIDIRECTIONAL); 71 dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, 72 DMA_TO_DEVICE); 73 kfree(sr->out.sgcomp); 74 sr->out.sg = NULL; 75 sr->out.sgmap_cnt = 0; 76 } 77 78 static void softreq_destroy(struct nitrox_softreq *sr) 79 { 80 softreq_unmap_sgbufs(sr); 81 kfree(sr); 82 } 83 84 /** 85 * create_sg_component - create SG componets for N5 device. 86 * @sr: Request structure 87 * @sgtbl: SG table 88 * @map_nents: number of dma mapped entries 89 * 90 * Component structure 91 * 92 * 63 48 47 32 31 16 15 0 93 * -------------------------------------- 94 * | LEN0 | LEN1 | LEN2 | LEN3 | 95 * |------------------------------------- 96 * | PTR0 | 97 * -------------------------------------- 98 * | PTR1 | 99 * -------------------------------------- 100 * | PTR2 | 101 * -------------------------------------- 102 * | PTR3 | 103 * -------------------------------------- 104 * 105 * Returns 0 if success or a negative errno code on error. 106 */ 107 static int create_sg_component(struct nitrox_softreq *sr, 108 struct nitrox_sgtable *sgtbl, int map_nents) 109 { 110 struct nitrox_device *ndev = sr->ndev; 111 struct nitrox_sgcomp *sgcomp; 112 struct scatterlist *sg; 113 dma_addr_t dma; 114 size_t sz_comp; 115 int i, j, nr_sgcomp; 116 117 nr_sgcomp = roundup(map_nents, 4) / 4; 118 119 /* each component holds 4 dma pointers */ 120 sz_comp = nr_sgcomp * sizeof(*sgcomp); 121 sgcomp = kzalloc(sz_comp, sr->gfp); 122 if (!sgcomp) 123 return -ENOMEM; 124 125 sgtbl->sgcomp = sgcomp; 126 127 sg = sgtbl->sg; 128 /* populate device sg component */ 129 for (i = 0; i < nr_sgcomp; i++) { 130 for (j = 0; j < 4 && sg; j++) { 131 sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg)); 132 sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg)); 133 sg = sg_next(sg); 134 } 135 } 136 /* map the device sg component */ 137 dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); 138 if (dma_mapping_error(DEV(ndev), dma)) { 139 kfree(sgtbl->sgcomp); 140 sgtbl->sgcomp = NULL; 141 return -ENOMEM; 142 } 143 144 sgtbl->sgcomp_dma = dma; 145 sgtbl->sgcomp_len = sz_comp; 146 147 return 0; 148 } 149 150 /** 151 * dma_map_inbufs - DMA map input sglist and creates sglist component 152 * for N5 device. 153 * @sr: Request structure 154 * @req: Crypto request structre 155 * 156 * Returns 0 if successful or a negative errno code on error. 157 */ 158 static int dma_map_inbufs(struct nitrox_softreq *sr, 159 struct se_crypto_request *req) 160 { 161 struct device *dev = DEV(sr->ndev); 162 struct scatterlist *sg; 163 int i, nents, ret = 0; 164 165 nents = dma_map_sg(dev, req->src, sg_nents(req->src), 166 DMA_BIDIRECTIONAL); 167 if (!nents) 168 return -EINVAL; 169 170 for_each_sg(req->src, sg, nents, i) 171 sr->in.total_bytes += sg_dma_len(sg); 172 173 sr->in.sg = req->src; 174 sr->in.sgmap_cnt = nents; 175 ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt); 176 if (ret) 177 goto incomp_err; 178 179 return 0; 180 181 incomp_err: 182 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); 183 sr->in.sgmap_cnt = 0; 184 return ret; 185 } 186 187 static int dma_map_outbufs(struct nitrox_softreq *sr, 188 struct se_crypto_request *req) 189 { 190 struct device *dev = DEV(sr->ndev); 191 int nents, ret = 0; 192 193 nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), 194 DMA_BIDIRECTIONAL); 195 if (!nents) 196 return -EINVAL; 197 198 sr->out.sg = req->dst; 199 sr->out.sgmap_cnt = nents; 200 ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt); 201 if (ret) 202 goto outcomp_map_err; 203 204 return 0; 205 206 outcomp_map_err: 207 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); 208 sr->out.sgmap_cnt = 0; 209 sr->out.sg = NULL; 210 return ret; 211 } 212 213 static inline int softreq_map_iobuf(struct nitrox_softreq *sr, 214 struct se_crypto_request *creq) 215 { 216 int ret; 217 218 ret = dma_map_inbufs(sr, creq); 219 if (ret) 220 return ret; 221 222 ret = dma_map_outbufs(sr, creq); 223 if (ret) 224 softreq_unmap_sgbufs(sr); 225 226 return ret; 227 } 228 229 static inline void backlog_list_add(struct nitrox_softreq *sr, 230 struct nitrox_cmdq *cmdq) 231 { 232 INIT_LIST_HEAD(&sr->backlog); 233 234 spin_lock_bh(&cmdq->backlog_qlock); 235 list_add_tail(&sr->backlog, &cmdq->backlog_head); 236 atomic_inc(&cmdq->backlog_count); 237 atomic_set(&sr->status, REQ_BACKLOG); 238 spin_unlock_bh(&cmdq->backlog_qlock); 239 } 240 241 static inline void response_list_add(struct nitrox_softreq *sr, 242 struct nitrox_cmdq *cmdq) 243 { 244 INIT_LIST_HEAD(&sr->response); 245 246 spin_lock_bh(&cmdq->resp_qlock); 247 list_add_tail(&sr->response, &cmdq->response_head); 248 spin_unlock_bh(&cmdq->resp_qlock); 249 } 250 251 static inline void response_list_del(struct nitrox_softreq *sr, 252 struct nitrox_cmdq *cmdq) 253 { 254 spin_lock_bh(&cmdq->resp_qlock); 255 list_del(&sr->response); 256 spin_unlock_bh(&cmdq->resp_qlock); 257 } 258 259 static struct nitrox_softreq * 260 get_first_response_entry(struct nitrox_cmdq *cmdq) 261 { 262 return list_first_entry_or_null(&cmdq->response_head, 263 struct nitrox_softreq, response); 264 } 265 266 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) 267 { 268 if (atomic_inc_return(&cmdq->pending_count) > qlen) { 269 atomic_dec(&cmdq->pending_count); 270 /* sync with other cpus */ 271 smp_mb__after_atomic(); 272 return true; 273 } 274 /* sync with other cpus */ 275 smp_mb__after_atomic(); 276 return false; 277 } 278 279 /** 280 * post_se_instr - Post SE instruction to Packet Input ring 281 * @sr: Request structure 282 * @cmdq: Command queue structure 283 * 284 * Returns 0 if successful or a negative error code, 285 * if no space in ring. 286 */ 287 static void post_se_instr(struct nitrox_softreq *sr, 288 struct nitrox_cmdq *cmdq) 289 { 290 struct nitrox_device *ndev = sr->ndev; 291 int idx; 292 u8 *ent; 293 294 spin_lock_bh(&cmdq->cmd_qlock); 295 296 idx = cmdq->write_idx; 297 /* copy the instruction */ 298 ent = cmdq->base + (idx * cmdq->instr_size); 299 memcpy(ent, &sr->instr, cmdq->instr_size); 300 301 atomic_set(&sr->status, REQ_POSTED); 302 response_list_add(sr, cmdq); 303 sr->tstamp = jiffies; 304 /* flush the command queue updates */ 305 dma_wmb(); 306 307 /* Ring doorbell with count 1 */ 308 writeq(1, cmdq->dbell_csr_addr); 309 310 cmdq->write_idx = incr_index(idx, 1, ndev->qlen); 311 312 spin_unlock_bh(&cmdq->cmd_qlock); 313 314 /* increment the posted command count */ 315 atomic64_inc(&ndev->stats.posted); 316 } 317 318 static int post_backlog_cmds(struct nitrox_cmdq *cmdq) 319 { 320 struct nitrox_device *ndev = cmdq->ndev; 321 struct nitrox_softreq *sr, *tmp; 322 int ret = 0; 323 324 if (!atomic_read(&cmdq->backlog_count)) 325 return 0; 326 327 spin_lock_bh(&cmdq->backlog_qlock); 328 329 list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { 330 /* submit until space available */ 331 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { 332 ret = -ENOSPC; 333 break; 334 } 335 /* delete from backlog list */ 336 list_del(&sr->backlog); 337 atomic_dec(&cmdq->backlog_count); 338 /* sync with other cpus */ 339 smp_mb__after_atomic(); 340 341 /* post the command */ 342 post_se_instr(sr, cmdq); 343 } 344 spin_unlock_bh(&cmdq->backlog_qlock); 345 346 return ret; 347 } 348 349 static int nitrox_enqueue_request(struct nitrox_softreq *sr) 350 { 351 struct nitrox_cmdq *cmdq = sr->cmdq; 352 struct nitrox_device *ndev = sr->ndev; 353 354 /* try to post backlog requests */ 355 post_backlog_cmds(cmdq); 356 357 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { 358 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 359 /* increment drop count */ 360 atomic64_inc(&ndev->stats.dropped); 361 return -ENOSPC; 362 } 363 /* add to backlog list */ 364 backlog_list_add(sr, cmdq); 365 return -EINPROGRESS; 366 } 367 post_se_instr(sr, cmdq); 368 369 return -EINPROGRESS; 370 } 371 372 /** 373 * nitrox_process_se_request - Send request to SE core 374 * @ndev: NITROX device 375 * @req: Crypto request 376 * @callback: Completion callback 377 * @cb_arg: Completion callback arguments 378 * 379 * Returns 0 on success, or a negative error code. 380 */ 381 int nitrox_process_se_request(struct nitrox_device *ndev, 382 struct se_crypto_request *req, 383 completion_t callback, 384 void *cb_arg) 385 { 386 struct nitrox_softreq *sr; 387 dma_addr_t ctx_handle = 0; 388 int qno, ret = 0; 389 390 if (!nitrox_ready(ndev)) 391 return -ENODEV; 392 393 sr = kzalloc(sizeof(*sr), req->gfp); 394 if (!sr) 395 return -ENOMEM; 396 397 sr->ndev = ndev; 398 sr->flags = req->flags; 399 sr->gfp = req->gfp; 400 sr->callback = callback; 401 sr->cb_arg = cb_arg; 402 403 atomic_set(&sr->status, REQ_NOT_POSTED); 404 405 sr->resp.orh = req->orh; 406 sr->resp.completion = req->comp; 407 408 ret = softreq_map_iobuf(sr, req); 409 if (ret) { 410 kfree(sr); 411 return ret; 412 } 413 414 /* get the context handle */ 415 if (req->ctx_handle) { 416 struct ctx_hdr *hdr; 417 u8 *ctx_ptr; 418 419 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; 420 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); 421 ctx_handle = hdr->ctx_dma; 422 } 423 424 /* select the queue */ 425 qno = smp_processor_id() % ndev->nr_queues; 426 427 sr->cmdq = &ndev->pkt_inq[qno]; 428 429 /* 430 * 64-Byte Instruction Format 431 * 432 * ---------------------- 433 * | DPTR0 | 8 bytes 434 * ---------------------- 435 * | PKT_IN_INSTR_HDR | 8 bytes 436 * ---------------------- 437 * | PKT_IN_HDR | 16 bytes 438 * ---------------------- 439 * | SLC_INFO | 16 bytes 440 * ---------------------- 441 * | Front data | 16 bytes 442 * ---------------------- 443 */ 444 445 /* fill the packet instruction */ 446 /* word 0 */ 447 sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); 448 449 /* word 1 */ 450 sr->instr.ih.value = 0; 451 sr->instr.ih.s.g = 1; 452 sr->instr.ih.s.gsz = sr->in.sgmap_cnt; 453 sr->instr.ih.s.ssz = sr->out.sgmap_cnt; 454 sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); 455 sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; 456 sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); 457 458 /* word 2 */ 459 sr->instr.irh.value[0] = 0; 460 sr->instr.irh.s.uddl = MIN_UDD_LEN; 461 /* context length in 64-bit words */ 462 sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); 463 /* offset from solicit base port 256 */ 464 sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; 465 sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; 466 sr->instr.irh.s.arg = req->ctrl.s.arg; 467 sr->instr.irh.s.opcode = req->opcode; 468 sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); 469 470 /* word 3 */ 471 sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); 472 473 /* word 4 */ 474 sr->instr.slc.value[0] = 0; 475 sr->instr.slc.s.ssz = sr->out.sgmap_cnt; 476 sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); 477 478 /* word 5 */ 479 sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); 480 481 /* 482 * No conversion for front data, 483 * It goes into payload 484 * put GP Header in front data 485 */ 486 sr->instr.fdata[0] = *((u64 *)&req->gph); 487 sr->instr.fdata[1] = 0; 488 489 ret = nitrox_enqueue_request(sr); 490 if (ret == -ENOSPC) 491 goto send_fail; 492 493 return ret; 494 495 send_fail: 496 softreq_destroy(sr); 497 return ret; 498 } 499 500 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) 501 { 502 return time_after_eq(jiffies, (tstamp + timeout)); 503 } 504 505 void backlog_qflush_work(struct work_struct *work) 506 { 507 struct nitrox_cmdq *cmdq; 508 509 cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); 510 post_backlog_cmds(cmdq); 511 } 512 513 static bool sr_completed(struct nitrox_softreq *sr) 514 { 515 u64 orh = READ_ONCE(*sr->resp.orh); 516 unsigned long timeout = jiffies + msecs_to_jiffies(1); 517 518 if ((orh != PENDING_SIG) && (orh & 0xff)) 519 return true; 520 521 while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { 522 if (time_after(jiffies, timeout)) { 523 pr_err("comp not done\n"); 524 return false; 525 } 526 } 527 528 return true; 529 } 530 531 /** 532 * process_response_list - process completed requests 533 * @cmdq: Command queue structure 534 * 535 * Returns the number of responses processed. 536 */ 537 static void process_response_list(struct nitrox_cmdq *cmdq) 538 { 539 struct nitrox_device *ndev = cmdq->ndev; 540 struct nitrox_softreq *sr; 541 int req_completed = 0, err = 0, budget; 542 completion_t callback; 543 void *cb_arg; 544 545 /* check all pending requests */ 546 budget = atomic_read(&cmdq->pending_count); 547 548 while (req_completed < budget) { 549 sr = get_first_response_entry(cmdq); 550 if (!sr) 551 break; 552 553 if (atomic_read(&sr->status) != REQ_POSTED) 554 break; 555 556 /* check orh and completion bytes updates */ 557 if (!sr_completed(sr)) { 558 /* request not completed, check for timeout */ 559 if (!cmd_timeout(sr->tstamp, ndev->timeout)) 560 break; 561 dev_err_ratelimited(DEV(ndev), 562 "Request timeout, orh 0x%016llx\n", 563 READ_ONCE(*sr->resp.orh)); 564 } 565 atomic_dec(&cmdq->pending_count); 566 atomic64_inc(&ndev->stats.completed); 567 /* sync with other cpus */ 568 smp_mb__after_atomic(); 569 /* remove from response list */ 570 response_list_del(sr, cmdq); 571 /* ORH error code */ 572 err = READ_ONCE(*sr->resp.orh) & 0xff; 573 callback = sr->callback; 574 cb_arg = sr->cb_arg; 575 softreq_destroy(sr); 576 if (callback) 577 callback(cb_arg, err); 578 579 req_completed++; 580 } 581 } 582 583 /* 584 * pkt_slc_resp_tasklet - post processing of SE responses 585 */ 586 void pkt_slc_resp_tasklet(unsigned long data) 587 { 588 struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); 589 struct nitrox_cmdq *cmdq = qvec->cmdq; 590 union nps_pkt_slc_cnts slc_cnts; 591 592 /* read completion count */ 593 slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); 594 /* resend the interrupt if more work to do */ 595 slc_cnts.s.resend = 1; 596 597 process_response_list(cmdq); 598 599 /* 600 * clear the interrupt with resend bit enabled, 601 * MSI-X interrupt generates if Completion count > Threshold 602 */ 603 writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); 604 605 if (atomic_read(&cmdq->backlog_count)) 606 schedule_work(&cmdq->backlog_qflush); 607 } 608