1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 Cavium, Inc. 4 */ 5 6 #include "cptvf.h" 7 #include "cptvf_algs.h" 8 #include "request_manager.h" 9 10 /** 11 * get_free_pending_entry - get free entry from pending queue 12 * @param pqinfo: pending_qinfo structure 13 * @param qno: queue number 14 */ 15 static struct pending_entry *get_free_pending_entry(struct pending_queue *q, 16 int qlen) 17 { 18 struct pending_entry *ent = NULL; 19 20 ent = &q->head[q->rear]; 21 if (unlikely(ent->busy)) { 22 ent = NULL; 23 goto no_free_entry; 24 } 25 26 q->rear++; 27 if (unlikely(q->rear == qlen)) 28 q->rear = 0; 29 30 no_free_entry: 31 return ent; 32 } 33 34 static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo, 35 int qno) 36 { 37 struct pending_queue *queue = &pqinfo->queue[qno]; 38 39 queue->front++; 40 if (unlikely(queue->front == pqinfo->qlen)) 41 queue->front = 0; 42 } 43 44 static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list, 45 int buf_count, u8 *buffer) 46 { 47 int ret = 0, i, j; 48 int components; 49 struct sglist_component *sg_ptr = NULL; 50 struct pci_dev *pdev = cptvf->pdev; 51 52 if (unlikely(!list)) { 53 dev_err(&pdev->dev, "Input List pointer is NULL\n"); 54 return -EFAULT; 55 } 56 57 for (i = 0; i < buf_count; i++) { 58 if (likely(list[i].vptr)) { 59 list[i].dma_addr = dma_map_single(&pdev->dev, 60 list[i].vptr, 61 list[i].size, 62 DMA_BIDIRECTIONAL); 63 if (unlikely(dma_mapping_error(&pdev->dev, 64 list[i].dma_addr))) { 65 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n", 66 i); 67 ret = -EIO; 68 goto sg_cleanup; 69 } 70 } 71 } 72 73 components = buf_count / 4; 74 sg_ptr = (struct sglist_component *)buffer; 75 for (i = 0; i < components; i++) { 76 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); 77 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); 78 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); 79 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size); 80 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 81 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 82 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 83 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); 84 sg_ptr++; 85 } 86 87 components = buf_count % 4; 88 89 switch (components) { 90 case 3: 91 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); 92 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 93 fallthrough; 94 case 2: 95 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); 96 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 97 fallthrough; 98 case 1: 99 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); 100 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 101 break; 102 default: 103 break; 104 } 105 106 return ret; 107 108 sg_cleanup: 109 for (j = 0; j < i; j++) { 110 if (list[j].dma_addr) { 111 dma_unmap_single(&pdev->dev, list[i].dma_addr, 112 list[i].size, DMA_BIDIRECTIONAL); 113 } 114 115 list[j].dma_addr = 0; 116 } 117 118 return ret; 119 } 120 121 static inline int setup_sgio_list(struct cpt_vf *cptvf, 122 struct cpt_info_buffer *info, 123 struct cpt_request_info *req) 124 { 125 u16 g_sz_bytes = 0, s_sz_bytes = 0; 126 int ret = 0; 127 struct pci_dev *pdev = cptvf->pdev; 128 129 if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) { 130 dev_err(&pdev->dev, "Request SG components are higher than supported\n"); 131 ret = -EINVAL; 132 goto scatter_gather_clean; 133 } 134 135 /* Setup gather (input) components */ 136 g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component); 137 info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); 138 if (!info->gather_components) { 139 ret = -ENOMEM; 140 goto scatter_gather_clean; 141 } 142 143 ret = setup_sgio_components(cptvf, req->in, 144 req->incnt, 145 info->gather_components); 146 if (ret) { 147 dev_err(&pdev->dev, "Failed to setup gather list\n"); 148 ret = -EFAULT; 149 goto scatter_gather_clean; 150 } 151 152 /* Setup scatter (output) components */ 153 s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component); 154 info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); 155 if (!info->scatter_components) { 156 ret = -ENOMEM; 157 goto scatter_gather_clean; 158 } 159 160 ret = setup_sgio_components(cptvf, req->out, 161 req->outcnt, 162 info->scatter_components); 163 if (ret) { 164 dev_err(&pdev->dev, "Failed to setup gather list\n"); 165 ret = -EFAULT; 166 goto scatter_gather_clean; 167 } 168 169 /* Create and initialize DPTR */ 170 info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; 171 info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); 172 if (!info->in_buffer) { 173 ret = -ENOMEM; 174 goto scatter_gather_clean; 175 } 176 177 ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt); 178 ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt); 179 ((__be16 *)info->in_buffer)[2] = 0; 180 ((__be16 *)info->in_buffer)[3] = 0; 181 182 memcpy(&info->in_buffer[8], info->gather_components, 183 g_sz_bytes); 184 memcpy(&info->in_buffer[8 + g_sz_bytes], 185 info->scatter_components, s_sz_bytes); 186 187 info->dptr_baddr = dma_map_single(&pdev->dev, 188 (void *)info->in_buffer, 189 info->dlen, 190 DMA_BIDIRECTIONAL); 191 if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) { 192 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen); 193 ret = -EIO; 194 goto scatter_gather_clean; 195 } 196 197 /* Create and initialize RPTR */ 198 info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); 199 if (!info->out_buffer) { 200 ret = -ENOMEM; 201 goto scatter_gather_clean; 202 } 203 204 *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT); 205 info->alternate_caddr = (u64 *)info->out_buffer; 206 info->rptr_baddr = dma_map_single(&pdev->dev, 207 (void *)info->out_buffer, 208 COMPLETION_CODE_SIZE, 209 DMA_BIDIRECTIONAL); 210 if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) { 211 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n", 212 COMPLETION_CODE_SIZE); 213 ret = -EIO; 214 goto scatter_gather_clean; 215 } 216 217 return 0; 218 219 scatter_gather_clean: 220 return ret; 221 } 222 223 static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, 224 u32 qno) 225 { 226 struct pci_dev *pdev = cptvf->pdev; 227 struct command_qinfo *qinfo = NULL; 228 struct command_queue *queue; 229 struct command_chunk *chunk; 230 u8 *ent; 231 int ret = 0; 232 233 if (unlikely(qno >= cptvf->nr_queues)) { 234 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n", 235 qno, cptvf->nr_queues); 236 return -EINVAL; 237 } 238 239 qinfo = &cptvf->cqinfo; 240 queue = &qinfo->queue[qno]; 241 /* lock commad queue */ 242 spin_lock(&queue->lock); 243 ent = &queue->qhead->head[queue->idx * qinfo->cmd_size]; 244 memcpy(ent, (void *)cmd, qinfo->cmd_size); 245 246 if (++queue->idx >= queue->qhead->size / 64) { 247 struct hlist_node *node; 248 249 hlist_for_each(node, &queue->chead) { 250 chunk = hlist_entry(node, struct command_chunk, 251 nextchunk); 252 if (chunk == queue->qhead) { 253 continue; 254 } else { 255 queue->qhead = chunk; 256 break; 257 } 258 } 259 queue->idx = 0; 260 } 261 /* make sure all memory stores are done before ringing doorbell */ 262 smp_wmb(); 263 cptvf_write_vq_doorbell(cptvf, 1); 264 /* unlock command queue */ 265 spin_unlock(&queue->lock); 266 267 return ret; 268 } 269 270 static void do_request_cleanup(struct cpt_vf *cptvf, 271 struct cpt_info_buffer *info) 272 { 273 int i; 274 struct pci_dev *pdev = cptvf->pdev; 275 struct cpt_request_info *req; 276 277 if (info->dptr_baddr) 278 dma_unmap_single(&pdev->dev, info->dptr_baddr, 279 info->dlen, DMA_BIDIRECTIONAL); 280 281 if (info->rptr_baddr) 282 dma_unmap_single(&pdev->dev, info->rptr_baddr, 283 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL); 284 285 if (info->comp_baddr) 286 dma_unmap_single(&pdev->dev, info->comp_baddr, 287 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL); 288 289 if (info->req) { 290 req = info->req; 291 for (i = 0; i < req->outcnt; i++) { 292 if (req->out[i].dma_addr) 293 dma_unmap_single(&pdev->dev, 294 req->out[i].dma_addr, 295 req->out[i].size, 296 DMA_BIDIRECTIONAL); 297 } 298 299 for (i = 0; i < req->incnt; i++) { 300 if (req->in[i].dma_addr) 301 dma_unmap_single(&pdev->dev, 302 req->in[i].dma_addr, 303 req->in[i].size, 304 DMA_BIDIRECTIONAL); 305 } 306 } 307 308 kfree_sensitive(info->scatter_components); 309 kfree_sensitive(info->gather_components); 310 kfree_sensitive(info->out_buffer); 311 kfree_sensitive(info->in_buffer); 312 kfree_sensitive((void *)info->completion_addr); 313 kfree_sensitive(info); 314 } 315 316 static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info) 317 { 318 struct pci_dev *pdev = cptvf->pdev; 319 320 if (!info) { 321 dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n"); 322 return; 323 } 324 325 do_request_cleanup(cptvf, info); 326 } 327 328 static inline void process_pending_queue(struct cpt_vf *cptvf, 329 struct pending_qinfo *pqinfo, 330 int qno) 331 { 332 struct pci_dev *pdev = cptvf->pdev; 333 struct pending_queue *pqueue = &pqinfo->queue[qno]; 334 struct pending_entry *pentry = NULL; 335 struct cpt_info_buffer *info = NULL; 336 union cpt_res_s *status = NULL; 337 unsigned char ccode; 338 339 while (1) { 340 spin_lock_bh(&pqueue->lock); 341 pentry = &pqueue->head[pqueue->front]; 342 if (unlikely(!pentry->busy)) { 343 spin_unlock_bh(&pqueue->lock); 344 break; 345 } 346 347 info = (struct cpt_info_buffer *)pentry->post_arg; 348 if (unlikely(!info)) { 349 dev_err(&pdev->dev, "Pending Entry post arg NULL\n"); 350 pending_queue_inc_front(pqinfo, qno); 351 spin_unlock_bh(&pqueue->lock); 352 continue; 353 } 354 355 status = (union cpt_res_s *)pentry->completion_addr; 356 ccode = status->s.compcode; 357 if ((status->s.compcode == CPT_COMP_E_FAULT) || 358 (status->s.compcode == CPT_COMP_E_SWERR)) { 359 dev_err(&pdev->dev, "Request failed with %s\n", 360 (status->s.compcode == CPT_COMP_E_FAULT) ? 361 "DMA Fault" : "Software error"); 362 pentry->completion_addr = NULL; 363 pentry->busy = false; 364 atomic64_dec((&pqueue->pending_count)); 365 pentry->post_arg = NULL; 366 pending_queue_inc_front(pqinfo, qno); 367 do_request_cleanup(cptvf, info); 368 spin_unlock_bh(&pqueue->lock); 369 break; 370 } else if (status->s.compcode == COMPLETION_CODE_INIT) { 371 /* check for timeout */ 372 if (time_after_eq(jiffies, 373 (info->time_in + 374 (CPT_COMMAND_TIMEOUT * HZ)))) { 375 dev_err(&pdev->dev, "Request timed out"); 376 pentry->completion_addr = NULL; 377 pentry->busy = false; 378 atomic64_dec((&pqueue->pending_count)); 379 pentry->post_arg = NULL; 380 pending_queue_inc_front(pqinfo, qno); 381 do_request_cleanup(cptvf, info); 382 spin_unlock_bh(&pqueue->lock); 383 break; 384 } else if ((*info->alternate_caddr == 385 (~COMPLETION_CODE_INIT)) && 386 (info->extra_time < TIME_IN_RESET_COUNT)) { 387 info->time_in = jiffies; 388 info->extra_time++; 389 spin_unlock_bh(&pqueue->lock); 390 break; 391 } 392 } 393 394 pentry->completion_addr = NULL; 395 pentry->busy = false; 396 pentry->post_arg = NULL; 397 atomic64_dec((&pqueue->pending_count)); 398 pending_queue_inc_front(pqinfo, qno); 399 spin_unlock_bh(&pqueue->lock); 400 401 do_post_process(info->cptvf, info); 402 /* 403 * Calling callback after we find 404 * that the request has been serviced 405 */ 406 pentry->callback(ccode, pentry->callback_arg); 407 } 408 } 409 410 int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) 411 { 412 int ret = 0, clear = 0, queue = 0; 413 struct cpt_info_buffer *info = NULL; 414 struct cptvf_request *cpt_req = NULL; 415 union ctrl_info *ctrl = NULL; 416 union cpt_res_s *result = NULL; 417 struct pending_entry *pentry = NULL; 418 struct pending_queue *pqueue = NULL; 419 struct pci_dev *pdev = cptvf->pdev; 420 u8 group = 0; 421 struct cpt_vq_command vq_cmd; 422 union cpt_inst_s cptinst; 423 424 info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); 425 if (unlikely(!info)) { 426 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n"); 427 return -ENOMEM; 428 } 429 430 cpt_req = (struct cptvf_request *)&req->req; 431 ctrl = (union ctrl_info *)&req->ctrl; 432 433 info->cptvf = cptvf; 434 group = ctrl->s.grp; 435 ret = setup_sgio_list(cptvf, info, req); 436 if (ret) { 437 dev_err(&pdev->dev, "Setting up SG list failed"); 438 goto request_cleanup; 439 } 440 441 cpt_req->dlen = info->dlen; 442 /* 443 * Get buffer for union cpt_res_s response 444 * structure and its physical address 445 */ 446 info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); 447 if (unlikely(!info->completion_addr)) { 448 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n"); 449 ret = -ENOMEM; 450 goto request_cleanup; 451 } 452 453 result = (union cpt_res_s *)info->completion_addr; 454 result->s.compcode = COMPLETION_CODE_INIT; 455 info->comp_baddr = dma_map_single(&pdev->dev, 456 (void *)info->completion_addr, 457 sizeof(union cpt_res_s), 458 DMA_BIDIRECTIONAL); 459 if (dma_mapping_error(&pdev->dev, info->comp_baddr)) { 460 dev_err(&pdev->dev, "mapping compptr Failed %lu\n", 461 sizeof(union cpt_res_s)); 462 ret = -EFAULT; 463 goto request_cleanup; 464 } 465 466 /* Fill the VQ command */ 467 vq_cmd.cmd.u64 = 0; 468 vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); 469 vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); 470 vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); 471 vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); 472 473 vq_cmd.dptr = info->dptr_baddr; 474 vq_cmd.rptr = info->rptr_baddr; 475 vq_cmd.cptr.u64 = 0; 476 vq_cmd.cptr.s.grp = group; 477 /* Get Pending Entry to submit command */ 478 /* Always queue 0, because 1 queue per VF */ 479 queue = 0; 480 pqueue = &cptvf->pqinfo.queue[queue]; 481 482 if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) { 483 dev_err(&pdev->dev, "pending threshold reached\n"); 484 process_pending_queue(cptvf, &cptvf->pqinfo, queue); 485 } 486 487 get_pending_entry: 488 spin_lock_bh(&pqueue->lock); 489 pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen); 490 if (unlikely(!pentry)) { 491 spin_unlock_bh(&pqueue->lock); 492 if (clear == 0) { 493 process_pending_queue(cptvf, &cptvf->pqinfo, queue); 494 clear = 1; 495 goto get_pending_entry; 496 } 497 dev_err(&pdev->dev, "Get free entry failed\n"); 498 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n", 499 queue, pqueue->rear, pqueue->front); 500 ret = -EFAULT; 501 goto request_cleanup; 502 } 503 504 pentry->completion_addr = info->completion_addr; 505 pentry->post_arg = (void *)info; 506 pentry->callback = req->callback; 507 pentry->callback_arg = req->callback_arg; 508 info->pentry = pentry; 509 pentry->busy = true; 510 atomic64_inc(&pqueue->pending_count); 511 512 /* Send CPT command */ 513 info->pentry = pentry; 514 info->time_in = jiffies; 515 info->req = req; 516 517 /* Create the CPT_INST_S type command for HW intrepretation */ 518 cptinst.s.doneint = true; 519 cptinst.s.res_addr = (u64)info->comp_baddr; 520 cptinst.s.tag = 0; 521 cptinst.s.grp = 0; 522 cptinst.s.wq_ptr = 0; 523 cptinst.s.ei0 = vq_cmd.cmd.u64; 524 cptinst.s.ei1 = vq_cmd.dptr; 525 cptinst.s.ei2 = vq_cmd.rptr; 526 cptinst.s.ei3 = vq_cmd.cptr.u64; 527 528 ret = send_cpt_command(cptvf, &cptinst, queue); 529 spin_unlock_bh(&pqueue->lock); 530 if (unlikely(ret)) { 531 dev_err(&pdev->dev, "Send command failed for AE\n"); 532 ret = -EFAULT; 533 goto request_cleanup; 534 } 535 536 return 0; 537 538 request_cleanup: 539 dev_dbg(&pdev->dev, "Failed to submit CPT command\n"); 540 do_request_cleanup(cptvf, info); 541 542 return ret; 543 } 544 545 void vq_post_process(struct cpt_vf *cptvf, u32 qno) 546 { 547 struct pci_dev *pdev = cptvf->pdev; 548 549 if (unlikely(qno > cptvf->nr_queues)) { 550 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n", 551 qno); 552 return; 553 } 554 555 process_pending_queue(cptvf, &cptvf->pqinfo, qno); 556 } 557 558 int cptvf_do_request(void *vfdev, struct cpt_request_info *req) 559 { 560 struct cpt_vf *cptvf = (struct cpt_vf *)vfdev; 561 struct pci_dev *pdev = cptvf->pdev; 562 563 if (!cpt_device_ready(cptvf)) { 564 dev_err(&pdev->dev, "CPT Device is not ready"); 565 return -ENODEV; 566 } 567 568 if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) { 569 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request", 570 cptvf->vfid); 571 return -EINVAL; 572 } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) { 573 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request", 574 cptvf->vfid); 575 return -EINVAL; 576 } 577 578 return process_request(cptvf, req); 579 } 580