1 /* 2 * Copyright (C) 2016 Cavium, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License 6 * as published by the Free Software Foundation. 7 */ 8 9 #include "cptvf.h" 10 #include "request_manager.h" 11 12 /** 13 * get_free_pending_entry - get free entry from pending queue 14 * @param pqinfo: pending_qinfo structure 15 * @param qno: queue number 16 */ 17 static struct pending_entry *get_free_pending_entry(struct pending_queue *q, 18 int qlen) 19 { 20 struct pending_entry *ent = NULL; 21 22 ent = &q->head[q->rear]; 23 if (unlikely(ent->busy)) { 24 ent = NULL; 25 goto no_free_entry; 26 } 27 28 q->rear++; 29 if (unlikely(q->rear == qlen)) 30 q->rear = 0; 31 32 no_free_entry: 33 return ent; 34 } 35 36 static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo, 37 int qno) 38 { 39 struct pending_queue *queue = &pqinfo->queue[qno]; 40 41 queue->front++; 42 if (unlikely(queue->front == pqinfo->qlen)) 43 queue->front = 0; 44 } 45 46 static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list, 47 int buf_count, u8 *buffer) 48 { 49 int ret = 0, i, j; 50 int components; 51 struct sglist_component *sg_ptr = NULL; 52 struct pci_dev *pdev = cptvf->pdev; 53 54 if (unlikely(!list)) { 55 dev_err(&pdev->dev, "Input List pointer is NULL\n"); 56 return -EFAULT; 57 } 58 59 for (i = 0; i < buf_count; i++) { 60 if (likely(list[i].vptr)) { 61 list[i].dma_addr = dma_map_single(&pdev->dev, 62 list[i].vptr, 63 list[i].size, 64 DMA_BIDIRECTIONAL); 65 if (unlikely(dma_mapping_error(&pdev->dev, 66 list[i].dma_addr))) { 67 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n", 68 i); 69 ret = -EIO; 70 goto sg_cleanup; 71 } 72 } 73 } 74 75 components = buf_count / 4; 76 sg_ptr = (struct sglist_component *)buffer; 77 for (i = 0; i < components; i++) { 78 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); 79 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); 80 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); 81 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size); 82 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 83 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 84 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 85 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); 86 sg_ptr++; 87 } 88 89 components = buf_count % 4; 90 91 switch (components) { 92 case 3: 93 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); 94 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 95 /* Fall through */ 96 case 2: 97 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); 98 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 99 /* Fall through */ 100 case 1: 101 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); 102 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 103 break; 104 default: 105 break; 106 } 107 108 return ret; 109 110 sg_cleanup: 111 for (j = 0; j < i; j++) { 112 if (list[j].dma_addr) { 113 dma_unmap_single(&pdev->dev, list[i].dma_addr, 114 list[i].size, DMA_BIDIRECTIONAL); 115 } 116 117 list[j].dma_addr = 0; 118 } 119 120 return ret; 121 } 122 123 static inline int setup_sgio_list(struct cpt_vf *cptvf, 124 struct cpt_info_buffer *info, 125 struct cpt_request_info *req) 126 { 127 u16 g_sz_bytes = 0, s_sz_bytes = 0; 128 int ret = 0; 129 struct pci_dev *pdev = cptvf->pdev; 130 131 if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) { 132 dev_err(&pdev->dev, "Request SG components are higher than supported\n"); 133 ret = -EINVAL; 134 goto scatter_gather_clean; 135 } 136 137 /* Setup gather (input) components */ 138 g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component); 139 info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL); 140 if (!info->gather_components) { 141 ret = -ENOMEM; 142 goto scatter_gather_clean; 143 } 144 145 ret = setup_sgio_components(cptvf, req->in, 146 req->incnt, 147 info->gather_components); 148 if (ret) { 149 dev_err(&pdev->dev, "Failed to setup gather list\n"); 150 ret = -EFAULT; 151 goto scatter_gather_clean; 152 } 153 154 /* Setup scatter (output) components */ 155 s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component); 156 info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL); 157 if (!info->scatter_components) { 158 ret = -ENOMEM; 159 goto scatter_gather_clean; 160 } 161 162 ret = setup_sgio_components(cptvf, req->out, 163 req->outcnt, 164 info->scatter_components); 165 if (ret) { 166 dev_err(&pdev->dev, "Failed to setup gather list\n"); 167 ret = -EFAULT; 168 goto scatter_gather_clean; 169 } 170 171 /* Create and initialize DPTR */ 172 info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; 173 info->in_buffer = kzalloc(info->dlen, GFP_KERNEL); 174 if (!info->in_buffer) { 175 ret = -ENOMEM; 176 goto scatter_gather_clean; 177 } 178 179 ((u16 *)info->in_buffer)[0] = req->outcnt; 180 ((u16 *)info->in_buffer)[1] = req->incnt; 181 ((u16 *)info->in_buffer)[2] = 0; 182 ((u16 *)info->in_buffer)[3] = 0; 183 *(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer); 184 185 memcpy(&info->in_buffer[8], info->gather_components, 186 g_sz_bytes); 187 memcpy(&info->in_buffer[8 + g_sz_bytes], 188 info->scatter_components, s_sz_bytes); 189 190 info->dptr_baddr = dma_map_single(&pdev->dev, 191 (void *)info->in_buffer, 192 info->dlen, 193 DMA_BIDIRECTIONAL); 194 if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) { 195 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen); 196 ret = -EIO; 197 goto scatter_gather_clean; 198 } 199 200 /* Create and initialize RPTR */ 201 info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL); 202 if (!info->out_buffer) { 203 ret = -ENOMEM; 204 goto scatter_gather_clean; 205 } 206 207 *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT); 208 info->alternate_caddr = (u64 *)info->out_buffer; 209 info->rptr_baddr = dma_map_single(&pdev->dev, 210 (void *)info->out_buffer, 211 COMPLETION_CODE_SIZE, 212 DMA_BIDIRECTIONAL); 213 if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) { 214 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n", 215 COMPLETION_CODE_SIZE); 216 ret = -EIO; 217 goto scatter_gather_clean; 218 } 219 220 return 0; 221 222 scatter_gather_clean: 223 return ret; 224 } 225 226 int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, 227 u32 qno) 228 { 229 struct pci_dev *pdev = cptvf->pdev; 230 struct command_qinfo *qinfo = NULL; 231 struct command_queue *queue; 232 struct command_chunk *chunk; 233 u8 *ent; 234 int ret = 0; 235 236 if (unlikely(qno >= cptvf->nr_queues)) { 237 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n", 238 qno, cptvf->nr_queues); 239 return -EINVAL; 240 } 241 242 qinfo = &cptvf->cqinfo; 243 queue = &qinfo->queue[qno]; 244 /* lock commad queue */ 245 spin_lock(&queue->lock); 246 ent = &queue->qhead->head[queue->idx * qinfo->cmd_size]; 247 memcpy(ent, (void *)cmd, qinfo->cmd_size); 248 249 if (++queue->idx >= queue->qhead->size / 64) { 250 struct hlist_node *node; 251 252 hlist_for_each(node, &queue->chead) { 253 chunk = hlist_entry(node, struct command_chunk, 254 nextchunk); 255 if (chunk == queue->qhead) { 256 continue; 257 } else { 258 queue->qhead = chunk; 259 break; 260 } 261 } 262 queue->idx = 0; 263 } 264 /* make sure all memory stores are done before ringing doorbell */ 265 smp_wmb(); 266 cptvf_write_vq_doorbell(cptvf, 1); 267 /* unlock command queue */ 268 spin_unlock(&queue->lock); 269 270 return ret; 271 } 272 273 void do_request_cleanup(struct cpt_vf *cptvf, 274 struct cpt_info_buffer *info) 275 { 276 int i; 277 struct pci_dev *pdev = cptvf->pdev; 278 struct cpt_request_info *req; 279 280 if (info->dptr_baddr) 281 dma_unmap_single(&pdev->dev, info->dptr_baddr, 282 info->dlen, DMA_BIDIRECTIONAL); 283 284 if (info->rptr_baddr) 285 dma_unmap_single(&pdev->dev, info->rptr_baddr, 286 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL); 287 288 if (info->comp_baddr) 289 dma_unmap_single(&pdev->dev, info->comp_baddr, 290 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL); 291 292 if (info->req) { 293 req = info->req; 294 for (i = 0; i < req->outcnt; i++) { 295 if (req->out[i].dma_addr) 296 dma_unmap_single(&pdev->dev, 297 req->out[i].dma_addr, 298 req->out[i].size, 299 DMA_BIDIRECTIONAL); 300 } 301 302 for (i = 0; i < req->incnt; i++) { 303 if (req->in[i].dma_addr) 304 dma_unmap_single(&pdev->dev, 305 req->in[i].dma_addr, 306 req->in[i].size, 307 DMA_BIDIRECTIONAL); 308 } 309 } 310 311 if (info->scatter_components) 312 kzfree(info->scatter_components); 313 314 if (info->gather_components) 315 kzfree(info->gather_components); 316 317 if (info->out_buffer) 318 kzfree(info->out_buffer); 319 320 if (info->in_buffer) 321 kzfree(info->in_buffer); 322 323 if (info->completion_addr) 324 kzfree((void *)info->completion_addr); 325 326 kzfree(info); 327 } 328 329 void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info) 330 { 331 struct pci_dev *pdev = cptvf->pdev; 332 333 if (!info) { 334 dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n"); 335 return; 336 } 337 338 do_request_cleanup(cptvf, info); 339 } 340 341 static inline void process_pending_queue(struct cpt_vf *cptvf, 342 struct pending_qinfo *pqinfo, 343 int qno) 344 { 345 struct pci_dev *pdev = cptvf->pdev; 346 struct pending_queue *pqueue = &pqinfo->queue[qno]; 347 struct pending_entry *pentry = NULL; 348 struct cpt_info_buffer *info = NULL; 349 union cpt_res_s *status = NULL; 350 unsigned char ccode; 351 352 while (1) { 353 spin_lock_bh(&pqueue->lock); 354 pentry = &pqueue->head[pqueue->front]; 355 if (unlikely(!pentry->busy)) { 356 spin_unlock_bh(&pqueue->lock); 357 break; 358 } 359 360 info = (struct cpt_info_buffer *)pentry->post_arg; 361 if (unlikely(!info)) { 362 dev_err(&pdev->dev, "Pending Entry post arg NULL\n"); 363 pending_queue_inc_front(pqinfo, qno); 364 spin_unlock_bh(&pqueue->lock); 365 continue; 366 } 367 368 status = (union cpt_res_s *)pentry->completion_addr; 369 ccode = status->s.compcode; 370 if ((status->s.compcode == CPT_COMP_E_FAULT) || 371 (status->s.compcode == CPT_COMP_E_SWERR)) { 372 dev_err(&pdev->dev, "Request failed with %s\n", 373 (status->s.compcode == CPT_COMP_E_FAULT) ? 374 "DMA Fault" : "Software error"); 375 pentry->completion_addr = NULL; 376 pentry->busy = false; 377 atomic64_dec((&pqueue->pending_count)); 378 pentry->post_arg = NULL; 379 pending_queue_inc_front(pqinfo, qno); 380 do_request_cleanup(cptvf, info); 381 spin_unlock_bh(&pqueue->lock); 382 break; 383 } else if (status->s.compcode == COMPLETION_CODE_INIT) { 384 /* check for timeout */ 385 if (time_after_eq(jiffies, 386 (info->time_in + 387 (CPT_COMMAND_TIMEOUT * HZ)))) { 388 dev_err(&pdev->dev, "Request timed out"); 389 pentry->completion_addr = NULL; 390 pentry->busy = false; 391 atomic64_dec((&pqueue->pending_count)); 392 pentry->post_arg = NULL; 393 pending_queue_inc_front(pqinfo, qno); 394 do_request_cleanup(cptvf, info); 395 spin_unlock_bh(&pqueue->lock); 396 break; 397 } else if ((*info->alternate_caddr == 398 (~COMPLETION_CODE_INIT)) && 399 (info->extra_time < TIME_IN_RESET_COUNT)) { 400 info->time_in = jiffies; 401 info->extra_time++; 402 spin_unlock_bh(&pqueue->lock); 403 break; 404 } 405 } 406 407 pentry->completion_addr = NULL; 408 pentry->busy = false; 409 pentry->post_arg = NULL; 410 atomic64_dec((&pqueue->pending_count)); 411 pending_queue_inc_front(pqinfo, qno); 412 spin_unlock_bh(&pqueue->lock); 413 414 do_post_process(info->cptvf, info); 415 /* 416 * Calling callback after we find 417 * that the request has been serviced 418 */ 419 pentry->callback(ccode, pentry->callback_arg); 420 } 421 } 422 423 int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) 424 { 425 int ret = 0, clear = 0, queue = 0; 426 struct cpt_info_buffer *info = NULL; 427 struct cptvf_request *cpt_req = NULL; 428 union ctrl_info *ctrl = NULL; 429 union cpt_res_s *result = NULL; 430 struct pending_entry *pentry = NULL; 431 struct pending_queue *pqueue = NULL; 432 struct pci_dev *pdev = cptvf->pdev; 433 u8 group = 0; 434 struct cpt_vq_command vq_cmd; 435 union cpt_inst_s cptinst; 436 437 info = kzalloc(sizeof(*info), GFP_KERNEL); 438 if (unlikely(!info)) { 439 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n"); 440 return -ENOMEM; 441 } 442 443 cpt_req = (struct cptvf_request *)&req->req; 444 ctrl = (union ctrl_info *)&req->ctrl; 445 446 info->cptvf = cptvf; 447 group = ctrl->s.grp; 448 ret = setup_sgio_list(cptvf, info, req); 449 if (ret) { 450 dev_err(&pdev->dev, "Setting up SG list failed"); 451 goto request_cleanup; 452 } 453 454 cpt_req->dlen = info->dlen; 455 /* 456 * Get buffer for union cpt_res_s response 457 * structure and its physical address 458 */ 459 info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL); 460 if (unlikely(!info->completion_addr)) { 461 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n"); 462 ret = -ENOMEM; 463 goto request_cleanup; 464 } 465 466 result = (union cpt_res_s *)info->completion_addr; 467 result->s.compcode = COMPLETION_CODE_INIT; 468 info->comp_baddr = dma_map_single(&pdev->dev, 469 (void *)info->completion_addr, 470 sizeof(union cpt_res_s), 471 DMA_BIDIRECTIONAL); 472 if (dma_mapping_error(&pdev->dev, info->comp_baddr)) { 473 dev_err(&pdev->dev, "mapping compptr Failed %lu\n", 474 sizeof(union cpt_res_s)); 475 ret = -EFAULT; 476 goto request_cleanup; 477 } 478 479 /* Fill the VQ command */ 480 vq_cmd.cmd.u64 = 0; 481 vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); 482 vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); 483 vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); 484 vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); 485 486 /* 64-bit swap for microcode data reads, not needed for addresses*/ 487 vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64); 488 vq_cmd.dptr = info->dptr_baddr; 489 vq_cmd.rptr = info->rptr_baddr; 490 vq_cmd.cptr.u64 = 0; 491 vq_cmd.cptr.s.grp = group; 492 /* Get Pending Entry to submit command */ 493 /* Always queue 0, because 1 queue per VF */ 494 queue = 0; 495 pqueue = &cptvf->pqinfo.queue[queue]; 496 497 if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) { 498 dev_err(&pdev->dev, "pending threshold reached\n"); 499 process_pending_queue(cptvf, &cptvf->pqinfo, queue); 500 } 501 502 get_pending_entry: 503 spin_lock_bh(&pqueue->lock); 504 pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen); 505 if (unlikely(!pentry)) { 506 spin_unlock_bh(&pqueue->lock); 507 if (clear == 0) { 508 process_pending_queue(cptvf, &cptvf->pqinfo, queue); 509 clear = 1; 510 goto get_pending_entry; 511 } 512 dev_err(&pdev->dev, "Get free entry failed\n"); 513 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n", 514 queue, pqueue->rear, pqueue->front); 515 ret = -EFAULT; 516 goto request_cleanup; 517 } 518 519 pentry->completion_addr = info->completion_addr; 520 pentry->post_arg = (void *)info; 521 pentry->callback = req->callback; 522 pentry->callback_arg = req->callback_arg; 523 info->pentry = pentry; 524 pentry->busy = true; 525 atomic64_inc(&pqueue->pending_count); 526 527 /* Send CPT command */ 528 info->pentry = pentry; 529 info->time_in = jiffies; 530 info->req = req; 531 532 /* Create the CPT_INST_S type command for HW intrepretation */ 533 cptinst.s.doneint = true; 534 cptinst.s.res_addr = (u64)info->comp_baddr; 535 cptinst.s.tag = 0; 536 cptinst.s.grp = 0; 537 cptinst.s.wq_ptr = 0; 538 cptinst.s.ei0 = vq_cmd.cmd.u64; 539 cptinst.s.ei1 = vq_cmd.dptr; 540 cptinst.s.ei2 = vq_cmd.rptr; 541 cptinst.s.ei3 = vq_cmd.cptr.u64; 542 543 ret = send_cpt_command(cptvf, &cptinst, queue); 544 spin_unlock_bh(&pqueue->lock); 545 if (unlikely(ret)) { 546 dev_err(&pdev->dev, "Send command failed for AE\n"); 547 ret = -EFAULT; 548 goto request_cleanup; 549 } 550 551 return 0; 552 553 request_cleanup: 554 dev_dbg(&pdev->dev, "Failed to submit CPT command\n"); 555 do_request_cleanup(cptvf, info); 556 557 return ret; 558 } 559 560 void vq_post_process(struct cpt_vf *cptvf, u32 qno) 561 { 562 struct pci_dev *pdev = cptvf->pdev; 563 564 if (unlikely(qno > cptvf->nr_queues)) { 565 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n", 566 qno); 567 return; 568 } 569 570 process_pending_queue(cptvf, &cptvf->pqinfo, qno); 571 } 572 573 int cptvf_do_request(void *vfdev, struct cpt_request_info *req) 574 { 575 struct cpt_vf *cptvf = (struct cpt_vf *)vfdev; 576 struct pci_dev *pdev = cptvf->pdev; 577 578 if (!cpt_device_ready(cptvf)) { 579 dev_err(&pdev->dev, "CPT Device is not ready"); 580 return -ENODEV; 581 } 582 583 if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) { 584 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request", 585 cptvf->vfid); 586 return -EINVAL; 587 } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) { 588 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request", 589 cptvf->vfid); 590 return -EINVAL; 591 } 592 593 return process_request(cptvf, req); 594 } 595