1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 Cavium, Inc. 4 */ 5 6 #include <linux/interrupt.h> 7 #include <linux/module.h> 8 9 #include "cptvf.h" 10 11 #define DRV_NAME "thunder-cptvf" 12 #define DRV_VERSION "1.0" 13 14 struct cptvf_wqe { 15 struct tasklet_struct twork; 16 void *cptvf; 17 u32 qno; 18 }; 19 20 struct cptvf_wqe_info { 21 struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF]; 22 }; 23 24 static void vq_work_handler(unsigned long data) 25 { 26 struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data; 27 struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0]; 28 29 vq_post_process(cwqe->cptvf, cwqe->qno); 30 } 31 32 static int init_worker_threads(struct cpt_vf *cptvf) 33 { 34 struct pci_dev *pdev = cptvf->pdev; 35 struct cptvf_wqe_info *cwqe_info; 36 int i; 37 38 cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL); 39 if (!cwqe_info) 40 return -ENOMEM; 41 42 if (cptvf->nr_queues) { 43 dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n", 44 cptvf->nr_queues); 45 } 46 47 for (i = 0; i < cptvf->nr_queues; i++) { 48 tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler, 49 (u64)cwqe_info); 50 cwqe_info->vq_wqe[i].qno = i; 51 cwqe_info->vq_wqe[i].cptvf = cptvf; 52 } 53 54 cptvf->wqe_info = cwqe_info; 55 56 return 0; 57 } 58 59 static void cleanup_worker_threads(struct cpt_vf *cptvf) 60 { 61 struct cptvf_wqe_info *cwqe_info; 62 struct pci_dev *pdev = cptvf->pdev; 63 int i; 64 65 cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; 66 if (!cwqe_info) 67 return; 68 69 if (cptvf->nr_queues) { 70 dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n", 71 cptvf->nr_queues); 72 } 73 74 for (i = 0; i < cptvf->nr_queues; i++) 75 tasklet_kill(&cwqe_info->vq_wqe[i].twork); 76 77 kfree_sensitive(cwqe_info); 78 cptvf->wqe_info = NULL; 79 } 80 81 static void free_pending_queues(struct pending_qinfo *pqinfo) 82 { 83 int i; 84 struct pending_queue *queue; 85 86 for_each_pending_queue(pqinfo, queue, i) { 87 if (!queue->head) 88 continue; 89 90 /* free single queue */ 91 kfree_sensitive((queue->head)); 92 93 queue->front = 0; 94 queue->rear = 0; 95 96 return; 97 } 98 99 pqinfo->qlen = 0; 100 pqinfo->nr_queues = 0; 101 } 102 103 static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen, 104 u32 nr_queues) 105 { 106 u32 i; 107 int ret; 108 struct pending_queue *queue = NULL; 109 110 pqinfo->nr_queues = nr_queues; 111 pqinfo->qlen = qlen; 112 113 for_each_pending_queue(pqinfo, queue, i) { 114 queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL); 115 if (!queue->head) { 116 ret = -ENOMEM; 117 goto pending_qfail; 118 } 119 120 queue->front = 0; 121 queue->rear = 0; 122 atomic64_set((&queue->pending_count), (0)); 123 124 /* init queue spin lock */ 125 spin_lock_init(&queue->lock); 126 } 127 128 return 0; 129 130 pending_qfail: 131 free_pending_queues(pqinfo); 132 133 return ret; 134 } 135 136 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) 137 { 138 struct pci_dev *pdev = cptvf->pdev; 139 int ret; 140 141 if (!nr_queues) 142 return 0; 143 144 ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues); 145 if (ret) { 146 dev_err(&pdev->dev, "failed to setup pending queues (%u)\n", 147 nr_queues); 148 return ret; 149 } 150 151 return 0; 152 } 153 154 static void cleanup_pending_queues(struct cpt_vf *cptvf) 155 { 156 struct pci_dev *pdev = cptvf->pdev; 157 158 if (!cptvf->nr_queues) 159 return; 160 161 dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n", 162 cptvf->nr_queues); 163 free_pending_queues(&cptvf->pqinfo); 164 } 165 166 static void free_command_queues(struct cpt_vf *cptvf, 167 struct command_qinfo *cqinfo) 168 { 169 int i; 170 struct command_queue *queue = NULL; 171 struct command_chunk *chunk = NULL; 172 struct pci_dev *pdev = cptvf->pdev; 173 struct hlist_node *node; 174 175 /* clean up for each queue */ 176 for (i = 0; i < cptvf->nr_queues; i++) { 177 queue = &cqinfo->queue[i]; 178 if (hlist_empty(&cqinfo->queue[i].chead)) 179 continue; 180 181 hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, 182 nextchunk) { 183 dma_free_coherent(&pdev->dev, 184 chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, 185 chunk->head, 186 chunk->dma_addr); 187 chunk->head = NULL; 188 chunk->dma_addr = 0; 189 hlist_del(&chunk->nextchunk); 190 kfree_sensitive(chunk); 191 } 192 193 queue->nchunks = 0; 194 queue->idx = 0; 195 } 196 197 /* common cleanup */ 198 cqinfo->cmd_size = 0; 199 } 200 201 static int alloc_command_queues(struct cpt_vf *cptvf, 202 struct command_qinfo *cqinfo, size_t cmd_size, 203 u32 qlen) 204 { 205 int i; 206 size_t q_size; 207 struct command_queue *queue = NULL; 208 struct pci_dev *pdev = cptvf->pdev; 209 210 /* common init */ 211 cqinfo->cmd_size = cmd_size; 212 /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */ 213 cptvf->qsize = min(qlen, cqinfo->qchunksize) * 214 CPT_NEXT_CHUNK_PTR_SIZE + 1; 215 /* Qsize in bytes to create space for alignment */ 216 q_size = qlen * cqinfo->cmd_size; 217 218 /* per queue initialization */ 219 for (i = 0; i < cptvf->nr_queues; i++) { 220 size_t c_size = 0; 221 size_t rem_q_size = q_size; 222 struct command_chunk *curr = NULL, *first = NULL, *last = NULL; 223 u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size; 224 225 queue = &cqinfo->queue[i]; 226 INIT_HLIST_HEAD(&cqinfo->queue[i].chead); 227 do { 228 curr = kzalloc(sizeof(*curr), GFP_KERNEL); 229 if (!curr) 230 goto cmd_qfail; 231 232 c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : 233 rem_q_size; 234 curr->head = dma_alloc_coherent(&pdev->dev, 235 c_size + CPT_NEXT_CHUNK_PTR_SIZE, 236 &curr->dma_addr, 237 GFP_KERNEL); 238 if (!curr->head) { 239 dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n", 240 i, queue->nchunks); 241 kfree(curr); 242 goto cmd_qfail; 243 } 244 245 curr->size = c_size; 246 if (queue->nchunks == 0) { 247 hlist_add_head(&curr->nextchunk, 248 &cqinfo->queue[i].chead); 249 first = curr; 250 } else { 251 hlist_add_behind(&curr->nextchunk, 252 &last->nextchunk); 253 } 254 255 queue->nchunks++; 256 rem_q_size -= c_size; 257 if (last) 258 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; 259 260 last = curr; 261 } while (rem_q_size); 262 263 /* Make the queue circular */ 264 /* Tie back last chunk entry to head */ 265 curr = first; 266 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; 267 queue->qhead = curr; 268 spin_lock_init(&queue->lock); 269 } 270 return 0; 271 272 cmd_qfail: 273 free_command_queues(cptvf, cqinfo); 274 return -ENOMEM; 275 } 276 277 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) 278 { 279 struct pci_dev *pdev = cptvf->pdev; 280 int ret; 281 282 /* setup AE command queues */ 283 ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE, 284 qlen); 285 if (ret) { 286 dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n", 287 cptvf->nr_queues); 288 return ret; 289 } 290 291 return ret; 292 } 293 294 static void cleanup_command_queues(struct cpt_vf *cptvf) 295 { 296 struct pci_dev *pdev = cptvf->pdev; 297 298 if (!cptvf->nr_queues) 299 return; 300 301 dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n", 302 cptvf->nr_queues); 303 free_command_queues(cptvf, &cptvf->cqinfo); 304 } 305 306 static void cptvf_sw_cleanup(struct cpt_vf *cptvf) 307 { 308 cleanup_worker_threads(cptvf); 309 cleanup_pending_queues(cptvf); 310 cleanup_command_queues(cptvf); 311 } 312 313 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) 314 { 315 struct pci_dev *pdev = cptvf->pdev; 316 int ret = 0; 317 u32 max_dev_queues = 0; 318 319 max_dev_queues = CPT_NUM_QS_PER_VF; 320 /* possible cpus */ 321 nr_queues = min_t(u32, nr_queues, max_dev_queues); 322 cptvf->nr_queues = nr_queues; 323 324 ret = init_command_queues(cptvf, qlen); 325 if (ret) { 326 dev_err(&pdev->dev, "Failed to setup command queues (%u)\n", 327 nr_queues); 328 return ret; 329 } 330 331 ret = init_pending_queues(cptvf, qlen, nr_queues); 332 if (ret) { 333 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n", 334 nr_queues); 335 goto setup_pqfail; 336 } 337 338 /* Create worker threads for BH processing */ 339 ret = init_worker_threads(cptvf); 340 if (ret) { 341 dev_err(&pdev->dev, "Failed to setup worker threads\n"); 342 goto init_work_fail; 343 } 344 345 return 0; 346 347 init_work_fail: 348 cleanup_worker_threads(cptvf); 349 cleanup_pending_queues(cptvf); 350 351 setup_pqfail: 352 cleanup_command_queues(cptvf); 353 354 return ret; 355 } 356 357 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) 358 { 359 irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); 360 free_cpumask_var(cptvf->affinity_mask[vec]); 361 } 362 363 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) 364 { 365 union cptx_vqx_ctl vqx_ctl; 366 367 vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0)); 368 vqx_ctl.s.ena = val; 369 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u); 370 } 371 372 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) 373 { 374 union cptx_vqx_doorbell vqx_dbell; 375 376 vqx_dbell.u = cpt_read_csr64(cptvf->reg_base, 377 CPTX_VQX_DOORBELL(0, 0)); 378 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ 379 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), 380 vqx_dbell.u); 381 } 382 383 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) 384 { 385 union cptx_vqx_inprog vqx_inprg; 386 387 vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); 388 vqx_inprg.s.inflight = val; 389 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u); 390 } 391 392 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) 393 { 394 union cptx_vqx_done_wait vqx_dwait; 395 396 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, 397 CPTX_VQX_DONE_WAIT(0, 0)); 398 vqx_dwait.s.num_wait = val; 399 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), 400 vqx_dwait.u); 401 } 402 403 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) 404 { 405 union cptx_vqx_done_wait vqx_dwait; 406 407 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, 408 CPTX_VQX_DONE_WAIT(0, 0)); 409 vqx_dwait.s.time_wait = time; 410 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), 411 vqx_dwait.u); 412 } 413 414 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) 415 { 416 union cptx_vqx_misc_ena_w1s vqx_misc_ena; 417 418 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, 419 CPTX_VQX_MISC_ENA_W1S(0, 0)); 420 /* Set mbox(0) interupts for the requested vf */ 421 vqx_misc_ena.s.swerr = 1; 422 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), 423 vqx_misc_ena.u); 424 } 425 426 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) 427 { 428 union cptx_vqx_misc_ena_w1s vqx_misc_ena; 429 430 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, 431 CPTX_VQX_MISC_ENA_W1S(0, 0)); 432 /* Set mbox(0) interupts for the requested vf */ 433 vqx_misc_ena.s.mbox = 1; 434 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), 435 vqx_misc_ena.u); 436 } 437 438 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) 439 { 440 union cptx_vqx_done_ena_w1s vqx_done_ena; 441 442 vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base, 443 CPTX_VQX_DONE_ENA_W1S(0, 0)); 444 /* Set DONE interrupt for the requested vf */ 445 vqx_done_ena.s.done = 1; 446 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), 447 vqx_done_ena.u); 448 } 449 450 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) 451 { 452 union cptx_vqx_misc_int vqx_misc_int; 453 454 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 455 CPTX_VQX_MISC_INT(0, 0)); 456 /* W1C for the VF */ 457 vqx_misc_int.s.dovf = 1; 458 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 459 vqx_misc_int.u); 460 } 461 462 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) 463 { 464 union cptx_vqx_misc_int vqx_misc_int; 465 466 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 467 CPTX_VQX_MISC_INT(0, 0)); 468 /* W1C for the VF */ 469 vqx_misc_int.s.irde = 1; 470 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 471 vqx_misc_int.u); 472 } 473 474 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) 475 { 476 union cptx_vqx_misc_int vqx_misc_int; 477 478 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 479 CPTX_VQX_MISC_INT(0, 0)); 480 /* W1C for the VF */ 481 vqx_misc_int.s.nwrp = 1; 482 cpt_write_csr64(cptvf->reg_base, 483 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); 484 } 485 486 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) 487 { 488 union cptx_vqx_misc_int vqx_misc_int; 489 490 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 491 CPTX_VQX_MISC_INT(0, 0)); 492 /* W1C for the VF */ 493 vqx_misc_int.s.mbox = 1; 494 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 495 vqx_misc_int.u); 496 } 497 498 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) 499 { 500 union cptx_vqx_misc_int vqx_misc_int; 501 502 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, 503 CPTX_VQX_MISC_INT(0, 0)); 504 /* W1C for the VF */ 505 vqx_misc_int.s.swerr = 1; 506 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), 507 vqx_misc_int.u); 508 } 509 510 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) 511 { 512 return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); 513 } 514 515 static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq) 516 { 517 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; 518 struct pci_dev *pdev = cptvf->pdev; 519 u64 intr; 520 521 intr = cptvf_read_vf_misc_intr_status(cptvf); 522 /*Check for MISC interrupt types*/ 523 if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { 524 dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", 525 intr, cptvf->vfid); 526 cptvf_handle_mbox_intr(cptvf); 527 cptvf_clear_mbox_intr(cptvf); 528 } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) { 529 cptvf_clear_dovf_intr(cptvf); 530 /*Clear doorbell count*/ 531 cptvf_write_vq_doorbell(cptvf, 0); 532 dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n", 533 intr, cptvf->vfid); 534 } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) { 535 cptvf_clear_irde_intr(cptvf); 536 dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n", 537 intr, cptvf->vfid); 538 } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) { 539 cptvf_clear_nwrp_intr(cptvf); 540 dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n", 541 intr, cptvf->vfid); 542 } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) { 543 cptvf_clear_swerr_intr(cptvf); 544 dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n", 545 intr, cptvf->vfid); 546 } else { 547 dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n", 548 cptvf->vfid); 549 } 550 551 return IRQ_HANDLED; 552 } 553 554 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, 555 int qno) 556 { 557 struct cptvf_wqe_info *nwqe_info; 558 559 if (unlikely(qno >= cptvf->nr_queues)) 560 return NULL; 561 nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; 562 563 return &nwqe_info->vq_wqe[qno]; 564 } 565 566 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) 567 { 568 union cptx_vqx_done vqx_done; 569 570 vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0)); 571 return vqx_done.s.done; 572 } 573 574 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, 575 u32 ackcnt) 576 { 577 union cptx_vqx_done_ack vqx_dack_cnt; 578 579 vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base, 580 CPTX_VQX_DONE_ACK(0, 0)); 581 vqx_dack_cnt.s.done_ack = ackcnt; 582 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), 583 vqx_dack_cnt.u); 584 } 585 586 static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq) 587 { 588 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; 589 struct pci_dev *pdev = cptvf->pdev; 590 /* Read the number of completions */ 591 u32 intr = cptvf_read_vq_done_count(cptvf); 592 593 if (intr) { 594 struct cptvf_wqe *wqe; 595 596 /* Acknowledge the number of 597 * scheduled completions for processing 598 */ 599 cptvf_write_vq_done_ack(cptvf, intr); 600 wqe = get_cptvf_vq_wqe(cptvf, 0); 601 if (unlikely(!wqe)) { 602 dev_err(&pdev->dev, "No work to schedule for VF (%d)", 603 cptvf->vfid); 604 return IRQ_NONE; 605 } 606 tasklet_hi_schedule(&wqe->twork); 607 } 608 609 return IRQ_HANDLED; 610 } 611 612 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) 613 { 614 struct pci_dev *pdev = cptvf->pdev; 615 int cpu; 616 617 if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], 618 GFP_KERNEL)) { 619 dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d", 620 cptvf->vfid); 621 return; 622 } 623 624 cpu = cptvf->vfid % num_online_cpus(); 625 cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), 626 cptvf->affinity_mask[vec]); 627 irq_set_affinity_hint(pci_irq_vector(pdev, vec), 628 cptvf->affinity_mask[vec]); 629 } 630 631 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) 632 { 633 union cptx_vqx_saddr vqx_saddr; 634 635 vqx_saddr.u = val; 636 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); 637 } 638 639 static void cptvf_device_init(struct cpt_vf *cptvf) 640 { 641 u64 base_addr = 0; 642 643 /* Disable the VQ */ 644 cptvf_write_vq_ctl(cptvf, 0); 645 /* Reset the doorbell */ 646 cptvf_write_vq_doorbell(cptvf, 0); 647 /* Clear inflight */ 648 cptvf_write_vq_inprog(cptvf, 0); 649 /* Write VQ SADDR */ 650 /* TODO: for now only one queue, so hard coded */ 651 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); 652 cptvf_write_vq_saddr(cptvf, base_addr); 653 /* Configure timerhold / coalescence */ 654 cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); 655 cptvf_write_vq_done_numwait(cptvf, 1); 656 /* Enable the VQ */ 657 cptvf_write_vq_ctl(cptvf, 1); 658 /* Flag the VF ready */ 659 cptvf->flags |= CPT_FLAG_DEVICE_READY; 660 } 661 662 static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 663 { 664 struct device *dev = &pdev->dev; 665 struct cpt_vf *cptvf; 666 int err; 667 668 cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); 669 if (!cptvf) 670 return -ENOMEM; 671 672 pci_set_drvdata(pdev, cptvf); 673 cptvf->pdev = pdev; 674 err = pci_enable_device(pdev); 675 if (err) { 676 dev_err(dev, "Failed to enable PCI device\n"); 677 pci_set_drvdata(pdev, NULL); 678 return err; 679 } 680 681 err = pci_request_regions(pdev, DRV_NAME); 682 if (err) { 683 dev_err(dev, "PCI request regions failed 0x%x\n", err); 684 goto cptvf_err_disable_device; 685 } 686 /* Mark as VF driver */ 687 cptvf->flags |= CPT_FLAG_VF_DRIVER; 688 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); 689 if (err) { 690 dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); 691 goto cptvf_err_release_regions; 692 } 693 694 /* MAP PF's configuration registers */ 695 cptvf->reg_base = pcim_iomap(pdev, 0, 0); 696 if (!cptvf->reg_base) { 697 dev_err(dev, "Cannot map config register space, aborting\n"); 698 err = -ENOMEM; 699 goto cptvf_err_release_regions; 700 } 701 702 cptvf->node = dev_to_node(&pdev->dev); 703 err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS, 704 CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX); 705 if (err < 0) { 706 dev_err(dev, "Request for #%d msix vectors failed\n", 707 CPT_VF_MSIX_VECTORS); 708 goto cptvf_err_release_regions; 709 } 710 711 err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), 712 cptvf_misc_intr_handler, 0, "CPT VF misc intr", 713 cptvf); 714 if (err) { 715 dev_err(dev, "Request misc irq failed"); 716 goto cptvf_free_vectors; 717 } 718 719 /* Enable mailbox interrupt */ 720 cptvf_enable_mbox_interrupts(cptvf); 721 cptvf_enable_swerr_interrupts(cptvf); 722 723 /* Check ready with PF */ 724 /* Gets chip ID / device Id from PF if ready */ 725 err = cptvf_check_pf_ready(cptvf); 726 if (err) { 727 dev_err(dev, "PF not responding to READY msg"); 728 goto cptvf_free_misc_irq; 729 } 730 731 /* CPT VF software resources initialization */ 732 cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; 733 err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); 734 if (err) { 735 dev_err(dev, "cptvf_sw_init() failed"); 736 goto cptvf_free_misc_irq; 737 } 738 /* Convey VQ LEN to PF */ 739 err = cptvf_send_vq_size_msg(cptvf); 740 if (err) { 741 dev_err(dev, "PF not responding to QLEN msg"); 742 goto cptvf_free_misc_irq; 743 } 744 745 /* CPT VF device initialization */ 746 cptvf_device_init(cptvf); 747 /* Send msg to PF to assign currnet Q to required group */ 748 cptvf->vfgrp = 1; 749 err = cptvf_send_vf_to_grp_msg(cptvf); 750 if (err) { 751 dev_err(dev, "PF not responding to VF_GRP msg"); 752 goto cptvf_free_misc_irq; 753 } 754 755 cptvf->priority = 1; 756 err = cptvf_send_vf_priority_msg(cptvf); 757 if (err) { 758 dev_err(dev, "PF not responding to VF_PRIO msg"); 759 goto cptvf_free_misc_irq; 760 } 761 762 err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), 763 cptvf_done_intr_handler, 0, "CPT VF done intr", 764 cptvf); 765 if (err) { 766 dev_err(dev, "Request done irq failed\n"); 767 goto cptvf_free_misc_irq; 768 } 769 770 /* Enable mailbox interrupt */ 771 cptvf_enable_done_interrupts(cptvf); 772 773 /* Set irq affinity masks */ 774 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); 775 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); 776 777 err = cptvf_send_vf_up(cptvf); 778 if (err) { 779 dev_err(dev, "PF not responding to UP msg"); 780 goto cptvf_free_irq_affinity; 781 } 782 err = cvm_crypto_init(cptvf); 783 if (err) { 784 dev_err(dev, "Algorithm register failed\n"); 785 goto cptvf_free_irq_affinity; 786 } 787 return 0; 788 789 cptvf_free_irq_affinity: 790 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); 791 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); 792 cptvf_free_misc_irq: 793 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); 794 cptvf_free_vectors: 795 pci_free_irq_vectors(cptvf->pdev); 796 cptvf_err_release_regions: 797 pci_release_regions(pdev); 798 cptvf_err_disable_device: 799 pci_disable_device(pdev); 800 pci_set_drvdata(pdev, NULL); 801 802 return err; 803 } 804 805 static void cptvf_remove(struct pci_dev *pdev) 806 { 807 struct cpt_vf *cptvf = pci_get_drvdata(pdev); 808 809 if (!cptvf) { 810 dev_err(&pdev->dev, "Invalid CPT-VF device\n"); 811 return; 812 } 813 814 /* Convey DOWN to PF */ 815 if (cptvf_send_vf_down(cptvf)) { 816 dev_err(&pdev->dev, "PF not responding to DOWN msg"); 817 } else { 818 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); 819 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); 820 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); 821 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); 822 pci_free_irq_vectors(cptvf->pdev); 823 cptvf_sw_cleanup(cptvf); 824 pci_set_drvdata(pdev, NULL); 825 pci_release_regions(pdev); 826 pci_disable_device(pdev); 827 cvm_crypto_exit(); 828 } 829 } 830 831 static void cptvf_shutdown(struct pci_dev *pdev) 832 { 833 cptvf_remove(pdev); 834 } 835 836 /* Supported devices */ 837 static const struct pci_device_id cptvf_id_table[] = { 838 {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0}, 839 { 0, } /* end of table */ 840 }; 841 842 static struct pci_driver cptvf_pci_driver = { 843 .name = DRV_NAME, 844 .id_table = cptvf_id_table, 845 .probe = cptvf_probe, 846 .remove = cptvf_remove, 847 .shutdown = cptvf_shutdown, 848 }; 849 850 module_pci_driver(cptvf_pci_driver); 851 852 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>"); 853 MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver"); 854 MODULE_LICENSE("GPL v2"); 855 MODULE_VERSION(DRV_VERSION); 856 MODULE_DEVICE_TABLE(pci, cptvf_id_table); 857