1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_inet.h" 31 #include "opt_inet6.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bitset.h> 36 #include <sys/bitstring.h> 37 #include <sys/buf_ring.h> 38 #include <sys/bus.h> 39 #include <sys/endian.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/pciio.h> 45 #include <sys/pcpu.h> 46 #include <sys/proc.h> 47 #include <sys/sockio.h> 48 #include <sys/socket.h> 49 #include <sys/stdatomic.h> 50 #include <sys/cpuset.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/smp.h> 54 #include <sys/taskqueue.h> 55 56 #include <vm/vm.h> 57 #include <vm/pmap.h> 58 59 #include <machine/bus.h> 60 #include <machine/vmparam.h> 61 62 #include <net/if.h> 63 #include <net/if_var.h> 64 #include <net/if_media.h> 65 #include <net/ifq.h> 66 #include <net/bpf.h> 67 #include <net/ethernet.h> 68 69 #include <netinet/in_systm.h> 70 #include <netinet/in.h> 71 #include <netinet/if_ether.h> 72 #include <netinet/ip.h> 73 #include <netinet/ip6.h> 74 #include <netinet/sctp.h> 75 #include <netinet/tcp.h> 76 #include <netinet/tcp_lro.h> 77 #include <netinet/udp.h> 78 79 #include <netinet6/ip6_var.h> 80 81 #include <dev/pci/pcireg.h> 82 #include <dev/pci/pcivar.h> 83 84 #include "thunder_bgx.h" 85 #include "nic_reg.h" 86 #include "nic.h" 87 #include "q_struct.h" 88 #include "nicvf_queues.h" 89 90 #define DEBUG 91 #undef DEBUG 92 93 #ifdef DEBUG 94 #define dprintf(dev, fmt, ...) device_printf(dev, fmt, ##__VA_ARGS__) 95 #else 96 #define dprintf(dev, fmt, ...) 97 #endif 98 99 MALLOC_DECLARE(M_NICVF); 100 101 static void nicvf_free_snd_queue(struct nicvf *, struct snd_queue *); 102 static struct mbuf * nicvf_get_rcv_mbuf(struct nicvf *, struct cqe_rx_t *); 103 static void nicvf_sq_disable(struct nicvf *, int); 104 static void nicvf_sq_enable(struct nicvf *, struct snd_queue *, int); 105 static void nicvf_put_sq_desc(struct snd_queue *, int); 106 static void nicvf_cmp_queue_config(struct nicvf *, struct queue_set *, int, 107 boolean_t); 108 static void nicvf_sq_free_used_descs(struct nicvf *, struct snd_queue *, int); 109 110 static int nicvf_tx_mbuf_locked(struct snd_queue *, struct mbuf **); 111 112 static void nicvf_rbdr_task(void *, int); 113 static void nicvf_rbdr_task_nowait(void *, int); 114 115 struct rbuf_info { 116 bus_dma_tag_t dmat; 117 bus_dmamap_t dmap; 118 struct mbuf * mbuf; 119 }; 120 121 #define GET_RBUF_INFO(x) ((struct rbuf_info *)((x) - NICVF_RCV_BUF_ALIGN_BYTES)) 122 123 /* Poll a register for a specific value */ 124 static int nicvf_poll_reg(struct nicvf *nic, int qidx, 125 uint64_t reg, int bit_pos, int bits, int val) 126 { 127 uint64_t bit_mask; 128 uint64_t reg_val; 129 int timeout = 10; 130 131 bit_mask = (1UL << bits) - 1; 132 bit_mask = (bit_mask << bit_pos); 133 134 while (timeout) { 135 reg_val = nicvf_queue_reg_read(nic, reg, qidx); 136 if (((reg_val & bit_mask) >> bit_pos) == val) 137 return (0); 138 139 DELAY(1000); 140 timeout--; 141 } 142 device_printf(nic->dev, "Poll on reg 0x%lx failed\n", reg); 143 return (ETIMEDOUT); 144 } 145 146 /* Callback for bus_dmamap_load() */ 147 static void 148 nicvf_dmamap_q_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 149 { 150 bus_addr_t *paddr; 151 152 KASSERT(nseg == 1, ("wrong number of segments, should be 1")); 153 paddr = arg; 154 *paddr = segs->ds_addr; 155 } 156 157 /* Allocate memory for a queue's descriptors */ 158 static int 159 nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem, 160 int q_len, int desc_size, int align_bytes) 161 { 162 int err, err_dmat __diagused; 163 164 /* Create DMA tag first */ 165 err = bus_dma_tag_create( 166 bus_get_dma_tag(nic->dev), /* parent tag */ 167 align_bytes, /* alignment */ 168 0, /* boundary */ 169 BUS_SPACE_MAXADDR, /* lowaddr */ 170 BUS_SPACE_MAXADDR, /* highaddr */ 171 NULL, NULL, /* filtfunc, filtfuncarg */ 172 (q_len * desc_size), /* maxsize */ 173 1, /* nsegments */ 174 (q_len * desc_size), /* maxsegsize */ 175 0, /* flags */ 176 NULL, NULL, /* lockfunc, lockfuncarg */ 177 &dmem->dmat); /* dmat */ 178 179 if (err != 0) { 180 device_printf(nic->dev, 181 "Failed to create busdma tag for descriptors ring\n"); 182 return (err); 183 } 184 185 /* Allocate segment of continuous DMA safe memory */ 186 err = bus_dmamem_alloc( 187 dmem->dmat, /* DMA tag */ 188 &dmem->base, /* virtual address */ 189 (BUS_DMA_NOWAIT | BUS_DMA_ZERO), /* flags */ 190 &dmem->dmap); /* DMA map */ 191 if (err != 0) { 192 device_printf(nic->dev, "Failed to allocate DMA safe memory for" 193 "descriptors ring\n"); 194 goto dmamem_fail; 195 } 196 197 err = bus_dmamap_load( 198 dmem->dmat, 199 dmem->dmap, 200 dmem->base, 201 (q_len * desc_size), /* allocation size */ 202 nicvf_dmamap_q_cb, /* map to DMA address cb. */ 203 &dmem->phys_base, /* physical address */ 204 BUS_DMA_NOWAIT); 205 if (err != 0) { 206 device_printf(nic->dev, 207 "Cannot load DMA map of descriptors ring\n"); 208 goto dmamap_fail; 209 } 210 211 dmem->q_len = q_len; 212 dmem->size = (desc_size * q_len); 213 214 return (0); 215 216 dmamap_fail: 217 bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap); 218 dmem->phys_base = 0; 219 dmamem_fail: 220 err_dmat = bus_dma_tag_destroy(dmem->dmat); 221 dmem->base = NULL; 222 KASSERT(err_dmat == 0, 223 ("%s: Trying to destroy BUSY DMA tag", __func__)); 224 225 return (err); 226 } 227 228 /* Free queue's descriptor memory */ 229 static void 230 nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem) 231 { 232 int err __diagused; 233 234 if ((dmem == NULL) || (dmem->base == NULL)) 235 return; 236 237 /* Unload a map */ 238 bus_dmamap_sync(dmem->dmat, dmem->dmap, BUS_DMASYNC_POSTREAD); 239 bus_dmamap_unload(dmem->dmat, dmem->dmap); 240 /* Free DMA memory */ 241 bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap); 242 /* Destroy DMA tag */ 243 err = bus_dma_tag_destroy(dmem->dmat); 244 245 KASSERT(err == 0, 246 ("%s: Trying to destroy BUSY DMA tag", __func__)); 247 248 dmem->phys_base = 0; 249 dmem->base = NULL; 250 } 251 252 /* 253 * Allocate buffer for packet reception 254 * HW returns memory address where packet is DMA'ed but not a pointer 255 * into RBDR ring, so save buffer address at the start of fragment and 256 * align the start address to a cache aligned address 257 */ 258 static __inline int 259 nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr, 260 bus_dmamap_t dmap, int mflags, uint32_t buf_len, bus_addr_t *rbuf) 261 { 262 struct mbuf *mbuf; 263 struct rbuf_info *rinfo; 264 bus_dma_segment_t segs[1]; 265 int nsegs; 266 int err; 267 268 mbuf = m_getjcl(mflags, MT_DATA, M_PKTHDR, MCLBYTES); 269 if (mbuf == NULL) 270 return (ENOMEM); 271 272 /* 273 * The length is equal to the actual length + one 128b line 274 * used as a room for rbuf_info structure. 275 */ 276 mbuf->m_len = mbuf->m_pkthdr.len = buf_len; 277 278 err = bus_dmamap_load_mbuf_sg(rbdr->rbdr_buff_dmat, dmap, mbuf, segs, 279 &nsegs, BUS_DMA_NOWAIT); 280 if (err != 0) { 281 device_printf(nic->dev, 282 "Failed to map mbuf into DMA visible memory, err: %d\n", 283 err); 284 m_freem(mbuf); 285 bus_dmamap_destroy(rbdr->rbdr_buff_dmat, dmap); 286 return (err); 287 } 288 if (nsegs != 1) 289 panic("Unexpected number of DMA segments for RB: %d", nsegs); 290 /* 291 * Now use the room for rbuf_info structure 292 * and adjust mbuf data and length. 293 */ 294 rinfo = (struct rbuf_info *)mbuf->m_data; 295 m_adj(mbuf, NICVF_RCV_BUF_ALIGN_BYTES); 296 297 rinfo->dmat = rbdr->rbdr_buff_dmat; 298 rinfo->dmap = dmap; 299 rinfo->mbuf = mbuf; 300 301 *rbuf = segs[0].ds_addr + NICVF_RCV_BUF_ALIGN_BYTES; 302 303 return (0); 304 } 305 306 /* Retrieve mbuf for received packet */ 307 static struct mbuf * 308 nicvf_rb_ptr_to_mbuf(struct nicvf *nic, bus_addr_t rb_ptr) 309 { 310 struct mbuf *mbuf; 311 struct rbuf_info *rinfo; 312 313 /* Get buffer start address and alignment offset */ 314 rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(rb_ptr)); 315 316 /* Now retrieve mbuf to give to stack */ 317 mbuf = rinfo->mbuf; 318 if (__predict_false(mbuf == NULL)) { 319 panic("%s: Received packet fragment with NULL mbuf", 320 device_get_nameunit(nic->dev)); 321 } 322 /* 323 * Clear the mbuf in the descriptor to indicate 324 * that this slot is processed and free to use. 325 */ 326 rinfo->mbuf = NULL; 327 328 bus_dmamap_sync(rinfo->dmat, rinfo->dmap, BUS_DMASYNC_POSTREAD); 329 bus_dmamap_unload(rinfo->dmat, rinfo->dmap); 330 331 return (mbuf); 332 } 333 334 /* Allocate RBDR ring and populate receive buffers */ 335 static int 336 nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, int ring_len, 337 int buf_size, int qidx) 338 { 339 bus_dmamap_t dmap; 340 bus_addr_t rbuf; 341 struct rbdr_entry_t *desc; 342 int idx; 343 int err; 344 345 /* Allocate rbdr descriptors ring */ 346 err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len, 347 sizeof(struct rbdr_entry_t), NICVF_RCV_BUF_ALIGN_BYTES); 348 if (err != 0) { 349 device_printf(nic->dev, 350 "Failed to create RBDR descriptors ring\n"); 351 return (err); 352 } 353 354 rbdr->desc = rbdr->dmem.base; 355 /* 356 * Buffer size has to be in multiples of 128 bytes. 357 * Make room for metadata of size of one line (128 bytes). 358 */ 359 rbdr->dma_size = buf_size - NICVF_RCV_BUF_ALIGN_BYTES; 360 rbdr->enable = TRUE; 361 rbdr->thresh = RBDR_THRESH; 362 rbdr->nic = nic; 363 rbdr->idx = qidx; 364 365 /* 366 * Create DMA tag for Rx buffers. 367 * Each map created using this tag is intended to store Rx payload for 368 * one fragment and one header structure containing rbuf_info (thus 369 * additional 128 byte line since RB must be a multiple of 128 byte 370 * cache line). 371 */ 372 if (buf_size > MCLBYTES) { 373 device_printf(nic->dev, 374 "Buffer size to large for mbuf cluster\n"); 375 return (EINVAL); 376 } 377 err = bus_dma_tag_create( 378 bus_get_dma_tag(nic->dev), /* parent tag */ 379 NICVF_RCV_BUF_ALIGN_BYTES, /* alignment */ 380 0, /* boundary */ 381 DMAP_MAX_PHYSADDR, /* lowaddr */ 382 DMAP_MIN_PHYSADDR, /* highaddr */ 383 NULL, NULL, /* filtfunc, filtfuncarg */ 384 roundup2(buf_size, MCLBYTES), /* maxsize */ 385 1, /* nsegments */ 386 roundup2(buf_size, MCLBYTES), /* maxsegsize */ 387 0, /* flags */ 388 NULL, NULL, /* lockfunc, lockfuncarg */ 389 &rbdr->rbdr_buff_dmat); /* dmat */ 390 391 if (err != 0) { 392 device_printf(nic->dev, 393 "Failed to create busdma tag for RBDR buffers\n"); 394 return (err); 395 } 396 397 rbdr->rbdr_buff_dmaps = malloc(sizeof(*rbdr->rbdr_buff_dmaps) * 398 ring_len, M_NICVF, (M_WAITOK | M_ZERO)); 399 400 for (idx = 0; idx < ring_len; idx++) { 401 err = bus_dmamap_create(rbdr->rbdr_buff_dmat, 0, &dmap); 402 if (err != 0) { 403 device_printf(nic->dev, 404 "Failed to create DMA map for RB\n"); 405 return (err); 406 } 407 rbdr->rbdr_buff_dmaps[idx] = dmap; 408 409 err = nicvf_alloc_rcv_buffer(nic, rbdr, dmap, M_WAITOK, 410 DMA_BUFFER_LEN, &rbuf); 411 if (err != 0) 412 return (err); 413 414 desc = GET_RBDR_DESC(rbdr, idx); 415 desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN); 416 } 417 418 /* Allocate taskqueue */ 419 TASK_INIT(&rbdr->rbdr_task, 0, nicvf_rbdr_task, rbdr); 420 TASK_INIT(&rbdr->rbdr_task_nowait, 0, nicvf_rbdr_task_nowait, rbdr); 421 rbdr->rbdr_taskq = taskqueue_create_fast("nicvf_rbdr_taskq", M_WAITOK, 422 taskqueue_thread_enqueue, &rbdr->rbdr_taskq); 423 taskqueue_start_threads(&rbdr->rbdr_taskq, 1, PI_NET, "%s: rbdr_taskq", 424 device_get_nameunit(nic->dev)); 425 426 return (0); 427 } 428 429 /* Free RBDR ring and its receive buffers */ 430 static void 431 nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr) 432 { 433 struct mbuf *mbuf; 434 struct queue_set *qs; 435 struct rbdr_entry_t *desc; 436 struct rbuf_info *rinfo; 437 bus_addr_t buf_addr; 438 int head, tail, idx; 439 int err __diagused; 440 441 qs = nic->qs; 442 443 if ((qs == NULL) || (rbdr == NULL)) 444 return; 445 446 rbdr->enable = FALSE; 447 if (rbdr->rbdr_taskq != NULL) { 448 /* Remove tasks */ 449 while (taskqueue_cancel(rbdr->rbdr_taskq, 450 &rbdr->rbdr_task_nowait, NULL) != 0) { 451 /* Finish the nowait task first */ 452 taskqueue_drain(rbdr->rbdr_taskq, 453 &rbdr->rbdr_task_nowait); 454 } 455 taskqueue_free(rbdr->rbdr_taskq); 456 rbdr->rbdr_taskq = NULL; 457 458 while (taskqueue_cancel(taskqueue_thread, 459 &rbdr->rbdr_task, NULL) != 0) { 460 /* Now finish the sleepable task */ 461 taskqueue_drain(taskqueue_thread, &rbdr->rbdr_task); 462 } 463 } 464 465 /* 466 * Free all of the memory under the RB descriptors. 467 * There are assumptions here: 468 * 1. Corresponding RBDR is disabled 469 * - it is safe to operate using head and tail indexes 470 * 2. All bffers that were received are properly freed by 471 * the receive handler 472 * - there is no need to unload DMA map and free MBUF for other 473 * descriptors than unused ones 474 */ 475 if (rbdr->rbdr_buff_dmat != NULL) { 476 head = rbdr->head; 477 tail = rbdr->tail; 478 while (head != tail) { 479 desc = GET_RBDR_DESC(rbdr, head); 480 buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; 481 rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr)); 482 bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap); 483 mbuf = rinfo->mbuf; 484 /* This will destroy everything including rinfo! */ 485 m_freem(mbuf); 486 head++; 487 head &= (rbdr->dmem.q_len - 1); 488 } 489 /* Free tail descriptor */ 490 desc = GET_RBDR_DESC(rbdr, tail); 491 buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; 492 rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr)); 493 bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap); 494 mbuf = rinfo->mbuf; 495 /* This will destroy everything including rinfo! */ 496 m_freem(mbuf); 497 498 /* Destroy DMA maps */ 499 for (idx = 0; idx < qs->rbdr_len; idx++) { 500 if (rbdr->rbdr_buff_dmaps[idx] == NULL) 501 continue; 502 err = bus_dmamap_destroy(rbdr->rbdr_buff_dmat, 503 rbdr->rbdr_buff_dmaps[idx]); 504 KASSERT(err == 0, 505 ("%s: Could not destroy DMA map for RB, desc: %d", 506 __func__, idx)); 507 rbdr->rbdr_buff_dmaps[idx] = NULL; 508 } 509 510 /* Now destroy the tag */ 511 err = bus_dma_tag_destroy(rbdr->rbdr_buff_dmat); 512 KASSERT(err == 0, 513 ("%s: Trying to destroy BUSY DMA tag", __func__)); 514 515 rbdr->head = 0; 516 rbdr->tail = 0; 517 } 518 519 /* Free RBDR ring */ 520 nicvf_free_q_desc_mem(nic, &rbdr->dmem); 521 } 522 523 /* 524 * Refill receive buffer descriptors with new buffers. 525 */ 526 static int 527 nicvf_refill_rbdr(struct rbdr *rbdr, int mflags) 528 { 529 struct nicvf *nic; 530 struct queue_set *qs; 531 int rbdr_idx; 532 int tail, qcount; 533 int refill_rb_cnt; 534 struct rbdr_entry_t *desc; 535 bus_dmamap_t dmap; 536 bus_addr_t rbuf; 537 boolean_t rb_alloc_fail; 538 int new_rb; 539 540 rb_alloc_fail = TRUE; 541 new_rb = 0; 542 nic = rbdr->nic; 543 qs = nic->qs; 544 rbdr_idx = rbdr->idx; 545 546 /* Check if it's enabled */ 547 if (!rbdr->enable) 548 return (0); 549 550 /* Get no of desc's to be refilled */ 551 qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx); 552 qcount &= 0x7FFFF; 553 /* Doorbell can be ringed with a max of ring size minus 1 */ 554 if (qcount >= (qs->rbdr_len - 1)) { 555 rb_alloc_fail = FALSE; 556 goto out; 557 } else 558 refill_rb_cnt = qs->rbdr_len - qcount - 1; 559 560 /* Start filling descs from tail */ 561 tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3; 562 while (refill_rb_cnt) { 563 tail++; 564 tail &= (rbdr->dmem.q_len - 1); 565 566 dmap = rbdr->rbdr_buff_dmaps[tail]; 567 if (nicvf_alloc_rcv_buffer(nic, rbdr, dmap, mflags, 568 DMA_BUFFER_LEN, &rbuf)) { 569 /* Something went wrong. Resign */ 570 break; 571 } 572 desc = GET_RBDR_DESC(rbdr, tail); 573 desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN); 574 refill_rb_cnt--; 575 new_rb++; 576 } 577 578 /* make sure all memory stores are done before ringing doorbell */ 579 wmb(); 580 581 /* Check if buffer allocation failed */ 582 if (refill_rb_cnt == 0) 583 rb_alloc_fail = FALSE; 584 585 /* Notify HW */ 586 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, 587 rbdr_idx, new_rb); 588 out: 589 if (!rb_alloc_fail) { 590 /* 591 * Re-enable RBDR interrupts only 592 * if buffer allocation is success. 593 */ 594 nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx); 595 596 return (0); 597 } 598 599 return (ENOMEM); 600 } 601 602 /* Refill RBs even if sleep is needed to reclaim memory */ 603 static void 604 nicvf_rbdr_task(void *arg, int pending) 605 { 606 struct rbdr *rbdr; 607 int err; 608 609 rbdr = (struct rbdr *)arg; 610 611 err = nicvf_refill_rbdr(rbdr, M_WAITOK); 612 if (__predict_false(err != 0)) { 613 panic("%s: Failed to refill RBs even when sleep enabled", 614 __func__); 615 } 616 } 617 618 /* Refill RBs as soon as possible without waiting */ 619 static void 620 nicvf_rbdr_task_nowait(void *arg, int pending) 621 { 622 struct rbdr *rbdr; 623 int err; 624 625 rbdr = (struct rbdr *)arg; 626 627 err = nicvf_refill_rbdr(rbdr, M_NOWAIT); 628 if (err != 0) { 629 /* 630 * Schedule another, sleepable kernel thread 631 * that will for sure refill the buffers. 632 */ 633 taskqueue_enqueue(taskqueue_thread, &rbdr->rbdr_task); 634 } 635 } 636 637 static int 638 nicvf_rcv_pkt_handler(struct nicvf *nic, struct cmp_queue *cq, 639 struct cqe_rx_t *cqe_rx, int cqe_type) 640 { 641 struct mbuf *mbuf; 642 struct rcv_queue *rq; 643 int rq_idx; 644 int err = 0; 645 646 rq_idx = cqe_rx->rq_idx; 647 rq = &nic->qs->rq[rq_idx]; 648 649 /* Check for errors */ 650 err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx); 651 if (err && !cqe_rx->rb_cnt) 652 return (0); 653 654 mbuf = nicvf_get_rcv_mbuf(nic, cqe_rx); 655 if (mbuf == NULL) { 656 dprintf(nic->dev, "Packet not received\n"); 657 return (0); 658 } 659 660 /* If error packet */ 661 if (err != 0) { 662 m_freem(mbuf); 663 return (0); 664 } 665 666 if (rq->lro_enabled && 667 ((cqe_rx->l3_type == L3TYPE_IPV4) && (cqe_rx->l4_type == L4TYPE_TCP)) && 668 (mbuf->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 669 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 670 /* 671 * At this point it is known that there are no errors in the 672 * packet. Attempt to LRO enqueue. Send to stack if no resources 673 * or enqueue error. 674 */ 675 if ((rq->lro.lro_cnt != 0) && 676 (tcp_lro_rx(&rq->lro, mbuf, 0) == 0)) 677 return (0); 678 } 679 /* 680 * Push this packet to the stack later to avoid 681 * unlocking completion task in the middle of work. 682 */ 683 err = buf_ring_enqueue(cq->rx_br, mbuf); 684 if (err != 0) { 685 /* 686 * Failed to enqueue this mbuf. 687 * We don't drop it, just schedule another task. 688 */ 689 return (err); 690 } 691 692 return (0); 693 } 694 695 static void 696 nicvf_snd_pkt_handler(struct nicvf *nic, struct cmp_queue *cq, 697 struct cqe_send_t *cqe_tx, int cqe_type) 698 { 699 bus_dmamap_t dmap; 700 struct mbuf *mbuf; 701 struct snd_queue *sq; 702 struct sq_hdr_subdesc *hdr; 703 704 mbuf = NULL; 705 sq = &nic->qs->sq[cqe_tx->sq_idx]; 706 707 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr); 708 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) 709 return; 710 711 dprintf(nic->dev, 712 "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n", 713 __func__, cqe_tx->sq_qs, cqe_tx->sq_idx, 714 cqe_tx->sqe_ptr, hdr->subdesc_cnt); 715 716 dmap = (bus_dmamap_t)sq->snd_buff[cqe_tx->sqe_ptr].dmap; 717 bus_dmamap_unload(sq->snd_buff_dmat, dmap); 718 719 mbuf = (struct mbuf *)sq->snd_buff[cqe_tx->sqe_ptr].mbuf; 720 if (mbuf != NULL) { 721 m_freem(mbuf); 722 sq->snd_buff[cqe_tx->sqe_ptr].mbuf = NULL; 723 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 724 } 725 726 nicvf_check_cqe_tx_errs(nic, cq, cqe_tx); 727 } 728 729 static int 730 nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx) 731 { 732 struct mbuf *mbuf; 733 if_t ifp; 734 int processed_cqe, tx_done = 0; 735 #ifdef DEBUG 736 int work_done = 0; 737 #endif 738 int cqe_count, cqe_head; 739 struct queue_set *qs = nic->qs; 740 struct cmp_queue *cq = &qs->cq[cq_idx]; 741 struct snd_queue *sq = &qs->sq[cq_idx]; 742 struct rcv_queue *rq; 743 struct cqe_rx_t *cq_desc; 744 struct lro_ctrl *lro; 745 int rq_idx; 746 int cmp_err; 747 748 NICVF_CMP_LOCK(cq); 749 cmp_err = 0; 750 processed_cqe = 0; 751 /* Get no of valid CQ entries to process */ 752 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); 753 cqe_count &= CQ_CQE_COUNT; 754 if (cqe_count == 0) 755 goto out; 756 757 /* Get head of the valid CQ entries */ 758 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9; 759 cqe_head &= 0xFFFF; 760 761 dprintf(nic->dev, "%s CQ%d cqe_count %d cqe_head %d\n", 762 __func__, cq_idx, cqe_count, cqe_head); 763 while (processed_cqe < cqe_count) { 764 /* Get the CQ descriptor */ 765 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head); 766 cqe_head++; 767 cqe_head &= (cq->dmem.q_len - 1); 768 /* Prefetch next CQ descriptor */ 769 __builtin_prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head)); 770 771 dprintf(nic->dev, "CQ%d cq_desc->cqe_type %d\n", cq_idx, 772 cq_desc->cqe_type); 773 switch (cq_desc->cqe_type) { 774 case CQE_TYPE_RX: 775 cmp_err = nicvf_rcv_pkt_handler(nic, cq, cq_desc, 776 CQE_TYPE_RX); 777 if (__predict_false(cmp_err != 0)) { 778 /* 779 * Ups. Cannot finish now. 780 * Let's try again later. 781 */ 782 goto done; 783 } 784 #ifdef DEBUG 785 work_done++; 786 #endif 787 break; 788 case CQE_TYPE_SEND: 789 nicvf_snd_pkt_handler(nic, cq, (void *)cq_desc, 790 CQE_TYPE_SEND); 791 tx_done++; 792 break; 793 case CQE_TYPE_INVALID: 794 case CQE_TYPE_RX_SPLIT: 795 case CQE_TYPE_RX_TCP: 796 case CQE_TYPE_SEND_PTP: 797 /* Ignore for now */ 798 break; 799 } 800 processed_cqe++; 801 } 802 done: 803 dprintf(nic->dev, 804 "%s CQ%d processed_cqe %d work_done %d\n", 805 __func__, cq_idx, processed_cqe, work_done); 806 807 /* Ring doorbell to inform H/W to reuse processed CQEs */ 808 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, cq_idx, processed_cqe); 809 810 if ((tx_done > 0) && 811 ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0)) { 812 /* Reenable TXQ if its stopped earlier due to SQ full */ 813 if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 814 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 815 } 816 out: 817 /* 818 * Flush any outstanding LRO work 819 */ 820 rq_idx = cq_idx; 821 rq = &nic->qs->rq[rq_idx]; 822 lro = &rq->lro; 823 tcp_lro_flush_all(lro); 824 825 NICVF_CMP_UNLOCK(cq); 826 827 ifp = nic->ifp; 828 /* Push received MBUFs to the stack */ 829 while (!buf_ring_empty(cq->rx_br)) { 830 mbuf = buf_ring_dequeue_mc(cq->rx_br); 831 if (__predict_true(mbuf != NULL)) 832 if_input(ifp, mbuf); 833 } 834 835 return (cmp_err); 836 } 837 838 /* 839 * Qset error interrupt handler 840 * 841 * As of now only CQ errors are handled 842 */ 843 static void 844 nicvf_qs_err_task(void *arg, int pending) 845 { 846 struct nicvf *nic; 847 struct queue_set *qs; 848 int qidx; 849 uint64_t status; 850 boolean_t enable = TRUE; 851 852 nic = (struct nicvf *)arg; 853 qs = nic->qs; 854 855 /* Deactivate network interface */ 856 if_setdrvflagbits(nic->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 857 858 /* Check if it is CQ err */ 859 for (qidx = 0; qidx < qs->cq_cnt; qidx++) { 860 status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, 861 qidx); 862 if ((status & CQ_ERR_MASK) == 0) 863 continue; 864 /* Process already queued CQEs and reconfig CQ */ 865 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 866 nicvf_sq_disable(nic, qidx); 867 (void)nicvf_cq_intr_handler(nic, qidx); 868 nicvf_cmp_queue_config(nic, qs, qidx, enable); 869 nicvf_sq_free_used_descs(nic, &qs->sq[qidx], qidx); 870 nicvf_sq_enable(nic, &qs->sq[qidx], qidx); 871 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 872 } 873 874 if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 875 /* Re-enable Qset error interrupt */ 876 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 877 } 878 879 static void 880 nicvf_cmp_task(void *arg, int pending) 881 { 882 struct cmp_queue *cq; 883 struct nicvf *nic; 884 int cmp_err; 885 886 cq = (struct cmp_queue *)arg; 887 nic = cq->nic; 888 889 /* Handle CQ descriptors */ 890 cmp_err = nicvf_cq_intr_handler(nic, cq->idx); 891 if (__predict_false(cmp_err != 0)) { 892 /* 893 * Schedule another thread here since we did not 894 * process the entire CQ due to Tx or Rx CQ parse error. 895 */ 896 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 897 } 898 899 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx); 900 /* Reenable interrupt (previously disabled in nicvf_intr_handler() */ 901 nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->idx); 902 903 } 904 905 /* Initialize completion queue */ 906 static int 907 nicvf_init_cmp_queue(struct nicvf *nic, struct cmp_queue *cq, int q_len, 908 int qidx) 909 { 910 int err; 911 912 /* Initizalize lock */ 913 snprintf(cq->mtx_name, sizeof(cq->mtx_name), "%s: CQ(%d) lock", 914 device_get_nameunit(nic->dev), qidx); 915 mtx_init(&cq->mtx, cq->mtx_name, NULL, MTX_DEF); 916 917 err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE, 918 NICVF_CQ_BASE_ALIGN_BYTES); 919 920 if (err != 0) { 921 device_printf(nic->dev, 922 "Could not allocate DMA memory for CQ\n"); 923 return (err); 924 } 925 926 cq->desc = cq->dmem.base; 927 cq->thresh = pass1_silicon(nic->dev) ? 0 : CMP_QUEUE_CQE_THRESH; 928 cq->nic = nic; 929 cq->idx = qidx; 930 nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1; 931 932 cq->rx_br = buf_ring_alloc(CMP_QUEUE_LEN * 8, M_DEVBUF, M_WAITOK, 933 &cq->mtx); 934 935 /* Allocate taskqueue */ 936 NET_TASK_INIT(&cq->cmp_task, 0, nicvf_cmp_task, cq); 937 cq->cmp_taskq = taskqueue_create_fast("nicvf_cmp_taskq", M_WAITOK, 938 taskqueue_thread_enqueue, &cq->cmp_taskq); 939 taskqueue_start_threads(&cq->cmp_taskq, 1, PI_NET, "%s: cmp_taskq(%d)", 940 device_get_nameunit(nic->dev), qidx); 941 942 return (0); 943 } 944 945 static void 946 nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq) 947 { 948 949 if (cq == NULL) 950 return; 951 /* 952 * The completion queue itself should be disabled by now 953 * (ref. nicvf_snd_queue_config()). 954 * Ensure that it is safe to disable it or panic. 955 */ 956 if (cq->enable) 957 panic("%s: Trying to free working CQ(%d)", __func__, cq->idx); 958 959 if (cq->cmp_taskq != NULL) { 960 /* Remove task */ 961 while (taskqueue_cancel(cq->cmp_taskq, &cq->cmp_task, NULL) != 0) 962 taskqueue_drain(cq->cmp_taskq, &cq->cmp_task); 963 964 taskqueue_free(cq->cmp_taskq); 965 cq->cmp_taskq = NULL; 966 } 967 /* 968 * Completion interrupt will possibly enable interrupts again 969 * so disable interrupting now after we finished processing 970 * completion task. It is safe to do so since the corresponding CQ 971 * was already disabled. 972 */ 973 nicvf_disable_intr(nic, NICVF_INTR_CQ, cq->idx); 974 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx); 975 976 NICVF_CMP_LOCK(cq); 977 nicvf_free_q_desc_mem(nic, &cq->dmem); 978 drbr_free(cq->rx_br, M_DEVBUF); 979 NICVF_CMP_UNLOCK(cq); 980 mtx_destroy(&cq->mtx); 981 memset(cq->mtx_name, 0, sizeof(cq->mtx_name)); 982 } 983 984 int 985 nicvf_xmit_locked(struct snd_queue *sq) 986 { 987 struct nicvf *nic; 988 if_t ifp; 989 struct mbuf *next; 990 int err; 991 992 NICVF_TX_LOCK_ASSERT(sq); 993 994 nic = sq->nic; 995 ifp = nic->ifp; 996 err = 0; 997 998 while ((next = drbr_peek(ifp, sq->br)) != NULL) { 999 /* Send a copy of the frame to the BPF listener */ 1000 ETHER_BPF_MTAP(ifp, next); 1001 1002 err = nicvf_tx_mbuf_locked(sq, &next); 1003 if (err != 0) { 1004 if (next == NULL) 1005 drbr_advance(ifp, sq->br); 1006 else 1007 drbr_putback(ifp, sq->br, next); 1008 1009 break; 1010 } 1011 drbr_advance(ifp, sq->br); 1012 } 1013 return (err); 1014 } 1015 1016 static void 1017 nicvf_snd_task(void *arg, int pending) 1018 { 1019 struct snd_queue *sq = (struct snd_queue *)arg; 1020 struct nicvf *nic; 1021 if_t ifp; 1022 int err; 1023 1024 nic = sq->nic; 1025 ifp = nic->ifp; 1026 1027 /* 1028 * Skip sending anything if the driver is not running, 1029 * SQ full or link is down. 1030 */ 1031 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1032 IFF_DRV_RUNNING) || !nic->link_up) 1033 return; 1034 1035 NICVF_TX_LOCK(sq); 1036 err = nicvf_xmit_locked(sq); 1037 NICVF_TX_UNLOCK(sq); 1038 /* Try again */ 1039 if (err != 0) 1040 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task); 1041 } 1042 1043 /* Initialize transmit queue */ 1044 static int 1045 nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len, 1046 int qidx) 1047 { 1048 size_t i; 1049 int err; 1050 1051 /* Initizalize TX lock for this queue */ 1052 snprintf(sq->mtx_name, sizeof(sq->mtx_name), "%s: SQ(%d) lock", 1053 device_get_nameunit(nic->dev), qidx); 1054 mtx_init(&sq->mtx, sq->mtx_name, NULL, MTX_DEF); 1055 1056 NICVF_TX_LOCK(sq); 1057 /* Allocate buffer ring */ 1058 sq->br = buf_ring_alloc(q_len / MIN_SQ_DESC_PER_PKT_XMIT, M_DEVBUF, 1059 M_NOWAIT, &sq->mtx); 1060 if (sq->br == NULL) { 1061 device_printf(nic->dev, 1062 "ERROR: Could not set up buf ring for SQ(%d)\n", qidx); 1063 err = ENOMEM; 1064 goto error; 1065 } 1066 1067 /* Allocate DMA memory for Tx descriptors */ 1068 err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE, 1069 NICVF_SQ_BASE_ALIGN_BYTES); 1070 if (err != 0) { 1071 device_printf(nic->dev, 1072 "Could not allocate DMA memory for SQ\n"); 1073 goto error; 1074 } 1075 1076 sq->desc = sq->dmem.base; 1077 sq->head = sq->tail = 0; 1078 atomic_store_rel_int(&sq->free_cnt, q_len - 1); 1079 sq->thresh = SND_QUEUE_THRESH; 1080 sq->idx = qidx; 1081 sq->nic = nic; 1082 1083 /* 1084 * Allocate DMA maps for Tx buffers 1085 */ 1086 1087 /* Create DMA tag first */ 1088 err = bus_dma_tag_create( 1089 bus_get_dma_tag(nic->dev), /* parent tag */ 1090 1, /* alignment */ 1091 0, /* boundary */ 1092 BUS_SPACE_MAXADDR, /* lowaddr */ 1093 BUS_SPACE_MAXADDR, /* highaddr */ 1094 NULL, NULL, /* filtfunc, filtfuncarg */ 1095 NICVF_TSO_MAXSIZE, /* maxsize */ 1096 NICVF_TSO_NSEGS, /* nsegments */ 1097 MCLBYTES, /* maxsegsize */ 1098 0, /* flags */ 1099 NULL, NULL, /* lockfunc, lockfuncarg */ 1100 &sq->snd_buff_dmat); /* dmat */ 1101 1102 if (err != 0) { 1103 device_printf(nic->dev, 1104 "Failed to create busdma tag for Tx buffers\n"); 1105 goto error; 1106 } 1107 1108 /* Allocate send buffers array */ 1109 sq->snd_buff = malloc(sizeof(*sq->snd_buff) * q_len, M_NICVF, 1110 (M_NOWAIT | M_ZERO)); 1111 if (sq->snd_buff == NULL) { 1112 device_printf(nic->dev, 1113 "Could not allocate memory for Tx buffers array\n"); 1114 err = ENOMEM; 1115 goto error; 1116 } 1117 1118 /* Now populate maps */ 1119 for (i = 0; i < q_len; i++) { 1120 err = bus_dmamap_create(sq->snd_buff_dmat, 0, 1121 &sq->snd_buff[i].dmap); 1122 if (err != 0) { 1123 device_printf(nic->dev, 1124 "Failed to create DMA maps for Tx buffers\n"); 1125 goto error; 1126 } 1127 } 1128 NICVF_TX_UNLOCK(sq); 1129 1130 /* Allocate taskqueue */ 1131 TASK_INIT(&sq->snd_task, 0, nicvf_snd_task, sq); 1132 sq->snd_taskq = taskqueue_create_fast("nicvf_snd_taskq", M_WAITOK, 1133 taskqueue_thread_enqueue, &sq->snd_taskq); 1134 taskqueue_start_threads(&sq->snd_taskq, 1, PI_NET, "%s: snd_taskq(%d)", 1135 device_get_nameunit(nic->dev), qidx); 1136 1137 return (0); 1138 error: 1139 NICVF_TX_UNLOCK(sq); 1140 return (err); 1141 } 1142 1143 static void 1144 nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) 1145 { 1146 struct queue_set *qs = nic->qs; 1147 size_t i; 1148 int err __diagused; 1149 1150 if (sq == NULL) 1151 return; 1152 1153 if (sq->snd_taskq != NULL) { 1154 /* Remove task */ 1155 while (taskqueue_cancel(sq->snd_taskq, &sq->snd_task, NULL) != 0) 1156 taskqueue_drain(sq->snd_taskq, &sq->snd_task); 1157 1158 taskqueue_free(sq->snd_taskq); 1159 sq->snd_taskq = NULL; 1160 } 1161 1162 NICVF_TX_LOCK(sq); 1163 if (sq->snd_buff_dmat != NULL) { 1164 if (sq->snd_buff != NULL) { 1165 for (i = 0; i < qs->sq_len; i++) { 1166 m_freem(sq->snd_buff[i].mbuf); 1167 sq->snd_buff[i].mbuf = NULL; 1168 1169 bus_dmamap_unload(sq->snd_buff_dmat, 1170 sq->snd_buff[i].dmap); 1171 err = bus_dmamap_destroy(sq->snd_buff_dmat, 1172 sq->snd_buff[i].dmap); 1173 /* 1174 * If bus_dmamap_destroy fails it can cause 1175 * random panic later if the tag is also 1176 * destroyed in the process. 1177 */ 1178 KASSERT(err == 0, 1179 ("%s: Could not destroy DMA map for SQ", 1180 __func__)); 1181 } 1182 } 1183 1184 free(sq->snd_buff, M_NICVF); 1185 1186 err = bus_dma_tag_destroy(sq->snd_buff_dmat); 1187 KASSERT(err == 0, 1188 ("%s: Trying to destroy BUSY DMA tag", __func__)); 1189 } 1190 1191 /* Free private driver ring for this send queue */ 1192 if (sq->br != NULL) 1193 drbr_free(sq->br, M_DEVBUF); 1194 1195 if (sq->dmem.base != NULL) 1196 nicvf_free_q_desc_mem(nic, &sq->dmem); 1197 1198 NICVF_TX_UNLOCK(sq); 1199 /* Destroy Tx lock */ 1200 mtx_destroy(&sq->mtx); 1201 memset(sq->mtx_name, 0, sizeof(sq->mtx_name)); 1202 } 1203 1204 static void 1205 nicvf_reclaim_snd_queue(struct nicvf *nic, struct queue_set *qs, int qidx) 1206 { 1207 1208 /* Disable send queue */ 1209 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0); 1210 /* Check if SQ is stopped */ 1211 if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01)) 1212 return; 1213 /* Reset send queue */ 1214 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); 1215 } 1216 1217 static void 1218 nicvf_reclaim_rcv_queue(struct nicvf *nic, struct queue_set *qs, int qidx) 1219 { 1220 union nic_mbx mbx = {}; 1221 1222 /* Make sure all packets in the pipeline are written back into mem */ 1223 mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC; 1224 nicvf_send_msg_to_pf(nic, &mbx); 1225 } 1226 1227 static void 1228 nicvf_reclaim_cmp_queue(struct nicvf *nic, struct queue_set *qs, int qidx) 1229 { 1230 1231 /* Disable timer threshold (doesn't get reset upon CQ reset */ 1232 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0); 1233 /* Disable completion queue */ 1234 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0); 1235 /* Reset completion queue */ 1236 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); 1237 } 1238 1239 static void 1240 nicvf_reclaim_rbdr(struct nicvf *nic, struct rbdr *rbdr, int qidx) 1241 { 1242 uint64_t tmp, fifo_state; 1243 int timeout = 10; 1244 1245 /* Save head and tail pointers for feeing up buffers */ 1246 rbdr->head = 1247 nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_HEAD, qidx) >> 3; 1248 rbdr->tail = 1249 nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, qidx) >> 3; 1250 1251 /* 1252 * If RBDR FIFO is in 'FAIL' state then do a reset first 1253 * before relaiming. 1254 */ 1255 fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx); 1256 if (((fifo_state >> 62) & 0x03) == 0x3) { 1257 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, 1258 qidx, NICVF_RBDR_RESET); 1259 } 1260 1261 /* Disable RBDR */ 1262 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0); 1263 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) 1264 return; 1265 while (1) { 1266 tmp = nicvf_queue_reg_read(nic, 1267 NIC_QSET_RBDR_0_1_PREFETCH_STATUS, qidx); 1268 if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF)) 1269 break; 1270 1271 DELAY(1000); 1272 timeout--; 1273 if (!timeout) { 1274 device_printf(nic->dev, 1275 "Failed polling on prefetch status\n"); 1276 return; 1277 } 1278 } 1279 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 1280 NICVF_RBDR_RESET); 1281 1282 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02)) 1283 return; 1284 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00); 1285 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) 1286 return; 1287 } 1288 1289 /* Configures receive queue */ 1290 static void 1291 nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, 1292 int qidx, bool enable) 1293 { 1294 union nic_mbx mbx = {}; 1295 struct rcv_queue *rq; 1296 struct rq_cfg rq_cfg; 1297 if_t ifp; 1298 struct lro_ctrl *lro; 1299 1300 ifp = nic->ifp; 1301 1302 rq = &qs->rq[qidx]; 1303 rq->enable = enable; 1304 1305 lro = &rq->lro; 1306 1307 /* Disable receive queue */ 1308 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); 1309 1310 if (!rq->enable) { 1311 nicvf_reclaim_rcv_queue(nic, qs, qidx); 1312 /* Free LRO memory */ 1313 tcp_lro_free(lro); 1314 rq->lro_enabled = FALSE; 1315 return; 1316 } 1317 1318 /* Configure LRO if enabled */ 1319 rq->lro_enabled = FALSE; 1320 if ((if_getcapenable(ifp) & IFCAP_LRO) != 0) { 1321 if (tcp_lro_init(lro) != 0) { 1322 device_printf(nic->dev, 1323 "Failed to initialize LRO for RXQ%d\n", qidx); 1324 } else { 1325 rq->lro_enabled = TRUE; 1326 lro->ifp = nic->ifp; 1327 } 1328 } 1329 1330 rq->cq_qs = qs->vnic_id; 1331 rq->cq_idx = qidx; 1332 rq->start_rbdr_qs = qs->vnic_id; 1333 rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1; 1334 rq->cont_rbdr_qs = qs->vnic_id; 1335 rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1; 1336 /* all writes of RBDR data to be loaded into L2 Cache as well*/ 1337 rq->caching = 1; 1338 1339 /* Send a mailbox msg to PF to config RQ */ 1340 mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; 1341 mbx.rq.qs_num = qs->vnic_id; 1342 mbx.rq.rq_num = qidx; 1343 mbx.rq.cfg = ((uint64_t)rq->caching << 26) | (rq->cq_qs << 19) | 1344 (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) | 1345 (rq->cont_qs_rbdr_idx << 8) | (rq->start_rbdr_qs << 1) | 1346 (rq->start_qs_rbdr_idx); 1347 nicvf_send_msg_to_pf(nic, &mbx); 1348 1349 mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG; 1350 mbx.rq.cfg = (1UL << 63) | (1UL << 62) | (qs->vnic_id << 0); 1351 nicvf_send_msg_to_pf(nic, &mbx); 1352 1353 /* 1354 * RQ drop config 1355 * Enable CQ drop to reserve sufficient CQEs for all tx packets 1356 */ 1357 mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG; 1358 mbx.rq.cfg = (1UL << 62) | (RQ_CQ_DROP << 8); 1359 nicvf_send_msg_to_pf(nic, &mbx); 1360 1361 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 0x00); 1362 1363 /* Enable Receive queue */ 1364 rq_cfg.ena = 1; 1365 rq_cfg.tcp_ena = 0; 1366 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 1367 *(uint64_t *)&rq_cfg); 1368 } 1369 1370 /* Configures completion queue */ 1371 static void 1372 nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, 1373 int qidx, boolean_t enable) 1374 { 1375 struct cmp_queue *cq; 1376 struct cq_cfg cq_cfg; 1377 1378 cq = &qs->cq[qidx]; 1379 cq->enable = enable; 1380 1381 if (!cq->enable) { 1382 nicvf_reclaim_cmp_queue(nic, qs, qidx); 1383 return; 1384 } 1385 1386 /* Reset completion queue */ 1387 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); 1388 1389 /* Set completion queue base address */ 1390 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, qidx, 1391 (uint64_t)(cq->dmem.phys_base)); 1392 1393 /* Enable Completion queue */ 1394 cq_cfg.ena = 1; 1395 cq_cfg.reset = 0; 1396 cq_cfg.caching = 0; 1397 cq_cfg.qsize = CMP_QSIZE; 1398 cq_cfg.avg_con = 0; 1399 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(uint64_t *)&cq_cfg); 1400 1401 /* Set threshold value for interrupt generation */ 1402 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh); 1403 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 1404 nic->cq_coalesce_usecs); 1405 } 1406 1407 /* Configures transmit queue */ 1408 static void 1409 nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx, 1410 boolean_t enable) 1411 { 1412 union nic_mbx mbx = {}; 1413 struct snd_queue *sq; 1414 struct sq_cfg sq_cfg; 1415 1416 sq = &qs->sq[qidx]; 1417 sq->enable = enable; 1418 1419 if (!sq->enable) { 1420 nicvf_reclaim_snd_queue(nic, qs, qidx); 1421 return; 1422 } 1423 1424 /* Reset send queue */ 1425 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); 1426 1427 sq->cq_qs = qs->vnic_id; 1428 sq->cq_idx = qidx; 1429 1430 /* Send a mailbox msg to PF to config SQ */ 1431 mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG; 1432 mbx.sq.qs_num = qs->vnic_id; 1433 mbx.sq.sq_num = qidx; 1434 mbx.sq.sqs_mode = nic->sqs_mode; 1435 mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx; 1436 nicvf_send_msg_to_pf(nic, &mbx); 1437 1438 /* Set queue base address */ 1439 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, qidx, 1440 (uint64_t)(sq->dmem.phys_base)); 1441 1442 /* Enable send queue & set queue size */ 1443 sq_cfg.ena = 1; 1444 sq_cfg.reset = 0; 1445 sq_cfg.ldwb = 0; 1446 sq_cfg.qsize = SND_QSIZE; 1447 sq_cfg.tstmp_bgx_intf = 0; 1448 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(uint64_t *)&sq_cfg); 1449 1450 /* Set threshold value for interrupt generation */ 1451 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh); 1452 } 1453 1454 /* Configures receive buffer descriptor ring */ 1455 static void 1456 nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, int qidx, 1457 boolean_t enable) 1458 { 1459 struct rbdr *rbdr; 1460 struct rbdr_cfg rbdr_cfg; 1461 1462 rbdr = &qs->rbdr[qidx]; 1463 nicvf_reclaim_rbdr(nic, rbdr, qidx); 1464 if (!enable) 1465 return; 1466 1467 /* Set descriptor base address */ 1468 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, qidx, 1469 (uint64_t)(rbdr->dmem.phys_base)); 1470 1471 /* Enable RBDR & set queue size */ 1472 /* Buffer size should be in multiples of 128 bytes */ 1473 rbdr_cfg.ena = 1; 1474 rbdr_cfg.reset = 0; 1475 rbdr_cfg.ldwb = 0; 1476 rbdr_cfg.qsize = RBDR_SIZE; 1477 rbdr_cfg.avg_con = 0; 1478 rbdr_cfg.lines = rbdr->dma_size / 128; 1479 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 1480 *(uint64_t *)&rbdr_cfg); 1481 1482 /* Notify HW */ 1483 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, qidx, 1484 qs->rbdr_len - 1); 1485 1486 /* Set threshold value for interrupt generation */ 1487 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, qidx, 1488 rbdr->thresh - 1); 1489 } 1490 1491 /* Requests PF to assign and enable Qset */ 1492 void 1493 nicvf_qset_config(struct nicvf *nic, boolean_t enable) 1494 { 1495 union nic_mbx mbx = {}; 1496 struct queue_set *qs; 1497 struct qs_cfg *qs_cfg; 1498 1499 qs = nic->qs; 1500 if (qs == NULL) { 1501 device_printf(nic->dev, 1502 "Qset is still not allocated, don't init queues\n"); 1503 return; 1504 } 1505 1506 qs->enable = enable; 1507 qs->vnic_id = nic->vf_id; 1508 1509 /* Send a mailbox msg to PF to config Qset */ 1510 mbx.qs.msg = NIC_MBOX_MSG_QS_CFG; 1511 mbx.qs.num = qs->vnic_id; 1512 1513 mbx.qs.cfg = 0; 1514 qs_cfg = (struct qs_cfg *)&mbx.qs.cfg; 1515 if (qs->enable) { 1516 qs_cfg->ena = 1; 1517 qs_cfg->vnic = qs->vnic_id; 1518 } 1519 nicvf_send_msg_to_pf(nic, &mbx); 1520 } 1521 1522 static void 1523 nicvf_free_resources(struct nicvf *nic) 1524 { 1525 int qidx; 1526 struct queue_set *qs; 1527 1528 qs = nic->qs; 1529 /* 1530 * Remove QS error task first since it has to be dead 1531 * to safely free completion queue tasks. 1532 */ 1533 if (qs->qs_err_taskq != NULL) { 1534 /* Shut down QS error tasks */ 1535 while (taskqueue_cancel(qs->qs_err_taskq, 1536 &qs->qs_err_task, NULL) != 0) { 1537 taskqueue_drain(qs->qs_err_taskq, &qs->qs_err_task); 1538 } 1539 taskqueue_free(qs->qs_err_taskq); 1540 qs->qs_err_taskq = NULL; 1541 } 1542 /* Free receive buffer descriptor ring */ 1543 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 1544 nicvf_free_rbdr(nic, &qs->rbdr[qidx]); 1545 1546 /* Free completion queue */ 1547 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 1548 nicvf_free_cmp_queue(nic, &qs->cq[qidx]); 1549 1550 /* Free send queue */ 1551 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1552 nicvf_free_snd_queue(nic, &qs->sq[qidx]); 1553 } 1554 1555 static int 1556 nicvf_alloc_resources(struct nicvf *nic) 1557 { 1558 struct queue_set *qs = nic->qs; 1559 int qidx; 1560 1561 /* Alloc receive buffer descriptor ring */ 1562 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1563 if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len, 1564 DMA_BUFFER_LEN, qidx)) 1565 goto alloc_fail; 1566 } 1567 1568 /* Alloc send queue */ 1569 for (qidx = 0; qidx < qs->sq_cnt; qidx++) { 1570 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx)) 1571 goto alloc_fail; 1572 } 1573 1574 /* Alloc completion queue */ 1575 for (qidx = 0; qidx < qs->cq_cnt; qidx++) { 1576 if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len, qidx)) 1577 goto alloc_fail; 1578 } 1579 1580 /* Allocate QS error taskqueue */ 1581 NET_TASK_INIT(&qs->qs_err_task, 0, nicvf_qs_err_task, nic); 1582 qs->qs_err_taskq = taskqueue_create_fast("nicvf_qs_err_taskq", M_WAITOK, 1583 taskqueue_thread_enqueue, &qs->qs_err_taskq); 1584 taskqueue_start_threads(&qs->qs_err_taskq, 1, PI_NET, "%s: qs_taskq", 1585 device_get_nameunit(nic->dev)); 1586 1587 return (0); 1588 alloc_fail: 1589 nicvf_free_resources(nic); 1590 return (ENOMEM); 1591 } 1592 1593 int 1594 nicvf_set_qset_resources(struct nicvf *nic) 1595 { 1596 struct queue_set *qs; 1597 1598 qs = malloc(sizeof(*qs), M_NICVF, (M_ZERO | M_WAITOK)); 1599 nic->qs = qs; 1600 1601 /* Set count of each queue */ 1602 qs->rbdr_cnt = RBDR_CNT; 1603 qs->rq_cnt = RCV_QUEUE_CNT; 1604 1605 qs->sq_cnt = SND_QUEUE_CNT; 1606 qs->cq_cnt = CMP_QUEUE_CNT; 1607 1608 /* Set queue lengths */ 1609 qs->rbdr_len = RCV_BUF_COUNT; 1610 qs->sq_len = SND_QUEUE_LEN; 1611 qs->cq_len = CMP_QUEUE_LEN; 1612 1613 nic->rx_queues = qs->rq_cnt; 1614 nic->tx_queues = qs->sq_cnt; 1615 1616 return (0); 1617 } 1618 1619 int 1620 nicvf_config_data_transfer(struct nicvf *nic, boolean_t enable) 1621 { 1622 boolean_t disable = FALSE; 1623 struct queue_set *qs; 1624 int qidx; 1625 1626 qs = nic->qs; 1627 if (qs == NULL) 1628 return (0); 1629 1630 if (enable) { 1631 if (nicvf_alloc_resources(nic) != 0) 1632 return (ENOMEM); 1633 1634 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1635 nicvf_snd_queue_config(nic, qs, qidx, enable); 1636 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 1637 nicvf_cmp_queue_config(nic, qs, qidx, enable); 1638 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 1639 nicvf_rbdr_config(nic, qs, qidx, enable); 1640 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1641 nicvf_rcv_queue_config(nic, qs, qidx, enable); 1642 } else { 1643 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1644 nicvf_rcv_queue_config(nic, qs, qidx, disable); 1645 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 1646 nicvf_rbdr_config(nic, qs, qidx, disable); 1647 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1648 nicvf_snd_queue_config(nic, qs, qidx, disable); 1649 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 1650 nicvf_cmp_queue_config(nic, qs, qidx, disable); 1651 1652 nicvf_free_resources(nic); 1653 } 1654 1655 return (0); 1656 } 1657 1658 /* 1659 * Get a free desc from SQ 1660 * returns descriptor ponter & descriptor number 1661 */ 1662 static __inline int 1663 nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt) 1664 { 1665 int qentry; 1666 1667 qentry = sq->tail; 1668 atomic_subtract_int(&sq->free_cnt, desc_cnt); 1669 sq->tail += desc_cnt; 1670 sq->tail &= (sq->dmem.q_len - 1); 1671 1672 return (qentry); 1673 } 1674 1675 /* Free descriptor back to SQ for future use */ 1676 static void 1677 nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt) 1678 { 1679 1680 atomic_add_int(&sq->free_cnt, desc_cnt); 1681 sq->head += desc_cnt; 1682 sq->head &= (sq->dmem.q_len - 1); 1683 } 1684 1685 static __inline int 1686 nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry) 1687 { 1688 qentry++; 1689 qentry &= (sq->dmem.q_len - 1); 1690 return (qentry); 1691 } 1692 1693 static void 1694 nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx) 1695 { 1696 uint64_t sq_cfg; 1697 1698 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); 1699 sq_cfg |= NICVF_SQ_EN; 1700 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); 1701 /* Ring doorbell so that H/W restarts processing SQEs */ 1702 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0); 1703 } 1704 1705 static void 1706 nicvf_sq_disable(struct nicvf *nic, int qidx) 1707 { 1708 uint64_t sq_cfg; 1709 1710 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); 1711 sq_cfg &= ~NICVF_SQ_EN; 1712 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); 1713 } 1714 1715 static void 1716 nicvf_sq_free_used_descs(struct nicvf *nic, struct snd_queue *sq, int qidx) 1717 { 1718 uint64_t head; 1719 struct snd_buff *snd_buff; 1720 struct sq_hdr_subdesc *hdr; 1721 1722 NICVF_TX_LOCK(sq); 1723 head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4; 1724 while (sq->head != head) { 1725 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head); 1726 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) { 1727 nicvf_put_sq_desc(sq, 1); 1728 continue; 1729 } 1730 snd_buff = &sq->snd_buff[sq->head]; 1731 if (snd_buff->mbuf != NULL) { 1732 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap); 1733 m_freem(snd_buff->mbuf); 1734 sq->snd_buff[sq->head].mbuf = NULL; 1735 } 1736 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 1737 } 1738 NICVF_TX_UNLOCK(sq); 1739 } 1740 1741 /* 1742 * Add SQ HEADER subdescriptor. 1743 * First subdescriptor for every send descriptor. 1744 */ 1745 static __inline int 1746 nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry, 1747 int subdesc_cnt, struct mbuf *mbuf, int len) 1748 { 1749 struct nicvf *nic; 1750 struct sq_hdr_subdesc *hdr; 1751 struct ether_vlan_header *eh; 1752 #ifdef INET 1753 struct ip *ip; 1754 struct tcphdr *th; 1755 #endif 1756 uint16_t etype; 1757 int ehdrlen, iphlen, poff, proto; 1758 1759 nic = sq->nic; 1760 1761 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry); 1762 sq->snd_buff[qentry].mbuf = mbuf; 1763 1764 memset(hdr, 0, SND_QUEUE_DESC_SIZE); 1765 hdr->subdesc_type = SQ_DESC_TYPE_HEADER; 1766 /* Enable notification via CQE after processing SQE */ 1767 hdr->post_cqe = 1; 1768 /* No of subdescriptors following this */ 1769 hdr->subdesc_cnt = subdesc_cnt; 1770 hdr->tot_len = len; 1771 1772 eh = mtod(mbuf, struct ether_vlan_header *); 1773 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1774 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1775 etype = ntohs(eh->evl_proto); 1776 } else { 1777 ehdrlen = ETHER_HDR_LEN; 1778 etype = ntohs(eh->evl_encap_proto); 1779 } 1780 1781 poff = proto = -1; 1782 switch (etype) { 1783 #ifdef INET6 1784 case ETHERTYPE_IPV6: 1785 if (mbuf->m_len < ehdrlen + sizeof(struct ip6_hdr)) { 1786 mbuf = m_pullup(mbuf, ehdrlen +sizeof(struct ip6_hdr)); 1787 sq->snd_buff[qentry].mbuf = NULL; 1788 if (mbuf == NULL) 1789 return (ENOBUFS); 1790 } 1791 poff = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &proto); 1792 if (poff < 0) 1793 return (ENOBUFS); 1794 poff += ehdrlen; 1795 break; 1796 #endif 1797 #ifdef INET 1798 case ETHERTYPE_IP: 1799 if (mbuf->m_len < ehdrlen + sizeof(struct ip)) { 1800 mbuf = m_pullup(mbuf, ehdrlen + sizeof(struct ip)); 1801 sq->snd_buff[qentry].mbuf = mbuf; 1802 if (mbuf == NULL) 1803 return (ENOBUFS); 1804 } 1805 if (mbuf->m_pkthdr.csum_flags & CSUM_IP) 1806 hdr->csum_l3 = 1; /* Enable IP csum calculation */ 1807 1808 ip = (struct ip *)(mbuf->m_data + ehdrlen); 1809 iphlen = ip->ip_hl << 2; 1810 poff = ehdrlen + iphlen; 1811 proto = ip->ip_p; 1812 break; 1813 #endif 1814 } 1815 1816 #if defined(INET6) || defined(INET) 1817 if (poff > 0 && mbuf->m_pkthdr.csum_flags != 0) { 1818 switch (proto) { 1819 case IPPROTO_TCP: 1820 if ((mbuf->m_pkthdr.csum_flags & CSUM_TCP) == 0) 1821 break; 1822 1823 if (mbuf->m_len < (poff + sizeof(struct tcphdr))) { 1824 mbuf = m_pullup(mbuf, poff + sizeof(struct tcphdr)); 1825 sq->snd_buff[qentry].mbuf = mbuf; 1826 if (mbuf == NULL) 1827 return (ENOBUFS); 1828 } 1829 hdr->csum_l4 = SEND_L4_CSUM_TCP; 1830 break; 1831 case IPPROTO_UDP: 1832 if ((mbuf->m_pkthdr.csum_flags & CSUM_UDP) == 0) 1833 break; 1834 1835 if (mbuf->m_len < (poff + sizeof(struct udphdr))) { 1836 mbuf = m_pullup(mbuf, poff + sizeof(struct udphdr)); 1837 sq->snd_buff[qentry].mbuf = mbuf; 1838 if (mbuf == NULL) 1839 return (ENOBUFS); 1840 } 1841 hdr->csum_l4 = SEND_L4_CSUM_UDP; 1842 break; 1843 case IPPROTO_SCTP: 1844 if ((mbuf->m_pkthdr.csum_flags & CSUM_SCTP) == 0) 1845 break; 1846 1847 if (mbuf->m_len < (poff + sizeof(struct sctphdr))) { 1848 mbuf = m_pullup(mbuf, poff + sizeof(struct sctphdr)); 1849 sq->snd_buff[qentry].mbuf = mbuf; 1850 if (mbuf == NULL) 1851 return (ENOBUFS); 1852 } 1853 hdr->csum_l4 = SEND_L4_CSUM_SCTP; 1854 break; 1855 default: 1856 break; 1857 } 1858 hdr->l3_offset = ehdrlen; 1859 hdr->l4_offset = poff; 1860 } 1861 1862 if ((mbuf->m_pkthdr.tso_segsz != 0) && nic->hw_tso) { 1863 th = (struct tcphdr *)((caddr_t)(mbuf->m_data + poff)); 1864 1865 hdr->tso = 1; 1866 hdr->tso_start = poff + (th->th_off * 4); 1867 hdr->tso_max_paysize = mbuf->m_pkthdr.tso_segsz; 1868 hdr->inner_l3_offset = ehdrlen - 2; 1869 nic->drv_stats.tx_tso++; 1870 } 1871 #endif 1872 1873 return (0); 1874 } 1875 1876 /* 1877 * SQ GATHER subdescriptor 1878 * Must follow HDR descriptor 1879 */ 1880 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, 1881 int size, uint64_t data) 1882 { 1883 struct sq_gather_subdesc *gather; 1884 1885 qentry &= (sq->dmem.q_len - 1); 1886 gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry); 1887 1888 memset(gather, 0, SND_QUEUE_DESC_SIZE); 1889 gather->subdesc_type = SQ_DESC_TYPE_GATHER; 1890 gather->ld_type = NIC_SEND_LD_TYPE_E_LDD; 1891 gather->size = size; 1892 gather->addr = data; 1893 } 1894 1895 /* Put an mbuf to a SQ for packet transfer. */ 1896 static int 1897 nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf **mbufp) 1898 { 1899 bus_dma_segment_t segs[256]; 1900 struct snd_buff *snd_buff; 1901 size_t seg; 1902 int nsegs, qentry; 1903 int subdesc_cnt; 1904 int err; 1905 1906 NICVF_TX_LOCK_ASSERT(sq); 1907 1908 if (sq->free_cnt == 0) 1909 return (ENOBUFS); 1910 1911 snd_buff = &sq->snd_buff[sq->tail]; 1912 1913 err = bus_dmamap_load_mbuf_sg(sq->snd_buff_dmat, snd_buff->dmap, 1914 *mbufp, segs, &nsegs, BUS_DMA_NOWAIT); 1915 if (__predict_false(err != 0)) { 1916 /* ARM64TODO: Add mbuf defragmenting if we lack maps */ 1917 m_freem(*mbufp); 1918 *mbufp = NULL; 1919 return (err); 1920 } 1921 1922 /* Set how many subdescriptors is required */ 1923 subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT + nsegs - 1; 1924 if (subdesc_cnt > sq->free_cnt) { 1925 /* ARM64TODO: Add mbuf defragmentation if we lack descriptors */ 1926 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap); 1927 return (ENOBUFS); 1928 } 1929 1930 qentry = nicvf_get_sq_desc(sq, subdesc_cnt); 1931 1932 /* Add SQ header subdesc */ 1933 err = nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, *mbufp, 1934 (*mbufp)->m_pkthdr.len); 1935 if (err != 0) { 1936 nicvf_put_sq_desc(sq, subdesc_cnt); 1937 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap); 1938 if (err == ENOBUFS) { 1939 m_freem(*mbufp); 1940 *mbufp = NULL; 1941 } 1942 return (err); 1943 } 1944 1945 /* Add SQ gather subdescs */ 1946 for (seg = 0; seg < nsegs; seg++) { 1947 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1948 nicvf_sq_add_gather_subdesc(sq, qentry, segs[seg].ds_len, 1949 segs[seg].ds_addr); 1950 } 1951 1952 /* make sure all memory stores are done before ringing doorbell */ 1953 bus_dmamap_sync(sq->dmem.dmat, sq->dmem.dmap, BUS_DMASYNC_PREWRITE); 1954 1955 dprintf(sq->nic->dev, "%s: sq->idx: %d, subdesc_cnt: %d\n", 1956 __func__, sq->idx, subdesc_cnt); 1957 /* Inform HW to xmit new packet */ 1958 nicvf_queue_reg_write(sq->nic, NIC_QSET_SQ_0_7_DOOR, 1959 sq->idx, subdesc_cnt); 1960 return (0); 1961 } 1962 1963 static __inline u_int 1964 frag_num(u_int i) 1965 { 1966 #if BYTE_ORDER == BIG_ENDIAN 1967 return ((i & ~3) + 3 - (i & 3)); 1968 #else 1969 return (i); 1970 #endif 1971 } 1972 1973 /* Returns MBUF for a received packet */ 1974 struct mbuf * 1975 nicvf_get_rcv_mbuf(struct nicvf *nic, struct cqe_rx_t *cqe_rx) 1976 { 1977 int frag; 1978 int payload_len = 0; 1979 struct mbuf *mbuf; 1980 struct mbuf *mbuf_frag; 1981 uint16_t *rb_lens = NULL; 1982 uint64_t *rb_ptrs = NULL; 1983 1984 mbuf = NULL; 1985 rb_lens = (uint16_t *)((uint8_t *)cqe_rx + (3 * sizeof(uint64_t))); 1986 rb_ptrs = (uint64_t *)((uint8_t *)cqe_rx + (6 * sizeof(uint64_t))); 1987 1988 dprintf(nic->dev, "%s rb_cnt %d rb0_ptr %lx rb0_sz %d\n", 1989 __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz); 1990 1991 for (frag = 0; frag < cqe_rx->rb_cnt; frag++) { 1992 payload_len = rb_lens[frag_num(frag)]; 1993 if (frag == 0) { 1994 /* First fragment */ 1995 mbuf = nicvf_rb_ptr_to_mbuf(nic, 1996 (*rb_ptrs - cqe_rx->align_pad)); 1997 mbuf->m_len = payload_len; 1998 mbuf->m_data += cqe_rx->align_pad; 1999 if_setrcvif(mbuf, nic->ifp); 2000 } else { 2001 /* Add fragments */ 2002 mbuf_frag = nicvf_rb_ptr_to_mbuf(nic, *rb_ptrs); 2003 m_append(mbuf, payload_len, mbuf_frag->m_data); 2004 m_freem(mbuf_frag); 2005 } 2006 /* Next buffer pointer */ 2007 rb_ptrs++; 2008 } 2009 2010 if (__predict_true(mbuf != NULL)) { 2011 m_fixhdr(mbuf); 2012 mbuf->m_pkthdr.flowid = cqe_rx->rq_idx; 2013 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE); 2014 if (__predict_true((if_getcapenable(nic->ifp) & IFCAP_RXCSUM) != 0)) { 2015 /* 2016 * HW by default verifies IP & TCP/UDP/SCTP checksums 2017 */ 2018 if (__predict_true(cqe_rx->l3_type == L3TYPE_IPV4)) { 2019 mbuf->m_pkthdr.csum_flags = 2020 (CSUM_IP_CHECKED | CSUM_IP_VALID); 2021 } 2022 2023 switch (cqe_rx->l4_type) { 2024 case L4TYPE_UDP: 2025 case L4TYPE_TCP: /* fall through */ 2026 mbuf->m_pkthdr.csum_flags |= 2027 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 2028 mbuf->m_pkthdr.csum_data = 0xffff; 2029 break; 2030 case L4TYPE_SCTP: 2031 mbuf->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 2032 break; 2033 default: 2034 break; 2035 } 2036 } 2037 } 2038 2039 return (mbuf); 2040 } 2041 2042 /* Enable interrupt */ 2043 void 2044 nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx) 2045 { 2046 uint64_t reg_val; 2047 2048 reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S); 2049 2050 switch (int_type) { 2051 case NICVF_INTR_CQ: 2052 reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2053 break; 2054 case NICVF_INTR_SQ: 2055 reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2056 break; 2057 case NICVF_INTR_RBDR: 2058 reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2059 break; 2060 case NICVF_INTR_PKT_DROP: 2061 reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT); 2062 break; 2063 case NICVF_INTR_TCP_TIMER: 2064 reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT); 2065 break; 2066 case NICVF_INTR_MBOX: 2067 reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT); 2068 break; 2069 case NICVF_INTR_QS_ERR: 2070 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT); 2071 break; 2072 default: 2073 device_printf(nic->dev, 2074 "Failed to enable interrupt: unknown type\n"); 2075 break; 2076 } 2077 2078 nicvf_reg_write(nic, NIC_VF_ENA_W1S, reg_val); 2079 } 2080 2081 /* Disable interrupt */ 2082 void 2083 nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx) 2084 { 2085 uint64_t reg_val = 0; 2086 2087 switch (int_type) { 2088 case NICVF_INTR_CQ: 2089 reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2090 break; 2091 case NICVF_INTR_SQ: 2092 reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2093 break; 2094 case NICVF_INTR_RBDR: 2095 reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2096 break; 2097 case NICVF_INTR_PKT_DROP: 2098 reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT); 2099 break; 2100 case NICVF_INTR_TCP_TIMER: 2101 reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT); 2102 break; 2103 case NICVF_INTR_MBOX: 2104 reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT); 2105 break; 2106 case NICVF_INTR_QS_ERR: 2107 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT); 2108 break; 2109 default: 2110 device_printf(nic->dev, 2111 "Failed to disable interrupt: unknown type\n"); 2112 break; 2113 } 2114 2115 nicvf_reg_write(nic, NIC_VF_ENA_W1C, reg_val); 2116 } 2117 2118 /* Clear interrupt */ 2119 void 2120 nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx) 2121 { 2122 uint64_t reg_val = 0; 2123 2124 switch (int_type) { 2125 case NICVF_INTR_CQ: 2126 reg_val = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2127 break; 2128 case NICVF_INTR_SQ: 2129 reg_val = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2130 break; 2131 case NICVF_INTR_RBDR: 2132 reg_val = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2133 break; 2134 case NICVF_INTR_PKT_DROP: 2135 reg_val = (1UL << NICVF_INTR_PKT_DROP_SHIFT); 2136 break; 2137 case NICVF_INTR_TCP_TIMER: 2138 reg_val = (1UL << NICVF_INTR_TCP_TIMER_SHIFT); 2139 break; 2140 case NICVF_INTR_MBOX: 2141 reg_val = (1UL << NICVF_INTR_MBOX_SHIFT); 2142 break; 2143 case NICVF_INTR_QS_ERR: 2144 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT); 2145 break; 2146 default: 2147 device_printf(nic->dev, 2148 "Failed to clear interrupt: unknown type\n"); 2149 break; 2150 } 2151 2152 nicvf_reg_write(nic, NIC_VF_INT, reg_val); 2153 } 2154 2155 /* Check if interrupt is enabled */ 2156 int 2157 nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx) 2158 { 2159 uint64_t reg_val; 2160 uint64_t mask = 0xff; 2161 2162 reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S); 2163 2164 switch (int_type) { 2165 case NICVF_INTR_CQ: 2166 mask = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2167 break; 2168 case NICVF_INTR_SQ: 2169 mask = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2170 break; 2171 case NICVF_INTR_RBDR: 2172 mask = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2173 break; 2174 case NICVF_INTR_PKT_DROP: 2175 mask = NICVF_INTR_PKT_DROP_MASK; 2176 break; 2177 case NICVF_INTR_TCP_TIMER: 2178 mask = NICVF_INTR_TCP_TIMER_MASK; 2179 break; 2180 case NICVF_INTR_MBOX: 2181 mask = NICVF_INTR_MBOX_MASK; 2182 break; 2183 case NICVF_INTR_QS_ERR: 2184 mask = NICVF_INTR_QS_ERR_MASK; 2185 break; 2186 default: 2187 device_printf(nic->dev, 2188 "Failed to check interrupt enable: unknown type\n"); 2189 break; 2190 } 2191 2192 return (reg_val & mask); 2193 } 2194 2195 void 2196 nicvf_update_rq_stats(struct nicvf *nic, int rq_idx) 2197 { 2198 struct rcv_queue *rq; 2199 2200 #define GET_RQ_STATS(reg) \ 2201 nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\ 2202 (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) 2203 2204 rq = &nic->qs->rq[rq_idx]; 2205 rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS); 2206 rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS); 2207 } 2208 2209 void 2210 nicvf_update_sq_stats(struct nicvf *nic, int sq_idx) 2211 { 2212 struct snd_queue *sq; 2213 2214 #define GET_SQ_STATS(reg) \ 2215 nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\ 2216 (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) 2217 2218 sq = &nic->qs->sq[sq_idx]; 2219 sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS); 2220 sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS); 2221 } 2222 2223 /* Check for errors in the receive cmp.queue entry */ 2224 int 2225 nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cmp_queue *cq, 2226 struct cqe_rx_t *cqe_rx) 2227 { 2228 struct nicvf_hw_stats *stats = &nic->hw_stats; 2229 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 2230 2231 if (!cqe_rx->err_level && !cqe_rx->err_opcode) { 2232 drv_stats->rx_frames_ok++; 2233 return (0); 2234 } 2235 2236 switch (cqe_rx->err_opcode) { 2237 case CQ_RX_ERROP_RE_PARTIAL: 2238 stats->rx_bgx_truncated_pkts++; 2239 break; 2240 case CQ_RX_ERROP_RE_JABBER: 2241 stats->rx_jabber_errs++; 2242 break; 2243 case CQ_RX_ERROP_RE_FCS: 2244 stats->rx_fcs_errs++; 2245 break; 2246 case CQ_RX_ERROP_RE_RX_CTL: 2247 stats->rx_bgx_errs++; 2248 break; 2249 case CQ_RX_ERROP_PREL2_ERR: 2250 stats->rx_prel2_errs++; 2251 break; 2252 case CQ_RX_ERROP_L2_MAL: 2253 stats->rx_l2_hdr_malformed++; 2254 break; 2255 case CQ_RX_ERROP_L2_OVERSIZE: 2256 stats->rx_oversize++; 2257 break; 2258 case CQ_RX_ERROP_L2_UNDERSIZE: 2259 stats->rx_undersize++; 2260 break; 2261 case CQ_RX_ERROP_L2_LENMISM: 2262 stats->rx_l2_len_mismatch++; 2263 break; 2264 case CQ_RX_ERROP_L2_PCLP: 2265 stats->rx_l2_pclp++; 2266 break; 2267 case CQ_RX_ERROP_IP_NOT: 2268 stats->rx_ip_ver_errs++; 2269 break; 2270 case CQ_RX_ERROP_IP_CSUM_ERR: 2271 stats->rx_ip_csum_errs++; 2272 break; 2273 case CQ_RX_ERROP_IP_MAL: 2274 stats->rx_ip_hdr_malformed++; 2275 break; 2276 case CQ_RX_ERROP_IP_MALD: 2277 stats->rx_ip_payload_malformed++; 2278 break; 2279 case CQ_RX_ERROP_IP_HOP: 2280 stats->rx_ip_ttl_errs++; 2281 break; 2282 case CQ_RX_ERROP_L3_PCLP: 2283 stats->rx_l3_pclp++; 2284 break; 2285 case CQ_RX_ERROP_L4_MAL: 2286 stats->rx_l4_malformed++; 2287 break; 2288 case CQ_RX_ERROP_L4_CHK: 2289 stats->rx_l4_csum_errs++; 2290 break; 2291 case CQ_RX_ERROP_UDP_LEN: 2292 stats->rx_udp_len_errs++; 2293 break; 2294 case CQ_RX_ERROP_L4_PORT: 2295 stats->rx_l4_port_errs++; 2296 break; 2297 case CQ_RX_ERROP_TCP_FLAG: 2298 stats->rx_tcp_flag_errs++; 2299 break; 2300 case CQ_RX_ERROP_TCP_OFFSET: 2301 stats->rx_tcp_offset_errs++; 2302 break; 2303 case CQ_RX_ERROP_L4_PCLP: 2304 stats->rx_l4_pclp++; 2305 break; 2306 case CQ_RX_ERROP_RBDR_TRUNC: 2307 stats->rx_truncated_pkts++; 2308 break; 2309 } 2310 2311 return (1); 2312 } 2313 2314 /* Check for errors in the send cmp.queue entry */ 2315 int 2316 nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cmp_queue *cq, 2317 struct cqe_send_t *cqe_tx) 2318 { 2319 struct cmp_queue_stats *stats = &cq->stats; 2320 2321 switch (cqe_tx->send_status) { 2322 case CQ_TX_ERROP_GOOD: 2323 stats->tx.good++; 2324 return (0); 2325 case CQ_TX_ERROP_DESC_FAULT: 2326 stats->tx.desc_fault++; 2327 break; 2328 case CQ_TX_ERROP_HDR_CONS_ERR: 2329 stats->tx.hdr_cons_err++; 2330 break; 2331 case CQ_TX_ERROP_SUBDC_ERR: 2332 stats->tx.subdesc_err++; 2333 break; 2334 case CQ_TX_ERROP_IMM_SIZE_OFLOW: 2335 stats->tx.imm_size_oflow++; 2336 break; 2337 case CQ_TX_ERROP_DATA_SEQUENCE_ERR: 2338 stats->tx.data_seq_err++; 2339 break; 2340 case CQ_TX_ERROP_MEM_SEQUENCE_ERR: 2341 stats->tx.mem_seq_err++; 2342 break; 2343 case CQ_TX_ERROP_LOCK_VIOL: 2344 stats->tx.lock_viol++; 2345 break; 2346 case CQ_TX_ERROP_DATA_FAULT: 2347 stats->tx.data_fault++; 2348 break; 2349 case CQ_TX_ERROP_TSTMP_CONFLICT: 2350 stats->tx.tstmp_conflict++; 2351 break; 2352 case CQ_TX_ERROP_TSTMP_TIMEOUT: 2353 stats->tx.tstmp_timeout++; 2354 break; 2355 case CQ_TX_ERROP_MEM_FAULT: 2356 stats->tx.mem_fault++; 2357 break; 2358 case CQ_TX_ERROP_CK_OVERLAP: 2359 stats->tx.csum_overlap++; 2360 break; 2361 case CQ_TX_ERROP_CK_OFLOW: 2362 stats->tx.csum_overflow++; 2363 break; 2364 } 2365 2366 return (1); 2367 } 2368