57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
108 static int enic_dev_open(struct enic *);
109 static int enic_dev_init(struct enic *);
119 static void enic_update_packet_filter(struct enic *enic);
144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0);
145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array);
146 MODULE_VERSION(enic, 2);
148 MODULE_DEPEND(enic, pci, 1, 1, 1);
149 MODULE_DEPEND(enic, ether, 1, 1, 1);
150 MODULE_DEPEND(enic, iflib, 1, 1, 1);
301 static void enic_set_rx_coal_setting(struct enic *enic)  in enic_set_rx_coal_setting()  argument
305 	struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting;  in enic_set_rx_coal_setting()
311 	speed = vnic_dev_port_speed(enic->vdev);  in enic_set_rx_coal_setting()
324 	for (index = 0; index < enic->rq_count; index++)  in enic_set_rx_coal_setting()
325 		enic->cq[index].cur_rx_coal_timeval =  in enic_set_rx_coal_setting()
326 		enic->config.intr_timer_usec;  in enic_set_rx_coal_setting()
337 	struct enic *enic;  in enic_attach_pre()  local
369 	enic = &softc->enic;  in enic_attach_pre()
370 	enic->softc = softc;  in enic_attach_pre()
373 	enic->vdev = vdev;  in enic_attach_pre()
374 	vdev->priv = enic;  in enic_attach_pre()
378 	enic->vdev = vdev;  in enic_attach_pre()
379 	vnic_dev_cmd_init(enic->vdev);  in enic_attach_pre()
389 	vnic_register_cbacks(enic->vdev, enic_alloc_consistent,  in enic_attach_pre()
397 	err = vnic_dev_alloc_stats_mem(enic->vdev);  in enic_attach_pre()
400 		dev_err(enic, "Failed to allocate cmd memory, aborting\n");  in enic_attach_pre()
403 	vnic_dev_stats_clear(enic->vdev);  in enic_attach_pre()
404 	ret = vnic_dev_stats_dump(enic->vdev, &stats);  in enic_attach_pre()
406 		dev_err(enic, "Error in getting stats\n");  in enic_attach_pre()
409 	err = vnic_dev_alloc_counter_mem(enic->vdev);  in enic_attach_pre()
411 		dev_err(enic, "Failed to allocate counter memory, aborting\n");  in enic_attach_pre()
416 	err = enic_dev_open(enic);  in enic_attach_pre()
418 		dev_err(enic, "vNIC dev open failed, aborting\n");  in enic_attach_pre()
423 	enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN;  in enic_attach_pre()
424 	enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN;  in enic_attach_pre()
425 	err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,  in enic_attach_pre()
426 						enic->ig_vlan_rewrite_mode);  in enic_attach_pre()
428 		dev_err(enic,  in enic_attach_pre()
441 	err = vnic_dev_init(enic->vdev, 0);  in enic_attach_pre()
443 		dev_err(enic, "vNIC dev init failed, aborting\n");  in enic_attach_pre()
447 	err = enic_dev_init(enic);  in enic_attach_pre()
449 		dev_err(enic, "Device initialization failed, aborting\n");  in enic_attach_pre()
454 	enic->port_mtu = vnic_dev_mtu(enic->vdev);  in enic_attach_pre()
462 	if_setmtu(softc->ifp, enic->config.mtu);  in enic_attach_pre()
463 	scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \  in enic_attach_pre()
465 	scctx->isc_nrxqsets_max = enic->conf_rq_count;  in enic_attach_pre()
466 	scctx->isc_ntxqsets_max = enic->conf_wq_count;  in enic_attach_pre()
467 	scctx->isc_nrxqsets = enic->conf_rq_count;  in enic_attach_pre()
468 	scctx->isc_ntxqsets = enic->conf_wq_count;  in enic_attach_pre()
469 	for (i = 0; i < enic->conf_wq_count; i++) {  in enic_attach_pre()
470 		scctx->isc_ntxd[i] = enic->config.wq_desc_count;  in enic_attach_pre()
473 		scctx->isc_ntxd[i + enic->conf_wq_count] =  in enic_attach_pre()
474 		    enic->config.wq_desc_count;  in enic_attach_pre()
475 		scctx->isc_txqsizes[i + enic->conf_wq_count] =  in enic_attach_pre()
477 		    enic->conf_wq_count];  in enic_attach_pre()
479 	for (i = 0; i < enic->conf_rq_count; i++) {  in enic_attach_pre()
480 		scctx->isc_nrxd[i] = enic->config.rq_desc_count;  in enic_attach_pre()
483 		scctx->isc_nrxd[i + enic->conf_rq_count] =  in enic_attach_pre()
484 		    enic->config.rq_desc_count;  in enic_attach_pre()
485 		scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct  in enic_attach_pre()
486 		    cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count];  in enic_attach_pre()
499 	if (softc->enic.cq == NULL)  in enic_attach_pre()
500 		softc->enic.cq = malloc(sizeof(struct vnic_cq) *  in enic_attach_pre()
501 		     softc->enic.wq_count + softc->enic.rq_count, M_DEVBUF,  in enic_attach_pre()
503 	if (softc->enic.cq == NULL)  in enic_attach_pre()
510 	err = vnic_dev_alloc_stats_mem(enic->vdev);  in enic_attach_pre()
512 		dev_err(enic, "Failed to allocate cmd memory, aborting\n");  in enic_attach_pre()
518 		dev_err(enic, "Failed to allocate MSIX, aborting\n");  in enic_attach_pre()
525 	vnic_dev_close(enic->vdev);  in enic_attach_pre()
526 	vnic_dev_deinit_devcmd2(enic->vdev);  in enic_attach_pre()
529 	free(softc->enic.intr_queues, M_DEVBUF);  in enic_attach_pre()
530 	free(softc->enic.cq, M_DEVBUF);  in enic_attach_pre()
543 	struct enic *enic;  in enic_msix_intr_assign()  local
551 	enic = &softc->enic;  in enic_msix_intr_assign()
555 	vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX);  in enic_msix_intr_assign()
558 	enic->intr_queues = malloc(sizeof(*enic->intr_queues) *  in enic_msix_intr_assign()
559 	    enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);  in enic_msix_intr_assign()
560 	enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT  in enic_msix_intr_assign()
567 		    &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX,  in enic_msix_intr_assign()
568 		    enic_rxq_intr, &enic->rq[i], i, irq_name);  in enic_msix_intr_assign()
574 		enic->intr[i].index = i;  in enic_msix_intr_assign()
575 		enic->intr[i].vdev = enic->vdev;  in enic_msix_intr_assign()
577 		enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,  in enic_msix_intr_assign()
579 		vnic_intr_mask(&enic->intr[i]);  in enic_msix_intr_assign()
588 		    &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX,  in enic_msix_intr_assign()
589 		    &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets,  in enic_msix_intr_assign()
592 		enic->intr[i].index = i;  in enic_msix_intr_assign()
593 		enic->intr[i].vdev = enic->vdev;  in enic_msix_intr_assign()
595 		enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,  in enic_msix_intr_assign()
597 		vnic_intr_mask(&enic->intr[i]);  in enic_msix_intr_assign()
610 	enic->intr[i].index = i;  in enic_msix_intr_assign()
611 	enic->intr[i].vdev = enic->vdev;  in enic_msix_intr_assign()
613 	enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,  in enic_msix_intr_assign()
615 	vnic_intr_mask(&enic->intr[i]);  in enic_msix_intr_assign()
626 	enic->intr[i].index = i;  in enic_msix_intr_assign()
627 	enic->intr[i].vdev = enic->vdev;  in enic_msix_intr_assign()
629 	enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,  in enic_msix_intr_assign()
631 	vnic_intr_mask(&enic->intr[i]);  in enic_msix_intr_assign()
634 	enic->intr_count = msix;  in enic_msix_intr_assign()
644 	struct enic    *enic;  in enic_free_irqs()  local
648 	enic = &softc->enic;  in enic_free_irqs()
651 		iflib_irq_free(softc->ctx, &enic->intr_queues[i].intr_irq);  in enic_free_irqs()
656 	free(enic->intr_queues, M_DEVBUF);  in enic_free_irqs()
657 	free(enic->intr, M_DEVBUF);  in enic_free_irqs()
663 	struct enic *enic;  in enic_attach_post()  local
668 	enic = &softc->enic;  in enic_attach_post()
672 	enic_init_vnic_resources(enic);  in enic_attach_post()
673 	enic_set_rx_coal_setting(enic);  in enic_attach_post()
674 	enic_setup_finish(enic);  in enic_attach_post()
686 	struct enic *enic;  in enic_detach()  local
689 	enic = &softc->enic;  in enic_detach()
691 	vnic_dev_notify_unset(enic->vdev);  in enic_detach()
696 	vnic_dev_deinit(enic->vdev);  in enic_detach()
697 	vnic_dev_close(enic->vdev);  in enic_detach()
698 	vnic_dev_deinit_devcmd2(enic->vdev);  in enic_detach()
716 	softc->enic.wq = malloc(sizeof(struct vnic_wq) *  in enic_tx_queues_alloc()
718 	if (softc->enic.wq == NULL)  in enic_tx_queues_alloc()
737 		wq = &softc->enic.wq[q];  in enic_tx_queues_alloc()
738 		cq_wq = enic_cq_wq(&softc->enic, q);  in enic_tx_queues_alloc()
739 		cq = &softc->enic.cq[cq_wq];  in enic_tx_queues_alloc()
742 		wq->vdev = softc->enic.vdev;  in enic_tx_queues_alloc()
744 		wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ,  in enic_tx_queues_alloc()
759 		cq->vdev = softc->enic.vdev;  in enic_tx_queues_alloc()
761 		cq->ctrl = vnic_dev_get_res(softc->enic.vdev,  in enic_tx_queues_alloc()
788 	softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF,  in enic_rx_queues_alloc()
790 	if (softc->enic.rq == NULL)  in enic_rx_queues_alloc()
809 		rq = &softc->enic.rq[q];  in enic_rx_queues_alloc()
810 		cq_rq = enic_cq_rq(&softc->enic, q);  in enic_rx_queues_alloc()
811 		cq = &softc->enic.cq[cq_rq];  in enic_rx_queues_alloc()
814 		cq->vdev = softc->enic.vdev;  in enic_rx_queues_alloc()
816 		cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ,  in enic_rx_queues_alloc()
826 		rq->vdev = softc->enic.vdev;  in enic_rx_queues_alloc()
829 		rq->ctrl = vnic_dev_get_res(softc->enic.vdev,  in enic_rx_queues_alloc()
853 	free(softc->enic.rq, M_DEVBUF);  in enic_queues_free()
854 	free(softc->enic.wq, M_DEVBUF);  in enic_queues_free()
855 	free(softc->enic.cq, M_DEVBUF);  in enic_queues_free()
876 	struct enic    *enic;  in enic_event_intr()  local
880 	enic = &softc->enic;  in enic_event_intr()
882 	mtu = vnic_dev_mtu(enic->vdev);  in enic_event_intr()
883 	if (mtu && mtu != enic->port_mtu) {  in enic_event_intr()
884 		enic->port_mtu = mtu;  in enic_event_intr()
909 	struct enic    *enic;  in enic_stop()  local
920 	enic = &softc->enic;  in enic_stop()
927 	enic_dev_disable(enic);  in enic_stop()
930 		enic_stop_wq(enic, index);  in enic_stop()
931 		vnic_wq_clean(&enic->wq[index]);  in enic_stop()
932 		vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]);  in enic_stop()
934 		wq = &softc->enic.wq[index];  in enic_stop()
940 		cq_wq = enic_cq_wq(&softc->enic, index);  in enic_stop()
941 		cq = &softc->enic.cq[cq_wq];  in enic_stop()
946 		enic_stop_rq(enic, index);  in enic_stop()
947 		vnic_rq_clean(&enic->rq[index]);  in enic_stop()
948 		vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]);  in enic_stop()
950 		rq = &softc->enic.rq[index];  in enic_stop()
951 		cq_rq = enic_cq_rq(&softc->enic, index);  in enic_stop()
952 		cq = &softc->enic.cq[cq_rq];  in enic_stop()
960 		vnic_intr_clean(&enic->intr[index]);  in enic_stop()
968 	struct enic *enic;  in enic_init()  local
974 	enic = &softc->enic;  in enic_init()
977 	enic_init_vnic_resources(enic);  in enic_init()
980 		enic_prep_wq_for_simple_tx(&softc->enic, index);  in enic_init()
983 		enic_start_wq(enic, index);  in enic_init()
986 		enic_start_rq(enic, index);  in enic_init()
993 	vnic_dev_enable_wait(enic->vdev);  in enic_init()
1003 	struct enic *enic;  in enic_del_mcast()  local
1006 	enic = &softc->enic;  in enic_del_mcast()
1008 		vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);  in enic_del_mcast()
1016 	struct enic *enic;  in enic_add_mcast()  local
1019 	enic = &softc->enic;  in enic_add_mcast()
1021 		vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);  in enic_add_mcast()
1065 	enic_update_packet_filter(&softc->enic);  in enic_multi_set()
1072 	struct enic *enic;  in enic_mtu_set()  local
1076 	enic = &softc->enic;  in enic_mtu_set()
1079 	if (mtu > enic->port_mtu){  in enic_mtu_set()
1083 	enic->config.mtu = mtu;  in enic_mtu_set()
1145 	enic_update_packet_filter(&softc->enic);  in enic_promisc_set()
1174 	struct enic *enic;  in enic_txq_timer()  local
1179 	enic = &softc->enic;  in enic_txq_timer()
1182 	ret = vnic_dev_stats_dump(enic->vdev, &stats);  in enic_txq_timer()
1185 		dev_err(enic, "Error in getting stats\n");  in enic_txq_timer()
1222 	struct enic *enic;  in enic_set_lladdr()  local
1223 	enic = &softc->enic;  in enic_set_lladdr()
1226 	vnic_dev_add_addr(enic->vdev, softc->lladdr);  in enic_set_lladdr()
1329 	enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child);  in enic_setup_queue_sysctl()
1330 	enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child);  in enic_setup_queue_sysctl()
1352 	struct enic *enic = &softc->enic;  in enic_enable_intr()  local
1354 	vnic_intr_unmask(&enic->intr[irq]);  in enic_enable_intr()
1355 	vnic_intr_return_all_credits(&enic->intr[irq]);  in enic_enable_intr()
1361 	struct enic *enic = &softc->enic;  in enic_disable_intr()  local
1363 	vnic_intr_mask(&enic->intr[irq]);  in enic_disable_intr()
1364 	vnic_intr_masked(&enic->intr[irq]);	/* flush write */  in enic_disable_intr()
1428 enic_dev_open(struct enic *enic)  in enic_dev_open()  argument
1433 	err = enic_dev_wait(enic->vdev, vnic_dev_open,  in enic_dev_open()
1436 		dev_err(enic_get_dev(enic),  in enic_dev_open()
1443 enic_dev_init(struct enic *enic)  in enic_dev_init()  argument
1447 	vnic_dev_intr_coal_timer_info_default(enic->vdev);  in enic_dev_init()
1452 	err = enic_get_vnic_config(enic);  in enic_dev_init()
1459 	enic_get_res_counts(enic);  in enic_dev_init()
1462 	enic->intr_queues = malloc(sizeof(*enic->intr_queues) *  in enic_dev_init()
1463 	    enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);  in enic_dev_init()
1465 	vnic_dev_set_reset_flag(enic->vdev, 0);  in enic_dev_init()
1466 	enic->max_flow_counter = -1;  in enic_dev_init()
1469 	vnic_dev_notify_set(enic->vdev, -1);	/* No Intr for notify */  in enic_dev_init()
1471 	enic->overlay_offload = false;  in enic_dev_init()
1472 	if (enic->disable_overlay && enic->vxlan) {  in enic_dev_init()
1477 		if (vnic_dev_overlay_offload_ctrl(enic->vdev,  in enic_dev_init()
1479 			dev_err(enic, "failed to disable overlay offload\n");  in enic_dev_init()
1481 			dev_info(enic, "Overlay offload is disabled\n");  in enic_dev_init()
1484 	if (!enic->disable_overlay && enic->vxlan &&  in enic_dev_init()
1486 	    vnic_dev_overlay_offload_ctrl(enic->vdev,  in enic_dev_init()
1488 		enic->overlay_offload = true;  in enic_dev_init()
1489 		enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;  in enic_dev_init()
1490 		dev_info(enic, "Overlay offload is enabled\n");  in enic_dev_init()
1495 		if (vnic_dev_overlay_offload_cfg(enic->vdev,  in enic_dev_init()
1497 			dev_err(enic, "failed to update vxlan port\n");  in enic_dev_init()
1510 	struct enic    *enic = (struct enic *)priv;  in enic_alloc_consistent()  local
1513 	rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT);  in enic_alloc_consistent()
1615 enic_init_vnic_resources(struct enic *enic)  in enic_init_vnic_resources()  argument
1627 	scctx = enic->softc->scctx;  in enic_init_vnic_resources()
1635 	for (index = 0; index < enic->intr_count; index++) {  in enic_init_vnic_resources()
1636 		vnic_intr_alloc(enic->vdev, &enic->intr[index], index);  in enic_init_vnic_resources()
1640 		cq_idx = enic_cq_rq(enic, index);  in enic_init_vnic_resources()
1642 		vnic_rq_clean(&enic->rq[index]);  in enic_init_vnic_resources()
1643 		vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable,  in enic_init_vnic_resources()
1646 		vnic_cq_clean(&enic->cq[cq_idx]);  in enic_init_vnic_resources()
1647 		vnic_cq_init(&enic->cq[cq_idx],  in enic_init_vnic_resources()
1663 		cq_idx = enic_cq_wq(enic, index);  in enic_init_vnic_resources()
1664 		vnic_wq_clean(&enic->wq[index]);  in enic_init_vnic_resources()
1665 		vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable,  in enic_init_vnic_resources()
1668 		enic->wq[index].tx_offload_notsup_mask = 0;  in enic_init_vnic_resources()
1670 		vnic_cq_clean(&enic->cq[cq_idx]);  in enic_init_vnic_resources()
1671 		vnic_cq_init(&enic->cq[cq_idx],  in enic_init_vnic_resources()
1685 	for (index = 0; index < enic->intr_count; index++) {  in enic_init_vnic_resources()
1686 		vnic_intr_init(&enic->intr[index], 125,  in enic_init_vnic_resources()
1687 		    enic->config.intr_timer_type, /* mask_on_assertion */ 1);  in enic_init_vnic_resources()
1692 enic_update_packet_filter(struct enic *enic)  in enic_update_packet_filter()  argument
1694 	struct enic_softc *softc = enic->softc;  in enic_update_packet_filter()
1697 	vnic_dev_packet_filter(enic->vdev,  in enic_update_packet_filter()
1717 enic_setup_finish(struct enic *enic)  in enic_setup_finish()  argument
1719 	struct enic_softc *softc = enic->softc;  in enic_setup_finish()
1727 	enic_update_packet_filter(enic);  in enic_setup_finish()