98 static int	lio_chip_specific_setup(struct octeon_device *oct);
100 static int	lio_load_firmware(struct octeon_device *oct);
101 static int	lio_nic_starter(struct octeon_device *oct);
102 static int	lio_init_nic_module(struct octeon_device *oct);
105 static void	lio_if_cfg_callback(struct octeon_device *oct, uint32_t status,
109 static int	lio_setup_glists(struct octeon_device *oct, struct lio *lio,
111 static void	lio_destroy_nic_device(struct octeon_device *oct, int ifidx);
115 static int	lio_stop_nic_module(struct octeon_device *oct);
116 static void	lio_destroy_resources(struct octeon_device *oct);
123 	lio_get_other_octeon_device(struct octeon_device *oct);
125 static int	lio_wait_for_oq_pkts(struct octeon_device *oct);
128 static int	lio_dbg_console_print(struct octeon_device *oct,
683  * @param oct octeon device
686 lio_pci_flr(struct octeon_device *oct)  in lio_pci_flr()  argument
690 	pci_find_cap(oct->device, PCIY_EXPRESS, &exppos);  in lio_pci_flr()
692 	pci_save_state(oct->device);  in lio_pci_flr()
695 	pci_write_config(oct->device, PCIR_COMMAND, PCIM_CMD_INTxDIS, 2);  in lio_pci_flr()
700 	status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2);  in lio_pci_flr()
702 		lio_dev_info(oct, "Function reset incomplete after 100ms, sleeping for 5 seconds\n");  in lio_pci_flr()
705 		status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2);  in lio_pci_flr()
707 			lio_dev_info(oct, "Function reset still incomplete after 5s, reset anyway\n");  in lio_pci_flr()
710 	pci_write_config(oct->device, exppos + PCIER_DEVICE_CTL, PCIEM_CTL_INITIATE_FLR, 2);  in lio_pci_flr()
713 	pci_restore_state(oct->device);  in lio_pci_flr()
730 lio_dbg_console_print(struct octeon_device *oct, uint32_t console_num,  in lio_dbg_console_print()  argument
735 		lio_dev_info(oct, "%u: %s%s\n", console_num, prefix, suffix);  in lio_dbg_console_print()
737 		lio_dev_info(oct, "%u: %s\n", console_num, prefix);  in lio_dbg_console_print()
739 		lio_dev_info(oct, "%u: %s\n", console_num, suffix);  in lio_dbg_console_print()
749 	struct octeon_device	*oct = param;  in lio_watchdog()  local
755 		kproc_suspend_check(oct->watchdog_task);  in lio_watchdog()
757 			(uint16_t)lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2);  in lio_watchdog()
762 			oct->cores_crashed = true;  in lio_watchdog()
763 			other_oct = lio_get_other_octeon_device(oct);  in lio_watchdog()
776 					lio_dev_err(oct,  in lio_watchdog()
777 						    "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n",  in lio_watchdog()
791 lio_chip_specific_setup(struct octeon_device *oct)  in lio_chip_specific_setup()  argument
797 	dev_id = lio_read_pci_cfg(oct, 0);  in lio_chip_specific_setup()
798 	oct->subdevice_id = pci_get_subdevice(oct->device);  in lio_chip_specific_setup()
802 		oct->chip_id = LIO_CN23XX_PF_VID;  in lio_chip_specific_setup()
803 		if (pci_get_function(oct->device) == 0) {  in lio_chip_specific_setup()
805 				lio_dev_info(oct, "Invalid num_queues_per_pf0: %d, Setting it to default\n",  in lio_chip_specific_setup()
810 			oct->sriov_info.num_pf_rings = num_queues_per_pf0;  in lio_chip_specific_setup()
813 				lio_dev_info(oct, "Invalid num_queues_per_pf1: %d, Setting it to default\n",  in lio_chip_specific_setup()
818 			oct->sriov_info.num_pf_rings = num_queues_per_pf1;  in lio_chip_specific_setup()
821 		ret = lio_cn23xx_pf_setup_device(oct);  in lio_chip_specific_setup()
827 		lio_dev_err(oct, "Unknown device found (dev_id: %x)\n", dev_id);  in lio_chip_specific_setup()
831 		lio_dev_info(oct, "%s PASS%d.%d %s Version: %s\n", s,  in lio_chip_specific_setup()
832 			     OCTEON_MAJOR_REV(oct), OCTEON_MINOR_REV(oct),  in lio_chip_specific_setup()
833 			     lio_get_conf(oct)->card_name, LIO_VERSION);  in lio_chip_specific_setup()
839 lio_get_other_octeon_device(struct octeon_device *oct)  in lio_get_other_octeon_device()  argument
843 	other_oct = lio_get_device(oct->octeon_id + 1);  in lio_get_other_octeon_device()
848 		oct_busnum = pci_get_bus(oct->device);  in lio_get_other_octeon_device()
854 			oct_slot = pci_get_slot(oct->device);  in lio_get_other_octeon_device()
866  * @param oct octeon device
871 lio_load_firmware(struct octeon_device *oct)  in lio_load_firmware()  argument
884 		lio_get_conf(oct)->card_name, tmp_fw_type, LIO_FW_NAME_SUFFIX);  in lio_load_firmware()
888 		lio_dev_err(oct, "Request firmware failed. Could not find file %s.\n",  in lio_load_firmware()
893 	ret = lio_download_firmware(oct, fw->data, fw->datasize);  in lio_load_firmware()
901 lio_nic_starter(struct octeon_device *oct)  in lio_nic_starter()  argument
905 	atomic_store_rel_int(&oct->status, LIO_DEV_RUNNING);  in lio_nic_starter()
907 	if (oct->app_mode && oct->app_mode == LIO_DRV_NIC_APP) {  in lio_nic_starter()
908 		if (lio_init_nic_module(oct)) {  in lio_nic_starter()
909 			lio_dev_err(oct, "NIC initialization failed\n");  in lio_nic_starter()
913 			if (octeon_enable_sriov(oct) < 0)  in lio_nic_starter()
918 		lio_dev_err(oct,  in lio_nic_starter()
920 			    oct->app_mode);  in lio_nic_starter()
928 lio_init_nic_module(struct octeon_device *oct)  in lio_init_nic_module()  argument
930 	int	num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(lio_get_conf(oct));  in lio_init_nic_module()
933 	lio_dev_dbg(oct, "Initializing network interfaces\n");  in lio_init_nic_module()
941 	oct->ifcount = num_nic_ports;  in lio_init_nic_module()
943 	bzero(&oct->props, sizeof(struct lio_if_props));  in lio_init_nic_module()
945 	oct->props.gmxport = -1;  in lio_init_nic_module()
947 	retval = lio_setup_nic_devices(oct);  in lio_init_nic_module()
949 		lio_dev_err(oct, "Setup NIC devices failed\n");  in lio_init_nic_module()
953 	lio_dev_dbg(oct, "Network interfaces ready\n");  in lio_init_nic_module()
959 	oct->ifcount = 0;  in lio_init_nic_module()
994 lio_get_media_subtype(struct octeon_device *oct)  in lio_get_media_subtype()  argument
997 	switch(oct->subdevice_id) {  in lio_get_media_subtype()
1012 lio_get_baudrate(struct octeon_device *oct)  in lio_get_baudrate()  argument
1015 	switch(oct->subdevice_id) {  in lio_get_baudrate()
1063 	struct octeon_device	*oct = lio->oct_dev;  in lio_get_counter()  local
1069 		for (i = 0; i < oct->num_oqs; i++) {  in lio_get_counter()
1071 			counter += oct->droq[q_no]->stats.rx_pkts_received;  in lio_get_counter()
1075 		for (i = 0; i < oct->num_iqs; i++) {  in lio_get_counter()
1077 			counter += oct->instr_queue[q_no]->stats.tx_done;  in lio_get_counter()
1081 		for (i = 0; i < oct->num_oqs; i++) {  in lio_get_counter()
1083 			counter += oct->droq[q_no]->stats.rx_bytes_received;  in lio_get_counter()
1087 		for (i = 0; i < oct->num_iqs; i++) {  in lio_get_counter()
1089 			counter += oct->instr_queue[q_no]->stats.tx_tot_bytes;  in lio_get_counter()
1093 		for (i = 0; i < oct->num_oqs; i++) {  in lio_get_counter()
1095 			counter += oct->droq[q_no]->stats.rx_dropped;  in lio_get_counter()
1099 		for (i = 0; i < oct->num_iqs; i++) {  in lio_get_counter()
1101 			counter += oct->instr_queue[q_no]->stats.tx_dropped;  in lio_get_counter()
1105 		counter = oct->link_stats.fromwire.total_mcst;  in lio_get_counter()
1108 		counter = oct->link_stats.fromhost.mcast_pkts_sent;  in lio_get_counter()
1111 		counter = oct->link_stats.fromhost.total_collisions;  in lio_get_counter()
1114 		counter = oct->link_stats.fromwire.fcs_err +  in lio_get_counter()
1115 		    oct->link_stats.fromwire.l2_err +  in lio_get_counter()
1116 		    oct->link_stats.fromwire.frame_err;  in lio_get_counter()
1128 	struct octeon_device	*oct = lio->oct_dev;  in lio_init_ifnet()  local
1137 		    (IFM_ETHER | IFM_FDX | lio_get_media_subtype(oct)),  in lio_init_ifnet()
1143 	lio_dev_dbg(oct, "IFMEDIA flags : %x\n", lio->ifmedia.ifm_media);  in lio_init_ifnet()
1145 	if_initname(ifp, device_get_name(oct->device),  in lio_init_ifnet()
1146 		    device_get_unit(oct->device));  in lio_init_ifnet()
1166 	if_setbaudrate(ifp, lio_get_baudrate(oct));  in lio_init_ifnet()
1486 	struct octeon_device	*oct = (struct octeon_device *)ptr;  in lio_link_info()  local
1491 	lio_dev_dbg(oct, "%s Called\n", __func__);  in lio_link_info()
1493 		lio_dev_err(oct, "Malformed NIC_INFO, len=%d, ifidx=%d\n",  in lio_link_info()
1504 	if (oct->props.gmxport == gmxport)  in lio_link_info()
1505 		lio_update_link_status(oct->props.ifp, ls);  in lio_link_info()
1528 	struct octeon_device	*oct;  in lio_free_sgmbuf()  local
1534 	oct = iq->oct_dev;  in lio_free_sgmbuf()
1535 	lio = if_getsoftc(oct->props.ifp);  in lio_free_sgmbuf()
1547 lio_if_cfg_callback(struct octeon_device *oct, uint32_t status, void *buf)  in lio_if_cfg_callback()  argument
1556 	oct = lio_get_device(ctx->octeon_id);  in lio_if_cfg_callback()
1558 		lio_dev_err(oct, "nic if cfg instruction failed. Status: %llx (0x%08x)\n",  in lio_if_cfg_callback()
1562 	snprintf(oct->fw_info.lio_firmware_version, 32, "%s",  in lio_if_cfg_callback()
1586 	struct octeon_device	*oct = lio->oct_dev;  in lio_open()  local
1595 	lio_dev_info(oct, "Interface Open, ready for traffic\n");  in lio_open()
1606 			lio_dev_err(oct, "MAC change failed, error: %d\n", ret);  in lio_open()
1612 	lio_dev_info(oct, "Interface is opened\n");  in lio_open()
1620 	struct octeon_device	*oct = lio->oct_dev;  in lio_set_rxcsum_command()  local
1633 		lio_dev_err(oct, "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",  in lio_set_rxcsum_command()
1641 lio_stop_nic_module(struct octeon_device *oct)  in lio_stop_nic_module()  argument
1646 	lio_dev_dbg(oct, "Stopping network interfaces\n");  in lio_stop_nic_module()
1647 	if (!oct->ifcount) {  in lio_stop_nic_module()
1648 		lio_dev_err(oct, "Init for Octeon was not completed\n");  in lio_stop_nic_module()
1652 	mtx_lock(&oct->cmd_resp_wqlock);  in lio_stop_nic_module()
1653 	oct->cmd_resp_state = LIO_DRV_OFFLINE;  in lio_stop_nic_module()
1654 	mtx_unlock(&oct->cmd_resp_wqlock);  in lio_stop_nic_module()
1656 	for (i = 0; i < oct->ifcount; i++) {  in lio_stop_nic_module()
1657 		lio = if_getsoftc(oct->props.ifp);  in lio_stop_nic_module()
1658 		for (j = 0; j < oct->num_oqs; j++)  in lio_stop_nic_module()
1659 			lio_unregister_droq_ops(oct,  in lio_stop_nic_module()
1665 	for (i = 0; i < oct->ifcount; i++)  in lio_stop_nic_module()
1666 		lio_destroy_nic_device(oct, i);  in lio_stop_nic_module()
1668 	lio_dev_dbg(oct, "Network interface stopped\n");  in lio_stop_nic_module()
1674 lio_delete_glists(struct octeon_device *oct, struct lio *lio)  in lio_delete_glists()  argument
1712 lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)  in lio_setup_glists()  argument
1741 		lio_delete_glists(oct, lio);  in lio_setup_glists()
1754 			lio_delete_glists(oct, lio);  in lio_setup_glists()
1772 			lio_delete_glists(oct, lio);  in lio_setup_glists()
1784 	struct octeon_device	*oct = lio->oct_dev;  in lio_stop()  local
1798 	lio_dev_info(oct, "Interface is stopped\n");  in lio_stop()
1805 	struct octeon_device *oct = lio->oct_dev;  in lio_check_rx_oom_status()  local
1809 	for (q = 0; q < oct->num_oqs; q++) {  in lio_check_rx_oom_status()
1811 		droq = oct->droq[q_no];  in lio_check_rx_oom_status()
1814 		if (lio_read_csr32(oct, droq->pkts_credit_reg) <= 0x40) {  in lio_check_rx_oom_status()
1816 			desc_refilled = lio_droq_refill(oct, droq);  in lio_check_rx_oom_status()
1823 			lio_write_csr32(oct, droq->pkts_credit_reg,  in lio_check_rx_oom_status()
1849 	struct octeon_device	*oct = lio->oct_dev;  in lio_setup_rx_oom_poll_fn()  local
1865 				oct->octeon_id);  in lio_setup_rx_oom_poll_fn()
1891 lio_destroy_nic_device(struct octeon_device *oct, int ifidx)  in lio_destroy_nic_device()  argument
1893 	if_t		ifp = oct->props.ifp;  in lio_destroy_nic_device()
1897 		lio_dev_err(oct, "%s No ifp ptr for index %d\n",  in lio_destroy_nic_device()
1906 	lio_dev_dbg(oct, "NIC device cleanup\n");  in lio_destroy_nic_device()
1911 	if (lio_wait_for_pending_requests(oct))  in lio_destroy_nic_device()
1912 		lio_dev_err(oct, "There were pending requests\n");  in lio_destroy_nic_device()
1914 	if (lio_wait_for_instr_fetch(oct))  in lio_destroy_nic_device()
1915 		lio_dev_err(oct, "IQ had pending instructions\n");  in lio_destroy_nic_device()
1917 	if (lio_wait_for_oq_pkts(oct))  in lio_destroy_nic_device()
1918 		lio_dev_err(oct, "OQ had pending packets\n");  in lio_destroy_nic_device()
1923 	lio_tcp_lro_free(oct, ifp);  in lio_destroy_nic_device()
1927 	lio_delete_glists(oct, lio);  in lio_destroy_nic_device()
1936 	oct->props.gmxport = -1;  in lio_destroy_nic_device()
1938 	oct->props.ifp = NULL;  in lio_destroy_nic_device()
1984 lio_rx_ctl_callback(struct octeon_device *oct, uint32_t status, void *buf)  in lio_rx_ctl_callback()  argument
1991 	oct = lio_get_device(ctx->octeon_id);  in lio_rx_ctl_callback()
1993 		lio_dev_err(oct, "rx ctl instruction failed. Status: %llx\n",  in lio_rx_ctl_callback()
2010 	struct octeon_device	*oct = (struct octeon_device *)lio->oct_dev;  in lio_send_rx_ctrl_cmd()  local
2014 	if (oct->props.rx_on == start_stop)  in lio_send_rx_ctrl_cmd()
2017 	sc = lio_alloc_soft_command(oct, OCTEON_CMD_SIZE, 16, ctx_size);  in lio_send_rx_ctrl_cmd()
2025 	ctx->octeon_id = lio_get_device_id(oct);  in lio_send_rx_ctrl_cmd()
2034 	lio_prepare_soft_command(oct, sc, LIO_OPCODE_NIC, LIO_OPCODE_NIC_CMD, 0,  in lio_send_rx_ctrl_cmd()
2041 	retval = lio_send_soft_command(oct, sc);  in lio_send_rx_ctrl_cmd()
2043 		lio_dev_err(oct, "Failed to send RX Control message\n");  in lio_send_rx_ctrl_cmd()
2049 		lio_sleep_cond(oct, &ctx->cond);  in lio_send_rx_ctrl_cmd()
2050 		oct->props.rx_on = start_stop;  in lio_send_rx_ctrl_cmd()
2053 	lio_free_soft_command(oct, sc);  in lio_send_rx_ctrl_cmd()
2061 	struct octeon_device	*oct = lio->oct_dev;  in lio_vlan_rx_add_vid()  local
2082 		lio_dev_err(oct, "Add VLAN filter failed in core (ret: 0x%x)\n",  in lio_vlan_rx_add_vid()
2092 	struct octeon_device	*oct = lio->oct_dev;  in lio_vlan_rx_kill_vid()  local
2113 		lio_dev_err(oct,  in lio_vlan_rx_kill_vid()
2120 lio_wait_for_oq_pkts(struct octeon_device *oct)  in lio_wait_for_oq_pkts()  argument
2127 		for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {  in lio_wait_for_oq_pkts()
2128 			if (!(oct->io_qmask.oq & BIT_ULL(i)))  in lio_wait_for_oq_pkts()
2131 			pkt_cnt = lio_droq_check_hw_for_pkts(oct->droq[i]);  in lio_wait_for_oq_pkts()
2134 				taskqueue_enqueue(oct->droq[i]->droq_taskqueue,  in lio_wait_for_oq_pkts()
2135 						  &oct->droq[i]->droq_task);  in lio_wait_for_oq_pkts()
2147 lio_destroy_resources(struct octeon_device *oct)  in lio_destroy_resources()  argument
2151 	switch (atomic_load_acq_int(&oct->status)) {  in lio_destroy_resources()
2155 		atomic_store_rel_int(&oct->status, LIO_DEV_IN_RESET);  in lio_destroy_resources()
2157 		oct->app_mode = LIO_DRV_INVALID_APP;  in lio_destroy_resources()
2158 		lio_dev_dbg(oct, "Device state is now %s\n",  in lio_destroy_resources()
2159 			    lio_get_state_string(&oct->status));  in lio_destroy_resources()
2169 		lio_remove_consoles(oct);  in lio_destroy_resources()
2173 		if (lio_wait_for_pending_requests(oct))  in lio_destroy_resources()
2174 			lio_dev_err(oct, "There were pending requests\n");  in lio_destroy_resources()
2176 		if (lio_wait_for_instr_fetch(oct))  in lio_destroy_resources()
2177 			lio_dev_err(oct, "IQ had pending instructions\n");  in lio_destroy_resources()
2184 		oct->fn_list.disable_io_queues(oct);  in lio_destroy_resources()
2186 		if (lio_wait_for_oq_pkts(oct))  in lio_destroy_resources()
2187 			lio_dev_err(oct, "OQ had pending packets\n");  in lio_destroy_resources()
2192 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);  in lio_destroy_resources()
2194 		if (oct->msix_on) {  in lio_destroy_resources()
2195 			for (i = 0; i < oct->num_msix_irqs - 1; i++) {  in lio_destroy_resources()
2196 				if (oct->ioq_vector[i].tag != NULL) {  in lio_destroy_resources()
2197 					bus_teardown_intr(oct->device,  in lio_destroy_resources()
2198 						  oct->ioq_vector[i].msix_res,  in lio_destroy_resources()
2199 						      oct->ioq_vector[i].tag);  in lio_destroy_resources()
2200 					oct->ioq_vector[i].tag = NULL;  in lio_destroy_resources()
2202 				if (oct->ioq_vector[i].msix_res != NULL) {  in lio_destroy_resources()
2203 					bus_release_resource(oct->device,  in lio_destroy_resources()
2205 						oct->ioq_vector[i].vector,  in lio_destroy_resources()
2206 						oct->ioq_vector[i].msix_res);  in lio_destroy_resources()
2207 					oct->ioq_vector[i].msix_res = NULL;  in lio_destroy_resources()
2210 			/* non-iov vector's argument is oct struct */  in lio_destroy_resources()
2211 			if (oct->tag != NULL) {  in lio_destroy_resources()
2212 				bus_teardown_intr(oct->device, oct->msix_res,  in lio_destroy_resources()
2213 						  oct->tag);  in lio_destroy_resources()
2214 				oct->tag = NULL;  in lio_destroy_resources()
2217 			if (oct->msix_res != NULL) {  in lio_destroy_resources()
2218 				bus_release_resource(oct->device, SYS_RES_IRQ,  in lio_destroy_resources()
2219 						     oct->aux_vector,  in lio_destroy_resources()
2220 						     oct->msix_res);  in lio_destroy_resources()
2221 				oct->msix_res = NULL;  in lio_destroy_resources()
2224 			pci_release_msi(oct->device);  in lio_destroy_resources()
2231 		for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {  in lio_destroy_resources()
2232 			if (!(oct->io_qmask.oq & BIT_ULL(i)))  in lio_destroy_resources()
2234 			lio_delete_droq(oct, i);  in lio_destroy_resources()
2240 			if (oct->droq[i] != NULL) {  in lio_destroy_resources()
2241 				free(oct->droq[i], M_DEVBUF);  in lio_destroy_resources()
2242 				oct->droq[i] = NULL;  in lio_destroy_resources()
2245 		lio_delete_response_list(oct);  in lio_destroy_resources()
2249 		for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {  in lio_destroy_resources()
2250 			if (!(oct->io_qmask.iq & BIT_ULL(i)))  in lio_destroy_resources()
2253 			lio_delete_instr_queue(oct, i);  in lio_destroy_resources()
2259 			if (oct->instr_queue[i] != NULL) {  in lio_destroy_resources()
2260 				free(oct->instr_queue[i], M_DEVBUF);  in lio_destroy_resources()
2261 				oct->instr_queue[i] = NULL;  in lio_destroy_resources()
2264 		lio_free_ioq_vector(oct);  in lio_destroy_resources()
2268 		lio_free_sc_buffer_pool(oct);  in lio_destroy_resources()
2272 		lio_delete_dispatch_list(oct);  in lio_destroy_resources()
2276 		refcount = lio_deregister_device(oct);  in lio_destroy_resources()
2279 			lio_pci_flr(oct);  in lio_destroy_resources()
2282 			oct->fn_list.soft_reset(oct);  in lio_destroy_resources()
2284 		lio_unmap_pci_barx(oct, 0);  in lio_destroy_resources()
2285 		lio_unmap_pci_barx(oct, 1);  in lio_destroy_resources()
2290 		pci_disable_busmaster(oct->device);  in lio_destroy_resources()
2295 	}	/* end switch (oct->status) */  in lio_destroy_resources()