205 __lio_retrieve_config_info(struct octeon_device *oct, uint16_t card_type)  in __lio_retrieve_config_info()  argument
208 	uint32_t	oct_id = oct->octeon_id;  in __lio_retrieve_config_info()
212 		if (oct->chip_id == LIO_CN23XX_PF_VID) {  in __lio_retrieve_config_info()
224 lio_get_config_info(struct octeon_device *oct, uint16_t card_type)  in lio_get_config_info()  argument
228 	conf = __lio_retrieve_config_info(oct, card_type);  in lio_get_config_info()
257 lio_free_device_mem(struct octeon_device *oct)  in lio_free_device_mem()  argument
261 	for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {  in lio_free_device_mem()
262 		if ((oct->io_qmask.oq & BIT_ULL(i)) && (oct->droq[i]))  in lio_free_device_mem()
263 			free(oct->droq[i], M_DEVBUF);  in lio_free_device_mem()
266 	for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {  in lio_free_device_mem()
267 		if ((oct->io_qmask.iq & BIT_ULL(i)) && (oct->instr_queue[i]))  in lio_free_device_mem()
268 			free(oct->instr_queue[i], M_DEVBUF);  in lio_free_device_mem()
271 	i = oct->octeon_id;  in lio_free_device_mem()
272 	free(oct->chip, M_DEVBUF);  in lio_free_device_mem()
281 	struct octeon_device	*oct;  in lio_allocate_device_mem()  local
306 	oct = (struct octeon_device *)device_get_softc(device);  in lio_allocate_device_mem()
307 	oct->chip = (void *)(buf);  in lio_allocate_device_mem()
308 	oct->dispatch.dlist = (struct lio_dispatch *)(buf + configsize);  in lio_allocate_device_mem()
310 	return (oct);  in lio_allocate_device_mem()
316 	struct octeon_device	*oct = NULL;  in lio_allocate_device()  local
326 		oct = lio_allocate_device_mem(device);  in lio_allocate_device()
327 		if (oct != NULL) {  in lio_allocate_device()
329 			octeon_device[oct_idx] = oct;  in lio_allocate_device()
335 	if (oct == NULL)  in lio_allocate_device()
338 	mtx_init(&oct->pci_win_lock, "pci_win_lock", NULL, MTX_DEF);  in lio_allocate_device()
339 	mtx_init(&oct->mem_access_lock, "mem_access_lock", NULL, MTX_DEF);  in lio_allocate_device()
341 	oct->octeon_id = oct_idx;  in lio_allocate_device()
342 	snprintf(oct->device_name, sizeof(oct->device_name), "%s%d",  in lio_allocate_device()
343 		 LIO_DRV_NAME, oct->octeon_id);  in lio_allocate_device()
345 	return (oct);  in lio_allocate_device()
350  *  @param oct        - pointer to the octeon device structure.
358 lio_register_device(struct octeon_device *oct, int bus, int dev, int func,  in lio_register_device()  argument
363 	oct->loc.bus = bus;  in lio_register_device()
364 	oct->loc.dev = dev;  in lio_register_device()
365 	oct->loc.func = func;  in lio_register_device()
367 	oct->adapter_refcount = &lio_adapter_refcounts[oct->octeon_id];  in lio_register_device()
368 	atomic_store_rel_int(oct->adapter_refcount, 0);  in lio_register_device()
371 	for (idx = (int)oct->octeon_id - 1; idx >= 0; idx--) {  in lio_register_device()
373 			lio_dev_err(oct, "%s: Internal driver error, missing dev\n",  in lio_register_device()
376 			atomic_add_int(oct->adapter_refcount, 1);  in lio_register_device()
383 			oct->adapter_refcount =  in lio_register_device()
391 	atomic_add_int(oct->adapter_refcount, 1);  in lio_register_device()
392 	refcount = atomic_load_acq_int(oct->adapter_refcount);  in lio_register_device()
394 	lio_dev_dbg(oct, "%s: %02x:%02x:%d refcount %u\n", __func__,  in lio_register_device()
395 		    oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);  in lio_register_device()
402  *  @param oct - pointer to the octeon device structure.
406 lio_deregister_device(struct octeon_device *oct)  in lio_deregister_device()  argument
410 	atomic_subtract_int(oct->adapter_refcount, 1);  in lio_deregister_device()
411 	refcount = atomic_load_acq_int(oct->adapter_refcount);  in lio_deregister_device()
413 	lio_dev_dbg(oct, "%s: %04d:%02d:%d refcount %u\n", __func__,  in lio_deregister_device()
414 		    oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);  in lio_deregister_device()
420 lio_allocate_ioq_vector(struct octeon_device *oct)  in lio_allocate_ioq_vector()  argument
425 	if (LIO_CN23XX_PF(oct))  in lio_allocate_ioq_vector()
426 		num_ioqs = oct->sriov_info.num_pf_rings;  in lio_allocate_ioq_vector()
430 	oct->ioq_vector = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);  in lio_allocate_ioq_vector()
431 	if (oct->ioq_vector == NULL)  in lio_allocate_ioq_vector()
435 		ioq_vector = &oct->ioq_vector[i];  in lio_allocate_ioq_vector()
436 		ioq_vector->oct_dev = oct;  in lio_allocate_ioq_vector()
441 		if (oct->chip_id == LIO_CN23XX_PF_VID)  in lio_allocate_ioq_vector()
442 			ioq_vector->ioq_num = i + oct->sriov_info.pf_srn;  in lio_allocate_ioq_vector()
450 lio_free_ioq_vector(struct octeon_device *oct)  in lio_free_ioq_vector()  argument
453 	free(oct->ioq_vector, M_DEVBUF);  in lio_free_ioq_vector()
454 	oct->ioq_vector = NULL;  in lio_free_ioq_vector()
459 lio_setup_instr_queue0(struct octeon_device *oct)  in lio_setup_instr_queue0()  argument
465 	if (LIO_CN23XX_PF(oct))  in lio_setup_instr_queue0()
467 			LIO_GET_NUM_DEF_TX_DESCS_CFG(LIO_CHIP_CONF(oct,  in lio_setup_instr_queue0()
470 	oct->num_iqs = 0;  in lio_setup_instr_queue0()
472 	oct->instr_queue[0]->q_index = 0;  in lio_setup_instr_queue0()
473 	oct->instr_queue[0]->app_ctx = (void *)(size_t)0;  in lio_setup_instr_queue0()
474 	oct->instr_queue[0]->ifidx = 0;  in lio_setup_instr_queue0()
477 	txpciq.s.pkind = oct->pfvf_hsword.pkind;  in lio_setup_instr_queue0()
480 	if (lio_init_instr_queue(oct, txpciq, num_descs)) {  in lio_setup_instr_queue0()
482 		lio_delete_instr_queue(oct, 0);  in lio_setup_instr_queue0()
486 	oct->num_iqs++;  in lio_setup_instr_queue0()
491 lio_setup_output_queue0(struct octeon_device *oct)  in lio_setup_output_queue0()  argument
495 	if (LIO_CN23XX_PF(oct)) {  in lio_setup_output_queue0()
497 			LIO_GET_NUM_DEF_RX_DESCS_CFG(LIO_CHIP_CONF(oct,  in lio_setup_output_queue0()
500 			LIO_GET_DEF_RX_BUF_SIZE_CFG(LIO_CHIP_CONF(oct,  in lio_setup_output_queue0()
504 	oct->num_oqs = 0;  in lio_setup_output_queue0()
506 	if (lio_init_droq(oct, oq_no, num_descs, desc_size, NULL)) {  in lio_setup_output_queue0()
510 	oct->num_oqs++;  in lio_setup_output_queue0()
516 lio_init_dispatch_list(struct octeon_device *oct)  in lio_init_dispatch_list()  argument
520 	oct->dispatch.count = 0;  in lio_init_dispatch_list()
523 		oct->dispatch.dlist[i].opcode = 0;  in lio_init_dispatch_list()
524 		STAILQ_INIT(&oct->dispatch.dlist[i].head);  in lio_init_dispatch_list()
527 	mtx_init(&oct->dispatch.lock, "dispatch_lock", NULL, MTX_DEF);  in lio_init_dispatch_list()
533 lio_delete_dispatch_list(struct octeon_device *oct)  in lio_delete_dispatch_list()  argument
541 	mtx_lock(&oct->dispatch.lock);  in lio_delete_dispatch_list()
546 		dispatch = &oct->dispatch.dlist[i].head;  in lio_delete_dispatch_list()
549 			STAILQ_REMOVE_HEAD(&oct->dispatch.dlist[i].head,  in lio_delete_dispatch_list()
554 		oct->dispatch.dlist[i].opcode = 0;  in lio_delete_dispatch_list()
557 	oct->dispatch.count = 0;  in lio_delete_dispatch_list()
559 	mtx_unlock(&oct->dispatch.lock);  in lio_delete_dispatch_list()
627 lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode,  in lio_register_dispatch_fn()  argument
636 	mtx_lock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
638 	if (oct->dispatch.dlist[idx].opcode == 0) {  in lio_register_dispatch_fn()
639 		oct->dispatch.dlist[idx].opcode = combined_opcode;  in lio_register_dispatch_fn()
640 		oct->dispatch.dlist[idx].dispatch_fn = fn;  in lio_register_dispatch_fn()
641 		oct->dispatch.dlist[idx].arg = fn_arg;  in lio_register_dispatch_fn()
642 		oct->dispatch.count++;  in lio_register_dispatch_fn()
643 		mtx_unlock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
647 	mtx_unlock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
653 	pfn = lio_get_dispatch(oct, opcode, subcode);  in lio_register_dispatch_fn()
657 		lio_dev_dbg(oct,  in lio_register_dispatch_fn()
663 			lio_dev_err(oct,  in lio_register_dispatch_fn()
676 		mtx_lock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
677 		STAILQ_INSERT_HEAD(&oct->dispatch.dlist[idx].head,  in lio_register_dispatch_fn()
679 		oct->dispatch.count++;  in lio_register_dispatch_fn()
680 		mtx_unlock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
683 		lio_dev_err(oct, "Found previously registered dispatch fn for opcode/subcode: %x/%x\n",  in lio_register_dispatch_fn()
694  *   oct       - octeon device
706 lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode,  in lio_unregister_dispatch_fn()  argument
717 	mtx_lock(&oct->dispatch.lock);  in lio_unregister_dispatch_fn()
719 	if (oct->dispatch.count == 0) {  in lio_unregister_dispatch_fn()
720 		mtx_unlock(&oct->dispatch.lock);  in lio_unregister_dispatch_fn()
721 		lio_dev_err(oct, "No dispatch functions registered for this device\n");  in lio_unregister_dispatch_fn()
724 	if (oct->dispatch.dlist[idx].opcode == combined_opcode) {  in lio_unregister_dispatch_fn()
725 		dispatch_head = &oct->dispatch.dlist[idx].head;  in lio_unregister_dispatch_fn()
728 			oct->dispatch.dlist[idx].opcode =  in lio_unregister_dispatch_fn()
730 			oct->dispatch.dlist[idx].dispatch_fn =  in lio_unregister_dispatch_fn()
732 			oct->dispatch.dlist[idx].arg =  in lio_unregister_dispatch_fn()
737 			oct->dispatch.dlist[idx].opcode = 0;  in lio_unregister_dispatch_fn()
738 			oct->dispatch.dlist[idx].dispatch_fn = NULL;  in lio_unregister_dispatch_fn()
739 			oct->dispatch.dlist[idx].arg = NULL;  in lio_unregister_dispatch_fn()
744 				    &oct->dispatch.dlist[idx].head,  in lio_unregister_dispatch_fn()
748 				STAILQ_REMOVE(&oct->dispatch.dlist[idx].head,  in lio_unregister_dispatch_fn()
758 		oct->dispatch.count--;  in lio_unregister_dispatch_fn()
760 	mtx_unlock(&oct->dispatch.lock);  in lio_unregister_dispatch_fn()
769 	struct octeon_device	*oct = (struct octeon_device *)buf;  in lio_core_drv_init()  local
776 	if (LIO_CN23XX_PF(oct))  in lio_core_drv_init()
778 					       LIO_CHIP_CONF(oct, cn23xx_pf));  in lio_core_drv_init()
780 	if (atomic_load_acq_int(&oct->status) >= LIO_DEV_RUNNING) {  in lio_core_drv_init()
781 		lio_dev_err(oct, "Received CORE OK when device state is 0x%x\n",  in lio_core_drv_init()
782 			    atomic_load_acq_int(&oct->status));  in lio_core_drv_init()
790 	oct->app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;  in lio_core_drv_init()
792 		oct->fw_info.max_nic_ports =  in lio_core_drv_init()
794 		oct->fw_info.num_gmx_ports =  in lio_core_drv_init()
798 	if (oct->fw_info.max_nic_ports < num_nic_ports) {  in lio_core_drv_init()
799 		lio_dev_err(oct, "Config has more ports than firmware allows (%d > %d).\n",  in lio_core_drv_init()
800 			    num_nic_ports, oct->fw_info.max_nic_ports);  in lio_core_drv_init()
804 	oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags;  in lio_core_drv_init()
805 	oct->fw_info.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;  in lio_core_drv_init()
806 	oct->pfvf_hsword.app_mode =  in lio_core_drv_init()
809 	oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind;  in lio_core_drv_init()
811 	for (i = 0; i < oct->num_iqs; i++)  in lio_core_drv_init()
812 		oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind;  in lio_core_drv_init()
814 	atomic_store_rel_int(&oct->status, LIO_DEV_CORE_OK);  in lio_core_drv_init()
816 	cs = &core_setup[oct->octeon_id];  in lio_core_drv_init()
819 		lio_dev_dbg(oct, "Core setup bytes expected %llu found %d\n",  in lio_core_drv_init()
826 	strncpy(oct->boardinfo.name, cs->boardname, LIO_BOARD_NAME);  in lio_core_drv_init()
827 	strncpy(oct->boardinfo.serial_number, cs->board_serial_number,  in lio_core_drv_init()
832 	oct->boardinfo.major = cs->board_rev_major;  in lio_core_drv_init()
833 	oct->boardinfo.minor = cs->board_rev_minor;  in lio_core_drv_init()
835 	lio_dev_info(oct, "Running %s (%llu Hz)\n", app_name,  in lio_core_drv_init()
847 lio_get_tx_qsize(struct octeon_device *oct, uint32_t q_no)  in lio_get_tx_qsize()  argument
850 	if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_INSTR_QUEUES(oct)) &&  in lio_get_tx_qsize()
851 	    (oct->io_qmask.iq & BIT_ULL(q_no)))  in lio_get_tx_qsize()
852 		return (oct->instr_queue[q_no]->max_count);  in lio_get_tx_qsize()
859 lio_get_rx_qsize(struct octeon_device *oct, uint32_t q_no)  in lio_get_rx_qsize()  argument
862 	if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_OUTPUT_QUEUES(oct)) &&  in lio_get_rx_qsize()
863 	    (oct->io_qmask.oq & BIT_ULL(q_no)))  in lio_get_rx_qsize()
864 		return (oct->droq[q_no]->max_count);  in lio_get_rx_qsize()
871 lio_get_conf(struct octeon_device *oct)  in lio_get_conf()  argument
879 	if (LIO_CN23XX_PF(oct)) {  in lio_get_conf()
881 					       LIO_CHIP_CONF(oct, cn23xx_pf));  in lio_get_conf()
904 lio_pci_readq(struct octeon_device *oct, uint64_t addr)  in lio_pci_readq()  argument
909 	mtx_lock(&oct->pci_win_lock);  in lio_pci_readq()
916 	if (oct->chip_id == LIO_CN23XX_PF_VID)  in lio_pci_readq()
918 	lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_hi, addrhi);  in lio_pci_readq()
921 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi);  in lio_pci_readq()
923 	lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_lo,  in lio_pci_readq()
925 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo);  in lio_pci_readq()
927 	val64 = lio_read_csr64(oct, oct->reg_list.pci_win_rd_data);  in lio_pci_readq()
929 	mtx_unlock(&oct->pci_win_lock);  in lio_pci_readq()
935 lio_pci_writeq(struct octeon_device *oct, uint64_t val, uint64_t addr)  in lio_pci_writeq()  argument
938 	mtx_lock(&oct->pci_win_lock);  in lio_pci_writeq()
940 	lio_write_csr64(oct, oct->reg_list.pci_win_wr_addr, addr);  in lio_pci_writeq()
943 	lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_hi, val >> 32);  in lio_pci_writeq()
945 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi);  in lio_pci_writeq()
947 	lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_lo,  in lio_pci_writeq()
950 	mtx_unlock(&oct->pci_win_lock);  in lio_pci_writeq()
954 lio_mem_access_ok(struct octeon_device *oct)  in lio_mem_access_ok()  argument
960 	if (LIO_CN23XX_PF(oct)) {  in lio_mem_access_ok()
961 		lmc0_reset_ctl = lio_pci_readq(oct, LIO_CN23XX_LMC0_RESET_CTL);  in lio_mem_access_ok()
970 lio_wait_for_ddr_init(struct octeon_device *oct, unsigned long *timeout)  in lio_wait_for_ddr_init()  argument
979 		ret = lio_mem_access_ok(oct);  in lio_wait_for_ddr_init()
1011 	struct octeon_device *oct = NULL;  in lio_enable_irq()  local
1017 		oct = droq->oct_dev;  in lio_enable_irq()
1020 		lio_write_csr32(oct, droq->pkts_sent_reg,  in lio_enable_irq()
1029 		oct = iq->oct_dev;  in lio_enable_irq()
1031 		lio_write_csr32(oct, iq->inst_cnt_reg, iq->pkt_in_done);  in lio_enable_irq()
1047 	if ((oct != NULL) && (LIO_CN23XX_PF(oct))) {  in lio_enable_irq()
1049 			lio_write_csr64(oct, droq->pkts_sent_reg,  in lio_enable_irq()
1054 			instr_cnt = lio_read_csr64(oct, iq->inst_cnt_reg);  in lio_enable_irq()
1055 			lio_write_csr64(oct, iq->inst_cnt_reg,  in lio_enable_irq()