40 mlxcx_wq_alloc_dma(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_wq_alloc_dma()  argument
52 		mlwq->mlwq_entshift = mlxp->mlx_props.mldp_sq_size_shift;  in mlxcx_wq_alloc_dma()
57 		mlwq->mlwq_entshift = mlxp->mlx_props.mldp_rq_size_shift;  in mlxcx_wq_alloc_dma()
67 	mlxcx_dma_acc_attr(mlxp, &acc);  in mlxcx_wq_alloc_dma()
68 	mlxcx_dma_queue_attr(mlxp, &attr);  in mlxcx_wq_alloc_dma()
70 	ret = mlxcx_dma_alloc(mlxp, &mlwq->mlwq_dma, &attr, &acc,  in mlxcx_wq_alloc_dma()
73 		mlxcx_warn(mlxp, "failed to allocate WQ memory");  in mlxcx_wq_alloc_dma()
83 	mlxcx_dma_acc_attr(mlxp, &acc);  in mlxcx_wq_alloc_dma()
84 	mlxcx_dma_qdbell_attr(mlxp, &attr);  in mlxcx_wq_alloc_dma()
86 	ret = mlxcx_dma_alloc(mlxp, &mlwq->mlwq_doorbell_dma, &attr, &acc,  in mlxcx_wq_alloc_dma()
89 		mlxcx_warn(mlxp, "failed to allocate WQ doorbell memory");  in mlxcx_wq_alloc_dma()
104 mlxcx_wq_rele_dma(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_wq_rele_dma()  argument
119 mlxcx_cq_alloc_dma(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq,  in mlxcx_cq_alloc_dma()  argument
134 	mlxcx_dma_acc_attr(mlxp, &acc);  in mlxcx_cq_alloc_dma()
135 	mlxcx_dma_queue_attr(mlxp, &attr);  in mlxcx_cq_alloc_dma()
137 	ret = mlxcx_dma_alloc(mlxp, &mlcq->mlcq_dma, &attr, &acc,  in mlxcx_cq_alloc_dma()
140 		mlxcx_warn(mlxp, "failed to allocate CQ memory");  in mlxcx_cq_alloc_dma()
151 	mlxcx_dma_acc_attr(mlxp, &acc);  in mlxcx_cq_alloc_dma()
152 	mlxcx_dma_qdbell_attr(mlxp, &attr);  in mlxcx_cq_alloc_dma()
154 	ret = mlxcx_dma_alloc(mlxp, &mlcq->mlcq_doorbell_dma, &attr, &acc,  in mlxcx_cq_alloc_dma()
157 		mlxcx_warn(mlxp, "failed to allocate CQ doorbell memory");  in mlxcx_cq_alloc_dma()
172 mlxcx_cq_rele_dma(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)  in mlxcx_cq_rele_dma()  argument
187 mlxcx_wq_teardown(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_wq_teardown()  argument
199 	list_remove(&mlxp->mlx_wqs, mlwq);  in mlxcx_wq_teardown()
205 		    !mlxcx_cmd_stop_rq(mlxp, mlwq)) {  in mlxcx_wq_teardown()
206 			mlxcx_warn(mlxp, "failed to stop "  in mlxcx_wq_teardown()
211 		    !mlxcx_cmd_stop_sq(mlxp, mlwq)) {  in mlxcx_wq_teardown()
212 			mlxcx_warn(mlxp, "failed to stop "  in mlxcx_wq_teardown()
216 		    !mlxcx_cmd_destroy_rq(mlxp, mlwq)) {  in mlxcx_wq_teardown()
217 			mlxcx_warn(mlxp, "failed to destroy "  in mlxcx_wq_teardown()
221 		    !mlxcx_cmd_destroy_sq(mlxp, mlwq)) {  in mlxcx_wq_teardown()
222 			mlxcx_warn(mlxp, "failed to destroy "  in mlxcx_wq_teardown()
227 		mlxcx_wq_rele_dma(mlxp, mlwq);  in mlxcx_wq_teardown()
248 mlxcx_cq_teardown(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)  in mlxcx_cq_teardown()  argument
262 	list_remove(&mlxp->mlx_cqs, mlcq);  in mlxcx_cq_teardown()
266 		if (!mlxcx_cmd_destroy_cq(mlxp, mlcq)) {  in mlxcx_cq_teardown()
267 			mlxcx_warn(mlxp, "failed to destroy "  in mlxcx_cq_teardown()
273 		mlxcx_cq_rele_dma(mlxp, mlcq);  in mlxcx_cq_teardown()
292 		mlxcx_buf_return_chain(mlxp, b, B_FALSE);  in mlxcx_cq_teardown()
296 		mlxcx_buf_return_chain(mlxp, b, B_FALSE);  in mlxcx_cq_teardown()
337 mlxcx_cq_setup(mlxcx_t *mlxp, mlxcx_event_queue_t *eq,  in mlxcx_cq_setup()  argument
344 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_cq_setup()
346 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_cq_setup()
348 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_cq_setup()
354 	cq->mlcq_mlx = mlxp;  in mlxcx_cq_setup()
355 	list_insert_tail(&mlxp->mlx_cqs, cq);  in mlxcx_cq_setup()
359 	if (!mlxcx_cq_alloc_dma(mlxp, cq, ent_shift)) {  in mlxcx_cq_setup()
367 	cq->mlcq_uar = &mlxp->mlx_uar;  in mlxcx_cq_setup()
370 	cq->mlcq_cqemod_period_usec = mlxp->mlx_props.mldp_cqemod_period_usec;  in mlxcx_cq_setup()
371 	cq->mlcq_cqemod_count = mlxp->mlx_props.mldp_cqemod_count;  in mlxcx_cq_setup()
373 	if (!mlxcx_cmd_create_cq(mlxp, cq)) {  in mlxcx_cq_setup()
386 	mlxcx_arm_cq(mlxp, cq);  in mlxcx_cq_setup()
396 mlxcx_rq_setup(mlxcx_t *mlxp, mlxcx_completion_queue_t *cq,  in mlxcx_rq_setup()  argument
400 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_rq_setup()
402 	list_insert_tail(&mlxp->mlx_wqs, wq);  in mlxcx_rq_setup()
406 	wq->mlwq_mlx = mlxp;  in mlxcx_rq_setup()
409 	wq->mlwq_pd = &mlxp->mlx_pd;  in mlxcx_rq_setup()
410 	wq->mlwq_uar = &mlxp->mlx_uar;  in mlxcx_rq_setup()
412 	wq->mlwq_bufs = mlxcx_mlbs_create(mlxp);  in mlxcx_rq_setup()
414 	if (!mlxcx_wq_alloc_dma(mlxp, wq)) {  in mlxcx_rq_setup()
419 	if (!mlxcx_cmd_create_rq(mlxp, wq)) {  in mlxcx_rq_setup()
440 mlxcx_sq_setup(mlxcx_t *mlxp, mlxcx_port_t *port, mlxcx_completion_queue_t *cq,  in mlxcx_sq_setup()  argument
444 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_sq_setup()
446 	list_insert_tail(&mlxp->mlx_wqs, wq);  in mlxcx_sq_setup()
450 	wq->mlwq_mlx = mlxp;  in mlxcx_sq_setup()
453 	wq->mlwq_pd = &mlxp->mlx_pd;  in mlxcx_sq_setup()
454 	wq->mlwq_uar = &mlxp->mlx_uar;  in mlxcx_sq_setup()
457 	wq->mlwq_bufs = mlxcx_mlbs_create(mlxp);  in mlxcx_sq_setup()
458 	wq->mlwq_foreign_bufs = mlxcx_mlbs_create(mlxp);  in mlxcx_sq_setup()
463 	if (!mlxcx_wq_alloc_dma(mlxp, wq)) {  in mlxcx_sq_setup()
468 	if (!mlxcx_cmd_create_sq(mlxp, wq)) {  in mlxcx_sq_setup()
493 mlxcx_quiesce_rx_cqs(mlxcx_t *mlxp, mlxcx_ring_group_t *g)  in mlxcx_quiesce_rx_cqs()  argument
518 mlxcx_teardown_rx_group(mlxcx_t *mlxp, mlxcx_ring_group_t *g)  in mlxcx_teardown_rx_group()  argument
531 		mlxcx_remove_all_umcast_entries(mlxp, g->mlg_port, g);  in mlxcx_teardown_rx_group()
534 			mlxcx_remove_all_vlan_entries(mlxp, g);  in mlxcx_teardown_rx_group()
536 		if (g == &mlxp->mlx_rx_groups[0]) {  in mlxcx_teardown_rx_group()
544 				    mlxp, fe);  in mlxcx_teardown_rx_group()
551 				    mlxp, fe);  in mlxcx_teardown_rx_group()
566 					    mlxp, fe);  in mlxcx_teardown_rx_group()
575 					    mlxp, fe);  in mlxcx_teardown_rx_group()
578 			mlxcx_teardown_flow_table(mlxp, g->mlg_rx_vlan_ft);  in mlxcx_teardown_rx_group()
585 		mlxcx_teardown_flow_table(mlxp, g->mlg_rx_hash_ft);  in mlxcx_teardown_rx_group()
597 			    !mlxcx_cmd_stop_rq(mlxp, wq)) {  in mlxcx_teardown_rx_group()
598 				mlxcx_warn(mlxp, "failed to stop rq %x",  in mlxcx_teardown_rx_group()
612 				if (!mlxcx_cmd_destroy_tir(mlxp, tir)) {  in mlxcx_teardown_rx_group()
613 					mlxcx_warn(mlxp,  in mlxcx_teardown_rx_group()
625 			if (!mlxcx_cmd_destroy_rqt(mlxp, g->mlg_rqt)) {  in mlxcx_teardown_rx_group()
626 				mlxcx_warn(mlxp, "failed to destroy rqt %u "  in mlxcx_teardown_rx_group()
641 		mlxcx_wq_teardown(mlxp, wq);  in mlxcx_teardown_rx_group()
643 			mlxcx_cq_teardown(mlxp, cq);  in mlxcx_teardown_rx_group()
659 mlxcx_teardown_tx_group(mlxcx_t *mlxp, mlxcx_ring_group_t *g)  in mlxcx_teardown_tx_group()  argument
673 			    !mlxcx_cmd_stop_sq(mlxp, wq)) {  in mlxcx_teardown_tx_group()
674 				mlxcx_warn(mlxp, "failed to stop sq %x",  in mlxcx_teardown_tx_group()
678 			mlxcx_wq_teardown(mlxp, wq);  in mlxcx_teardown_tx_group()
680 				mlxcx_cq_teardown(mlxp, cq);  in mlxcx_teardown_tx_group()
691 		if (!mlxcx_cmd_destroy_tis(mlxp, &g->mlg_tis)) {  in mlxcx_teardown_tx_group()
692 			mlxcx_warn(mlxp, "failed to destroy tis %u for tx ring",  in mlxcx_teardown_tx_group()
705 mlxcx_teardown_groups(mlxcx_t *mlxp)  in mlxcx_teardown_groups()  argument
710 	for (i = 0; i < mlxp->mlx_rx_ngroups; ++i) {  in mlxcx_teardown_groups()
711 		g = &mlxp->mlx_rx_groups[i];  in mlxcx_teardown_groups()
715 		mlxcx_quiesce_rx_cqs(mlxp, g);  in mlxcx_teardown_groups()
718 	for (i = 0; i < mlxp->mlx_rx_ngroups; ++i) {  in mlxcx_teardown_groups()
719 		g = &mlxp->mlx_rx_groups[i];  in mlxcx_teardown_groups()
722 		mlxcx_teardown_rx_group(mlxp, g);  in mlxcx_teardown_groups()
725 	kmem_free(mlxp->mlx_rx_groups, mlxp->mlx_rx_groups_size);  in mlxcx_teardown_groups()
726 	mlxp->mlx_rx_groups = NULL;  in mlxcx_teardown_groups()
728 	for (i = 0; i < mlxp->mlx_tx_ngroups; ++i) {  in mlxcx_teardown_groups()
729 		g = &mlxp->mlx_tx_groups[i];  in mlxcx_teardown_groups()
733 		mlxcx_teardown_tx_group(mlxp, g);  in mlxcx_teardown_groups()
736 	kmem_free(mlxp->mlx_tx_groups, mlxp->mlx_tx_groups_size);  in mlxcx_teardown_groups()
737 	mlxp->mlx_tx_groups = NULL;  in mlxcx_teardown_groups()
741 mlxcx_rx_group_setup(mlxcx_t *mlxp, mlxcx_ring_group_t *g)  in mlxcx_rx_group_setup()  argument
755 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_rx_group_setup()
757 	g->mlg_mlx = mlxp;  in mlxcx_rx_group_setup()
759 	g->mlg_port = &mlxp->mlx_ports[0];  in mlxcx_rx_group_setup()
762 	g->mlg_nwqs = mlxp->mlx_props.mldp_rx_nrings_per_small_group;  in mlxcx_rx_group_setup()
763 	i = g - &mlxp->mlx_rx_groups[0];  in mlxcx_rx_group_setup()
764 	if (i < mlxp->mlx_props.mldp_rx_ngroups_large)  in mlxcx_rx_group_setup()
765 		g->mlg_nwqs = mlxp->mlx_props.mldp_rx_nrings_per_large_group;  in mlxcx_rx_group_setup()
783 			eq = &mlxp->mlx_eqs[mlxp->mlx_next_eq++];  in mlxcx_rx_group_setup()
784 			if (mlxp->mlx_next_eq >= mlxp->mlx_intr_count)  in mlxcx_rx_group_setup()
785 				mlxp->mlx_next_eq = mlxp->mlx_intr_cq0;  in mlxcx_rx_group_setup()
798 		ent_shift = MIN(mlxp->mlx_props.mldp_cq_size_shift,  in mlxcx_rx_group_setup()
799 		    mlxp->mlx_props.mldp_rq_size_shift);  in mlxcx_rx_group_setup()
800 		if (!mlxcx_cq_setup(mlxp, eq, &cq, ent_shift)) {  in mlxcx_rx_group_setup()
808 		if (!mlxcx_rq_setup(mlxp, cq, rq)) {  in mlxcx_rx_group_setup()
821 	if (!mlxcx_cmd_create_rqt(mlxp, g->mlg_rqt)) {  in mlxcx_rx_group_setup()
828 		tir->mltir_tdom = &mlxp->mlx_tdom;  in mlxcx_rx_group_setup()
884 		if (!mlxcx_cmd_create_tir(mlxp, tir)) {  in mlxcx_rx_group_setup()
899 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_rx_group_setup()
923 	if (!mlxcx_cmd_create_flow_table(mlxp, ft)) {  in mlxcx_rx_group_setup()
934 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
945 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
956 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
967 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
978 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
989 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
1000 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1011 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
1022 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1032 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
1043 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1053 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
1063 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1072 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
1088 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_rx_group_setup()
1097 	ft->mlft_entshift = mlxp->mlx_props.mldp_ftbl_vlan_size_shift;  in mlxcx_rx_group_setup()
1112 	if (!mlxcx_cmd_create_flow_table(mlxp, ft)) {  in mlxcx_rx_group_setup()
1126 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1141 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1147 	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_setup()
1162 	if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {  in mlxcx_rx_group_setup()
1169 	fe->mlfe_dest[0].mlfed_flow = mlxp->mlx_rx_groups[0].mlg_rx_hash_ft;  in mlxcx_rx_group_setup()
1179 mlxcx_rx_ring_start(mlxcx_t *mlxp, mlxcx_ring_group_t *g,  in mlxcx_rx_ring_start()  argument
1197 		if (!mlxcx_rx_group_start(mlxp, g))  in mlxcx_rx_ring_start()
1216 	if (!mlxcx_cmd_start_rq(mlxp, rq)) {  in mlxcx_rx_ring_start()
1230 		if (!mlxcx_buf_create(mlxp, rq->mlwq_bufs, &b))  in mlxcx_rx_ring_start()
1232 		mlxcx_buf_return(mlxp, b);  in mlxcx_rx_ring_start()
1235 		if (!mlxcx_buf_create(mlxp, rq->mlwq_bufs, &b))  in mlxcx_rx_ring_start()
1237 		mlxcx_buf_return(mlxp, b);  in mlxcx_rx_ring_start()
1240 	mlxcx_rq_refill(mlxp, rq);  in mlxcx_rx_ring_start()
1250 mlxcx_rx_group_start(mlxcx_t *mlxp, mlxcx_ring_group_t *g)  in mlxcx_rx_group_start()  argument
1269 	    ddi_driver_name(mlxp->mlx_dip), mlxp->mlx_inst,  in mlxcx_rx_group_start()
1270 	    g - &mlxp->mlx_rx_groups[0]);  in mlxcx_rx_group_start()
1280 		mlxcx_warn(mlxp, "failed to create rq refill task queue");  in mlxcx_rx_group_start()
1285 	if (g == &mlxp->mlx_rx_groups[0]) {  in mlxcx_rx_group_start()
1296 		if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {  in mlxcx_rx_group_start()
1321 mlxcx_tx_group_setup(mlxcx_t *mlxp, mlxcx_ring_group_t *g)  in mlxcx_tx_group_setup()  argument
1331 	    DDI_INTR_PRI(mlxp->mlx_intr_pri));  in mlxcx_tx_group_setup()
1335 	g->mlg_mlx = mlxp;  in mlxcx_tx_group_setup()
1337 	g->mlg_port = &mlxp->mlx_ports[0];  in mlxcx_tx_group_setup()
1339 	g->mlg_nwqs = mlxp->mlx_props.mldp_tx_nrings_per_group;  in mlxcx_tx_group_setup()
1344 	g->mlg_tis.mltis_tdom = &mlxp->mlx_tdom;  in mlxcx_tx_group_setup()
1346 	if (!mlxcx_cmd_create_tis(mlxp, &g->mlg_tis)) {  in mlxcx_tx_group_setup()
1356 			eq = &mlxp->mlx_eqs[mlxp->mlx_next_eq++];  in mlxcx_tx_group_setup()
1357 			if (mlxp->mlx_next_eq >= mlxp->mlx_intr_count)  in mlxcx_tx_group_setup()
1358 				mlxp->mlx_next_eq = mlxp->mlx_intr_cq0;  in mlxcx_tx_group_setup()
1366 		if (!mlxcx_cq_setup(mlxp, eq, &cq,  in mlxcx_tx_group_setup()
1367 		    mlxp->mlx_props.mldp_cq_size_shift))  in mlxcx_tx_group_setup()
1373 		if (!mlxcx_sq_setup(mlxp, g->mlg_port, cq, &g->mlg_tis, sq)) {  in mlxcx_tx_group_setup()
1386 mlxcx_tx_ring_start(mlxcx_t *mlxp, mlxcx_ring_group_t *g,  in mlxcx_tx_ring_start()  argument
1409 		if (!mlxcx_buf_create_foreign(mlxp, sq->mlwq_foreign_bufs, &b))  in mlxcx_tx_ring_start()
1411 		mlxcx_buf_return(mlxp, b);  in mlxcx_tx_ring_start()
1414 		if (!mlxcx_buf_create_foreign(mlxp, sq->mlwq_foreign_bufs, &b))  in mlxcx_tx_ring_start()
1416 		mlxcx_buf_return(mlxp, b);  in mlxcx_tx_ring_start()
1419 		if (!mlxcx_buf_create(mlxp, sq->mlwq_bufs, &b))  in mlxcx_tx_ring_start()
1421 		mlxcx_buf_return(mlxp, b);  in mlxcx_tx_ring_start()
1428 	if (!mlxcx_cmd_start_sq(mlxp, sq)) {  in mlxcx_tx_ring_start()
1436 	(void) mlxcx_sq_add_nop(mlxp, sq);  in mlxcx_tx_ring_start()
1446 mlxcx_sq_ring_dbell(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, uint_t first)  in mlxcx_sq_ring_dbell()  argument
1478 	mlxcx_put64(mlxp, bf->mbf_even, from_be64(  in mlxcx_sq_ring_dbell()
1480 	ddi_fm_acc_err_get(mlxp->mlx_regs_handle, &err,  in mlxcx_sq_ring_dbell()
1485 		ddi_fm_acc_err_clear(mlxp->mlx_regs_handle, DDI_FME_VERSION);  in mlxcx_sq_ring_dbell()
1490 	ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);  in mlxcx_sq_ring_dbell()
1495 mlxcx_sq_add_nop(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_sq_add_nop()  argument
1533 	if (!mlxcx_sq_ring_dbell(mlxp, mlwq, index)) {  in mlxcx_sq_add_nop()
1540 mlxcx_sq_add_buffer(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,  in mlxcx_sq_add_buffer()  argument
1591 		ASSERT(mlxp->mlx_caps->mlc_checksum);  in mlxcx_sq_add_buffer()
1596 		ASSERT(mlxp->mlx_caps->mlc_checksum);  in mlxcx_sq_add_buffer()
1636 			seg->mlds_lkey = to_be32(mlxp->mlx_rsvd_lkey);  in mlxcx_sq_add_buffer()
1704 	if ((rv = mlxcx_sq_ring_dbell(mlxp, mlwq, first))) {  in mlxcx_sq_add_buffer()
1716 mlxcx_rq_add_buffer(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,  in mlxcx_rq_add_buffer()  argument
1719 	return (mlxcx_rq_add_buffers(mlxp, mlwq, &buf, 1));  in mlxcx_rq_add_buffer()
1723 mlxcx_rq_add_buffers(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,  in mlxcx_rq_add_buffers()  argument
1762 			seg->mlds_lkey = to_be32(mlxp->mlx_rsvd_lkey);  in mlxcx_rq_add_buffers()
1816 	mlxcx_t *mlxp = wq->mlwq_mlx;  in mlxcx_rq_refill_task()  local
1844 			mlxcx_rq_refill(mlxp, wq);  in mlxcx_rq_refill_task()
1860 mlxcx_rq_refill(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_rq_refill()  argument
1892 		n = mlxcx_buf_take_n(mlxp, mlwq, b, MLXCX_RQ_REFILL_STEP);  in mlxcx_rq_refill()
1914 				mlxcx_buf_return(mlxp, b[i]);  in mlxcx_rq_refill()
1917 		if (!mlxcx_rq_add_buffers(mlxp, mlwq, b, n)) {  in mlxcx_rq_refill()
1926 					mlxcx_buf_return(mlxp, b[i]);  in mlxcx_rq_refill()
1965 mlxcx_fm_cqe_ereport(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq,  in mlxcx_fm_cqe_ereport()  argument
1972 	if (!DDI_FM_EREPORT_CAP(mlxp->mlx_fm_caps))  in mlxcx_fm_cqe_ereport()
1979 	ddi_fm_ereport_post(mlxp->mlx_dip, buf, ena, DDI_NOSLEEP,  in mlxcx_fm_cqe_ereport()
1991 	ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_DEGRADED);  in mlxcx_fm_cqe_ereport()
1995 mlxcx_tx_completion(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq,  in mlxcx_tx_completion()  argument
2002 		mlxcx_fm_cqe_ereport(mlxp, mlcq, eent);  in mlxcx_tx_completion()
2003 		mlxcx_buf_return_chain(mlxp, buf, B_FALSE);  in mlxcx_tx_completion()
2005 		mlxcx_check_sq(mlxp, mlcq->mlcq_wq);  in mlxcx_tx_completion()
2011 		mlxcx_warn(mlxp, "!got weird cq opcode: %x", ent->mlcqe_opcode);  in mlxcx_tx_completion()
2012 		mlxcx_buf_return_chain(mlxp, buf, B_FALSE);  in mlxcx_tx_completion()
2017 		mlxcx_warn(mlxp, "!got weird cq wqe opcode: %x",  in mlxcx_tx_completion()
2019 		mlxcx_buf_return_chain(mlxp, buf, B_FALSE);  in mlxcx_tx_completion()
2024 		mlxcx_warn(mlxp, "!got weird cq format: %x", ent->mlcqe_format);  in mlxcx_tx_completion()
2025 		mlxcx_buf_return_chain(mlxp, buf, B_FALSE);  in mlxcx_tx_completion()
2029 	mlxcx_buf_return_chain(mlxp, buf, B_FALSE);  in mlxcx_tx_completion()
2033 mlxcx_rx_completion(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq,  in mlxcx_rx_completion()  argument
2046 		mlxcx_fm_cqe_ereport(mlxp, mlcq, eent);  in mlxcx_rx_completion()
2047 		mlxcx_buf_return(mlxp, buf);  in mlxcx_rx_completion()
2049 		mlxcx_check_rq(mlxp, mlcq->mlcq_wq);  in mlxcx_rx_completion()
2055 		mlxcx_warn(mlxp, "!got weird cq opcode: %x", ent->mlcqe_opcode);  in mlxcx_rx_completion()
2056 		mlxcx_buf_return(mlxp, buf);  in mlxcx_rx_completion()
2061 		mlxcx_warn(mlxp, "!got weird cq format: %x", ent->mlcqe_format);  in mlxcx_rx_completion()
2062 		mlxcx_buf_return(mlxp, buf);  in mlxcx_rx_completion()
2077 		mlxcx_buf_return(mlxp, buf);  in mlxcx_rx_completion()
2091 	if (mlxcx_buf_loan(mlxp, buf)) {  in mlxcx_rx_completion()
2103 			mlxcx_buf_return(mlxp, buf);  in mlxcx_rx_completion()
2110 		mlxcx_buf_return(mlxp, buf);  in mlxcx_rx_completion()
2138 			mlxcx_rq_refill(mlxp, wq);  in mlxcx_rx_completion()
2149 	mlxcx_t *mlxp = b->mlb_mlx;  in mlxcx_buf_mp_return()  local
2155 		mlxcx_buf_return(mlxp, b);  in mlxcx_buf_mp_return()
2159 mlxcx_buf_create(mlxcx_t *mlxp, mlxcx_buf_shard_t *shard, mlxcx_buffer_t **bp)  in mlxcx_buf_create()  argument
2166 	b = kmem_cache_alloc(mlxp->mlx_bufs_cache, KM_SLEEP);  in mlxcx_buf_create()
2170 	mlxcx_dma_acc_attr(mlxp, &acc);  in mlxcx_buf_create()
2171 	mlxcx_dma_buf_attr(mlxp, &attr);  in mlxcx_buf_create()
2173 	ret = mlxcx_dma_alloc_offset(mlxp, &b->mlb_dma, &attr, &acc,  in mlxcx_buf_create()
2174 	    B_FALSE, mlxp->mlx_ports[0].mlp_mtu, 2, B_TRUE);  in mlxcx_buf_create()
2176 		kmem_cache_free(mlxp->mlx_bufs_cache, b);  in mlxcx_buf_create()
2191 mlxcx_buf_create_foreign(mlxcx_t *mlxp, mlxcx_buf_shard_t *shard,  in mlxcx_buf_create_foreign()  argument
2198 	b = kmem_cache_alloc(mlxp->mlx_bufs_cache, KM_SLEEP);  in mlxcx_buf_create_foreign()
2202 	mlxcx_dma_buf_attr(mlxp, &attr);  in mlxcx_buf_create_foreign()
2204 	ret = mlxcx_dma_init(mlxp, &b->mlb_dma, &attr, B_TRUE);  in mlxcx_buf_create_foreign()
2206 		kmem_cache_free(mlxp->mlx_bufs_cache, b);  in mlxcx_buf_create_foreign()
2216 mlxcx_buf_take_foreign(mlxcx_t *mlxp, mlxcx_work_queue_t *wq)  in mlxcx_buf_take_foreign()  argument
2239 mlxcx_copy_data(mlxcx_t *mlxp, mlxcx_work_queue_t *wq, uint8_t *rptr, size_t sz)  in mlxcx_copy_data()  argument
2246 	if ((b = mlxcx_buf_take(mlxp, wq)) == NULL)  in mlxcx_copy_data()
2259 		mlxcx_buf_return(mlxp, b);  in mlxcx_copy_data()
2270 mlxcx_bind_or_copy_mblk(mlxcx_t *mlxp, mlxcx_work_queue_t *wq,  in mlxcx_bind_or_copy_mblk()  argument
2290 	if (sz < mlxp->mlx_props.mldp_tx_bind_threshold) {  in mlxcx_bind_or_copy_mblk()
2291 		b = mlxcx_copy_data(mlxp, wq, rptr, sz);  in mlxcx_bind_or_copy_mblk()
2293 		b = mlxcx_buf_take_foreign(mlxp, wq);  in mlxcx_bind_or_copy_mblk()
2297 		ret = mlxcx_dma_bind_mblk(mlxp, &b->mlb_dma, mp, off,  in mlxcx_bind_or_copy_mblk()
2301 			mlxcx_buf_return(mlxp, b);  in mlxcx_bind_or_copy_mblk()
2303 			b = mlxcx_copy_data(mlxp, wq, rptr, sz);  in mlxcx_bind_or_copy_mblk()
2311 mlxcx_buf_bind_or_copy(mlxcx_t *mlxp, mlxcx_work_queue_t *wq,  in mlxcx_buf_bind_or_copy()  argument
2323 		b = mlxcx_bind_or_copy_mblk(mlxp, wq, mp, offset);  in mlxcx_buf_bind_or_copy()
2354 		DTRACE_PROBE4(pullup, mlxcx_t *, mlxp, mlxcx_work_queue_t *, wq,  in mlxcx_buf_bind_or_copy()
2358 			mlxcx_buf_return_chain(mlxp, b0, B_TRUE);  in mlxcx_buf_bind_or_copy()
2363 		b0 = mlxcx_bind_or_copy_mblk(mlxp, wq, mp, off);  in mlxcx_buf_bind_or_copy()
2383 		mlxcx_buf_return_chain(mlxp, b0, B_TRUE);  in mlxcx_buf_bind_or_copy()
2389 mlxcx_buf_take(mlxcx_t *mlxp, mlxcx_work_queue_t *wq)  in mlxcx_buf_take()  argument
2411 mlxcx_buf_take_n(mlxcx_t *mlxp, mlxcx_work_queue_t *wq, mlxcx_buffer_t **bp,  in mlxcx_buf_take_n()  argument
2437 mlxcx_buf_loan(mlxcx_t *mlxp, mlxcx_buffer_t *b)  in mlxcx_buf_loan()  argument
2442 	ASSERT3P(b->mlb_mlx, ==, mlxp);  in mlxcx_buf_loan()
2455 	    b->mlb_used < mlxp->mlx_props.mldp_rx_p50_loan_min_size) {  in mlxcx_buf_loan()
2474 mlxcx_buf_return_chain(mlxcx_t *mlxp, mlxcx_buffer_t *b0, boolean_t keepmp)  in mlxcx_buf_return_chain()  argument
2479 		mlxcx_buf_return(mlxp, b0);  in mlxcx_buf_return_chain()
2484 		mlxcx_buf_return(mlxp, b);  in mlxcx_buf_return_chain()
2490 	mlxcx_buf_return(mlxp, b0);  in mlxcx_buf_return_chain()
2502 mlxcx_buf_return(mlxcx_t *mlxp, mlxcx_buffer_t *b)  in mlxcx_buf_return()  argument
2510 	ASSERT3P(b->mlb_mlx, ==, mlxp);  in mlxcx_buf_return()
2527 			mlxcx_dma_unbind(mlxp, &b->mlb_dma);  in mlxcx_buf_return()
2551 			mlxcx_buf_destroy(mlxp, b);  in mlxcx_buf_return()
2586 mlxcx_buf_destroy(mlxcx_t *mlxp, mlxcx_buffer_t *b)  in mlxcx_buf_destroy()  argument
2613 	kmem_cache_free(mlxp->mlx_bufs_cache, b);  in mlxcx_buf_destroy()